diff --git a/backend/src/__tests__/blueprints-git-managed-gates.test.ts b/backend/src/__tests__/blueprints-git-managed-gates.test.ts index 877ead45..b399dbb4 100644 --- a/backend/src/__tests__/blueprints-git-managed-gates.test.ts +++ b/backend/src/__tests__/blueprints-git-managed-gates.test.ts @@ -185,6 +185,82 @@ describe('Git-managed Blueprint fail-closed gates', () => { } }); + it('refuses a malformed recoveryBinding on apply-local', async () => { + const { blueprint, applicationId } = converted(); + const token = jwt.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' }); + const res = await request(app) + .post('/api/blueprints/apply-local') + .set('Authorization', `Bearer ${token}`) + .send({ + stackName: blueprint.name, + composeContent: 'services:\n fromgen:\n image: alpine:3.20\n', + markerContent: JSON.stringify({ + blueprintId: blueprint.id, + revision: blueprint.revision, + lastApplied: Date.now(), + applicationId, + }), + allowGitManagedContent: true, + captureRecovery: true, + recoveryBinding: { generationId: 123 }, + }); + expect(res.status).toBe(400); + }); + + it('refuses a non-boolean captureRecovery on apply-local', async () => { + const { blueprint, applicationId } = converted(); + const token = jwt.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' }); + const res = await request(app) + .post('/api/blueprints/apply-local') + .set('Authorization', `Bearer ${token}`) + .send({ + stackName: blueprint.name, + composeContent: 'services:\n fromgen:\n image: alpine:3.20\n', + markerContent: JSON.stringify({ + blueprintId: blueprint.id, + revision: blueprint.revision, + lastApplied: Date.now(), + applicationId, + }), + allowGitManagedContent: true, + captureRecovery: 'yes', + }); + expect(res.status).toBe(400); + }); + + it('does not capture a recovery point for a cookie session', async () => { + const { blueprint, applicationId } = converted(); + const { BlueprintService } = await import('../services/BlueprintService'); + const applySpy = vi.spyOn(BlueprintService.prototype, 'applyLocalUnderLock').mockResolvedValue({ ran: true }); + try { + const res = await request(app) + .post('/api/blueprints/apply-local') + .set('Cookie', adminCookie) + .send({ + stackName: blueprint.name, + composeContent: 'services:\n fromgen:\n image: alpine:3.20\n', + markerContent: JSON.stringify({ + blueprintId: blueprint.id, + revision: blueprint.revision, + lastApplied: Date.now(), + applicationId, + }), + allowGitManagedContent: true, + captureRecovery: true, + recoveryBinding: { + generationId: 'gen-1', + artifactSetId: null, + sourceAcceptanceRef: null, + }, + }); + expect(res.status).toBe(200); + const options = applySpy.mock.calls[0][5] as { captureRecovery?: boolean }; + expect(options.captureRecovery).toBe(false); + } finally { + applySpy.mockRestore(); + } + }); + it('honors allowGitManagedContent on apply-local for node_proxy', async () => { const { blueprint, applicationId } = converted(); const { ComposeService } = await import('../services/ComposeService'); diff --git a/backend/src/__tests__/blueprints-remote-deploy.test.ts b/backend/src/__tests__/blueprints-remote-deploy.test.ts index 7a5eed43..f1d26bb2 100644 --- a/backend/src/__tests__/blueprints-remote-deploy.test.ts +++ b/backend/src/__tests__/blueprints-remote-deploy.test.ts @@ -155,6 +155,67 @@ describe('BlueprintService remote deploy', () => { expect(dep?.applied_revision).toBe(bpObj.revision); }); + it('asks the remote to capture a recovery point when the rollout requests one', async () => { + const node = seedRemoteNode(); + const bp = seedBlueprint([node.id]); + const nodeObj = DatabaseService.getInstance().getNode(node.id)!; + const bpObj = DatabaseService.getInstance().getBlueprint(bp.id)!; + + vi.spyOn(axios, 'get').mockResolvedValue({ status: 200, data: [] }); + const postSpy = vi.spyOn(axios, 'post').mockResolvedValue({ status: 200, data: { deployed: true } }); + + const result = await BlueprintService.getInstance().deployAuthorizedMaterialization({ + blueprint: bpObj, + node: nodeObj, + composeContent: bpObj.compose_content, + marker: { + blueprintId: bpObj.id, + revision: bpObj.revision, + lastApplied: Date.now(), + applicationId: 'app-1', + bindingRevision: 'intent-1', + }, + auditPath: `/api/blueprints/${bpObj.id}/rollout/app-1`, + captureRecovery: true, + recoveryBinding: { + gitops_generation_id: 'gen-prior', + gitops_artifact_set_id: 'art-prior', + gitops_source_acceptance_ref: 'acc-prior', + }, + }); + + expect(result.status).toBe('active'); + const payload = postSpy.mock.calls[0][1] as { + captureRecovery?: boolean; + recoveryBinding?: Record; + }; + expect(payload.captureRecovery).toBe(true); + expect(payload.recoveryBinding).toEqual({ + generationId: 'gen-prior', + artifactSetId: 'art-prior', + sourceAcceptanceRef: 'acc-prior', + }); + + // A rollout that did not ask for a capture sends neither field. + postSpy.mockClear(); + await BlueprintService.getInstance().deployAuthorizedMaterialization({ + blueprint: bpObj, + node: nodeObj, + composeContent: bpObj.compose_content, + marker: { + blueprintId: bpObj.id, + revision: bpObj.revision, + lastApplied: Date.now(), + applicationId: 'app-1', + bindingRevision: 'intent-1', + }, + auditPath: `/api/blueprints/${bpObj.id}/rollout/app-1`, + }); + const plainPayload = postSpy.mock.calls[0][1] as Record; + expect(plainPayload.captureRecovery).toBeUndefined(); + expect(plainPayload.recoveryBinding).toBeUndefined(); + }); + it('fails closed when the remote lacks apply-local (404); no legacy mutations', async () => { const node = seedRemoteNode(); const bp = seedBlueprint([node.id]); diff --git a/backend/src/__tests__/blueprints.test.ts b/backend/src/__tests__/blueprints.test.ts index fff08a0d..b721b3b0 100644 --- a/backend/src/__tests__/blueprints.test.ts +++ b/backend/src/__tests__/blueprints.test.ts @@ -399,6 +399,187 @@ describe('BlueprintReconciler developer-mode diagnostics', () => { }); describe('BlueprintService per-stack lock', () => { + it('captures the recovery point before the new compose reaches disk', async () => { + const nodeId = seedNode(); + const bp = seedBlueprint({ classification: 'stateless', nodeIds: [nodeId] }); + const { FileSystemService } = await import('../services/FileSystemService'); + const { ComposeService } = await import('../services/ComposeService'); + const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService'); + + const svc = BlueprintService.getInstance(); + type PrivateProbe = { + stackDirExists: (nodeId: number, stackName: string) => Promise; + readLocalMarkerFromDisk: (nodeId: number, stackName: string) => Promise; + }; + vi.spyOn(svc as unknown as PrivateProbe, 'stackDirExists').mockResolvedValue(true); + vi.spyOn(svc as unknown as PrivateProbe, 'readLocalMarkerFromDisk').mockResolvedValue({ + kind: 'ok', + marker: { blueprintId: bp.id, revision: bp.revision, lastApplied: 1 }, + }); + vi.spyOn(FileSystemService.prototype, 'readStackFile').mockResolvedValue({ + content: 'services:\n app:\n image: nginx:previous\n', + oversized: false, + binary: false, + } as never); + const writeSpy = vi.spyOn(FileSystemService.prototype, 'writeStackFile').mockResolvedValue(undefined); + vi.spyOn(FileSystemService.prototype, 'removeAlternateRootComposeFiles').mockResolvedValue(undefined); + vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null, gitopsOperationId: null }); + + const recovery = StackUpdateRecoveryService.getInstance(); + const captureSpy = vi.spyOn(recovery, 'captureCandidate').mockResolvedValue({ id: 'rec-1' } as never); + vi.spyOn(recovery, 'markAcquired').mockReturnValue(true); + vi.spyOn(recovery, 'handoff').mockReturnValue(true); + vi.spyOn(recovery, 'markReconciling').mockReturnValue(true); + vi.spyOn(recovery, 'markImmediateVerified').mockReturnValue(true); + + const outcome = await svc.applyLocalUnderLock( + nodeId, + bp.name, + 'services:\n app:\n image: nginx:next\n', + JSON.stringify({ blueprintId: bp.id, revision: bp.revision, lastApplied: Date.now() }), + `/api/blueprints/${bp.id}/rollout/app-1`, + { + captureRecovery: true, + recoveryBinding: { + gitops_generation_id: 'gen-prior', + gitops_artifact_set_id: null, + gitops_source_acceptance_ref: null, + }, + }, + ); + + expect(outcome).toEqual({ ran: true }); + expect(captureSpy).toHaveBeenCalledWith(expect.objectContaining({ + gitopsBinding: { + gitops_generation_id: 'gen-prior', + gitops_artifact_set_id: null, + gitops_source_acceptance_ref: null, + }, + })); + // The capture must see the project as it was: before compose.yaml is + // overwritten with the generation being deployed. + const composeWrite = writeSpy.mock.calls.findIndex(([name, file]) => name === bp.name && file === 'compose.yaml'); + expect(composeWrite).toBeGreaterThanOrEqual(0); + const captureOrder = captureSpy.mock.invocationCallOrder[0]; + const composeWriteOrder = writeSpy.mock.invocationCallOrder[composeWrite]; + expect(captureOrder).toBeLessThan(composeWriteOrder); + }); + + type CaptureApply = { + apply: () => Promise; + captureSpy: ReturnType; + abandonSpy: ReturnType; + deploySpy: ReturnType; + compensateSpy: ReturnType; + }; + + /** A Blueprint apply with capture requested and every primitive mocked. */ + async function setupCaptureApply(opts: { stackExists?: boolean } = {}): Promise { + const nodeId = seedNode(); + const bp = seedBlueprint({ classification: 'stateless', nodeIds: [nodeId] }); + const { FileSystemService } = await import('../services/FileSystemService'); + const { ComposeService } = await import('../services/ComposeService'); + const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService'); + const svc = BlueprintService.getInstance(); + type PrivateProbe = { + stackDirExists: (nodeId: number, stackName: string) => Promise; + readLocalMarkerFromDisk: (nodeId: number, stackName: string) => Promise; + }; + vi.spyOn(svc as unknown as PrivateProbe, 'stackDirExists').mockResolvedValue(opts.stackExists ?? true); + vi.spyOn(svc as unknown as PrivateProbe, 'readLocalMarkerFromDisk').mockResolvedValue({ + kind: 'ok', + marker: { blueprintId: bp.id, revision: bp.revision, lastApplied: 1 }, + }); + vi.spyOn(FileSystemService.prototype, 'createStack').mockResolvedValue(undefined); + vi.spyOn(FileSystemService.prototype, 'readStackFile').mockResolvedValue({ + content: 'services:\n app:\n image: nginx:previous\n', + oversized: false, + binary: false, + } as never); + vi.spyOn(FileSystemService.prototype, 'writeStackFile').mockResolvedValue(undefined); + vi.spyOn(FileSystemService.prototype, 'removeAlternateRootComposeFiles').mockResolvedValue(undefined); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ + recoveryId: null, + deployedGenerationId: null, + gitopsOperationId: null, + }); + const recovery = StackUpdateRecoveryService.getInstance(); + const captureSpy = vi.spyOn(recovery, 'captureCandidate').mockResolvedValue({ id: 'rec-1' } as never); + const abandonSpy = vi.spyOn(recovery, 'abandon').mockResolvedValue(true); + vi.spyOn(recovery, 'markAcquired').mockReturnValue(true); + vi.spyOn(recovery, 'handoff').mockReturnValue(true); + vi.spyOn(recovery, 'markReconciling').mockReturnValue(true); + vi.spyOn(recovery, 'markImmediateVerified').mockReturnValue(true); + const compensateSpy = vi.spyOn(recovery, 'compensateWithCandidate').mockResolvedValue(true); + const apply = (): Promise => svc.applyLocalUnderLock( + nodeId, + bp.name, + 'services:\n app:\n image: nginx:next\n', + JSON.stringify({ blueprintId: bp.id, revision: bp.revision, lastApplied: Date.now() }), + `/api/blueprints/${bp.id}/rollout/app-1`, + { + captureRecovery: true, + recoveryBinding: { + gitops_generation_id: 'gen-prior', + gitops_artifact_set_id: null, + gitops_source_acceptance_ref: null, + }, + }, + ); + return { apply, captureSpy, abandonSpy, deploySpy, compensateSpy }; + } + + it('fails the apply before any mutation when the capture rejects', async () => { + const { apply, captureSpy, deploySpy } = await setupCaptureApply(); + captureSpy.mockRejectedValue(new Error('capture refused: mixed replica images')); + await expect(apply()).rejects.toThrow(/capture refused/); + expect(deploySpy).not.toHaveBeenCalled(); + }); + + it('abandons the recovery row when it cannot be acquired or handed off', async () => { + const { apply, abandonSpy } = await setupCaptureApply(); + const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService'); + vi.spyOn(StackUpdateRecoveryService.getInstance(), 'markAcquired').mockReturnValue(false); + await expect(apply()).rejects.toThrow(/Failed to acquire/); + expect(abandonSpy).toHaveBeenCalledWith('rec-1'); + }); + + it('abandons the recovery row when the handoff fails', async () => { + const { apply, abandonSpy } = await setupCaptureApply(); + const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService'); + vi.spyOn(StackUpdateRecoveryService.getInstance(), 'handoff').mockReturnValue(false); + await expect(apply()).rejects.toThrow(/Failed to hand off/); + expect(abandonSpy).toHaveBeenCalledWith('rec-1'); + }); + + it('compensates a handed-off recovery row when the deploy fails', async () => { + const { apply, deploySpy, compensateSpy } = await setupCaptureApply(); + deploySpy.mockRejectedValue(new Error('compose up failed')); + await expect(apply()).rejects.toThrow(/compose up failed/); + expect(compensateSpy).toHaveBeenCalledWith('rec-1', expect.any(Function)); + }); + + it('compensates when the recovery row cannot be marked reconciling', async () => { + const { apply, compensateSpy } = await setupCaptureApply(); + const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService'); + vi.spyOn(StackUpdateRecoveryService.getInstance(), 'markReconciling').mockReturnValue(false); + await expect(apply()).rejects.toThrow(/reconciling/); + expect(compensateSpy).toHaveBeenCalledWith('rec-1', expect.any(Function)); + }); + + it('still surfaces the original failure when compensation cannot complete', async () => { + const { apply, deploySpy, compensateSpy } = await setupCaptureApply(); + deploySpy.mockRejectedValue(new Error('compose up failed')); + compensateSpy.mockResolvedValue(false); + await expect(apply()).rejects.toThrow(/compose up failed/); + }); + + it('does not capture for a newly created stack', async () => { + const { apply, captureSpy } = await setupCaptureApply({ stackExists: false }); + await expect(apply()).resolves.toEqual({ ran: true }); + expect(captureSpy).not.toHaveBeenCalled(); + }); + it('deploy under a free lock writes compose, cleans siblings, deploys, then writes the marker', async () => { const nodeId = seedNode(); const bp = seedBlueprint({ classification: 'stateless', nodeIds: [nodeId] }); diff --git a/backend/src/__tests__/gitops-blueprint-rollout-authorization.test.ts b/backend/src/__tests__/gitops-blueprint-rollout-authorization.test.ts index 05d69009..0d3c8f29 100644 --- a/backend/src/__tests__/gitops-blueprint-rollout-authorization.test.ts +++ b/backend/src/__tests__/gitops-blueprint-rollout-authorization.test.ts @@ -288,6 +288,141 @@ describe('BlueprintTargetAdapter unlock', () => { }); }); +describe('rollout pause holds execution', () => { + function pauseEnvelope() { + return { operationId: randomUUID(), actor: 'tester', trigger: 'test', at: Date.now() }; + } + + /** + * Reconstruction walks every authorized application in the DB, and earlier + * tests leave their fixtures behind, so a count assertion needs only this + * test's application to exist. + */ + function clearGitOpsState(): void { + const db = DatabaseService.getInstance().getDb(); + for (const table of [ + 'gitops_target_current', 'gitops_history', 'gitops_approvals', 'gitops_rollout_generations', + 'gitops_rollout_candidates', 'gitops_artifact_sets', 'gitops_generations', + 'gitops_intent_revisions', 'gitops_applications', + ]) { + db.prepare(`DELETE FROM ${table}`).run(); + } + } + + it('blocks a dispatch while the application is paused', async () => { + clearGitOpsState(); + const fixture = seedAuthorizedReadyApp(); + await writeAppliedCompose(fixture.applicationId, fixture.generationId, 'services:\n web:\n image: alpine:3.20\n'); + authorize(fixture.applicationId); + GitOpsTransitions.getInstance().rolloutPaused( + fixture.applicationId, null, 'wait for the window', pauseEnvelope(), + ); + const deploySpy = vi.spyOn(BlueprintService.getInstance(), 'deployAuthorizedMaterialization') + .mockResolvedValue({ status: 'active' }); + const store = GitOpsStore.getInstance(); + const before = store.getApplication(fixture.applicationId)!; + + const gen = buildAcceptedGeneration(store.getGeneration(fixture.generationId)!); + const result = await new BlueprintTargetAdapter().dispatch(gen, { + targetMode: 'blueprint', + nodeId: fixture.nodeId, + bindingRevision: null, + }); + expect(result.status).toBe('blocked'); + if (result.status === 'blocked') expect(result.reason).toMatch(/paused/i); + expect(deploySpy).not.toHaveBeenCalled(); + // The hold also prevents minting or superseding an authorization during + // the preflight window. + const after = store.getApplication(fixture.applicationId)!; + expect(after.rollout_authorization_ref).toBe(before.rollout_authorization_ref); + expect(after.rollout_generation_id).toBe(before.rollout_generation_id); + expect(after.latest_preflight_evidence_json).toBe(before.latest_preflight_evidence_json); + }); + + it('does not resume a paused rollout on restart', async () => { + clearGitOpsState(); + const fixture = seedAuthorizedReadyApp({ nodeCount: 2 }); + await writeAppliedCompose(fixture.applicationId, fixture.generationId, 'services:\n web:\n image: alpine:3.20\n'); + authorize(fixture.applicationId); + GitOpsTransitions.getInstance().rolloutPaused( + fixture.applicationId, null, 'wait for the window', pauseEnvelope(), + ); + const deploySpy = vi.spyOn(BlueprintService.getInstance(), 'deployAuthorizedMaterialization') + .mockResolvedValue({ status: 'active' }); + const dispatchSpy = vi.spyOn(BlueprintTargetAdapter.prototype, 'dispatch'); + + const resumed = await reconstructBlueprintRolloutQueue(); + expect(resumed).toBe(0); + expect(dispatchSpy).not.toHaveBeenCalled(); + expect(deploySpy).not.toHaveBeenCalled(); + }); + + it('continues the queue but skips an individually paused target', async () => { + clearGitOpsState(); + const fixture = seedAuthorizedReadyApp({ nodeCount: 3 }); + await writeAppliedCompose(fixture.applicationId, fixture.generationId, 'services:\n web:\n image: alpine:3.20\n'); + authorize(fixture.applicationId); + const store = GitOpsStore.getInstance(); + const app = store.getApplication(fixture.applicationId)!; + const binding = store.currentAuthorizationBinding(app)!; + const [firstNode, pausedNode, lastNode] = fixture.nodeIds; + + store.upsertTarget({ + ...emptyTarget(fixture.applicationId, firstNode!), + intent_revision_id: binding.intentRevisionId, + rollout_candidate_id: binding.rolloutCandidateId, + applied_generation_id: binding.acceptedGenerationId, + desired_generation_id: binding.acceptedGenerationId, + rollout_authorization_ref: app.rollout_authorization_ref, + latest_stage: 'blueprint_ack_recorded', + }); + store.upsertTarget(emptyTarget(fixture.applicationId, pausedNode!)); + GitOpsTransitions.getInstance().rolloutPaused( + fixture.applicationId, pausedNode!, 'node maintenance', pauseEnvelope(), + ); + store.upsertTarget(emptyTarget(fixture.applicationId, lastNode!)); + + const deploySpy = vi.spyOn(BlueprintService.getInstance(), 'deployAuthorizedMaterialization') + .mockResolvedValue({ status: 'active' }); + const resumed = await reconstructBlueprintRolloutQueue(); + expect(resumed).toBe(1); + expect(deploySpy).toHaveBeenCalledTimes(1); + expect(deploySpy.mock.calls[0]![0].node.id).toBe(lastNode); + expect(store.getTarget(fixture.applicationId, pausedNode!)?.pause_at).not.toBeNull(); + }); + + it('reports a hold instead of progress when every remaining target is paused', async () => { + clearGitOpsState(); + const fixture = seedAuthorizedReadyApp({ nodeCount: 2 }); + await writeAppliedCompose(fixture.applicationId, fixture.generationId, 'services:\n web:\n image: alpine:3.20\n'); + authorize(fixture.applicationId); + const store = GitOpsStore.getInstance(); + const app = store.getApplication(fixture.applicationId)!; + const binding = store.currentAuthorizationBinding(app)!; + const [firstNode, pausedNode] = fixture.nodeIds; + + store.upsertTarget({ + ...emptyTarget(fixture.applicationId, firstNode!), + intent_revision_id: binding.intentRevisionId, + rollout_candidate_id: binding.rolloutCandidateId, + applied_generation_id: binding.acceptedGenerationId, + desired_generation_id: binding.acceptedGenerationId, + rollout_authorization_ref: app.rollout_authorization_ref, + latest_stage: 'blueprint_ack_recorded', + }); + store.upsertTarget(emptyTarget(fixture.applicationId, pausedNode!)); + GitOpsTransitions.getInstance().rolloutPaused( + fixture.applicationId, pausedNode!, 'node maintenance', pauseEnvelope(), + ); + + const deploySpy = vi.spyOn(BlueprintService.getInstance(), 'deployAuthorizedMaterialization') + .mockResolvedValue({ status: 'active' }); + const resumed = await reconstructBlueprintRolloutQueue(); + expect(resumed).toBe(0); + expect(deploySpy).not.toHaveBeenCalled(); + }); +}); + describe('derive facets for authorization and convergence', () => { it('flips produced placement and rollout statuses to current', () => { expect(FACET_EVIDENCE_SOURCE.placement.rollout_authorization_pending).toBe('current'); diff --git a/backend/src/__tests__/gitops-deferred.test.ts b/backend/src/__tests__/gitops-deferred.test.ts index c0008110..ff5c0b08 100644 --- a/backend/src/__tests__/gitops-deferred.test.ts +++ b/backend/src/__tests__/gitops-deferred.test.ts @@ -217,6 +217,7 @@ describe('gitops deferred state', () => { applicationId: 'app-rb-done', nodeId: 1, recoveryRef: 'rb-3', + recoveryGenerationId: generationId, capturedArtifactSetId: null, capturedSourceAcceptanceRef: null, envelope: env('op-rb-done-early'), @@ -233,6 +234,7 @@ describe('gitops deferred state', () => { applicationId: 'app-rb-done', nodeId: 1, recoveryRef: 'rb-3', + recoveryGenerationId: generationId, capturedArtifactSetId: null, capturedSourceAcceptanceRef: null, envelope: env('op-rb-done'), @@ -263,6 +265,7 @@ describe('gitops deferred state', () => { applicationId: 'app-rb-foreign', nodeId: 1, recoveryRef: 'rb-4', + recoveryGenerationId: 'gen-app-rb-owner', capturedArtifactSetId: null, capturedSourceAcceptanceRef: null, envelope: env('op-rb-foreign'), diff --git a/backend/src/__tests__/gitops-recovery-capture.test.ts b/backend/src/__tests__/gitops-recovery-capture.test.ts index c339b3ba..5e967e21 100644 --- a/backend/src/__tests__/gitops-recovery-capture.test.ts +++ b/backend/src/__tests__/gitops-recovery-capture.test.ts @@ -1,9 +1,10 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { DatabaseService } from '../services/DatabaseService'; import { captureGitOpsRecoveryBinding } from '../services/gitops/recoveryCapture'; -import { GitOpsStore } from '../services/gitops/store'; +import { emptyTargetRow, GitOpsStore } from '../services/gitops/store'; import { GitOpsTransitions, type EventEnvelope } from '../services/gitops/transitions'; -import type { GitOpsApplicationRow, GitOpsGenerationRow } from '../services/gitops/types'; +import type { GitOpsApplicationRow, GitOpsGenerationRow, GitOpsIntentRevisionRow } from '../services/gitops/types'; describe('gitops recovery capture', () => { let tmpDir: string; @@ -51,6 +52,23 @@ describe('gitops recovery capture', () => { }); }); + it('reads the deployed pointer, not the applied one, for a Direct stack', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-mode', 'mode-web'), nodeId: 1, envelope: env('op-act-mode') }); + store.insertGeneration(gen('gen-deployed', 'app-mode')); + store.insertGeneration(gen('gen-applied', 'app-mode')); + const target = store.getTarget('app-mode', 1)!; + store.upsertTarget({ + ...target, + desired_generation_id: 'gen-applied', + applied_generation_id: 'gen-applied', + deployed_generation_id: 'gen-deployed', + }); + + expect(captureGitOpsRecoveryBinding('mode-web', 1).gitops_generation_id).toBe('gen-deployed'); + }); + it('does not capture a newer generation acceptance for an older deployed generation', () => { const store = GitOpsStore.getInstance(); const tx = GitOpsTransitions.getInstance(); @@ -100,8 +118,123 @@ describe('gitops recovery capture', () => { expect(captured.gitops_artifact_set_id).toBe('art-old'); expect(captured.gitops_source_acceptance_ref).toBe('acc-old'); }); + + it('captures the deployed generation for a Blueprint-managed stack', () => { + const store = GitOpsStore.getInstance(); + const blueprintId = 9001; + const row: GitOpsApplicationRow = { + ...app('app-bp-cap', 'unused'), + target_mode: 'blueprint', + lifecycle_key: `blueprint:${blueprintId}`, + stack_name: null, + configured_source_stack_name: null, + blueprint_id: blueprintId, + intent_revision_id: 'intent-bp-cap', + }; + store.insertApplication(row); + store.insertIntentRevision(intent('intent-bp-cap', 'app-bp-cap', blueprintId, 'bp-cap-stack')); + store.insertGeneration(gen('gen-bp', 'app-bp-cap')); + store.insertArtifactSet({ + id: 'art-bp', + generation_id: 'gen-bp', + evidence_version: 1, + authoritative: 0, + qualification: 'exact', + evidence_json: JSON.stringify({ kind: 'exact', identity: 'sha256:deadbeef' }), + created_at: 1, + }); + store.insertApproval({ + id: 'acc-bp', + kind: 'source_acceptance', + authority: 'operator', + authoritative: 1, + application_id: 'app-bp-cap', + generation_id: 'gen-bp', + intent_revision_id: null, + artifact_set_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + required_targets_json: null, + preflight_fingerprint: null, + fingerprint: null, + blast_json: null, + policy_provenance_json: null, + actor: 'tester', + created_at: 1, + }); + store.upsertTarget({ + ...emptyTargetRow('app-bp-cap', 2, 1), + desired_generation_id: 'gen-bp', + // Blueprint mode has no deploy-bound writer, so the ack's applied + // pointer is the identity a recovery point binds. + applied_generation_id: 'gen-bp', + expected_artifact_set_id: 'art-bp', + }); + + expect(captureGitOpsRecoveryBinding('bp-cap-stack', 2)).toEqual({ + gitops_generation_id: 'gen-bp', + gitops_artifact_set_id: 'art-bp', + gitops_source_acceptance_ref: 'acc-bp', + }); + }); + + it('resolves a Blueprint application only through its current deploy stack', () => { + const store = GitOpsStore.getInstance(); + const blueprintId = 9002; + const applicationId = 'app-bp-retarget'; + const row: GitOpsApplicationRow = { + ...app(applicationId, 'unused-retarget'), + target_mode: 'blueprint', + lifecycle_key: `blueprint:${blueprintId}`, + stack_name: null, + configured_source_stack_name: null, + blueprint_id: blueprintId, + intent_revision_id: 'intent-bp-retarget', + }; + store.insertApplication(row); + store.insertIntentRevision(intent('intent-bp-retarget', applicationId, blueprintId, 'bp-retarget-stack')); + + expect(store.getLiveBlueprintApplicationByDeployStack('bp-retarget-stack')?.id).toBe(applicationId); + + // The current intent no longer names the old stack: a superseded intent's + // stack must not resolve to an application that no longer asks for it. + store.insertIntentRevision(intent('intent-bp-retarget-2', applicationId, blueprintId, 'bp-other-stack')); + const db = DatabaseService.getInstance().getDb(); + db.prepare('UPDATE gitops_applications SET intent_revision_id = ? WHERE id = ?') + .run('intent-bp-retarget-2', applicationId); + expect(store.getLiveBlueprintApplicationByDeployStack('bp-retarget-stack')).toBeUndefined(); + expect(store.getLiveBlueprintApplicationByDeployStack('bp-other-stack')?.id).toBe(applicationId); + + // A non-live application is not the owner. + db.prepare('UPDATE gitops_applications SET lifecycle_status = ? WHERE id = ?') + .run('detached', applicationId); + expect(store.getLiveBlueprintApplicationByDeployStack('bp-other-stack')).toBeUndefined(); + }); }); +function intent(id: string, applicationId: string, blueprintId: number, deployStackName: string): GitOpsIntentRevisionRow { + return { + id, + application_id: applicationId, + blueprint_id: blueprintId, + compose_content_sha256: 'c'.repeat(64), + blueprint_revision: 1, + deploy_stack_name: deployStackName, + selector_json: '{}', + pinned_node_id: null, + cordon_implications_json: '[]', + rollout_strategy_json: '{}', + runtime_drift_policy: null, + stateful_policy_json: null, + health_failure_rollback_policy_json: null, + operation_id: `op-${id}`, + actor: 'tester', + created_at: 1, + }; +} + function env(operationId: string): EventEnvelope { return { operationId, actor: 'tester', trigger: 'manual', at: 1 }; } diff --git a/backend/src/__tests__/gitops-rollout-lifecycle-route.test.ts b/backend/src/__tests__/gitops-rollout-lifecycle-route.test.ts new file mode 100644 index 00000000..575fad90 --- /dev/null +++ b/backend/src/__tests__/gitops-rollout-lifecycle-route.test.ts @@ -0,0 +1,793 @@ +/** + * Rollout lifecycle controls: pause, resume, replan, supersede, rollback. + * + * The routes decide and record; the node restores. These tests seed an + * authorized Git-managed Blueprint rollout and assert the recorded state and + * the refusals that keep an operator from recovering against evidence that + * does not name the generation being restored. + */ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { randomUUID } from 'crypto'; +import bcrypt from 'bcrypt'; +import request from 'supertest'; +import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb'; +import { directApplicationFixture } from './helpers/gitopsFixtures'; +import { commitBlueprintCreate } from '../services/gitops/blueprintProducers'; +import { emptyTargetRow } from '../services/gitops/store'; +import { encodeGitOpsRequiredTargetsJson } from '../services/gitops/json'; +import { ensureRolloutAuthorization } from '../services/gitops/handoff'; +import { projectApplication } from '../services/gitops/derive'; +import type { + GitOpsApplicationRow, + GitOpsArtifactSetRow, + GitOpsGenerationRow, + GitOpsIntentRevisionRow, + GitOpsRolloutCandidateRow, +} from '../services/gitops/types'; + +let tmpDir: string; +let app: import('express').Express; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; +let LicenseService: typeof import('../services/LicenseService').LicenseService; +let GitOpsStore: typeof import('../services/gitops/store').GitOpsStore; +let StackUpdateRecoveryService: typeof import('../services/StackUpdateRecoveryService').StackUpdateRecoveryService; +let setRegistryReadinessDepsForTests: typeof import('../services/gitops/handoff').setRegistryReadinessDepsForTests; +let adminCookie: string; +let viewerCookie: string; +let deployerCookie: string; + +function registryReadyTestDeps() { + return { + probeRemoteCapability: vi.fn(async () => ({ kind: 'supported' as const })), + probeManifestAnonymous: vi.fn(async () => ({ classification: 'public' as const, status: 200 })), + resolveHubDockerConfigForHost: vi.fn(async () => ({ state: 'missing' as const })), + discoverOnTarget: vi.fn(async () => ({ + contractVersion: 1 as const, + referencedHosts: [] as string[], + referencedPullRefs: [] as string[], + coveredHosts: [] as string[], + sourceHash: 's', + actionSetHash: 'a', + deliverySourceId: 'd', + attestation: 'tok', + })), + isControlNode: () => true, + nowMs: () => 1_000_000, + }; +} + +function insertGeneration(id: string, applicationId: string): GitOpsGenerationRow { + const row: GitOpsGenerationRow = { + id, + application_id: applicationId, + commit_sha: 'a'.repeat(40), + repo_url: 'https://github.com/example/repo.git', + configured_ref: 'main', + resolved_ref_kind: 'branch', + repo_identity_json: '{"host":"github.com","pathname":"/example/repo.git"}', + manifest_version: 1, + candidate_dir: `generations/candidate-${id}`, + applied_dir: `generations/applied-${id}-0`, + expected_invocation_json: '{}', + materialization_fingerprint: 'b'.repeat(64), + validation_ok: 1, + plan_blocked: 0, + change_plan_fingerprint: null, + operation_id: `op-${id}`, + trigger: 'manual', + actor: 'tester', + previous_generation_id: null, + redacted_limitations_json: '[]', + portable_manifest_json: '{"files":[{"path":"compose.yaml","role":"primary"}]}', + compose_inputs_json: '{"composeFileOrder":["compose.yaml"]}', + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, + secret_capability_json: null, + created_at: 1, + }; + GitOpsStore.getInstance().insertGeneration(row); + return row; +} + +function intentRow(id: string, applicationId: string, blueprintId: number, stackName: string): GitOpsIntentRevisionRow { + return { + id, + application_id: applicationId, + blueprint_id: blueprintId, + compose_content_sha256: 'c'.repeat(64), + blueprint_revision: 1, + deploy_stack_name: stackName, + selector_json: '{}', + pinned_node_id: null, + cordon_implications_json: '[]', + rollout_strategy_json: '{}', + runtime_drift_policy: null, + stateful_policy_json: null, + health_failure_rollback_policy_json: null, + operation_id: 'op-intent', + actor: 'tester', + created_at: 1, + }; +} + +function candidateRow( + id: string, + applicationId: string, + intentRevisionId: string, + nodeIds: number[], +): GitOpsRolloutCandidateRow { + return { + id, + application_id: applicationId, + intent_revision_id: intentRevisionId, + compose_content_sha256: 'c'.repeat(64), + accepted_generation_id: null, + artifact_set_id: null, + required_targets_json: encodeGitOpsRequiredTargetsJson(nodeIds), + authoritative: 1, + provenance: 'intent_change', + operation_id: 'op-cand', + created_at: 1, + }; +} + +function artifactRow(id: string, generationId: string): GitOpsArtifactSetRow { + return { + id, + generation_id: generationId, + evidence_version: 1, + authoritative: 0, + qualification: 'exact', + evidence_json: JSON.stringify({ kind: 'exact', identity: 'sha256:deadbeef' }), + created_at: 1, + }; +} + +function insertNode(name: string): number { + const result = DatabaseService.getInstance().getDb().prepare( + `INSERT INTO nodes (name, type, mode, compose_dir, is_default, status, created_at) + VALUES (?, 'local', 'proxy', '/tmp/compose', 0, 'online', ?)`, + ).run(name, Date.now()); + return result.lastInsertRowid as number; +} + +type Seeded = { + blueprintId: number; + applicationId: string; + intentId: string; + candidateId: string; + generationId: string; + stackName: string; + nodeIds: number[]; +}; + +function seedGitManagedBlueprint(opts: { nodeCount?: number } = {}): Seeded { + const store = GitOpsStore.getInstance(); + const nodeIds: number[] = []; + for (let i = 0; i < (opts.nodeCount ?? 1); i += 1) { + nodeIds.push(insertNode(`lifecycle-node-${randomUUID().slice(0, 8)}`)); + } + const applicationId = `app-${randomUUID().slice(0, 8)}`; + const intentId = `intent-${randomUUID().slice(0, 8)}`; + const candidateId = `cand-${randomUUID().slice(0, 8)}`; + const generationId = `gen-${randomUUID().slice(0, 8)}`; + const stackName = `bp-${applicationId}`; + const blueprint = DatabaseService.getInstance().createBlueprint({ + name: `bp-lifecycle-${randomUUID().slice(0, 8)}`, + description: null, + compose_content: 'services:\n app:\n image: nginx:1.27\n', + selector: { type: 'nodes', ids: nodeIds }, + drift_mode: 'suggest', + classification: 'stateless', + classification_reasons: [], + enabled: true, + created_by: 'tester', + }); + DatabaseService.getInstance().updateBlueprintContentOrigin(blueprint.id, 'git', applicationId); + + const artifactId = `art-${randomUUID().slice(0, 8)}`; + const acceptanceId = `acc-${randomUUID().slice(0, 8)}`; + const row: GitOpsApplicationRow = { + ...directApplicationFixture(applicationId, `src-${applicationId}`), + target_mode: 'blueprint', + lifecycle_key: `blueprint:${blueprint.id}`, + stack_name: null, + configured_source_stack_name: `src-${applicationId}`, + blueprint_id: blueprint.id, + intent_revision_id: intentId, + rollout_candidate_id: candidateId, + candidate_generation_id: null, + accepted_generation_id: generationId, + artifact_set_id: artifactId, + latest_artifact_set_id: artifactId, + source_acceptance_ref: acceptanceId, + review_required: 0, + source_policy: 'manual', + }; + store.insertApplication(row); + store.insertIntentRevision(intentRow(intentId, applicationId, blueprint.id, stackName)); + store.insertRolloutCandidate(candidateRow(candidateId, applicationId, intentId, nodeIds)); + insertGeneration(generationId, applicationId); + store.insertArtifactSet(artifactRow(artifactId, generationId)); + for (const nodeId of nodeIds) { + store.upsertTarget(emptyTargetRow(applicationId, nodeId, 1)); + } + store.insertApproval({ + id: acceptanceId, + kind: 'source_acceptance', + authority: 'operator', + authoritative: 1, + application_id: applicationId, + generation_id: generationId, + intent_revision_id: null, + artifact_set_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + required_targets_json: null, + preflight_fingerprint: null, + fingerprint: null, + blast_json: null, + policy_provenance_json: null, + actor: 'tester', + created_at: 1, + }); + return { blueprintId: blueprint.id, applicationId, intentId, candidateId, generationId, stackName, nodeIds }; +} + +/** Record placement approval and mint a live operator rollout authorization. */ +async function authorizeRollout(seeded: Seeded): Promise { + const transitions = (await import('../services/gitops/transitions')).GitOpsTransitions.getInstance(); + const { encodeGitOpsApprovedTargetEffectJson } = await import('../services/gitops/json'); + const envelope = { operationId: randomUUID(), actor: 'tester', trigger: 'test', at: Date.now() }; + transitions.placementApproved({ + applicationId: seeded.applicationId, + approvalId: `place-${randomUUID().slice(0, 8)}`, + intentRevisionId: seeded.intentId, + blastJson: encodeGitOpsApprovedTargetEffectJson( + seeded.nodeIds.map(nodeId => ({ nodeId, outcome: 'place' as const })), + ), + requiredNodeIds: seeded.nodeIds, + fingerprint: null, + actor: 'tester', + envelope, + rolloutGenerationId: `rgen-${randomUUID().slice(0, 8)}`, + candidateId: seeded.candidateId, + provenance: 'placement_approval', + }); + const result = await ensureRolloutAuthorization(seeded.applicationId, 'tester', 'test', undefined, 'operator'); + expect(result.ok).toBe(true); +} + +/** A current recovery point bound to one application generation. */ +function insertRecoveryPoint(seeded: Seeded, nodeId: number, gitopsGenerationId: string): void { + DatabaseService.getInstance().insertStackUpdateRecoveryGeneration({ + id: `rec-${randomUUID().slice(0, 8)}`, + node_id: nodeId, + stack_name: seeded.stackName, + status: 'active', + phase: 'handoff_committed', + is_current: 1, + backup_slot_id: null, + content_path: null, + operation_kind: 'manual_backup', + override_path: '/tmp/rollback-override.yaml', + services_json: '[]', + health_gate_id: null, + gate_retain_until: null, + artifact_expires_at: null, + operation_lease_expires_at: null, + created_at: 1, + updated_at: 1, + created_by: 'tester', + artifacts_retired: 0, + released_at: null, + released_by: null, + gitops_generation_id: gitopsGenerationId, + }); +} + +async function seedAndLoginRole(username: string, password: string, role: 'viewer' | 'deployer'): Promise { + const passwordHash = await bcrypt.hash(password, 1); + DatabaseService.getInstance().addUser({ username, password_hash: passwordHash, role }); + const res = await request(app).post('/api/auth/login').send({ username, password }); + const cookies = res.headers['set-cookie'] as string | string[]; + return Array.isArray(cookies) ? cookies[0] : cookies; +} + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ DatabaseService } = await import('../services/DatabaseService')); + ({ LicenseService } = await import('../services/LicenseService')); + ({ GitOpsStore } = await import('../services/gitops/store')); + ({ StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService')); + ({ setRegistryReadinessDepsForTests } = await import('../services/gitops/handoff')); + ({ app } = await import('../index')); + adminCookie = await loginAsTestAdmin(app); + viewerCookie = await seedAndLoginRole('lifecycle-viewer', 'lifecycle-viewer-pass', 'viewer'); + deployerCookie = await seedAndLoginRole('lifecycle-deployer', 'lifecycle-deployer-pass', 'deployer'); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); +}); + +afterAll(() => cleanupTestDb(tmpDir)); + +beforeEach(() => { + vi.restoreAllMocks(); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); + GitOpsStore.resetForTests(); + setRegistryReadinessDepsForTests(registryReadyTestDeps()); +}); + +afterEach(() => { + setRegistryReadinessDepsForTests(null); +}); + +describe('POST /api/gitops/applications/:id/rollout/pause', () => { + it('pauses the rollout with the operator reason', async () => { + const seeded = seedGitManagedBlueprint(); + await authorizeRollout(seeded); + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/pause`) + .set('Cookie', adminCookie) + .send({ reason: 'waiting for the maintenance window' }); + expect(res.status).toBe(200); + const row = GitOpsStore.getInstance().getApplication(seeded.applicationId)!; + expect(row.pause_reason).toBe('waiting for the maintenance window'); + expect(projectApplication(seeded.applicationId, false).facets?.rollout.status).toBe('rollout_paused'); + }); + + it('requires a reason', async () => { + const seeded = seedGitManagedBlueprint(); + await authorizeRollout(seeded); + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/pause`) + .set('Cookie', adminCookie) + .send({}); + expect(res.status).toBe(400); + expect(res.body.code).toBe('CONFIRM_REQUIRED'); + }); + + it('refuses to pause when there is no rollout to hold', async () => { + const seeded = seedGitManagedBlueprint(); + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/pause`) + .set('Cookie', adminCookie) + .send({ reason: 'wait for the window' }); + expect(res.status).toBe(409); + expect(res.body.code).toBe('ROLLOUT_PAUSE_REFUSED'); + expect(GitOpsStore.getInstance().getApplication(seeded.applicationId)!.pause_at).toBeNull(); + }); + + it('rejects a caller without stack:deploy', async () => { + const seeded = seedGitManagedBlueprint(); + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/pause`) + .set('Cookie', viewerCookie) + .send({ reason: 'nope' }); + expect(res.status).toBe(403); + }); +}); + +describe('POST /api/gitops/applications/:id/rollout/resume', () => { + it('resumes a paused rollout and continues the authorized queue', async () => { + const seeded = seedGitManagedBlueprint(); + await authorizeRollout(seeded); + const transitions = (await import('../services/gitops/transitions')).GitOpsTransitions.getInstance(); + transitions.rolloutPaused(seeded.applicationId, null, 'hold', { + operationId: randomUUID(), actor: 'tester', trigger: 'test', at: Date.now(), + }); + const dispatchSpy = vi + .spyOn((await import('../services/GitSourceService')).GitSourceService.getInstance(), 'dispatchAcceptedGeneration') + .mockResolvedValue({ status: 'dispatched' }); + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/resume`) + .set('Cookie', adminCookie) + .send({}); + expect(res.status).toBe(200); + expect(res.body).toMatchObject({ ok: true, dispatched: true }); + expect(dispatchSpy).toHaveBeenCalledTimes(1); + expect(GitOpsStore.getInstance().getApplication(seeded.applicationId)!.pause_at).toBeNull(); + }); + + it('reports that nothing started when no authorization is live', async () => { + const seeded = seedGitManagedBlueprint(); + const transitions = (await import('../services/gitops/transitions')).GitOpsTransitions.getInstance(); + transitions.rolloutPaused(seeded.applicationId, null, 'hold', { + operationId: randomUUID(), actor: 'tester', trigger: 'test', at: Date.now(), + }); + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/resume`) + .set('Cookie', adminCookie) + .send({}); + expect(res.status).toBe(200); + expect(res.body.dispatched).toBe(false); + expect(res.body.note).toContain('no live authorization'); + expect(GitOpsStore.getInstance().getApplication(seeded.applicationId)!.pause_at).toBeNull(); + }); + + it('refuses to resume a rollout that is not paused', async () => { + const seeded = seedGitManagedBlueprint(); + await authorizeRollout(seeded); + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/resume`) + .set('Cookie', adminCookie) + .send({}); + expect(res.status).toBe(409); + expect(res.body.code).toBe('ROLLOUT_RESUME_REFUSED'); + }); + + it('pauses and resumes a single target without touching the application pause', async () => { + const seeded = seedGitManagedBlueprint(); + await authorizeRollout(seeded); + const store = GitOpsStore.getInstance(); + const pause = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/pause`) + .set('Cookie', adminCookie) + .send({ reason: 'node maintenance', nodeId: seeded.nodeIds[0] }); + expect(pause.status).toBe(200); + expect(store.getTarget(seeded.applicationId, seeded.nodeIds[0])!.pause_at).not.toBeNull(); + expect(store.getApplication(seeded.applicationId)!.pause_at).toBeNull(); + + // A target resume clears that target's hold and continues the queue, so + // the still-authorized rollout is dispatched again. + const dispatchSpy = vi + .spyOn((await import('../services/GitSourceService')).GitSourceService.getInstance(), 'dispatchAcceptedGeneration') + .mockResolvedValue({ status: 'dispatched' }); + const resume = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/resume`) + .set('Cookie', adminCookie) + .send({ nodeId: seeded.nodeIds[0] }); + expect(resume.status).toBe(200); + expect(resume.body).toMatchObject({ ok: true, dispatched: true, note: null }); + expect(dispatchSpy).toHaveBeenCalledTimes(1); + expect(store.getTarget(seeded.applicationId, seeded.nodeIds[0])!.pause_at).toBeNull(); + }); + + it('refuses an unusable nodeId', async () => { + const seeded = seedGitManagedBlueprint(); + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/pause`) + .set('Cookie', adminCookie) + .send({ reason: 'node maintenance', nodeId: 'one' }); + expect(res.status).toBe(400); + }); +}); + +describe('POST /api/gitops/applications/:id/rollout/supersede', () => { + it('withdraws the live authorization and reports the abandoned rollout', async () => { + const seeded = seedGitManagedBlueprint(); + await authorizeRollout(seeded); + const store = GitOpsStore.getInstance(); + const generationId = store.getApplication(seeded.applicationId)!.rollout_generation_id!; + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/supersede`) + .set('Cookie', adminCookie) + .send({}); + expect(res.status).toBe(200); + const row = store.getApplication(seeded.applicationId)!; + expect(row.rollout_authorization_ref).toBeNull(); + expect(store.getRolloutGeneration(generationId)?.superseded_at).toBeTruthy(); + expect(projectApplication(seeded.applicationId, false).facets?.rollout.status).toBe('rollout_superseded'); + }); + + it('refuses when no authorized rollout exists', async () => { + const seeded = seedGitManagedBlueprint(); + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/supersede`) + .set('Cookie', adminCookie) + .send({}); + expect(res.status).toBe(409); + expect(res.body.code).toBe('ROLLOUT_SUPERSEDE_REFUSED'); + }); + + it('refuses a placement generation that was never authorized', async () => { + const seeded = seedGitManagedBlueprint(); + const transitions = (await import('../services/gitops/transitions')).GitOpsTransitions.getInstance(); + const { encodeGitOpsApprovedTargetEffectJson } = await import('../services/gitops/json'); + transitions.placementApproved({ + applicationId: seeded.applicationId, + approvalId: `place-${randomUUID().slice(0, 8)}`, + intentRevisionId: seeded.intentId, + blastJson: encodeGitOpsApprovedTargetEffectJson( + seeded.nodeIds.map(nodeId => ({ nodeId, outcome: 'place' as const })), + ), + requiredNodeIds: seeded.nodeIds, + fingerprint: null, + actor: 'tester', + envelope: { operationId: randomUUID(), actor: 'tester', trigger: 'test', at: Date.now() }, + rolloutGenerationId: `rgen-${randomUUID().slice(0, 8)}`, + candidateId: seeded.candidateId, + provenance: 'placement_approval', + }); + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/supersede`) + .set('Cookie', adminCookie) + .send({}); + expect(res.status).toBe(409); + expect(res.body.code).toBe('ROLLOUT_SUPERSEDE_REFUSED'); + }); +}); + +describe('POST /api/gitops/applications/:id/rollout/replan', () => { + it('mints a fresh intent and candidate when placement moved', async () => { + const seeded = seedGitManagedBlueprint(); + const store = GitOpsStore.getInstance(); + const extraNode = insertNode(`replan-node-${randomUUID().slice(0, 8)}`); + DatabaseService.getInstance().updateBlueprint(seeded.blueprintId, { + selector: { type: 'nodes', ids: [...seeded.nodeIds, extraNode] }, + }); + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/replan`) + .set('Cookie', adminCookie) + .send({}); + expect(res.status).toBe(200); + const row = store.getApplication(seeded.applicationId)!; + expect(row.intent_revision_id).not.toBe(seeded.intentId); + expect(row.rollout_candidate_id).not.toBe(seeded.candidateId); + const candidate = store.getRolloutCandidate(row.rollout_candidate_id!)!; + expect(JSON.parse(candidate.required_targets_json).nodeIds).toContain(extraNode); + }); + + it('refuses when the current placement already matches', async () => { + const seeded = seedGitManagedBlueprint(); + // The seeded intent describes another stack, so it is not the Blueprint's + // current intent and replan has a real question to ask. Re-seed that + // relationship by minting once, then replanning again with nothing moved. + const first = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/replan`) + .set('Cookie', adminCookie) + .send({}); + expect(first.status).toBe(200); + const second = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/replan`) + .set('Cookie', adminCookie) + .send({}); + expect(second.status).toBe(409); + expect(second.body.code).toBe('REPLAN_UNAVAILABLE'); + }); + + it('refuses when the application has no candidate to replan', async () => { + const seeded = seedGitManagedBlueprint(); + DatabaseService.getInstance().getDb() + .prepare('UPDATE gitops_applications SET rollout_candidate_id = NULL WHERE id = ?') + .run(seeded.applicationId); + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/replan`) + .set('Cookie', adminCookie) + .send({}); + expect(res.status).toBe(409); + expect(res.body.code).toBe('REPLAN_UNAVAILABLE'); + }); + + it('rejects a caller without stack:create', async () => { + const seeded = seedGitManagedBlueprint(); + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/replan`) + .set('Cookie', viewerCookie) + .send({}); + expect(res.status).toBe(403); + }); + + it('lets a deployer pause without stack:create but not replan', async () => { + const seeded = seedGitManagedBlueprint(); + await authorizeRollout(seeded); + const pause = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/pause`) + .set('Cookie', deployerCookie) + .send({ reason: 'waiting' }); + expect(pause.status).toBe(200); + const replan = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/replan`) + .set('Cookie', deployerCookie) + .send({}); + expect(replan.status).toBe(403); + }); +}); + +describe('POST /api/gitops/applications/:id/rollout/rollback', () => { + it('restores the selected generation from the node recovery point', async () => { + const seeded = seedGitManagedBlueprint(); + await authorizeRollout(seeded); + insertRecoveryPoint(seeded, seeded.nodeIds[0], seeded.generationId); + const compensateSpy = vi + .spyOn(StackUpdateRecoveryService.getInstance(), 'compensateWithCandidate') + .mockResolvedValue(true); + + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/rollback`) + .set('Cookie', adminCookie) + .send({ generationId: seeded.generationId, scope: { kind: 'target', nodeId: seeded.nodeIds[0] } }); + expect(res.status).toBe(200); + expect(res.body).toMatchObject({ ok: true, results: [{ nodeId: seeded.nodeIds[0], status: 'restored' }] }); + expect(compensateSpy).toHaveBeenCalledTimes(1); + + const store = GitOpsStore.getInstance(); + const target = store.getTarget(seeded.applicationId, seeded.nodeIds[0])!; + expect(target.applied_generation_id).toBe(seeded.generationId); + expect(target.recovery_phase).toBe('complete'); + expect(store.getApplication(seeded.applicationId)!.rollout_authorization_ref).toBeNull(); + expect(projectApplication(seeded.applicationId, false).facets?.rollout.status).toBe('rollout_superseded'); + // The strongest evidence for the restored generation is rebound; the + // acceptance that authorized exactly that generation survives. + expect(target.expected_artifact_set_id).toBeTruthy(); + expect(target.latest_artifact_set_id).toBe(target.expected_artifact_set_id); + expect(target.source_acceptance_ref).toBeTruthy(); + }); + + it('fails a target that has no recovery point and reports a partial rollback', async () => { + const seeded = seedGitManagedBlueprint(); + await authorizeRollout(seeded); + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/rollback`) + .set('Cookie', adminCookie) + .send({ generationId: seeded.generationId, scope: { kind: 'all_changed' } }); + expect(res.status).toBe(200); + expect(res.body.ok).toBe(false); + expect(res.body.results[0].status).toBe('failed'); + expect(projectApplication(seeded.applicationId, false).facets?.rollout.status).toBe('rollback_partial_failed'); + }); + + it('restores the matching targets and fails the rest in one scope', async () => { + const seeded = seedGitManagedBlueprint({ nodeCount: 2 }); + await authorizeRollout(seeded); + insertRecoveryPoint(seeded, seeded.nodeIds[0], seeded.generationId); + vi.spyOn(StackUpdateRecoveryService.getInstance(), 'compensateWithCandidate').mockResolvedValue(true); + + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/rollback`) + .set('Cookie', adminCookie) + .send({ generationId: seeded.generationId, scope: { kind: 'all_changed' } }); + expect(res.status).toBe(200); + expect(res.body.ok).toBe(false); + expect(res.body.results).toEqual([ + { nodeId: seeded.nodeIds[0], status: 'restored' }, + { nodeId: seeded.nodeIds[1], status: 'failed', error: 'The node holds no recovery point for "' + seeded.stackName + '".' }, + ]); + const store = GitOpsStore.getInstance(); + expect(store.getTarget(seeded.applicationId, seeded.nodeIds[0])!.recovery_phase).toBe('complete'); + expect(store.getTarget(seeded.applicationId, seeded.nodeIds[1])!.recovery_phase).toBe('failed'); + expect(projectApplication(seeded.applicationId, false).facets?.rollout.status).toBe('rollback_partial_failed'); + }); + + it('reports a partial failure when the recovery point names another generation', async () => { + const seeded = seedGitManagedBlueprint(); + await authorizeRollout(seeded); + insertRecoveryPoint(seeded, seeded.nodeIds[0], 'gen-someone-else'); + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/rollback`) + .set('Cookie', adminCookie) + .send({ generationId: seeded.generationId, scope: { kind: 'target', nodeId: seeded.nodeIds[0] } }); + expect(res.status).toBe(200); + expect(res.body.ok).toBe(false); + expect(res.body.results[0]).toMatchObject({ nodeId: seeded.nodeIds[0], status: 'failed' }); + const projection = projectApplication(seeded.applicationId, false); + expect(projection.facets?.rollout.status).toBe('rollback_partial_failed'); + }); + + it('reports a target whose restore completed but could not be recorded', async () => { + const seeded = seedGitManagedBlueprint(); + await authorizeRollout(seeded); + insertRecoveryPoint(seeded, seeded.nodeIds[0], seeded.generationId); + vi.spyOn(StackUpdateRecoveryService.getInstance(), 'compensateWithCandidate').mockResolvedValue(true); + const { GitOpsTransitions } = await import('../services/gitops/transitions'); + vi.spyOn(GitOpsTransitions.getInstance(), 'rollbackCompleted').mockImplementation(() => { + throw new Error('db write failed'); + }); + + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/rollback`) + .set('Cookie', adminCookie) + .send({ generationId: seeded.generationId, scope: { kind: 'all_changed' } }); + expect(res.status).toBe(200); + expect(res.body.ok).toBe(false); + expect(res.body.results[0]).toEqual({ + nodeId: seeded.nodeIds[0], + status: 'failed', + error: 'The restore completed but its result could not be recorded.', + }); + expect(projectApplication(seeded.applicationId, false).facets?.rollout.status).toBe('rollback_partial_failed'); + }); + + it('refuses a failed-target scope when nothing failed', async () => { + const seeded = seedGitManagedBlueprint(); + await authorizeRollout(seeded); + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/rollback`) + .set('Cookie', adminCookie) + .send({ generationId: seeded.generationId, scope: { kind: 'failed' } }); + expect(res.status).toBe(409); + expect(res.body.code).toBe('NO_FAILED_TARGETS'); + }); + + it('refuses a generation another application owns', async () => { + const seeded = seedGitManagedBlueprint(); + await authorizeRollout(seeded); + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/rollback`) + .set('Cookie', adminCookie) + .send({ generationId: randomUUID(), scope: { kind: 'all_changed' } }); + expect(res.status).toBe(409); + expect(res.body.code).toBe('ROLLBACK_REFUSED'); + }); + + it('refuses a disabled Blueprint before restoring anything', async () => { + const seeded = seedGitManagedBlueprint(); + await authorizeRollout(seeded); + DatabaseService.getInstance().updateBlueprint(seeded.blueprintId, { enabled: false }); + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/rollback`) + .set('Cookie', adminCookie) + .send({ generationId: seeded.generationId, scope: { kind: 'all_changed' } }); + expect(res.status).toBe(409); + expect(res.body.code).toBe('BLUEPRINT_DISABLED'); + }); + + it('exposes the prior rollout generations on the application detail read', async () => { + const seeded = seedGitManagedBlueprint(); + await authorizeRollout(seeded); + const store = GitOpsStore.getInstance(); + const priorGenerationId = `gen-prior-${randomUUID().slice(0, 8)}`; + insertGeneration(priorGenerationId, seeded.applicationId); + DatabaseService.getInstance().getDb().prepare( + `UPDATE gitops_rollout_generations SET accepted_generation_id = ?, created_at = 1 + WHERE application_id = ?`, + ).run(priorGenerationId, seeded.applicationId); + const currentGenerationId = `gen-current-${randomUUID().slice(0, 8)}`; + insertGeneration(currentGenerationId, seeded.applicationId); + DatabaseService.getInstance().getDb().prepare( + 'UPDATE gitops_applications SET accepted_generation_id = ? WHERE id = ?', + ).run(currentGenerationId, seeded.applicationId); + + const res = await request(app) + .get(`/api/gitops/applications/bp:${seeded.blueprintId}`) + .set('Cookie', adminCookie); + expect(res.status).toBe(200); + expect(res.body.rollbackCandidates).toEqual([ + expect.objectContaining({ generationId: priorGenerationId }), + ]); + expect(store.getApplication(seeded.applicationId)).toBeTruthy(); + }); + + it('rejects a caller without stack:deploy', async () => { + const seeded = seedGitManagedBlueprint(); + await authorizeRollout(seeded); + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/rollback`) + .set('Cookie', viewerCookie) + .send({ generationId: seeded.generationId, scope: { kind: 'all_changed' } }); + expect(res.status).toBe(403); + }); + + it('rejects an unparseable scope', async () => { + const seeded = seedGitManagedBlueprint(); + const res = await request(app) + .post(`/api/gitops/applications/bp:${seeded.blueprintId}/rollout/rollback`) + .set('Cookie', adminCookie) + .send({ generationId: seeded.generationId, scope: { kind: 'everything' } }); + expect(res.status).toBe(400); + expect(res.body.code).toBe('CONFIRM_REQUIRED'); + }); + + it('refuses an Inline Blueprint application', async () => { + const blueprint = commitBlueprintCreate({ + name: `bp-inline-lifecycle-${randomUUID().slice(0, 8)}`, + description: null, + compose_content: 'services:\n app:\n image: nginx\n', + selector: { type: 'nodes', ids: [1] }, + drift_mode: 'suggest', + classification: 'stateless', + classification_reasons: [], + enabled: true, + created_by: 'tester', + }, () => [1]); + const res = await request(app) + .post(`/api/gitops/applications/bp:${blueprint.id}/rollout/rollback`) + .set('Cookie', adminCookie) + .send({ generationId: randomUUID(), scope: { kind: 'all_changed' } }); + expect(res.status).toBe(409); + expect(res.body.code).toBe('NOT_GIT_MANAGED'); + }); +}); diff --git a/backend/src/__tests__/gitops-rollout-recovery.test.ts b/backend/src/__tests__/gitops-rollout-recovery.test.ts new file mode 100644 index 00000000..824e3095 --- /dev/null +++ b/backend/src/__tests__/gitops-rollout-recovery.test.ts @@ -0,0 +1,402 @@ +/** + * Rollout recovery coordination: which targets a scope resolves to, and how a + * remote target's restore request is formed and interpreted. + * + * The route suite covers the recorded transitions; these tests pin the + * evidence rules underneath them. + */ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import axios from 'axios'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { directApplicationFixture } from './helpers/gitopsFixtures'; +import { emptyTargetRow, GitOpsStore } from '../services/gitops/store'; +import { encodeGitOpsRequiredTargetsJson } from '../services/gitops/json'; +import { + resolveRollbackTargets, + restoreTargetToGeneration, + rollbackCandidatesForApplication, +} from '../services/gitops/rolloutRecovery'; +import { NodeRegistry } from '../services/NodeRegistry'; +import { PROXY_SCOPED_STACK_ACTIONS_HEADER, PROXY_SCOPED_STACK_NAME_HEADER } from '../services/license-headers'; +import type { GitOpsApplicationRow, GitOpsIntentRevisionRow, GitOpsRolloutCandidateRow, GitOpsRolloutGenerationRow } from '../services/gitops/types'; + +let tmpDir: string; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; +let counter = 0; + +function insertNode(type: 'local' | 'remote', apiUrl?: string): number { + counter += 1; + return DatabaseService.getInstance().addNode({ + name: `rollout-recovery-${counter}`, + type, + mode: 'proxy', + compose_dir: '/tmp/compose', + is_default: false, + api_url: apiUrl, + api_token: apiUrl ? 'remote-tok' : undefined, + }); +} + +function intentRow(id: string, applicationId: string, blueprintId: number, stackName: string): GitOpsIntentRevisionRow { + return { + id, + application_id: applicationId, + blueprint_id: blueprintId, + compose_content_sha256: 'c'.repeat(64), + blueprint_revision: 1, + deploy_stack_name: stackName, + selector_json: '{}', + pinned_node_id: null, + cordon_implications_json: '[]', + rollout_strategy_json: '{}', + runtime_drift_policy: null, + stateful_policy_json: null, + health_failure_rollback_policy_json: null, + operation_id: 'op-intent', + actor: 'tester', + created_at: 1, + }; +} + +function seedApplication(opts: { nodeIds: number[]; stackName: string; candidateRequired?: number[] }): { + app: GitOpsApplicationRow; +} { + const store = GitOpsStore.getInstance(); + counter += 1; + const blueprintId = 9000 + counter; + const applicationId = `app-${Math.random().toString(36).slice(2, 10)}`; + const intentId = `intent-${applicationId}`; + const candidateId = `cand-${applicationId}`; + const app: GitOpsApplicationRow = { + ...directApplicationFixture(applicationId, `src-${applicationId}`), + target_mode: 'blueprint', + lifecycle_key: `blueprint:${applicationId}`, + stack_name: null, + configured_source_stack_name: null, + blueprint_id: blueprintId, + intent_revision_id: intentId, + rollout_candidate_id: candidateId, + }; + store.insertApplication(app); + store.insertIntentRevision(intentRow(intentId, applicationId, blueprintId, opts.stackName)); + const candidate: GitOpsRolloutCandidateRow = { + id: candidateId, + application_id: applicationId, + intent_revision_id: intentId, + compose_content_sha256: 'c'.repeat(64), + accepted_generation_id: null, + artifact_set_id: null, + required_targets_json: encodeGitOpsRequiredTargetsJson(opts.candidateRequired ?? opts.nodeIds), + authoritative: 1, + provenance: 'intent_change', + operation_id: 'op-cand', + created_at: 1, + }; + store.insertRolloutCandidate(candidate); + for (const nodeId of opts.nodeIds) { + store.upsertTarget(emptyTargetRow(applicationId, nodeId, 1)); + } + return { app: store.getApplication(applicationId)! }; +} + +function seedRolloutGeneration(app: GitOpsApplicationRow, nodeIds: number[]): GitOpsRolloutGenerationRow { + counter += 1; + const row: GitOpsRolloutGenerationRow = { + id: `rgen-${counter}`, + application_id: app.id, + intent_revision_id: app.intent_revision_id!, + rollout_candidate_id: app.rollout_candidate_id!, + accepted_generation_id: null, + artifact_set_id: null, + placement_approval_ref: null, + source_acceptance_ref: null, + rollout_authorization_ref: null, + required_targets_json: encodeGitOpsRequiredTargetsJson(nodeIds), + preflight_fingerprint: null, + preflight_evidence_json: null, + rollout_strategy_json: '{}', + provenance: 'placement_approval', + supersedes_generation_id: null, + superseded_at: null, + operation_id: 'op-rgen', + actor: 'tester', + trigger: 'test', + created_at: 1, + }; + GitOpsStore.getInstance().insertRolloutGeneration(row); + DatabaseService.getInstance().getDb() + .prepare('UPDATE gitops_applications SET rollout_generation_id = ? WHERE id = ?') + .run(row.id, app.id); + return row; +} + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ DatabaseService } = await import('../services/DatabaseService')); +}); + +afterAll(() => cleanupTestDb(tmpDir)); + +beforeEach(() => { + vi.restoreAllMocks(); + GitOpsStore.resetForTests(); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe('resolveRollbackTargets', () => { + it('resolves all changed targets from the last rollout generation when no authorization is live', () => { + const { app } = seedApplication({ nodeIds: [1, 2], stackName: 'bp-stack' }); + seedRolloutGeneration(app, [1, 2]); + const resolved = resolveRollbackTargets(GitOpsStore.getInstance().getApplication(app.id)!, { kind: 'all_changed' }, 'gen-restore'); + expect(resolved).toEqual({ ok: true, nodeIds: [1, 2] }); + }); + + it('returns only the failing target for the failed scope', () => { + const { app } = seedApplication({ nodeIds: [1, 2], stackName: 'bp-stack' }); + seedRolloutGeneration(app, [1, 2]); + const store = GitOpsStore.getInstance(); + const failing = store.getTarget(app.id, 2)!; + store.upsertTarget({ ...failing, failure_stage: 'blueprint_deploy' }); + + const resolved = resolveRollbackTargets(store.getApplication(app.id)!, { kind: 'failed' }, 'gen-restore'); + expect(resolved).toEqual({ ok: true, nodeIds: [2] }); + }); + + it('refuses the failed scope when nothing failed', () => { + const { app } = seedApplication({ nodeIds: [1, 2], stackName: 'bp-stack' }); + seedRolloutGeneration(app, [1, 2]); + const resolved = resolveRollbackTargets(GitOpsStore.getInstance().getApplication(app.id)!, { kind: 'failed' }, 'gen-restore'); + expect(resolved.ok).toBe(false); + if (!resolved.ok) expect(resolved.code).toBe('NO_FAILED_TARGETS'); + }); + + it('refuses a named target that is not in the frozen set', () => { + const { app } = seedApplication({ nodeIds: [1, 2], stackName: 'bp-stack' }); + seedRolloutGeneration(app, [1, 2]); + const resolved = resolveRollbackTargets(GitOpsStore.getInstance().getApplication(app.id)!, { + kind: 'target', + nodeId: 99, + }, 'gen-restore'); + expect(resolved.ok).toBe(false); + if (!resolved.ok) expect(resolved.code).toBe('TARGET_NOT_IN_ROLLOUT'); + }); + + it('refuses when the frozen set has no live targets left', () => { + const { app } = seedApplication({ nodeIds: [1], stackName: 'bp-stack' }); + seedRolloutGeneration(app, [1]); + const store = GitOpsStore.getInstance(); + const target = store.getTarget(app.id, 1)!; + store.upsertTarget({ ...target, target_status: 'tombstoned' }); + const resolved = resolveRollbackTargets(store.getApplication(app.id)!, { kind: 'all_changed' }, 'gen-restore'); + expect(resolved.ok).toBe(false); + if (!resolved.ok) expect(resolved.code).toBe('ROLLBACK_UNAVAILABLE'); + }); + + it('refuses when there is no frozen set at all', () => { + const { app } = seedApplication({ nodeIds: [1], stackName: 'bp-stack' }); + const resolved = resolveRollbackTargets(GitOpsStore.getInstance().getApplication(app.id)!, { kind: 'all_changed' }, 'gen-restore'); + expect(resolved.ok).toBe(false); + if (!resolved.ok) expect(resolved.code).toBe('ROLLBACK_UNAVAILABLE'); + }); + + it('skips targets already running the selected generation', () => { + const { app } = seedApplication({ nodeIds: [1, 2], stackName: 'bp-stack' }); + seedRolloutGeneration(app, [1, 2]); + const store = GitOpsStore.getInstance(); + const current = store.getTarget(app.id, 1)!; + store.upsertTarget({ ...current, applied_generation_id: 'gen-restore' }); + + const all = resolveRollbackTargets(store.getApplication(app.id)!, { kind: 'all_changed' }, 'gen-restore'); + expect(all).toEqual({ ok: true, nodeIds: [2] }); + + const single = resolveRollbackTargets(store.getApplication(app.id)!, { kind: 'target', nodeId: 1 }, 'gen-restore'); + expect(single.ok).toBe(false); + if (!single.ok) expect(single.code).toBe('TARGET_ALREADY_CURRENT'); + }); + + it('returns no candidates for a Direct application or one with no rollout history', () => { + const { app } = seedApplication({ nodeIds: [1], stackName: 'bp-stack' }); + expect(rollbackCandidatesForApplication(app.id)).toEqual([]); + + const directId = `direct-${app.id}`; + GitOpsStore.getInstance().insertApplication({ + ...app, + id: directId, + target_mode: 'direct', + lifecycle_key: `direct:${directId}`, + stack_name: directId, + blueprint_id: null, + configured_source_stack_name: directId, + }); + expect(rollbackCandidatesForApplication(directId)).toEqual([]); + }); + + it('lists prior generations newest first and excludes the current one', () => { + const { app } = seedApplication({ nodeIds: [1], stackName: 'bp-stack' }); + const store = GitOpsStore.getInstance(); + const db = DatabaseService.getInstance().getDb(); + db.prepare('UPDATE gitops_applications SET accepted_generation_id = ? WHERE id = ?') + .run('gen-current', app.id); + const older = seedRolloutGeneration(store.getApplication(app.id)!, [1]); + const current = seedRolloutGeneration(store.getApplication(app.id)!, [1]); + db.prepare('UPDATE gitops_rollout_generations SET accepted_generation_id = ?, created_at = 1 WHERE id = ?') + .run('gen-older', older.id); + db.prepare('UPDATE gitops_rollout_generations SET accepted_generation_id = ?, created_at = 2 WHERE id = ?') + .run('gen-current', current.id); + + expect(rollbackCandidatesForApplication(app.id)).toEqual([ + { generationId: 'gen-older', rolloutGenerationId: older.id, createdAt: 1 }, + ]); + }); +}); + +describe('restoreTargetToGeneration on a remote node', () => { + function remoteApp(nodeId: number): GitOpsApplicationRow { + const { app } = seedApplication({ nodeIds: [nodeId], stackName: 'bp-stack' }); + return GitOpsStore.getInstance().getApplication(app.id)!; + } + + it('posts the expected generation with the acting user credential', async () => { + const nodeId = insertNode('remote', 'https://remote.example.com:1852'); + const app = remoteApp(nodeId); + const postSpy = vi.spyOn(axios, 'post').mockResolvedValue({ + status: 200, + data: { message: 'rolled back', gitopsGenerationId: 'gen-restore' }, + }); + + const outcome = await restoreTargetToGeneration({ + app, + stackName: 'bp-stack', + nodeId, + generationId: 'gen-restore', + actor: 'operator', + role: 'deployer', + scopedActions: ['stack:deploy'], + }); + + expect(outcome).toEqual({ ok: true }); + const [url, body, config] = postSpy.mock.calls[0] as [string, unknown, { headers: Record }]; + expect(url).toBe('https://remote.example.com:1852/api/stacks/bp-stack/rollback'); + expect(body).toEqual({ expectedGitopsGenerationId: 'gen-restore' }); + expect(config.headers.Authorization).toBe('Bearer remote-tok'); + expect(config.headers[PROXY_SCOPED_STACK_NAME_HEADER]).toBe('bp-stack'); + expect(config.headers[PROXY_SCOPED_STACK_ACTIONS_HEADER]).toContain('stack:deploy'); + }); + + it('carries the node refusal code and message back', async () => { + const nodeId = insertNode('remote', 'https://remote.example.com:1852'); + const app = remoteApp(nodeId); + vi.spyOn(axios, 'post').mockResolvedValue({ + status: 409, + data: { error: 'The current recovery point does not restore the requested application generation.', code: 'RECOVERY_POINT_MISMATCH' }, + }); + + const outcome = await restoreTargetToGeneration({ + app, + stackName: 'bp-stack', + nodeId, + generationId: 'gen-restore', + actor: null, + role: 'admin', + scopedActions: ['stack:deploy'], + }); + + expect(outcome).toEqual({ + ok: false, + code: 'RECOVERY_POINT_MISMATCH', + error: 'The current recovery point does not restore the requested application generation.', + }); + }); + + it('fails a 2xx that does not echo the requested generation', async () => { + const nodeId = insertNode('remote', 'https://remote.example.com:1852'); + const app = remoteApp(nodeId); + const postSpy = vi.spyOn(axios, 'post').mockResolvedValue({ + status: 200, + data: { message: 'rolled back' }, + }); + + const first = await restoreTargetToGeneration({ + app, + stackName: 'bp-stack', + nodeId, + generationId: 'gen-restore', + actor: null, + role: 'admin', + scopedActions: ['stack:deploy'], + }); + expect(first.ok).toBe(false); + if (!first.ok) expect(first.code).toBe('RECOVERY_POINT_MISMATCH'); + + postSpy.mockResolvedValue({ status: 200, data: { gitopsGenerationId: 'gen-someone-else' } }); + const second = await restoreTargetToGeneration({ + app, + stackName: 'bp-stack', + nodeId, + generationId: 'gen-restore', + actor: null, + role: 'admin', + scopedActions: ['stack:deploy'], + }); + expect(second.ok).toBe(false); + if (!second.ok) expect(second.code).toBe('RECOVERY_POINT_MISMATCH'); + }); + + it('maps a bare 403 to PERMISSION_DENIED', async () => { + const nodeId = insertNode('remote', 'https://remote.example.com:1852'); + const app = remoteApp(nodeId); + vi.spyOn(axios, 'post').mockResolvedValue({ status: 403, data: {} }); + + const outcome = await restoreTargetToGeneration({ + app, + stackName: 'bp-stack', + nodeId, + generationId: 'gen-restore', + actor: null, + role: 'viewer', + scopedActions: [], + }); + + expect(outcome.ok).toBe(false); + if (!outcome.ok) expect(outcome.code).toBe('PERMISSION_DENIED'); + }); + + it('fails closed when the node cannot be reached', async () => { + const nodeId = insertNode('remote', 'https://remote.example.com:1852'); + const app = remoteApp(nodeId); + vi.spyOn(axios, 'post').mockRejectedValue(new Error('ECONNREFUSED')); + + const outcome = await restoreTargetToGeneration({ + app, + stackName: 'bp-stack', + nodeId, + generationId: 'gen-restore', + actor: null, + role: 'admin', + scopedActions: ['stack:deploy'], + }); + + expect(outcome).toEqual({ ok: false, code: 'NODE_UNREACHABLE', error: 'The restore request to the owning node failed.' }); + }); + + it('fails closed when the node has no proxy target', async () => { + const nodeId = insertNode('remote'); + const app = remoteApp(nodeId); + vi.spyOn(NodeRegistry.getInstance(), 'getProxyTarget').mockReturnValue(null); + + const outcome = await restoreTargetToGeneration({ + app, + stackName: 'bp-stack', + nodeId, + generationId: 'gen-restore', + actor: null, + role: 'admin', + scopedActions: ['stack:deploy'], + }); + + expect(outcome).toEqual({ ok: false, code: 'NODE_UNREACHABLE', error: 'The owning node is unreachable.' }); + }); +}); diff --git a/backend/src/__tests__/stacks-failure-notifications.test.ts b/backend/src/__tests__/stacks-failure-notifications.test.ts index 7412fd87..81b5cb4a 100644 --- a/backend/src/__tests__/stacks-failure-notifications.test.ts +++ b/backend/src/__tests__/stacks-failure-notifications.test.ts @@ -544,7 +544,7 @@ describe('deploy_failure notification on /update error', () => { }); describe('generation rollback error mapping', () => { - function stubCurrentGeneration(id: string): StackUpdateRecoveryGenerationRow { + function stubCurrentGeneration(id: string, gitopsGenerationId?: string): StackUpdateRecoveryGenerationRow { return { id, node_id: 1, @@ -567,6 +567,7 @@ describe('generation rollback error mapping', () => { artifacts_retired: 0, released_at: null, released_by: null, + gitops_generation_id: gitopsGenerationId, }; } @@ -611,6 +612,62 @@ describe('generation rollback error mapping', () => { expect(res.body).toMatchObject({ error: 'Rollback restore did not complete.' }); expect(res.body.code).toBeUndefined(); }); + + it('restores the requested generation when the recovery point names it', async () => { + const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService'); + const svc = StackUpdateRecoveryService.getInstance(); + const getSpy = vi.spyOn(svc, 'getCurrent').mockReturnValue(stubCurrentGeneration('gen-1', 'gen-app-1')); + const compensateSpy = vi.spyOn(svc, 'compensateWithCandidate').mockResolvedValue(true); + try { + const res = await request(app) + .post('/api/stacks/myapp/rollback') + .set('Cookie', authCookie) + .send({ expectedGitopsGenerationId: 'gen-app-1' }); + expect(res.status).toBe(200); + expect(compensateSpy).toHaveBeenCalledTimes(1); + } finally { + getSpy.mockRestore(); + compensateSpy.mockRestore(); + } + }); + + it('refuses when the stack has no recovery point at all', async () => { + const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService'); + const svc = StackUpdateRecoveryService.getInstance(); + const getSpy = vi.spyOn(svc, 'getCurrent').mockReturnValue(undefined); + const compensateSpy = vi.spyOn(svc, 'compensateWithCandidate').mockResolvedValue(true); + try { + const res = await request(app) + .post('/api/stacks/myapp/rollback') + .set('Cookie', authCookie) + .send({ expectedGitopsGenerationId: 'gen-app-1' }); + expect(res.status).toBe(409); + expect(res.body.code).toBe('NO_RECOVERY_POINT'); + expect(compensateSpy).not.toHaveBeenCalled(); + } finally { + getSpy.mockRestore(); + compensateSpy.mockRestore(); + } + }); + + it('refuses when the recovery point names a different generation', async () => { + const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService'); + const svc = StackUpdateRecoveryService.getInstance(); + const getSpy = vi.spyOn(svc, 'getCurrent').mockReturnValue(stubCurrentGeneration('gen-1', 'gen-app-1')); + const compensateSpy = vi.spyOn(svc, 'compensateWithCandidate').mockResolvedValue(true); + try { + const res = await request(app) + .post('/api/stacks/myapp/rollback') + .set('Cookie', authCookie) + .send({ expectedGitopsGenerationId: 'gen-app-2' }); + expect(res.status).toBe(409); + expect(res.body.code).toBe('RECOVERY_POINT_MISMATCH'); + expect(compensateSpy).not.toHaveBeenCalled(); + } finally { + getSpy.mockRestore(); + compensateSpy.mockRestore(); + } + }); }); describe('rollback file-revert safety on a policy-blocked rollback', () => { diff --git a/backend/src/routes/blueprints.ts b/backend/src/routes/blueprints.ts index 79de930c..8caf573f 100644 --- a/backend/src/routes/blueprints.ts +++ b/backend/src/routes/blueprints.ts @@ -34,6 +34,7 @@ import { commitBlueprintUpdate, } from '../services/gitops/blueprintProducers'; import { projectBlueprintRevision, projectCommittedRevision } from '../helpers/gitopsResponse'; +import { rollbackCandidatesForApplication } from '../services/gitops/rolloutRecovery'; import { isValidStackName } from '../utils/validation'; import { parseIntParam } from '../utils/parseIntParam'; import { auditActorUsername } from '../helpers/auditActor'; @@ -58,6 +59,7 @@ import { GitOpsBindingService, isGitManagedBlueprint, } from '../services/gitops/binding'; +import type { GitOpsRecoveryCapture } from '../services/gitops/recoveryCapture'; export const blueprintsRouter = Router(); @@ -119,6 +121,31 @@ function respondBindingFailure(res: Response, error: unknown, context: string, c res.status(500).json({ error: clientMessage }); } +/** + * A GitOps recovery binding as the hub sends it to a leaf on apply-local. + * + * Strict on shape so a malformed binding cannot reach the recovery row: each + * field may be absent, null, or a non-empty string, and nothing else is + * carried. + */ +function parseRecoveryBinding(value: unknown): GitOpsRecoveryCapture | null { + if (typeof value !== 'object' || value === null) return null; + const raw = value as { generationId?: unknown; artifactSetId?: unknown; sourceAcceptanceRef?: unknown }; + const field = (candidate: unknown): string | null | undefined => { + if (candidate === null || candidate === undefined) return null; + return typeof candidate === 'string' && candidate.length > 0 ? candidate : undefined; + }; + const generationId = field(raw.generationId); + const artifactSetId = field(raw.artifactSetId); + const sourceAcceptanceRef = field(raw.sourceAcceptanceRef); + if (generationId === undefined || artifactSetId === undefined || sourceAcceptanceRef === undefined) return null; + return { + gitops_generation_id: generationId, + gitops_artifact_set_id: artifactSetId, + gitops_source_acceptance_ref: sourceAcceptanceRef, + }; +} + async function respondBindingPreview( req: Request, res: Response, @@ -245,6 +272,7 @@ function summarizeBlueprint(blueprintId: number) { counts[dep.status] = (counts[dep.status] ?? 0) + 1; } const auth = evaluateLightweightEffectiveApproval(blueprintId); + const application = GitOpsStore.getInstance().getLiveBlueprintApplication(blueprintId); return { blueprint, deployments, @@ -252,6 +280,10 @@ function summarizeBlueprint(blueprintId: number) { effectiveApproval: auth?.effectiveApproval ?? 'pending', unauthorizedActions: auth?.unauthorizedActions ?? [], gitopsRevision: projectBlueprintRevision(blueprintId), + // The prior generations a rollout rollback can select. Read from the + // hub's own rollout-generation history: the projection has no + // generation list, and the node's recovery points live on the node. + rollbackCandidates: application ? rollbackCandidatesForApplication(application.id) : [], }; } @@ -500,6 +532,8 @@ blueprintsRouter.post('/apply-local', async (req: Request, res: Response): Promi markerContent?: unknown; allowGitManagedContent?: unknown; digestPins?: unknown; + captureRecovery?: unknown; + recoveryBinding?: unknown; }; if (typeof body.stackName !== 'string' || !isValidStackName(body.stackName)) { res.status(400).json({ error: 'Invalid stack name' }); @@ -521,6 +555,19 @@ blueprintsRouter.post('/apply-local', async (req: Request, res: Response): Promi res.status(400).json({ error: 'Invalid blueprint marker' }); return; } + if (body.captureRecovery !== undefined && typeof body.captureRecovery !== 'boolean') { + res.status(400).json({ error: 'captureRecovery must be a boolean' }); + return; + } + const recoveryBinding = body.recoveryBinding === undefined + ? null + : parseRecoveryBinding(body.recoveryBinding); + if (body.recoveryBinding !== undefined && recoveryBinding === null) { + res.status(400).json({ + error: 'recoveryBinding must be an object of generationId, artifactSetId, and sourceAcceptanceRef, each absent, null, or a non-empty string', + }); + return; + } let digestPins: DigestPinsMap | undefined; if (body.digestPins !== undefined) { if (!isDigestPinsMap(body.digestPins)) { @@ -537,6 +584,12 @@ blueprintsRouter.post('/apply-local', async (req: Request, res: Response): Promi // Hub-to-leaf proxy only. Browser sessions and opaque API tokens must // not opt a git-managed Blueprint into snapshot compose writes. && req.machineAuthScope === 'node_proxy'; + // A recovery capture on a Blueprint apply is a hub-driven rollout feature: + // a leaf has no GitOps rows of its own, so the hub both asks for the + // capture and supplies the binding it should record. Only the machine-auth + // hub hop may request it, on the same rule as git-managed content. + const captureRecovery = body.captureRecovery === true + && req.machineAuthScope === 'node_proxy'; try { const outcome = await BlueprintService.getInstance().applyLocalUnderLock( req.nodeId, @@ -544,7 +597,12 @@ blueprintsRouter.post('/apply-local', async (req: Request, res: Response): Promi body.composeContent, body.markerContent, '/api/blueprints/apply-local', - { allowGitManaged, digestPins }, + { + allowGitManaged, + digestPins, + captureRecovery, + recoveryBinding: recoveryBinding ?? undefined, + }, ); if (!outcome.ran) { res.status(409).json({ diff --git a/backend/src/routes/gitopsApplications.ts b/backend/src/routes/gitopsApplications.ts index 84b96155..0b9e61e2 100644 --- a/backend/src/routes/gitopsApplications.ts +++ b/backend/src/routes/gitopsApplications.ts @@ -18,7 +18,8 @@ import { Router, type Request, type Response } from 'express'; import { DatabaseService, type Blueprint } from '../services/DatabaseService'; import { NodeRegistry } from '../services/NodeRegistry'; -import { checkPermission, requirePermission } from '../middleware/permissions'; +import { checkPermission, requirePermission, scopedActionsForStack } from '../middleware/permissions'; +import { BlueprintReconciler } from '../services/BlueprintReconciler'; import { projectApplication } from '../services/gitops/derive'; import { latestTransitionByApplication } from '../services/gitops/history'; import { classifySourceRow, satisfiesGitOpsRead } from '../services/gitops/readAuth'; @@ -35,6 +36,15 @@ import { decodeGitOpsRequiredTargetsJson, encodeGitOpsApprovedTargetEffectJson, } from '../services/gitops/json'; +import { candidateRowFor, intentRowFor } from '../services/gitops/blueprintProducers'; +import { + resolveRollbackTargets, + restoreTargetToGeneration, + rollbackCandidatesForApplication, + type RestoreTargetOutcome, + type RolloutRollbackScope, + type RolloutRollbackTargetResult, +} from '../services/gitops/rolloutRecovery'; import { placementEffectCompatible } from '../services/gitops/store'; import { GitOpsTransitions, GitOpsTransitionError } from '../services/gitops/transitions'; import { newGitOpsId } from '../services/gitops/directApplication'; @@ -401,6 +411,64 @@ function actorFromRequest(req: Request): string | null { return req.user?.username ?? null; } +/** One envelope per operator action, so every row it writes shares an operation. */ +function authorityEnvelope(req: Request): { operationId: string; actor: string | null; trigger: string; at: number } { + return { operationId: newGitOpsId(), actor: actorFromRequest(req), trigger: 'manual', at: Date.now() }; +} + +type AuthorityEnvelope = ReturnType; + +/** `null` when absent, `'invalid'` when present but unusable. */ +function parseOptionalNodeId(raw: unknown): number | null | 'invalid' { + if (raw === undefined) return null; + return typeof raw === 'number' && Number.isSafeInteger(raw) && raw >= 1 ? raw : 'invalid'; +} + +/** + * The stack a Blueprint's targets run, from its current intent. + * + * Permissions and node-side lookups are both stack-scoped, and the deploy + * stack name is frozen on the intent rather than copied onto the application. + */ +function deployStackNameFor(app: { intent_revision_id: string | null }): string | null { + if (!app.intent_revision_id) return null; + return GitOpsStore.getInstance().getIntentRevision(app.intent_revision_id)?.deploy_stack_name ?? null; +} + +/** Exact deploy grant on one target's stack, checked before any target work starts. */ +function requireDeployOnTarget(req: Request, res: Response, stackName: string, nodeId: number): boolean { + if (checkPermission(req, 'stack:deploy', 'stack', stackName, nodeId)) return true; + res.status(403).json({ + error: `Permission denied for the target on node ${nodeId}.`, + code: 'PERMISSION_DENIED', + }); + return false; +} + +type ScopeParse = { ok: true; scope: RolloutRollbackScope } | { ok: false; message: string }; + +function parseRolloutScope(raw: unknown): ScopeParse { + if (typeof raw !== 'object' || raw === null) { + return { ok: false, message: "scope must be { kind: 'target' | 'failed' | 'all_changed' }" }; + } + const { kind, nodeId } = raw as { kind?: unknown; nodeId?: unknown }; + if (kind === 'target') { + if (typeof nodeId !== 'number' || !Number.isSafeInteger(nodeId) || nodeId < 1) { + return { ok: false, message: 'scope.nodeId must be a positive integer for a target rollback' }; + } + return { ok: true, scope: { kind: 'target', nodeId } }; + } + if (kind === 'failed' || kind === 'all_changed') return { ok: true, scope: { kind } }; + return { ok: false, message: "scope.kind must be 'target', 'failed', or 'all_changed'" }; +} + +function sameNodeSet(a: readonly number[], b: readonly number[]): boolean { + if (a.length !== b.length) return false; + const left = [...a].sort((x, y) => x - y); + const right = [...b].sort((x, y) => x - y); + return left.every((value, index) => value === right[index]); +} + gitopsApplicationsRouter.get('/:id', async (req: Request, res: Response): Promise => { try { const rawParam: unknown = req.params.id; @@ -445,6 +513,10 @@ gitopsApplicationsRouter.get('/:id', async (req: Request, res: Response): Promis nodeNames, }), projection, + rollbackCandidates: application.target_mode === 'blueprint' + ? rollbackCandidatesForApplication(application.id) + : undefined, + blueprintEnabled: blueprint?.enabled ?? null, }; res.json(response); return; @@ -814,3 +886,543 @@ gitopsApplicationsRouter.post('/:id/rollout/authorize', async (req: Request, res } res.json({ ok: true, dispatched, note }); }); + +/** + * Pause the rollout, application-wide or on one target. + * + * A pause is a statement about future execution, never about health: what was + * deployed stays deployed, and the projection reports the rollout as paused + * rather than converged. The reason is required because a paused rollout + * outlives the session that paused it. + */ +gitopsApplicationsRouter.post('/:id/rollout/pause', (req: Request, res: Response): void => { + const target = resolveAuthorityTarget(req, res); + if (!target) return; + const reason = typeof req.body?.reason === 'string' ? req.body.reason.trim() : ''; + if (reason.length === 0 || reason.length > 280) { + res.status(400).json({ error: 'A pause reason of 1 to 280 characters is required', code: 'CONFIRM_REQUIRED' }); + return; + } + const nodeId = parseOptionalNodeId(req.body?.nodeId); + if (nodeId === 'invalid') { + res.status(400).json({ error: 'nodeId must be a positive integer' }); + return; + } + const stackName = deployStackNameFor(target.application); + if (nodeId !== null) { + // A single target's pause is an exact stack action; the caller may hold the + // grant for that target without holding it fleet-wide. + if (!stackName) { + res.status(409).json({ error: 'The deploy stack identity could not be resolved', code: 'ROLLOUT_PAUSE_REFUSED' }); + return; + } + if (!requireDeployOnTarget(req, res, stackName, nodeId)) return; + } else if (!requirePermission(req, res, 'stack:deploy')) { + return; + } + if (!target.blueprint.enabled) { + res.status(409).json({ + error: 'The Blueprint is disabled. Enable it before pausing the rollout.', + code: 'BLUEPRINT_DISABLED', + }); + return; + } + // A pause holds a rollout that exists. With no authorization and no partial + // record there is nothing to hold, and recording a pause would only make the + // surface read as held when it never had anything running. + const hasRollout = !!target.application.rollout_authorization_ref + || !!target.application.partial_json; + if (!hasRollout) { + res.status(409).json({ + error: 'There is no rollout to pause.', + code: 'ROLLOUT_PAUSE_REFUSED', + }); + return; + } + + try { + GitOpsTransitions.getInstance().rolloutPaused( + target.application.id, + nodeId, + reason, + authorityEnvelope(req), + ); + } catch (error) { + if (error instanceof GitOpsTransitionError) { + res.status(409).json({ error: error.message, code: 'ROLLOUT_PAUSE_REFUSED' }); + return; + } + console.error( + '[GitOps authority] Rollout pause failed:', + sanitizeForLog(error instanceof Error ? error.message : String(error)), + ); + res.status(500).json({ error: 'Failed to pause the rollout' }); + return; + } + res.json({ ok: true }); +}); + +/** + * Resume a paused rollout and continue the queue when one is still authorized. + * + * Unpausing is the decision; continuing is the executor's answer. A resume + * with no live authorization reports that nothing started rather than + * minting authority the operator never granted. + */ +gitopsApplicationsRouter.post('/:id/rollout/resume', async (req: Request, res: Response): Promise => { + const target = resolveAuthorityTarget(req, res); + if (!target) return; + const nodeId = parseOptionalNodeId(req.body?.nodeId); + if (nodeId === 'invalid') { + res.status(400).json({ error: 'nodeId must be a positive integer' }); + return; + } + const stackName = deployStackNameFor(target.application); + if (nodeId !== null) { + if (!stackName) { + res.status(409).json({ error: 'The deploy stack identity could not be resolved', code: 'ROLLOUT_RESUME_REFUSED' }); + return; + } + if (!requireDeployOnTarget(req, res, stackName, nodeId)) return; + } else if (!requirePermission(req, res, 'stack:deploy')) { + return; + } + if (!target.blueprint.enabled) { + res.status(409).json({ + error: 'The Blueprint is disabled. Enable it before resuming the rollout.', + code: 'BLUEPRINT_DISABLED', + }); + return; + } + + try { + GitOpsTransitions.getInstance().rolloutUnpaused(target.application.id, nodeId, authorityEnvelope(req)); + } catch (error) { + if (error instanceof GitOpsTransitionError) { + res.status(409).json({ error: error.message, code: 'ROLLOUT_RESUME_REFUSED' }); + return; + } + console.error( + '[GitOps authority] Rollout resume failed:', + sanitizeForLog(error instanceof Error ? error.message : String(error)), + ); + res.status(500).json({ error: 'Failed to resume the rollout' }); + return; + } + + // Resuming continues the queue in both scopes: an application resume clears + // the fleet hold, and a target resume clears that target's hold so the + // dispatch below reaches it. Targets still individually paused are skipped + // by the adapter. + const store = GitOpsStore.getInstance(); + const app = store.getApplication(target.application.id) ?? target.application; + const binding = store.currentAuthorizationBinding(app); + if (!binding) { + res.json({ + ok: true, + dispatched: false, + note: 'The rollout is resumed, but no live authorization exists; authorize the rollout to start it.', + }); + return; + } + const genRow = store.getGeneration(binding.acceptedGenerationId); + if (!genRow) { + res.json({ + ok: true, + dispatched: false, + note: 'The authorized generation could not be read; refresh the application and authorize again.', + }); + return; + } + + const actor = actorFromRequest(req); + let dispatched = false; + let note: string | null = null; + try { + const result = await GitSourceService.getInstance().dispatchAcceptedGeneration( + buildAcceptedGeneration(genRow), + GitSourceService.dispatchContextFor(app), + { trigger: 'manual', actor: actor ?? 'operator' }, + ); + dispatched = result.status === 'dispatched'; + if (result.status === 'blocked') note = result.reason; + } catch (error) { + console.error( + '[GitOps authority] Rollout dispatch failed after resume:', + sanitizeForLog(error instanceof Error ? error.message : String(error)), + ); + note = 'The rollout is resumed but could not start; the authorization stands and the rollout remains queued.'; + } + res.json({ ok: true, dispatched, note }); +}); + +/** + * Re-derive placement for the current Blueprint and open a fresh review. + * + * Replanning is an operator statement that the current placement question is + * no longer the right one (a node was rebuilt, a cordon moved, a plan was + * reviewed against stale intent). It mints intent and candidate through the + * same producers an edit uses, which invalidates the old placement approval + * and authorization. Nothing moved means nothing is written: re-opening a + * review that already describes the Blueprint would invalidate approvals that + * are still accurate. + */ +gitopsApplicationsRouter.post('/:id/rollout/replan', (req: Request, res: Response): void => { + if (!requirePermission(req, res, 'stack:create')) return; + const target = resolveAuthorityTarget(req, res); + if (!target) return; + const store = GitOpsStore.getInstance(); + const app = target.application; + const candidate = app.rollout_candidate_id ? store.getRolloutCandidate(app.rollout_candidate_id) : undefined; + if (!candidate) { + res.status(409).json({ + error: 'There is no current placement to replan.', + code: 'REPLAN_UNAVAILABLE', + }); + return; + } + const intent = app.intent_revision_id ? store.getIntentRevision(app.intent_revision_id) : undefined; + let frozenNodeIds: number[]; + try { + frozenNodeIds = decodeGitOpsRequiredTargetsJson(candidate.required_targets_json).nodeIds; + } catch (error) { + console.error( + '[GitOps authority] Rollout candidate frozen set is unreadable:', + sanitizeForLog(error instanceof Error ? error.message : String(error)), + ); + res.status(409).json({ + error: 'The current placement could not be read; refresh the application and try again.', + code: 'REPLAN_UNAVAILABLE', + }); + return; + } + const desiredNodeIds = BlueprintReconciler.getInstance() + .listDesiredNodes(target.blueprint, DatabaseService.getInstance().getNodes()) + .map(node => node.id); + const intentIsCurrent = !!intent + && intent.blueprint_revision === target.blueprint.revision + && intent.pinned_node_id === target.blueprint.pinned_node_id + && intent.deploy_stack_name === target.blueprint.name + && intent.selector_json === JSON.stringify(target.blueprint.selector); + if (intentIsCurrent && sameNodeSet(desiredNodeIds, frozenNodeIds)) { + res.status(409).json({ + error: 'The current placement already matches the Blueprint; there is nothing to replan.', + code: 'REPLAN_UNAVAILABLE', + }); + return; + } + + const envelope = authorityEnvelope(req); + try { + DatabaseService.getInstance().getDb().transaction(() => { + const tx = GitOpsTransitions.getInstance(); + const nextIntent = intentRowFor(app.id, target.blueprint, envelope.operationId, envelope.actor, envelope.at); + tx.intentRevised({ applicationId: app.id, intent: nextIntent, envelope }); + tx.rolloutCandidateOpened({ + applicationId: app.id, + candidate: candidateRowFor( + app.id, + nextIntent, + desiredNodeIds, + 'roster_change', + envelope.operationId, + envelope.at, + ), + envelope, + }); + })(); + } catch (error) { + if (error instanceof GitOpsTransitionError) { + res.status(409).json({ error: error.message, code: 'REPLAN_UNAVAILABLE' }); + return; + } + console.error( + '[GitOps authority] Rollout replan failed:', + sanitizeForLog(error instanceof Error ? error.message : String(error)), + ); + res.status(500).json({ error: 'Failed to replan the rollout' }); + return; + } + res.json({ ok: true }); +}); + +/** + * Withdraw the live rollout authorization on the operator's decision. + * + * The abandoned generation stays visible as superseded, and the authorization + * is cleared so the dispatch boundary cannot start it again. A later rollout + * authorization mints a new generation over the same placement and accepted + * source. + */ +gitopsApplicationsRouter.post('/:id/rollout/supersede', (req: Request, res: Response): void => { + if (!requirePermission(req, res, 'stack:deploy')) return; + const target = resolveAuthorityTarget(req, res); + if (!target) return; + if (!target.blueprint.enabled) { + res.status(409).json({ + error: 'The Blueprint is disabled. Enable it before superseding the rollout.', + code: 'BLUEPRINT_DISABLED', + }); + return; + } + try { + GitOpsTransitions.getInstance().rolloutSuperseded({ + applicationId: target.application.id, + envelope: authorityEnvelope(req), + }); + } catch (error) { + if (error instanceof GitOpsTransitionError) { + res.status(409).json({ error: error.message, code: 'ROLLOUT_SUPERSEDE_REFUSED' }); + return; + } + console.error( + '[GitOps authority] Rollout supersede failed:', + sanitizeForLog(error instanceof Error ? error.message : String(error)), + ); + res.status(500).json({ error: 'Failed to supersede the rollout' }); + return; + } + res.json({ ok: true }); +}); + +/** + * Restore one target and record what happened, whatever path the restore took. + * + * Every exit attempts to record a terminal state for the target: a restore + * that cannot be opened, cannot be requested, or cannot have its result + * recorded is reported as a failed target rather than a clean restore. If the + * terminal write itself fails it is logged; the boot-time reclassification + * settles a row left in `restoring`. + */ +async function runRollbackTarget( + ctx: { + app: GitOpsApplicationRow; + stackName: string; + generationId: string; + recoveryRef: string; + envelope: AuthorityEnvelope; + actor: string | null; + role: string; + userId: number; + }, + nodeId: number, +): Promise { + const tx = GitOpsTransitions.getInstance(); + try { + tx.rollbackInProgress({ + applicationId: ctx.app.id, + nodeId, + recoveryRef: ctx.recoveryRef, + recoveryGenerationId: ctx.generationId, + envelope: ctx.envelope, + }); + } catch (error) { + return { + nodeId, + status: 'failed', + error: error instanceof Error ? error.message : 'The rollback could not be opened.', + }; + } + + let outcome: RestoreTargetOutcome; + try { + outcome = await restoreTargetToGeneration({ + app: ctx.app, + stackName: ctx.stackName, + nodeId, + generationId: ctx.generationId, + actor: ctx.actor, + role: ctx.role, + scopedActions: scopedActionsForStack(ctx.userId, nodeId, ctx.stackName), + }); + } catch (error) { + console.error( + '[GitOps authority] Rollout restore request failed on node %s:', + sanitizeForLog(nodeId), + sanitizeForLog(error instanceof Error ? error.message : String(error)), + ); + outcome = { ok: false, code: 'ROLLBACK_FAILED', error: 'The restore request failed.' }; + } + + if (outcome.ok) { + const store = GitOpsStore.getInstance(); + try { + tx.rollbackCompleted({ + applicationId: ctx.app.id, + nodeId, + recoveryRef: ctx.recoveryRef, + recoveryGenerationId: ctx.generationId, + // The strongest evidence recorded for the restored generation. The + // node's own capture is not read here; restoreArtifactPointers drops + // anything this generation does not own and records the limitation. + capturedArtifactSetId: store.newestArtifactSetIdForGeneration(ctx.generationId), + capturedSourceAcceptanceRef: store.newestSourceAcceptanceId(ctx.app.id, ctx.generationId), + envelope: ctx.envelope, + }); + return { nodeId, status: 'restored' }; + } catch (error) { + console.error( + '[GitOps authority] Restore succeeded but its completion could not be recorded:', + sanitizeForLog(error instanceof Error ? error.message : String(error)), + ); + outcome = { ok: false, code: 'RECORD_FAILED', error: 'The restore completed but its result could not be recorded.' }; + } + } + + console.error( + '[GitOps authority] Rollout rollback failed on node %s:', + sanitizeForLog(nodeId), + sanitizeForLog(outcome.error), + ); + try { + tx.rollbackPartialFailed({ + applicationId: ctx.app.id, + nodeId, + recoveryRef: ctx.recoveryRef, + failureClass: 'partial', + envelope: ctx.envelope, + }); + } catch (error) { + console.error( + '[GitOps authority] Could not record the failed rollback target:', + sanitizeForLog(error instanceof Error ? error.message : String(error)), + ); + } + return { nodeId, status: 'failed', error: outcome.error }; +} + +/** + * Roll a rollout back to an explicit prior application generation. + * + * The scope names which targets recover: one target, the targets recorded as + * failed or unreachable, or every changed target in the frozen set. Each + * target is restored from its own captured recovery point, and that point must + * name the selected generation, so a rollback that cannot prove what it + * restored reports a partial failure instead of a completion. Configuration is + * restored; application data is untouched, and the restored generation's + * artifact expectation is rebound only from evidence that generation owns. + */ +gitopsApplicationsRouter.post('/:id/rollout/rollback', async (req: Request, res: Response): Promise => { + // Fleet-wide on purpose: a rollback withdraws the application's rollout + // authorization, which is broader than any one target. The per-target + // checks below then pin the exact stack and node of every restore. + if (!requirePermission(req, res, 'stack:deploy')) return; + const target = resolveAuthorityTarget(req, res); + if (!target) return; + const body = (req.body ?? {}) as { generationId?: unknown; scope?: unknown }; + const generationId = typeof body.generationId === 'string' ? body.generationId : ''; + if (generationId.length === 0) { + res.status(400).json({ error: 'generationId is required', code: 'CONFIRM_REQUIRED' }); + return; + } + const parsedScope = parseRolloutScope(body.scope); + if (!parsedScope.ok) { + res.status(400).json({ error: parsedScope.message, code: 'CONFIRM_REQUIRED' }); + return; + } + if (!target.blueprint.enabled) { + res.status(409).json({ + error: 'The Blueprint is disabled. Enable it before rolling back a rollout.', + code: 'BLUEPRINT_DISABLED', + }); + return; + } + + const app = target.application; + const store = GitOpsStore.getInstance(); + const generation = store.getGeneration(generationId); + if (!generation || generation.application_id !== app.id) { + res.status(409).json({ + error: 'The selected application generation is not part of this application.', + code: 'ROLLBACK_REFUSED', + }); + return; + } + const stackName = deployStackNameFor(app); + if (!stackName) { + res.status(409).json({ + error: 'The deploy stack identity could not be resolved.', + code: 'ROLLBACK_REFUSED', + }); + return; + } + const resolved = resolveRollbackTargets(app, parsedScope.scope, generationId); + if (!resolved.ok) { + res.status(409).json({ error: resolved.error, code: resolved.code }); + return; + } + // Exact authorization on every resolved target, before any restore starts: + // the application spans nodes, so a fleet-wide grant is not what this action + // needs, and a bulk action that half-runs on a permission failure would + // leave a partial recovery nobody asked for. + for (const nodeId of resolved.nodeIds) { + if (!requireDeployOnTarget(req, res, stackName, nodeId)) return; + } + + // The abandoned rollout must not be dispatchable while its targets are being + // restored. Only an authorized rollout has a dispatch to stop; a placement + // generation that never authorized cannot be superseded and does not need to + // be. + const liveRolloutGeneration = app.rollout_generation_id + ? store.getRolloutGeneration(app.rollout_generation_id) + : undefined; + if (app.rollout_generation_id && !liveRolloutGeneration) { + res.status(409).json({ + error: 'The current rollout generation could not be read; refresh the application and try again.', + code: 'ROLLBACK_REFUSED', + }); + return; + } + const envelope = authorityEnvelope(req); + if (liveRolloutGeneration?.provenance === 'rollout_authorization') { + try { + GitOpsTransitions.getInstance().rolloutSuperseded({ applicationId: app.id, envelope }); + } catch (error) { + console.error( + '[GitOps authority] Could not withdraw the rollout before rolling back:', + sanitizeForLog(error instanceof Error ? error.message : String(error)), + ); + res.status(409).json({ + error: 'The live rollout authorization could not be withdrawn; refresh the application and try again.', + code: 'ROLLBACK_REFUSED', + }); + return; + } + } + + const ctx = { + app, + stackName, + generationId, + recoveryRef: newGitOpsId(), + envelope, + actor: actorFromRequest(req), + role: req.user?.role ?? 'viewer', + userId: req.user?.userId ?? 0, + }; + const results: RolloutRollbackTargetResult[] = []; + for (const nodeId of resolved.nodeIds) { + results.push(await runRollbackTarget(ctx, nodeId)); + } + + const failedAny = results.some(result => result.status === 'failed'); + if (failedAny) { + // Lift the application out of `restoring` so the projection reports the + // partial failure rather than an in-flight rollback that nothing drives. + try { + GitOpsTransitions.getInstance().rollbackPartialFailed({ + applicationId: app.id, + nodeId: null, + recoveryRef: ctx.recoveryRef, + failureClass: 'partial', + envelope, + }); + } catch (error) { + console.error( + '[GitOps authority] Could not record the partial rollback failure:', + sanitizeForLog(error instanceof Error ? error.message : String(error)), + ); + } + } + res.json({ ok: !failedAny, results }); +}); diff --git a/backend/src/routes/stacks.ts b/backend/src/routes/stacks.ts index c95c4380..995c237d 100644 --- a/backend/src/routes/stacks.ts +++ b/backend/src/routes/stacks.ts @@ -2599,6 +2599,30 @@ stacksRouter.post('/:stackName/rollback', async (req: Request, res: Response) => const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService'); const recoverySvc = StackUpdateRecoveryService.getInstance(); const currentGen = recoverySvc.getCurrent(req.nodeId, stackName); + // A caller that names the application generation it means to restore + // (the GitOps rollout rollback path) must be answered against that + // generation, never against whatever recovery point happens to be current. + // Restoring the wrong point under a rollback claim would report a rollout + // recovery that never happened. + const expectedGitopsGenerationId = typeof req.body?.expectedGitopsGenerationId === 'string' + && req.body.expectedGitopsGenerationId.length > 0 + ? req.body.expectedGitopsGenerationId + : null; + if (expectedGitopsGenerationId !== null && !currentGen) { + res.status(409).json({ + error: 'This stack has no recovery point to restore.', + code: 'NO_RECOVERY_POINT', + }); + return; + } + if (expectedGitopsGenerationId !== null && currentGen + && currentGen.gitops_generation_id !== expectedGitopsGenerationId) { + res.status(409).json({ + error: 'The current recovery point does not restore the requested application generation.', + code: 'RECOVERY_POINT_MISMATCH', + }); + return; + } // Any current recovery row uses compensateWithCandidate. Policy is evaluated // against the restored target inside compensate, not the live pre-restore project. if (currentGen) { @@ -2651,7 +2675,14 @@ stacksRouter.post('/:stackName/rollback', async (req: Request, res: Response) => } invalidateNodeCaches(req.nodeId); dlog(`[Stacks] Rollback completed: ${sanitizeForLog(stackName)}`); - res.json({ message: 'Stack rolled back from recovery generation.', recoveryId: currentGen.id }); + // Echo the generation the point was bound to. A hub-driven rollout + // rollback requires this to equal the generation it asked for, so a + // node that ignored the request cannot be reported as a restore. + res.json({ + message: 'Stack rolled back from recovery generation.', + recoveryId: currentGen.id, + gitopsGenerationId: currentGen.gitops_generation_id ?? null, + }); notifyActionSuccess('deploy_success', `${stackName} rolled back`, stackName, req.user?.username ?? 'system'); return; } diff --git a/backend/src/services/BlueprintService.ts b/backend/src/services/BlueprintService.ts index c01b2f56..2a5f45c1 100644 --- a/backend/src/services/BlueprintService.ts +++ b/backend/src/services/BlueprintService.ts @@ -35,6 +35,7 @@ import { type BlueprintMarker, } from '../helpers/blueprintMarker'; import { throwIfGitManagedDeploy } from './gitops/gitManaged'; +import type { GitOpsRecoveryCapture } from './gitops/recoveryCapture'; import { commitBlueprintDeploymentCause, commitBlueprintDeploymentRemoved, @@ -281,6 +282,13 @@ export class BlueprintService { * Pass `lockHeld: true` when the caller already acquired the lock (so * deploy-started is not recorded before the lock is held). The caller * releases in that case; otherwise this method acquires and releases. + * + * `captureRecovery` asks the node to capture a recovery generation of the + * pre-deploy state before it deploys, which is what lets a Git-managed + * rollout be rolled back later. `recoveryBinding` names the GitOps + * generation that state belongs to, because a remote node cannot resolve a + * hub-owned GitOps application itself. Both travel to the remote in the + * apply-local request so the capture happens where the files are. */ async deployAuthorizedMaterialization(args: { blueprint: Blueprint; @@ -290,6 +298,8 @@ export class BlueprintService { auditPath: string; lockHeld?: boolean; digestPins?: DigestPinsMap; + captureRecovery?: boolean; + recoveryBinding?: GitOpsRecoveryCapture; }): Promise { const manageLock = !args.lockHeld; if (manageLock && !this.acquireLock(args.blueprint.id, args.node.id)) { @@ -311,7 +321,12 @@ export class BlueprintService { args.composeContent, markerContent, args.auditPath, - { allowGitManaged: true, digestPins: args.digestPins }, + { + allowGitManaged: true, + digestPins: args.digestPins, + captureRecovery: args.captureRecovery, + recoveryBinding: args.recoveryBinding, + }, ); if (!outcome.ran) { throw new Error(stackOpSkipMessage(args.blueprint.name, outcome.existingAction)); @@ -323,6 +338,8 @@ export class BlueprintService { args.composeContent, markerContent, args.digestPins, + args.captureRecovery === true, + args.recoveryBinding, ); } this.setStatus(args.blueprint.id, args.node.id, 'active', 'deploy_ack', { @@ -362,6 +379,8 @@ export class BlueprintService { composeContent: string, markerContent: string, digestPins?: DigestPinsMap, + captureRecovery = false, + recoveryBinding?: GitOpsRecoveryCapture, ): Promise { const target = NodeRegistry.getInstance().getProxyTarget(node.id); if (!target) throw new Error(`Remote node "${node.name}" has no proxy target configured`); @@ -376,6 +395,16 @@ export class BlueprintService { if (digestPins) { applyBody.digestPins = digestPins; } + if (captureRecovery) { + applyBody.captureRecovery = true; + if (recoveryBinding) { + applyBody.recoveryBinding = { + generationId: recoveryBinding.gitops_generation_id, + artifactSetId: recoveryBinding.gitops_artifact_set_id, + sourceAcceptanceRef: recoveryBinding.gitops_source_acceptance_ref, + }; + } + } const augmented = await prepareOutboundRegistryDeliveryBody({ method: 'POST', apiPath: '/api/blueprints/apply-local', @@ -948,7 +977,14 @@ export class BlueprintService { composeContent: string, markerContent: string, auditPath: string, - options: { allowGitManaged?: boolean; digestPins?: DigestPinsMap } = {}, + options: { + allowGitManaged?: boolean; + digestPins?: DigestPinsMap; + /** Capture a recovery generation of the pre-deploy state (atomic deploy). */ + captureRecovery?: boolean; + /** GitOps binding to record on that recovery generation, when known. */ + recoveryBinding?: GitOpsRecoveryCapture; + } = {}, ): Promise<{ ran: true } | { ran: false; existingAction: StackOpAction }> { const expected = parseBlueprintMarker(markerContent); if (!expected) { @@ -988,10 +1024,45 @@ export class BlueprintService { } previousComposeContent = prior.content; } - await fs.writeStackFile(stackName, COMPOSE_FILENAME, composeContent); - // Clear lower-priority compose siblings so discovery cannot shadow compose.yaml. - await fs.removeAlternateRootComposeFiles(stackName); + // The recovery point must describe the project as it stands + // before this apply, so it is captured before the new compose + // is written. A newly created stack has no prior state and no + // rollback to offer. + let recoveryId: string | null = null; + if (options.captureRecovery === true && !createdStack) { + const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); + const recoverySvc = StackUpdateRecoveryService.getInstance(); + const candidate = await recoverySvc.captureCandidate({ + nodeId, + stackName, + createdBy: 'blueprint-rollout', + operationKind: 'deployment', + gitopsBinding: options.recoveryBinding, + }); + if (!recoverySvc.markAcquired(candidate.id)) { + await recoverySvc.abandon(candidate.id); + throw new Error('Failed to acquire the recovery generation for this Blueprint apply'); + } + if (!recoverySvc.handoff(candidate.id, nodeId, stackName)) { + await recoverySvc.abandon(candidate.id); + throw new Error('Failed to hand off the recovery generation for this Blueprint apply'); + } + // From here the row is the stack's current recovery point, + // so every later failure path must settle it: the catch + // below compensates it rather than leaving a current row + // nothing owns. + recoveryId = candidate.id; + } try { + if (recoveryId !== null) { + const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); + if (!StackUpdateRecoveryService.getInstance().markReconciling(recoveryId)) { + throw new Error('Failed to mark the recovery generation reconciling for this Blueprint apply'); + } + } + await fs.writeStackFile(stackName, COMPOSE_FILENAME, composeContent); + // Clear lower-priority compose siblings so discovery cannot shadow compose.yaml. + await fs.removeAlternateRootComposeFiles(stackName); if (options.digestPins) { const model = await buildEffectiveServiceModel(nodeId, stackName); if (!model.renderable) { @@ -1021,8 +1092,44 @@ export class BlueprintService { digestPins: options.digestPins, }, ); + if (recoveryId !== null) { + const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); + if (!StackUpdateRecoveryService.getInstance().markImmediateVerified(recoveryId)) { + console.warn( + '[BlueprintService] Could not mark recovery %s immediate_verified', + sanitizeForLog(recoveryId), + ); + } + } await fs.writeStackFile(stackName, BLUEPRINT_MARKER_FILENAME, markerContent); } catch (err) { + // A failed apply attempts to restore the project and the + // runtime the recovery point captured, so the node is not + // left running a half-applied generation. A restore that + // cannot complete is logged and the original failure still + // surfaces. + if (recoveryId !== null) { + try { + const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); + const recovered = await StackUpdateRecoveryService.getInstance().compensateWithCandidate( + recoveryId, + (overridePath, invocation, overlay) => ComposeService.getInstance(nodeId) + .composeUpWithRecoveryOverride(stackName, overridePath, undefined, invocation, overlay), + ); + if (!recovered) { + console.warn( + '[BlueprintService] Recovery compensation did not complete for "%s"', + sanitizeForLog(stackName), + ); + } + } catch (compError) { + console.error( + '[BlueprintService] Recovery compensation failed for "%s": %s', + sanitizeForLog(stackName), + sanitizeForLog(BlueprintService.formatError(compError)), + ); + } + } if (createdStack) { try { await fs.deleteStack(stackName); diff --git a/backend/src/services/StackUpdateRecoveryService.ts b/backend/src/services/StackUpdateRecoveryService.ts index 2d39dd4d..43e4d307 100644 --- a/backend/src/services/StackUpdateRecoveryService.ts +++ b/backend/src/services/StackUpdateRecoveryService.ts @@ -250,6 +250,15 @@ export interface CaptureStackUpdateInput { createdBy: string | null; /** Capture trigger; defaults to 'update'. */ operationKind?: RollbackOperationKind; + /** + * GitOps binding to record on the recovery row, when the caller knows it. + * + * A remote node has no GitOps application rows (they are hub-owned), so it + * cannot resolve this binding itself. The hub supplies the generation the + * target was running before the deploy it is asking for, and only that hub + * reaches this field. + */ + gitopsBinding?: GitOpsRecoveryCapture; } function yamlQuote(value: string): string { @@ -578,7 +587,7 @@ export class StackUpdateRecoveryService { artifacts_retired: 0, released_at: null, released_by: null, - ...this.captureGitOpsBindingOrEmpty(stackName, nodeId), + ...(input.gitopsBinding ?? this.captureGitOpsBindingOrEmpty(stackName, nodeId)), }; DatabaseService.getInstance().insertStackUpdateRecoveryGeneration(row); return row; @@ -614,16 +623,32 @@ export class StackUpdateRecoveryService { * than swallowed, and the degraded shape is the same one legacy rows carry. */ private captureGitOpsBindingOrEmpty(stackName: string, nodeId: number): GitOpsRecoveryCapture { + let resolved: GitOpsRecoveryCapture | null = null; try { - return captureGitOpsRecoveryBinding(stackName, nodeId); + resolved = captureGitOpsRecoveryBinding(stackName, nodeId); } catch (error) { console.warn( '[StackUpdateRecovery] Could not capture the GitOps binding for %s; recovery point stored without it: %s', sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(error, 'unknown')), ); - return { ...EMPTY_GITOPS_RECOVERY_CAPTURE }; } + if (resolved?.gitops_generation_id) return resolved; + // A node without the GitOps rows (a leaf keeping a Blueprint-managed + // stack) cannot resolve the owning application. The row being replaced + // still describes the generation this node is running, and a re-capture + // (backup, atomic deploy) does not change what is running, so the binding + // carries forward rather than dropping to null and stranding a later + // rollback. + const previous = this.getCurrent(nodeId, stackName); + if (previous?.gitops_generation_id) { + return { + gitops_generation_id: previous.gitops_generation_id, + gitops_artifact_set_id: previous.gitops_artifact_set_id ?? null, + gitops_source_acceptance_ref: previous.gitops_source_acceptance_ref ?? null, + }; + } + return resolved ?? { ...EMPTY_GITOPS_RECOVERY_CAPTURE }; } /** diff --git a/backend/src/services/gitops/handoff.ts b/backend/src/services/gitops/handoff.ts index b9ec9373..21a70912 100644 --- a/backend/src/services/gitops/handoff.ts +++ b/backend/src/services/gitops/handoff.ts @@ -25,6 +25,7 @@ import { } from './registryReadiness'; import { stackManagedRoot } from './directApplication'; import { decodeGitOpsRequiredTargetsJson } from './json'; +import { recoveryBindingForTarget } from './recoveryCapture'; import { DatabaseService } from '../DatabaseService'; import { BlueprintService } from '../BlueprintService'; import { buildBlueprintMarker } from '../../helpers/blueprintMarker'; @@ -238,6 +239,12 @@ function liveRolloutBinding(app: GitOpsApplicationRow): FutureRolloutAuthorizati * * `authority` names who is minting: the automatic handoff acts on the * configured policy, an explicit operator authorization on the operator. + * + * `shouldAbort` lets a dispatch caller refuse to mint while a hold is placed. + * It is re-read after the preflight await, because a pause that lands during + * the evaluation must not mint or supersede an authorization the pause is + * holding. Explicit operator authorizations do not pass it: the operator's own + * decision is the authority there. */ export async function ensureRolloutAuthorization( applicationId: string, @@ -245,6 +252,7 @@ export async function ensureRolloutAuthorization( trigger = 'blueprint_dispatch', depsPartial?: Partial, authority: 'operator' | 'configured_policy' = 'configured_policy', + shouldAbort?: () => boolean, ): Promise<{ ok: true; binding: FutureRolloutAuthorizationBinding } | { ok: false; reason: string }> { return withSerializedEvaluation(applicationId, async () => { const store = GitOpsStore.getInstance(); @@ -253,6 +261,7 @@ export async function ensureRolloutAuthorization( if (app.target_mode !== 'blueprint') { return { ok: false, reason: 'Rollout authorization is only for Blueprint target mode.' }; } + if (shouldAbort?.()) return { ok: false, reason: 'The rollout is paused.' }; const ingredients = store.authorizationIngredients(app); if (!ingredients) { @@ -297,6 +306,9 @@ export async function ensureRolloutAuthorization( }, deps, ); + // Re-read after the await: a pause placed while preflight ran must not + // mint, supersede, or invalidate the authorization it is holding. + if (shouldAbort?.()) return { ok: false, reason: 'The rollout is paused.' }; const preflightEvidenceJson = encodePreflightEvidenceJson(preflight); const preflightFingerprint = fingerprintPreflightEvidence(preflight); const transitions = GitOpsTransitions.getInstance(); @@ -450,9 +462,22 @@ export class BlueprintTargetAdapter implements TargetAdapter { if (blueprintId === null) { return { status: 'blocked', reason: 'No Blueprint is bound to this application.' }; } + // A pause is an execution hold, not a rendering state: nothing dispatches + // while it is set, including the boot-time reconstruction that resumes + // authorized rollouts. + if (app.pause_at) { + return { status: 'blocked', reason: 'The rollout is paused.' }; + } const trigger = generation.trigger || 'blueprint_dispatch'; - const auth = await ensureRolloutAuthorization(app.id, generation.actor ?? null, trigger); + const auth = await ensureRolloutAuthorization( + app.id, + generation.actor ?? null, + trigger, + undefined, + 'configured_policy', + () => !!store.getApplication(app.id)?.pause_at, + ); if (!auth.ok) return { status: 'blocked', reason: auth.reason }; const binding = auth.binding; @@ -470,6 +495,9 @@ export class BlueprintTargetAdapter implements TargetAdapter { if (!resolvesRolloutAuthorization(liveApp, binding)) { return { status: 'blocked', reason: 'The current rollout authorization no longer matches the live binding.' }; } + if (liveApp.pause_at) { + return { status: 'blocked', reason: 'The rollout is paused.' }; + } const liveAuthorizationRef = liveApp.rollout_authorization_ref; const genRow = store.getGeneration(binding.acceptedGenerationId); @@ -491,12 +519,26 @@ export class BlueprintTargetAdapter implements TargetAdapter { const tx = GitOpsTransitions.getInstance(); const svc = BlueprintService.getInstance(); + const pausedTargets: number[] = []; + let anyDispatched = false; for (const nodeId of binding.requiredNodeIds) { + // Re-read before every target: a pause that lands while this loop runs + // stops the rollout at the next target rather than after the fleet. + if (store.getApplication(liveApp.id)?.pause_at) { + return { status: 'blocked', reason: 'The rollout was paused while it was running.' }; + } const target = store.getTarget(liveApp.id, nodeId); if (targetAlreadyAcked(target, binding, liveAuthorizationRef)) { continue; } + // A per-target pause holds that target alone: the rest of the queue + // continues, and this target is retried by the dispatch a later resume + // triggers. + if (target?.pause_at) { + pausedTargets.push(nodeId); + continue; + } const node = DatabaseService.getInstance().getNode(nodeId); if (!node) { @@ -548,6 +590,10 @@ export class BlueprintTargetAdapter implements TargetAdapter { bindingRevision: binding.intentRevisionId, }); + // Capture the pre-deploy state so this rollout can be rolled back + // later. The binding names the generation the target is running now, + // which is the one a rollback would restore from here. + const recoveryTarget = store.getTarget(liveApp.id, nodeId); const outcome = await svc.deployAuthorizedMaterialization({ blueprint, node, @@ -555,6 +601,8 @@ export class BlueprintTargetAdapter implements TargetAdapter { marker, auditPath: `/api/blueprints/${blueprint.id}/rollout/${liveApp.id}`, lockHeld: true, + captureRecovery: true, + recoveryBinding: recoveryBindingForTarget(liveApp, recoveryTarget), }); if (outcome.status !== 'active') { @@ -595,11 +643,20 @@ export class BlueprintTargetAdapter implements TargetAdapter { reason: `Could not record ack for node ${nodeId}: ${errorMessage(err)}`, }; } + anyDispatched = true; } finally { svc.releaseAuthorizedDeployLock(blueprint.id, nodeId); } } + // Nothing ran because every remaining target is paused: report the hold + // rather than a dispatch, so the caller does not count it as progress. + if (!anyDispatched && pausedTargets.length > 0) { + return { + status: 'blocked', + reason: `The rollout is paused on ${pausedTargets.length} target${pausedTargets.length === 1 ? '' : 's'}.`, + }; + } return { status: 'dispatched' }; } } @@ -615,6 +672,9 @@ export async function reconstructBlueprintRolloutQueue(): Promise { let resumed = 0; for (const app of apps) { if (!app.accepted_generation_id || !app.rollout_authorization_ref) continue; + // A pause survives a restart: reconstruction resumes only rollouts that + // nothing is holding. A per-target pause is honored inside the adapter. + if (app.pause_at) continue; if (!app.latest_preflight_evidence_json) continue; const stored = decodePreflightEvidenceJson(app.latest_preflight_evidence_json); if (isPreflightBlocked(stored)) continue; diff --git a/backend/src/services/gitops/portfolioTypes.ts b/backend/src/services/gitops/portfolioTypes.ts index 207cbfe7..4cce2147 100644 --- a/backend/src/services/gitops/portfolioTypes.ts +++ b/backend/src/services/gitops/portfolioTypes.ts @@ -173,12 +173,31 @@ export type GitOpsPortfolioResponse = { truncated: boolean; }; +/** + * A prior application generation a rollout rollback can select, from the + * hub's own rollout-generation history. Optional so a node or build that does + * not compute it omits the field rather than claiming an empty list. + */ +export type GitOpsRollbackCandidate = { + generationId: string; + rolloutGenerationId: string; + createdAt: number; +}; + export type GitOpsPortfolioDetailResponse = { schemaVersion: 1; generatedAt: number; application: GitOpsPortfolioRow; /** The full canonical projection for the application (the SEN-510 read shape). */ projection: GitOpsRevisionProjection; + /** For a Git-managed Blueprint: the prior generations its rollback can select. */ + rollbackCandidates?: GitOpsRollbackCandidate[]; + /** + * The bound Blueprint's enabled state, when there is one. Execution + * controls are withheld on a disabled Blueprint, and this read is where the + * application view learns that. + */ + blueprintEnabled?: boolean | null; }; /** Query filters accepted by the list route. Unknown values are rejected, not dropped. */ diff --git a/backend/src/services/gitops/recoveryCapture.ts b/backend/src/services/gitops/recoveryCapture.ts index 5e659689..f580c05a 100644 --- a/backend/src/services/gitops/recoveryCapture.ts +++ b/backend/src/services/gitops/recoveryCapture.ts @@ -1,3 +1,4 @@ +import type { GitOpsApplicationRow, GitOpsTargetCurrentRow } from './types'; import { GitOpsStore } from './store'; export type GitOpsRecoveryCapture = { @@ -13,22 +14,28 @@ export const EMPTY_GITOPS_RECOVERY_CAPTURE: GitOpsRecoveryCapture = { }; /** - * Bind a rollback point to the generation that is actually deployed on this - * target. A generation that was applied but never deployed is not rollback - * identity, so only `deployed_generation_id` is read. + * Bind a recovery point to the generation the node is actually running. + * + * Direct mode reads `deployed_generation_id`, because a generation that was + * applied but never deployed is not rollback identity there. Blueprint mode + * has no deploy-bound writer (nothing resolves a Direct application for a + * Blueprint-managed stack), so the ack's `applied_generation_id` is the + * pointer that names what the node acknowledged running. * * The acceptance reference prefers the one recorded on the target and * otherwise falls back to the newest acceptance *of that same generation*, so * an acceptance belonging to a later generation can never be captured. Any - * candidate that does not resolve against the deployed generation is stored as - * null rather than as a reference the restore path would have to trust. + * candidate that does not resolve against that generation is stored as null + * rather than as a reference the restore path would have to trust. */ -export function captureGitOpsRecoveryBinding(stackName: string, nodeId: number): GitOpsRecoveryCapture { +export function recoveryBindingForTarget( + application: GitOpsApplicationRow, + target: GitOpsTargetCurrentRow | undefined, +): GitOpsRecoveryCapture { const store = GitOpsStore.getInstance(); - const application = store.getLiveDirectApplication(stackName); - if (!application) return { ...EMPTY_GITOPS_RECOVERY_CAPTURE }; - const target = store.getTarget(application.id, nodeId); - const generationId = target?.deployed_generation_id ?? null; + const generationId = application.target_mode === 'direct' + ? (target?.deployed_generation_id ?? null) + : (target?.applied_generation_id ?? null); if (!generationId) return { ...EMPTY_GITOPS_RECOVERY_CAPTURE }; let artifactSetId: string | null = null; @@ -60,3 +67,20 @@ export function captureGitOpsRecoveryBinding(stackName: string, nodeId: number): gitops_source_acceptance_ref: sourceAcceptanceRef, }; } + +/** + * The binding for the stack a node reports, resolved from the application that + * owns it. + * + * Both modes resolve: a Direct stack by its own name, and a Blueprint-managed + * stack through the live application whose current intent deploys it. A + * Blueprint target's rollback point has to name the generation the node was + * running before the deploy, or a rollout rollback cannot select it. + */ +export function captureGitOpsRecoveryBinding(stackName: string, nodeId: number): GitOpsRecoveryCapture { + const store = GitOpsStore.getInstance(); + const application = store.getLiveDirectApplication(stackName) + ?? store.getLiveBlueprintApplicationByDeployStack(stackName); + if (!application) return { ...EMPTY_GITOPS_RECOVERY_CAPTURE }; + return recoveryBindingForTarget(application, store.getTarget(application.id, nodeId)); +} diff --git a/backend/src/services/gitops/rolloutRecovery.ts b/backend/src/services/gitops/rolloutRecovery.ts new file mode 100644 index 00000000..523df444 --- /dev/null +++ b/backend/src/services/gitops/rolloutRecovery.ts @@ -0,0 +1,376 @@ +/** + * Rollout-scoped recovery execution for Blueprint targets. + * + * The GitOps model owns the decision state (which generation, which targets), + * and the node owns the restore evidence: its captured recovery point is what + * proves the node was running the generation a rollback selects. This module + * therefore never decides what to restore. It selects the recovery point that + * names the requested application generation and asks the owning node to + * compensate from it, then reports what actually happened per target. + * + * A local target is restored in-process under the same stack operation lock + * the stacks rollback route holds. A remote target is restored through that + * node's own `POST /api/stacks/:stackName/rollback` with the same + * generation-selection contract, so both paths enforce identical evidence + * rules. + */ +import axios from 'axios'; +import type { PermissionAction } from '../../middleware/permissions'; +import { DatabaseService } from '../DatabaseService'; +import { NodeRegistry } from '../NodeRegistry'; +import { LicenseService } from '../LicenseService'; +import { StackUpdateRecoveryService } from '../StackUpdateRecoveryService'; +import { StackOpLockService } from '../StackOpLockService'; +import { ComposeService } from '../ComposeService'; +import { sanitizeForLog } from '../../utils/safeLog'; +import { safeAxiosTransport } from '../../utils/outboundTarget'; +import { formatScopedStackActionsHeader } from '../../helpers/stackRouteAuth'; +import { + PROXY_DEPLOY_ACTOR_HEADER, + PROXY_DEPLOY_SOURCE_HEADER, + PROXY_ROLE_HEADER, + PROXY_SCOPED_STACK_ACTIONS_HEADER, + PROXY_SCOPED_STACK_NAME_HEADER, + PROXY_TIER_HEADER, +} from '../license-headers'; +import type { PolicyEnforcementOptions } from '../PolicyEnforcement'; +import type { GitOpsApplicationRow, GitOpsTargetCurrentRow } from './types'; +import { GitOpsStore } from './store'; +import { decodeGitOpsRequiredTargetsJson } from './json'; + +const REMOTE_ROLLBACK_TIMEOUT_MS = 10 * 60_000; + +export type RolloutRollbackScope = + | { kind: 'target'; nodeId: number } + | { kind: 'failed' } + | { kind: 'all_changed' }; + +export type RolloutRollbackTargetResult = { + nodeId: number; + status: 'restored' | 'failed'; + error?: string; +}; + +export type RestoreTargetOutcome = + | { ok: true } + | { ok: false; code: string; error: string }; + +/** + * The frozen target set the current rollout was authorized against. + * + * The live authorization binding is authoritative while it resolves. A + * superseded generation whose pointer is still on the application keeps + * naming the set the operator reviewed, so a rollback can act on the targets + * that actually moved. An invalidation clears the pointer, and then there is + * no set to recover. + */ +export function rolloutTargetSet(app: GitOpsApplicationRow): { + nodeIds: number[]; + acceptedGenerationId: string | null; +} | null { + const store = GitOpsStore.getInstance(); + const binding = store.currentAuthorizationBinding(app); + if (binding) { + return { nodeIds: [...binding.requiredNodeIds], acceptedGenerationId: binding.acceptedGenerationId }; + } + if (!app.rollout_generation_id) return null; + const generation = store.getRolloutGeneration(app.rollout_generation_id); + if (!generation) return null; + try { + return { + nodeIds: decodeGitOpsRequiredTargetsJson(generation.required_targets_json).nodeIds, + acceptedGenerationId: generation.accepted_generation_id, + }; + } catch (error) { + console.error( + '[GitOps recovery] Rollout generation required set is unreadable:', + sanitizeForLog(app.id), + sanitizeForLog(error instanceof Error ? error.message : String(error)), + ); + return null; + } +} + +/** Targets whose last recorded state names a failure rather than a result. */ +function isFailedTarget(target: GitOpsTargetCurrentRow): boolean { + return target.failure_stage === 'blueprint_deploy' + || target.recovery_phase === 'failed' + || target.connectivity === 'unreachable' + || target.connectivity === 'stale'; +} + +/** + * Prior application generations a rollback can select, from the hub's own + * rollout-generation history. + * + * The newest first, excluding the generation the application currently asks + * for: there is nothing to restore when a target already runs it. A lifecycle + * that only ever authorized one generation has no candidates, which is the + * truthful answer for a first rollout. + */ +export function rollbackCandidatesForApplication(applicationId: string): Array<{ + generationId: string; + rolloutGenerationId: string; + createdAt: number; +}> { + const store = GitOpsStore.getInstance(); + const app = store.getApplication(applicationId); + if (!app || app.target_mode !== 'blueprint') return []; + // No frozen set means the backend would refuse every rollback, so offering + // the generations would only produce a 409. + if (!rolloutTargetSet(app)) return []; + const seen = new Set(); + if (app.accepted_generation_id) seen.add(app.accepted_generation_id); + const candidates: Array<{ generationId: string; rolloutGenerationId: string; createdAt: number }> = []; + for (const generation of store.listRolloutGenerationsForApplication(applicationId)) { + const generationId = generation.accepted_generation_id; + if (!generationId || seen.has(generationId)) continue; + seen.add(generationId); + candidates.push({ + generationId, + rolloutGenerationId: generation.id, + createdAt: generation.created_at, + }); + if (candidates.length >= 10) break; + } + return candidates; +} + +/** + * Resolve the requested scope against the frozen set. + * + * Returns the node ids in the order the frozen set lists them, so a sequential + * restore follows the same order the rollout did. A target already running the + * generation the rollback selects is not a recovery, so it is not in scope. + */ +export function resolveRollbackTargets( + app: GitOpsApplicationRow, + scope: RolloutRollbackScope, + restoreGenerationId: string, +): { ok: true; nodeIds: number[] } | { ok: false; code: string; error: string } { + const store = GitOpsStore.getInstance(); + const frozen = rolloutTargetSet(app); + if (!frozen || frozen.nodeIds.length === 0) { + return { + ok: false, + code: 'ROLLBACK_UNAVAILABLE', + error: 'There is no current rollout target set to recover.', + }; + } + const live = frozen.nodeIds + .map((nodeId) => store.getTarget(app.id, nodeId)) + .filter((target): target is GitOpsTargetCurrentRow => !!target + && target.target_status === 'active' + && target.applied_generation_id !== restoreGenerationId); + + if (scope.kind === 'target') { + const match = live.find((target) => target.node_id === scope.nodeId); + if (!match) { + const targetRow = store.getTarget(app.id, scope.nodeId); + const isActive = !!targetRow && targetRow.target_status === 'active'; + if (isActive && frozen.nodeIds.includes(scope.nodeId)) { + return { + ok: false, + code: 'TARGET_ALREADY_CURRENT', + error: `Node ${scope.nodeId} already runs the selected generation.`, + }; + } + return { + ok: false, + code: 'TARGET_NOT_IN_ROLLOUT', + error: `Node ${scope.nodeId} is not an active target of the current rollout.`, + }; + } + return { ok: true, nodeIds: [scope.nodeId] }; + } + + if (scope.kind === 'failed') { + const failed = live.filter(isFailedTarget).map((target) => target.node_id); + if (failed.length === 0) { + return { + ok: false, + code: 'NO_FAILED_TARGETS', + error: 'No target in the current rollout is recorded as failed, unreachable, or stale.', + }; + } + return { ok: true, nodeIds: frozen.nodeIds.filter((nodeId) => failed.includes(nodeId)) }; + } + + const changed = frozen.nodeIds.filter((nodeId) => live.some((target) => target.node_id === nodeId)); + if (changed.length === 0) { + return { + ok: false, + code: 'ROLLBACK_UNAVAILABLE', + error: 'No live target in the current rollout needs the selected generation.', + }; + } + return { ok: true, nodeIds: changed }; +} + +function policyOptions(actor: string | null, applicationId: string): PolicyEnforcementOptions { + return { + bypass: false, + actor: actor ?? 'system:gitops-rollback', + auditMethod: 'POST', + auditPath: `/api/gitops/applications/${applicationId}/rollout/rollback`, + }; +} + +function errorCode(error: unknown): string { + const code = (error as { code?: unknown } | null)?.code; + return typeof code === 'string' && code.length > 0 ? code : 'ROLLBACK_FAILED'; +} + +function errorText(error: unknown): string { + return error instanceof Error && error.message.length > 0 ? error.message : 'The restore did not complete.'; +} + +function restoreLocal(args: { + nodeId: number; + stackName: string; + generationId: string; + app: GitOpsApplicationRow; + actor: string | null; +}): Promise { + return StackOpLockService.getInstance() + .runExclusive(args.nodeId, args.stackName, 'rollback', args.actor ?? 'system', async () => { + const recovery = StackUpdateRecoveryService.getInstance(); + const current = recovery.getCurrent(args.nodeId, args.stackName); + if (!current) { + return { + ok: false as const, + code: 'NO_RECOVERY_POINT', + error: `The node holds no recovery point for "${args.stackName}".`, + }; + } + if (current.gitops_generation_id !== args.generationId) { + return { + ok: false as const, + code: 'RECOVERY_POINT_MISMATCH', + error: 'The recovery point on this node restores a different generation than the one selected.', + }; + } + try { + const rolledBack = await recovery.compensateWithCandidate( + current.id, + (overridePath, invocation, overlay) => ComposeService.getInstance(args.nodeId) + .composeUpWithRecoveryOverride(args.stackName, overridePath, undefined, invocation, overlay), + policyOptions(args.actor, args.app.id), + ); + if (!rolledBack) { + return { + ok: false as const, + code: 'ROLLBACK_FAILED', + error: 'The restore did not complete.', + }; + } + return { ok: true as const }; + } catch (error) { + console.error( + '[GitOps recovery] Local restore failed for %s:', + sanitizeForLog(args.stackName), + sanitizeForLog(error instanceof Error ? error.message : String(error)), + ); + return { ok: false as const, code: errorCode(error), error: errorText(error) }; + } + }) + .then((result) => { + if (!result.ran) { + return { + ok: false as const, + code: 'STACK_BUSY', + error: `Another operation is already running on "${args.stackName}".`, + }; + } + return result.result; + }); +} + +async function restoreRemote(args: { + nodeId: number; + stackName: string; + generationId: string; + actor: string | null; + role: string; + scopedActions: readonly PermissionAction[]; +}): Promise { + const target = NodeRegistry.getInstance().getProxyTarget(args.nodeId); + if (!target) { + return { ok: false, code: 'NODE_UNREACHABLE', error: 'The owning node is unreachable.' }; + } + const baseUrl = target.apiUrl.replace(/\/$/, ''); + const headers: Record = { + [PROXY_TIER_HEADER]: LicenseService.getInstance().getProxyHeaders().tier, + [PROXY_ROLE_HEADER]: args.role, + [PROXY_DEPLOY_SOURCE_HEADER]: 'manual', + [PROXY_SCOPED_STACK_NAME_HEADER]: args.stackName, + [PROXY_SCOPED_STACK_ACTIONS_HEADER]: formatScopedStackActionsHeader(args.scopedActions), + }; + if (args.actor) headers[PROXY_DEPLOY_ACTOR_HEADER] = args.actor; + if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`; + + try { + const res = await axios.post( + `${baseUrl}/api/stacks/${encodeURIComponent(args.stackName)}/rollback`, + { expectedGitopsGenerationId: args.generationId }, + { + ...safeAxiosTransport(target.trustedLoopback), + headers, + timeout: REMOTE_ROLLBACK_TIMEOUT_MS, + validateStatus: () => true, + }, + ); + const body = res.data as { error?: unknown; code?: unknown; gitopsGenerationId?: unknown } | null; + if (res.status >= 200 && res.status < 300) { + // The node must confirm the restored generation. A node that does not + // know the guard (an older build) would ignore it and answer 200 for + // whatever its current recovery point is, so an absent or different + // echo fails closed rather than claiming the requested restore. + if (body?.gitopsGenerationId !== args.generationId) { + return { + ok: false, + code: 'RECOVERY_POINT_MISMATCH', + error: 'The node did not confirm restoring the requested application generation.', + }; + } + return { ok: true }; + } + const code = typeof body?.code === 'string' && body.code.length > 0 + ? body.code + : res.status === 403 ? 'PERMISSION_DENIED' : 'ROLLBACK_FAILED'; + const message = typeof body?.error === 'string' && body.error.length > 0 + ? body.error + : `The node answered HTTP ${res.status} to the restore request.`; + return { ok: false, code, error: message }; + } catch (error) { + console.error( + '[GitOps recovery] Remote restore request failed for node %s:', + sanitizeForLog(args.nodeId), + sanitizeForLog(error instanceof Error ? error.message : String(error)), + ); + return { ok: false, code: 'NODE_UNREACHABLE', error: 'The restore request to the owning node failed.' }; + } +} + +/** + * Restore one target to the selected application generation. + * + * `role` and `scopedActions` are the acting user's credential for the remote + * hop; the local path runs under the same stack operation lock a local + * operator's rollback would hold. + */ +export async function restoreTargetToGeneration(args: { + app: GitOpsApplicationRow; + stackName: string; + nodeId: number; + generationId: string; + actor: string | null; + role: string; + scopedActions: readonly PermissionAction[]; +}): Promise { + const node = DatabaseService.getInstance().getNode(args.nodeId); + if (node?.type === 'local') { + return restoreLocal(args); + } + return restoreRemote(args); +} diff --git a/backend/src/services/gitops/store.ts b/backend/src/services/gitops/store.ts index dd62d609..29c61364 100644 --- a/backend/src/services/gitops/store.ts +++ b/backend/src/services/gitops/store.ts @@ -143,6 +143,25 @@ export class GitOpsStore { ).get(blueprintId) as GitOpsApplicationRow | undefined; } + /** + * The live Blueprint application whose current intent deploys this stack. + * + * A Blueprint-managed stack carries no application id on disk, so the stack + * name is the only identity the owning node has. The lookup joins through the + * current intent, which is where the deploy stack name is frozen, so a stack + * left behind by a superseded intent does not resolve to the application that + * no longer asks for it. + */ + getLiveBlueprintApplicationByDeployStack(stackName: string): GitOpsApplicationRow | undefined { + return this.db().prepare( + `SELECT a.* FROM gitops_applications a + JOIN gitops_intent_revisions i ON i.id = a.intent_revision_id + WHERE i.deploy_stack_name = ? + AND a.target_mode IN ('inline_blueprint','blueprint') + AND a.lifecycle_status IN ('active','creating')`, + ).get(stackName) as GitOpsApplicationRow | undefined; + } + /** * The live Blueprint-mode (or demoted inline) application that still holds * the original Direct stack as its credential carrier. @@ -319,6 +338,22 @@ export class GitOpsStore { return this.db().prepare('SELECT * FROM gitops_artifact_sets WHERE id = ?').get(id) as GitOpsArtifactSetRow | undefined; } + /** + * The newest artifact evidence recorded for one generation, if any. + * + * Used when a restore has to rebind an expectation to a generation that is + * no longer the application's current one. The reader decides whether the row + * is usable for that generation; this only answers what exists. + */ + newestArtifactSetIdForGeneration(generationId: string): string | null { + const row = this.db().prepare( + `SELECT id FROM gitops_artifact_sets + WHERE generation_id = ? + ORDER BY evidence_version DESC LIMIT 1`, + ).get(generationId) as { id: string } | undefined; + return row?.id ?? null; + } + getIntentRevision(id: string): GitOpsIntentRevisionRow | undefined { return this.db().prepare('SELECT * FROM gitops_intent_revisions WHERE id = ?').get(id) as GitOpsIntentRevisionRow | undefined; } @@ -331,6 +366,13 @@ export class GitOpsStore { return this.db().prepare('SELECT * FROM gitops_rollout_generations WHERE id = ?').get(id) as GitOpsRolloutGenerationRow | undefined; } + /** Every rollout generation the application ever opened, newest first. */ + listRolloutGenerationsForApplication(applicationId: string): GitOpsRolloutGenerationRow[] { + return this.db().prepare( + 'SELECT * FROM gitops_rollout_generations WHERE application_id = ? ORDER BY created_at DESC', + ).all(applicationId) as GitOpsRolloutGenerationRow[]; + } + getApproval(id: string): GitOpsApprovalRow | undefined { return this.db().prepare('SELECT * FROM gitops_approvals WHERE id = ?').get(id) as GitOpsApprovalRow | undefined; } diff --git a/backend/src/services/gitops/transitions.ts b/backend/src/services/gitops/transitions.ts index 0d41bbd1..3e6a9c50 100644 --- a/backend/src/services/gitops/transitions.ts +++ b/backend/src/services/gitops/transitions.ts @@ -1846,6 +1846,50 @@ export class GitOpsTransitions { ); } + /** + * Withdraw a live rollout authorization on the operator's decision. + * + * The generation stays on the application as the record that a rollout was + * abandoned, while the authorization itself is cleared: a superseded + * generation whose authorization still resolved would let the dispatch + * boundary start it again. A later authorization mints a new generation over + * the same placement and accepted source, which is the only way forward from + * here. + */ + rolloutSuperseded(args: { + applicationId: string; + envelope: EventEnvelope; + }): TransitionResult { + // Read before the transaction only to annotate the history row; the + // guards below re-read and remain authoritative if the row moved. + const named = this.store().getApplication(args.applicationId)?.rollout_generation_id ?? null; + return this.mutateApp( + args.applicationId, + args.envelope, + 'rollout_generation_superseded', + 'superseded', + (app) => { + if (app.lifecycle_status !== 'active') throw new GitOpsTransitionError('application is not live'); + if (!app.rollout_generation_id) { + throw new GitOpsTransitionError('there is no live rollout generation to supersede'); + } + const live = this.store().getRolloutGeneration(app.rollout_generation_id); + if (!live || live.application_id !== app.id) { + throw new GitOpsTransitionError('the live rollout generation could not be read'); + } + if (live.provenance !== 'rollout_authorization') { + throw new GitOpsTransitionError('only an authorized rollout can be superseded'); + } + app.rollout_authorization_ref = null; + app.preflight_fingerprint = null; + // The generation pointer stays: the projection reports the abandoned + // rollout rather than pretending the application never had one. + this.store().markRolloutGenerationSuperseded(app.rollout_generation_id, args.envelope.at); + }, + named ? { rolloutGenerationId: named } : {}, + ); + } + /** * A Blueprint deploy was handed to one node. * @@ -2242,8 +2286,9 @@ export class GitOpsTransitions { * A rollout-scoped rollback started. * * The same columns and the same rules as `recovery_started`; only the trigger - * differs. Direct Git recovery emits the `recovery_*` names, and a later - * rollout producer emits these. Nothing in this PR writes them. + * differs. Direct Git recovery emits the `recovery_*` names; the rollout + * rollback route emits these, because a rollout recovery is not a stack + * update recovery even when it restores the same kind of state. */ rollbackInProgress(args: { applicationId: string; @@ -2344,6 +2389,7 @@ export class GitOpsTransitions { applicationId: string; nodeId: number; recoveryRef: string; + recoveryGenerationId: string; capturedArtifactSetId: string | null; capturedSourceAcceptanceRef: string | null; envelope: EventEnvelope; @@ -2353,7 +2399,7 @@ export class GitOpsTransitions { args.nodeId, args.envelope, 'rollback_completed', - null, + args.recoveryGenerationId, (target) => { const restored = target.recovery_generation_id; if (!restored) { @@ -2649,11 +2695,13 @@ export class GitOpsTransitions { /** * Rebind the artifact expectation to the restored generation. * - * The expectation comes from what the recovery point captured, never from - * what the application expects now: those describe different generations - * after a restore. Latest becomes the newest evidence for the restored - * generation, which is a statement about what has been seen, not an - * acceptance of it. + * The expectation is the captured set when the caller has it (a node + * recovery point), otherwise the strongest set recorded for the restored + * generation. Either way it is never what the application expects now: those + * describe different generations after a restore. A set the generation does + * not own is dropped with a limitation. Latest becomes the newest evidence + * for the restored generation, which is a statement about what has been + * seen, not an acceptance of it. */ private restoreArtifactPointers( target: GitOpsTargetCurrentRow, @@ -2672,12 +2720,8 @@ export class GitOpsTransitions { capturedArtifactSetId && !usable ? capturedArtifactSetId : null, ); - const newest = this.raw().prepare( - `SELECT id FROM gitops_artifact_sets - WHERE generation_id = ? - ORDER BY evidence_version DESC LIMIT 1`, - ).get(generationId) as { id: string } | undefined; - target.latest_artifact_set_id = newest?.id ?? target.expected_artifact_set_id; + target.latest_artifact_set_id = this.store().newestArtifactSetIdForGeneration(generationId) + ?? target.expected_artifact_set_id; } /** @@ -3226,6 +3270,7 @@ export class GitOpsTransitions { generationId?: string; artifactSetId?: string; sourceAcceptanceRef?: string; + rolloutGenerationId?: string; } = {}, ): TransitionResult { return this.raw().transaction(() => { @@ -3243,6 +3288,7 @@ export class GitOpsTransitions { generationId: extraHistory.generationId, artifactSetId: extraHistory.artifactSetId, sourceAcceptanceRef: extraHistory.sourceAcceptanceRef, + rolloutGenerationId: extraHistory.rolloutGenerationId, before, after: snapshotApp(app), }); diff --git a/docs/features/blueprint-model.mdx b/docs/features/blueprint-model.mdx index 9e623b3d..fc85f184 100644 --- a/docs/features/blueprint-model.mdx +++ b/docs/features/blueprint-model.mdx @@ -227,6 +227,27 @@ A disabled Blueprint withholds placement approval and rollout authorization unti The **Apply now** flow stays Inline-only. For a Git-managed Blueprint the rollout preview is a placement review: its confirm writes the placement approval instead of applying the frozen snapshot. +### Pause, resume, replan, or supersede a rollout + +A live rollout also offers lifecycle controls on the same GitOps section, on the **Rollout controls** row: + +- **Pause rollout** stops the rollout from advancing and asks for a reason. Nodes keep running what they already deployed, and a paused rollout never reads as converged. +- **Resume rollout** clears the pause and continues the sequential rollout when it is still authorized. If the authorization is no longer live, the surface says so instead of starting anything. +- **Replan placement** re-derives the Blueprint's target nodes and opens a fresh placement review, which withdraws the existing placement approval and rollout authorization. It appears in the overflow menu and refuses when the current placement already matches the Blueprint, so a review that is still accurate is not invalidated for nothing. +- **Supersede rollout** withdraws the rollout authorization and marks that rollout generation superseded. Nodes that already received it keep running it; the rollout reads as superseded until you authorize a new one. + +Rollout authorization needs resolved executable artifact evidence for the accepted generation. While that evidence is unresolved, the authorization is refused with the artifact identity as the reason, and the lifecycle controls do not work around it. + +### Roll back a rollout + +Every Git-managed rollout deploy captures the node's previous project as a recovery generation before it applies anything, which is what a later rollback restores from. **Roll back rollout** lists the application's earlier generations, newest first, and asks for the scope: one target, the targets recorded as failed, unreachable, or stale, or every changed target in the rollout. A target already running the selected generation is skipped. + +Each node in the scope must hold a recovery generation that names the selected application generation. A node keeps the generation captured immediately before its latest rollout, so an older selection has no recovery generation to restore from and is reported as a failed target; the rollout then reads as a partially failed rollback rather than a completion. + +Rollback restores the authored project. It does not restore application data, and a locally built image is rebuilt from its authored inputs unless the node's recovery generation still holds the previous image. The per-target result states what actually happened. + +A Blueprint with drift mode **enforce** keeps enforcing its current revision, so it will re-apply the accepted generation on the next reconciliation after a rollback. Switch that Blueprint to observe or suggest while a rollback should stand. + ### Detach Git-managed content **Detach Git** restores Inline editing from the frozen snapshot. Later Git commits are not written back. Detach is refused when that snapshot is empty (no generation exists to restore as Inline content). After detach, **Apply now** and compose editing return. diff --git a/docs/features/gitops-portfolio.mdx b/docs/features/gitops-portfolio.mdx index 2a39ddd8..61685e49 100644 --- a/docs/features/gitops-portfolio.mdx +++ b/docs/features/gitops-portfolio.mdx @@ -48,7 +48,7 @@ Selecting an application name in the list, or an entry in the attention queue, o - **Identity**: the target (Direct node or Blueprint), the repository and ref, the desired and fetched commits, the candidate and accepted source generations, and for a Blueprint application the rollout generation. - **Attention**: every reason the application is waiting on an operator, each with its one-line explanation. -- **Application state**: the three authority steps as separate chips (**source accepted**, **placement approved**, **rollout authorized**, plus a **legacy combined approval** chip when one was recorded), then the source, artifact, placement, and rollout state cards. For a Git-managed Blueprint application, the step that is outstanding appears here as an action: **Accept source**, **Review and approve placement**, or **Authorize rollout**. +- **Application state**: the three authority steps as separate chips (**source accepted**, **placement approved**, **rollout authorized**, plus a **legacy combined approval** chip when one was recorded), then the source, artifact, placement, and rollout state cards. For a Git-managed Blueprint application, the step that is outstanding appears here as an action: **Accept source**, **Review and approve placement**, or **Authorize rollout**. A live rollout also offers **Pause rollout** or **Resume rollout**, with replan, supersede, and rollback in the overflow menu. - **Targets**: one card per node, with its runtime state, health, connectivity, last known good state, the deployed generation, and the artifact identity observed running there. A target is never folded into an aggregate; a node reporting a state this Sencho build does not recognize still gets a card that says so. - **Drift**: each confirmed divergence, expected versus observed. - Anything Sencho could not prove about the state shown, stated last. @@ -73,4 +73,4 @@ The portfolio refreshes on GitOps transitions, not on a timer. When a node canno - Verifying a rollout or webhook pipeline end to end, across stacks and nodes. - Finding drift concentrated on a node or in one class, for example all runtime drift across the fleet at once. -Single-application actions (reviewing a candidate, authorizing a rollout, recovering) keep their home on the owning stack or Blueprint surfaces; the application view shows the full state and links there instead of duplicating them. +A Git-managed Blueprint application can be acted on in place from its state section: the outstanding authority step, the rollout lifecycle controls, and rollout rollback. Everything else keeps its home on the owning stack or Blueprint surface, and the view links there instead of duplicating it. diff --git a/frontend/src/components/blueprints/BlueprintDetail.tsx b/frontend/src/components/blueprints/BlueprintDetail.tsx index db6807fb..5757f520 100644 --- a/frontend/src/components/blueprints/BlueprintDetail.tsx +++ b/frontend/src/components/blueprints/BlueprintDetail.tsx @@ -4,6 +4,7 @@ import { SystemSheet, SheetSection } from '@/components/ui/system-sheet'; import GitOpsStateCard, { GitOpsFaultCard } from '@/components/gitops/GitOpsStateCard'; import GitOpsApprovalChips from '@/components/gitops/GitOpsApprovalChips'; import GitOpsAuthorityActions from '@/components/gitops/GitOpsAuthorityActions'; +import GitOpsRolloutControls from '@/components/gitops/GitOpsRolloutControls'; import GitOpsCaveats from '@/components/gitops/GitOpsCaveats'; import { blueprintApplicationId } from '@/lib/gitopsAuthorityApi'; import { ROLLOUT_STATE_LOOKUP, absentFault, liveCaveats, livePlacementFacet, liveRolloutFacet, placementStateMeta } from '@/lib/gitopsState'; @@ -389,6 +390,14 @@ export function BlueprintDetail({ blueprintId, open, onOpenChange, onChanged, ca can={can} blueprintEnabled={blueprint.enabled} /> + {gitopsFaults.length > 0 && } diff --git a/frontend/src/components/gitops/GitOpsRolloutControls.test.tsx b/frontend/src/components/gitops/GitOpsRolloutControls.test.tsx new file mode 100644 index 00000000..3792289f --- /dev/null +++ b/frontend/src/components/gitops/GitOpsRolloutControls.test.tsx @@ -0,0 +1,353 @@ +/** + * The rollout lifecycle control row: which control each rollout facet offers, + * who may see it, and what the confirmed writes report back. + */ +import type { ReactNode } from 'react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { render, screen, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { facets, liveRevision, noApprovals, plainSource, target } from '@/__tests__/gitopsFixtures'; +import type { GitOpsRevisionLive, RolloutFacet } from '@/types/gitops'; +import type { PermissionAction } from '@/context/AuthContext'; + +// The overflow menu is the design's home for the rare and destructive +// controls; this suite drives the controls themselves, so the menu renders its +// items inline instead of negotiating Radix pointer events in jsdom. +vi.mock('@/components/ui/dropdown-menu', () => ({ + DropdownMenu: ({ children }: { children: ReactNode }) =>
{children}
, + DropdownMenuTrigger: ({ children }: { children: ReactNode }) => <>{children}, + DropdownMenuContent: ({ children }: { children: ReactNode }) =>
{children}
, + DropdownMenuItem: ({ children, onSelect, ...rest }: { children: ReactNode; onSelect?: () => void }) => ( + + ), +})); + +vi.mock('@/lib/gitopsAuthorityApi', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + pauseGitOpsRollout: vi.fn(), + resumeGitOpsRollout: vi.fn(), + replanGitOpsRollout: vi.fn(), + supersedeGitOpsRollout: vi.fn(), + rollbackGitOpsRollout: vi.fn(), + }; +}); + +vi.mock('@/components/ui/toast-store', () => ({ + toast: { error: vi.fn(), success: vi.fn(), warning: vi.fn(), info: vi.fn(), loading: vi.fn(), dismiss: vi.fn() }, +})); + +import { + pauseGitOpsRollout, + replanGitOpsRollout, + resumeGitOpsRollout, + rollbackGitOpsRollout, + supersedeGitOpsRollout, +} from '@/lib/gitopsAuthorityApi'; +import { toast } from '@/components/ui/toast-store'; +import GitOpsRolloutControls from './GitOpsRolloutControls'; + +function blueprintRevision(rollout: RolloutFacet, targets = [target()]): GitOpsRevisionLive { + return liveRevision({ + targetMode: 'blueprint', + applicationId: 'app-1', + stackName: null, + blueprintId: 5, + approvals: noApprovals, + targets, + facets: facets({ + source: plainSource('application_generation_accepted', { candidateGenerationId: null }), + placement: { status: 'blueprint_bound', completion: 'unknown' }, + rollout, + }), + }); +} + +function renderControls( + projection: GitOpsRevisionLive, + can: (action: PermissionAction) => boolean = () => true, + blueprintEnabled = true, + onChanged: () => void = () => {}, + rollbackGenerations?: Array<{ generationId: string }>, +) { + return render( + , + ); +} + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe('GitOpsRolloutControls', () => { + it('pauses a queued rollout with the operator reason', async () => { + const user = userEvent.setup(); + const onChanged = vi.fn(); + vi.mocked(pauseGitOpsRollout).mockResolvedValue(undefined); + renderControls( + blueprintRevision({ status: 'rollout_queued', rolloutGenerationId: 'rgen-1' }), + () => true, + true, + onChanged, + ); + + await user.click(screen.getByTestId('gitops-action-pause-rollout')); + await user.type(screen.getByTestId('gitops-pause-reason'), 'maintenance window'); + await user.click(screen.getByTestId('gitops-confirm-pause')); + + await waitFor(() => expect(pauseGitOpsRollout).toHaveBeenCalledWith('bp:5', { reason: 'maintenance window' })); + expect(toast.success).toHaveBeenCalledWith('Rollout paused'); + expect(onChanged).toHaveBeenCalled(); + }); + + it('resumes a paused rollout and reports a started dispatch', async () => { + const user = userEvent.setup(); + const onChanged = vi.fn(); + vi.mocked(resumeGitOpsRollout).mockResolvedValue({ dispatched: true, note: null }); + renderControls( + blueprintRevision({ status: 'rollout_paused', pauseAt: 1, pauseReason: 'hold' }), + () => true, + true, + onChanged, + ); + + await user.click(screen.getByTestId('gitops-action-resume-rollout')); + await waitFor(() => expect(resumeGitOpsRollout).toHaveBeenCalledWith('bp:5')); + expect(toast.success).toHaveBeenCalledWith('Rollout resumed and started'); + expect(onChanged).toHaveBeenCalled(); + }); + + it('keeps a resumed rollout visible when nothing started', async () => { + const user = userEvent.setup(); + vi.mocked(resumeGitOpsRollout).mockResolvedValue({ + dispatched: false, + note: 'The rollout is resumed, but no live authorization exists; authorize the rollout to start it.', + }); + renderControls(blueprintRevision({ status: 'rollout_paused', pauseAt: 1, pauseReason: 'hold' })); + await user.click(screen.getByTestId('gitops-action-resume-rollout')); + await waitFor(() => expect(toast.warning).toHaveBeenCalledWith( + 'The rollout is resumed, but no live authorization exists; authorize the rollout to start it.', + )); + }); + + it('replans placement through its confirmation', async () => { + const user = userEvent.setup(); + const onChanged = vi.fn(); + vi.mocked(replanGitOpsRollout).mockResolvedValue(undefined); + renderControls( + blueprintRevision({ status: 'rollout_not_executable', rolloutCandidateId: 'cand-1' }), + () => true, + true, + onChanged, + ); + + await user.click(screen.getByTestId('gitops-action-replan')); + await user.click(screen.getByRole('button', { name: 'Replan placement' })); + await waitFor(() => expect(replanGitOpsRollout).toHaveBeenCalledWith('bp:5')); + expect(toast.success).toHaveBeenCalledWith('Placement review reopened'); + }); + + it('supersedes the live rollout through its destructive confirmation', async () => { + const user = userEvent.setup(); + const onChanged = vi.fn(); + vi.mocked(supersedeGitOpsRollout).mockResolvedValue(undefined); + renderControls( + blueprintRevision({ status: 'rollout_queued', rolloutGenerationId: 'rgen-1' }), + () => true, + true, + onChanged, + ); + + await user.click(screen.getByTestId('gitops-action-supersede')); + await user.click(screen.getByRole('button', { name: 'Supersede rollout' })); + await waitFor(() => expect(supersedeGitOpsRollout).toHaveBeenCalledWith('bp:5')); + expect(onChanged).toHaveBeenCalled(); + }); + + it('rolls back to the hub-known prior generation', async () => { + const user = userEvent.setup(); + const onChanged = vi.fn(); + vi.mocked(rollbackGitOpsRollout).mockResolvedValue({ + ok: true, + results: [{ nodeId: 1, status: 'restored' }], + }); + renderControls( + blueprintRevision({ status: 'rollout_queued', rolloutGenerationId: 'rgen-1' }), + () => true, + true, + onChanged, + [{ generationId: 'gen-lkg' }], + ); + + await user.click(screen.getByTestId('gitops-action-rollback')); + expect(screen.getByTestId('gitops-rollback-caveats')).toHaveTextContent( + /Only the generation captured immediately before each node's latest rollout can be restored there/, + ); + await user.click(screen.getByTestId('gitops-confirm-rollback')); + await waitFor(() => expect(rollbackGitOpsRollout).toHaveBeenCalledWith('bp:5', { + generationId: 'gen-lkg', + scope: { kind: 'all_changed' }, + })); + expect(toast.success).toHaveBeenCalledWith('Rolled back 1 target'); + expect(onChanged).toHaveBeenCalled(); + }); + + it('reports a partial rollback failure truthfully', async () => { + const user = userEvent.setup(); + vi.mocked(rollbackGitOpsRollout).mockResolvedValue({ + ok: false, + results: [{ nodeId: 1, status: 'failed', error: 'The node holds no recovery point for "bp-app".' }], + }); + renderControls( + blueprintRevision({ status: 'rollout_queued', rolloutGenerationId: 'rgen-1' }), + () => true, + true, + () => {}, + [{ generationId: 'gen-lkg' }], + ); + + await user.click(screen.getByTestId('gitops-action-rollback')); + await user.click(screen.getByTestId('gitops-confirm-rollback')); + await waitFor(() => expect(toast.warning).toHaveBeenCalledWith( + 'Rollback failed on 1 of 1 targets: The node holds no recovery point for "bp-app".', + )); + }); + + it('scopes a rollback to the failed targets', async () => { + const user = userEvent.setup(); + vi.mocked(rollbackGitOpsRollout).mockResolvedValue({ + ok: true, + results: [{ nodeId: 1, status: 'restored' }], + }); + renderControls( + blueprintRevision( + { status: 'rollout_queued', rolloutGenerationId: 'rgen-1' }, + [target({ + runtime: { + status: 'recovery_failed', + recoveryRef: 'rec-1', + recoveryGenerationId: 'gen-lkg', + failureClass: 'partial', + failureAt: 1, + }, + })], + ), + () => true, + true, + () => {}, + [{ generationId: 'gen-lkg' }], + ); + + await user.click(screen.getByTestId('gitops-action-rollback')); + await user.click(screen.getByRole('radio', { name: 'Failed targets' })); + await user.click(screen.getByTestId('gitops-confirm-rollback')); + await waitFor(() => expect(rollbackGitOpsRollout).toHaveBeenCalledWith('bp:5', { + generationId: 'gen-lkg', + scope: { kind: 'failed' }, + })); + }); + + it('scopes a rollback to one target', async () => { + const user = userEvent.setup(); + vi.mocked(rollbackGitOpsRollout).mockResolvedValue({ + ok: true, + results: [{ nodeId: 1, status: 'restored' }], + }); + renderControls( + blueprintRevision({ status: 'rollout_queued', rolloutGenerationId: 'rgen-1' }), + () => true, + true, + () => {}, + [{ generationId: 'gen-lkg' }], + ); + + await user.click(screen.getByTestId('gitops-action-rollback')); + await user.click(screen.getByRole('radio', { name: 'One target' })); + await user.click(screen.getByTestId('gitops-confirm-rollback')); + await waitFor(() => expect(rollbackGitOpsRollout).toHaveBeenCalledWith('bp:5', { + generationId: 'gen-lkg', + scope: { kind: 'target', nodeId: 1 }, + })); + }); + + it('reports a rejected control with the server message', async () => { + const user = userEvent.setup(); + vi.mocked(pauseGitOpsRollout).mockRejectedValue(new Error('The Blueprint is disabled.')); + renderControls(blueprintRevision({ status: 'rollout_queued', rolloutGenerationId: 'rgen-1' })); + + await user.click(screen.getByTestId('gitops-action-pause-rollout')); + await user.type(screen.getByTestId('gitops-pause-reason'), 'waiting'); + await user.click(screen.getByTestId('gitops-confirm-pause')); + await waitFor(() => expect(toast.error).toHaveBeenCalledWith('The Blueprint is disabled.')); + }); + + it('hides replan without stack:create but keeps the deploy controls', () => { + renderControls( + blueprintRevision({ status: 'rollout_queued', rolloutGenerationId: 'rgen-1' }), + action => action !== 'stack:create', + true, + () => {}, + [{ generationId: 'gen-lkg' }], + ); + expect(screen.getByTestId('gitops-action-pause-rollout')).toBeInTheDocument(); + expect(screen.getByTestId('gitops-action-supersede')).toBeInTheDocument(); + expect(screen.getByTestId('gitops-action-rollback')).toBeInTheDocument(); + expect(screen.queryByTestId('gitops-action-replan')).toBeNull(); + }); + + it('offers the hub-known prior generations even when no target reports one', async () => { + const user = userEvent.setup(); + vi.mocked(rollbackGitOpsRollout).mockResolvedValue({ + ok: true, + results: [{ nodeId: 1, status: 'restored' }], + }); + renderControls( + blueprintRevision({ status: 'rollout_queued', rolloutGenerationId: 'rgen-1' }, [target({ lkgGenerationId: null })]), + () => true, + true, + () => {}, + [{ generationId: 'gen-hub-prior' }], + ); + + await user.click(screen.getByTestId('gitops-action-rollback')); + await user.click(screen.getByTestId('gitops-confirm-rollback')); + await waitFor(() => expect(rollbackGitOpsRollout).toHaveBeenCalledWith('bp:5', { + generationId: 'gen-hub-prior', + scope: { kind: 'all_changed' }, + })); + }); + + it('offers no controls without stack:deploy', () => { + renderControls( + blueprintRevision({ status: 'rollout_queued', rolloutGenerationId: 'rgen-1' }), + action => action !== 'stack:deploy' && action !== 'stack:create', + ); + expect(screen.queryByTestId('gitops-rollout-controls')).toBeNull(); + }); + + it('withholds execution controls for a disabled Blueprint', () => { + renderControls( + blueprintRevision({ status: 'rollout_queued', rolloutGenerationId: 'rgen-1' }), + () => true, + false, + () => {}, + [{ generationId: 'gen-lkg' }], + ); + expect(screen.queryByTestId('gitops-action-pause-rollout')).toBeNull(); + expect(screen.queryByTestId('gitops-action-supersede')).toBeNull(); + expect(screen.queryByTestId('gitops-action-rollback')).toBeNull(); + }); + + it('offers no controls for a Direct application', () => { + renderControls(liveRevision({ facets: facets({ placement: { status: 'unbound_direct' } }) })); + expect(screen.queryByTestId('gitops-rollout-controls')).toBeNull(); + }); +}); diff --git a/frontend/src/components/gitops/GitOpsRolloutControls.tsx b/frontend/src/components/gitops/GitOpsRolloutControls.tsx new file mode 100644 index 00000000..e1333963 --- /dev/null +++ b/frontend/src/components/gitops/GitOpsRolloutControls.tsx @@ -0,0 +1,513 @@ +import { useMemo, useState } from 'react'; +import { CirclePause, CirclePlay, MoreHorizontal, Undo2 } from 'lucide-react'; + +import { Button } from '@/components/ui/button'; +import { Combobox } from '@/components/ui/combobox'; +import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from '@/components/ui/dropdown-menu'; +import { Input } from '@/components/ui/input'; +import { Label } from '@/components/ui/label'; +import { ConfirmModal, Modal, ModalBody, ModalFooter, ModalHeader } from '@/components/ui/modal'; +import { SegmentedControl } from '@/components/ui/segmented-control'; +import { toast } from '@/components/ui/toast-store'; +import { + pauseGitOpsRollout, + replanGitOpsRollout, + resumeGitOpsRollout, + rollbackGitOpsRollout, + supersedeGitOpsRollout, + type RolloutRollbackScope, +} from '@/lib/gitopsAuthorityApi'; +import { livePlacementFacet, liveRolloutFacet } from '@/lib/gitopsState'; +import type { PermissionAction } from '@/context/AuthContext'; +import type { GitOpsRevisionProjection, GitOpsTargetProjection } from '@/types/gitops'; + +interface GitOpsRolloutControlsProps { + /** The portfolio identity (`bp:`) the controls write to. */ + applicationId: string; + projection: GitOpsRevisionProjection; + /** Refresh the surface after a write. */ + onChanged: () => void; + can?: (action: PermissionAction) => boolean; + /** + * A disabled Blueprint withholds the execution controls (pause, resume, + * supersede, rollback), the same way it withholds placement approval and + * rollout authorization. Replanning placement stays available: it re-opens a + * review and changes nothing on the fleet. + */ + blueprintEnabled?: boolean; + /** Display name for a target node; falls back to `node `. */ + nodeLabel?: (nodeId: number) => string; + /** + * Prior generations the hub knows for this application, from the + * rollout-generation history on the detail payload. Rollback is offered only + * when this is present and non-empty: the projection carries no generation + * list of its own. + */ + rollbackGenerations?: Array<{ generationId: string }>; +} + +type PendingAction = 'pause' | 'resume'; + +const PAUSABLE_ROLLOUT_STATES = new Set([ + 'rollout_queued', + 'canary_in_progress', + 'batch_in_progress', + 'partially_rolled_out', +]); + +const SUPERSEDABLE_ROLLOUT_STATES = new Set([ + 'rollout_queued', + 'canary_in_progress', + 'batch_in_progress', + 'partially_rolled_out', + 'rollout_paused', +]); + +/** Runtime states that read as a failed target rather than a result. */ +const FAILED_RUNTIME_STATES = new Set([ + 'recovery_failed', + 'recovery_required', + 'failed_previous_workload_intact', + 'failed_after_mutation', +]); + +function errorMessage(error: unknown, fallback: string): string { + return error instanceof Error && error.message.length > 0 ? error.message : fallback; +} + +function targetLooksFailed(target: GitOpsTargetProjection): boolean { + return FAILED_RUNTIME_STATES.has(target.runtime.status) + || target.connectivity === 'unreachable' + || target.connectivity === 'stale'; +} + +/** + * The rollout lifecycle controls for one Git-managed Blueprint application. + * + * Pause and resume are the only first-row actions: they are reversible and + * apply to the rollout that is running now. Replan, supersede, and rollback are + * rare or consequential, so they live behind the overflow menu and each opens + * an explicit confirmation. Every control is offered only from the projection's + * own statuses, and each action is gated by the permission the route requires. + */ +export default function GitOpsRolloutControls({ + applicationId, + projection, + onChanged, + can, + blueprintEnabled = true, + nodeLabel, + rollbackGenerations, +}: GitOpsRolloutControlsProps) { + const allowed = (action: PermissionAction): boolean => can?.(action) === true; + const live = projection.targetMode === 'blueprint' ? projection : null; + const rollout = liveRolloutFacet(projection); + const placement = livePlacementFacet(projection); + const targets = useMemo(() => live?.targets ?? [], [live]); + const [pending, setPending] = useState(null); + const [pauseOpen, setPauseOpen] = useState(false); + const [replanOpen, setReplanOpen] = useState(false); + const [supersedeOpen, setSupersedeOpen] = useState(false); + const [rollbackOpen, setRollbackOpen] = useState(false); + + const canPause = !!live + && blueprintEnabled + && rollout !== null + && PAUSABLE_ROLLOUT_STATES.has(rollout.status) + && allowed('stack:deploy'); + const canResume = !!live + && blueprintEnabled + && rollout?.status === 'rollout_paused' + && allowed('stack:deploy'); + const canReplan = !!live + && placement !== null + && placement.status !== 'unbound_direct' + && allowed('stack:create'); + const canSupersede = !!live + && blueprintEnabled + && rollout !== null + && SUPERSEDABLE_ROLLOUT_STATES.has(rollout.status) + && allowed('stack:deploy'); + const generations = useMemo( + () => (rollbackGenerations ?? []).map(candidate => ({ generationId: candidate.generationId, nodeIds: [] })), + [rollbackGenerations], + ); + const canRollback = !!live + && blueprintEnabled + && generations.length > 0 + && allowed('stack:deploy'); + + if (!canPause && !canResume && !canReplan && !canSupersede && !canRollback) return null; + + async function handlePause(reason: string): Promise { + setPending('pause'); + try { + await pauseGitOpsRollout(applicationId, { reason }); + toast.success('Rollout paused'); + setPauseOpen(false); + onChanged(); + } catch (error) { + toast.error(errorMessage(error, 'Failed to pause the rollout')); + } finally { + setPending(null); + } + } + + async function handleResume(): Promise { + setPending('resume'); + try { + const result = await resumeGitOpsRollout(applicationId); + if (result.dispatched) { + toast.success('Rollout resumed and started'); + } else { + toast.warning(result.note ?? 'Rollout resumed; nothing was dispatched yet.'); + } + onChanged(); + } catch (error) { + toast.error(errorMessage(error, 'Failed to resume the rollout')); + } finally { + setPending(null); + } + } + + async function handleReplan(): Promise { + try { + await replanGitOpsRollout(applicationId); + toast.success('Placement review reopened'); + setReplanOpen(false); + onChanged(); + } catch (error) { + toast.error(errorMessage(error, 'Failed to replan the rollout')); + } + } + + async function handleSupersede(): Promise { + try { + await supersedeGitOpsRollout(applicationId); + toast.success('Rollout superseded'); + setSupersedeOpen(false); + onChanged(); + } catch (error) { + toast.error(errorMessage(error, 'Failed to supersede the rollout')); + } + } + + return ( + <> +
+ + Rollout controls + + {canPause && ( + + )} + {canResume && ( + + )} + {(canReplan || canSupersede || canRollback) && ( + + + + + + {canReplan && ( + setReplanOpen(true)} data-testid="gitops-action-replan"> + Replan placement + + )} + {canSupersede && ( + setSupersedeOpen(true)} data-testid="gitops-action-supersede"> + Supersede rollout + + )} + {canRollback && ( + setRollbackOpen(true)} data-testid="gitops-action-rollback"> + + Roll back rollout + + )} + + + )} +
+ + + + + + + ); +} + +type RollbackCandidate = { generationId: string; nodeIds: number[] }; + +function shortId(id: string): string { + return id.length > 8 ? id.slice(0, 8) : id; +} + +function PauseRolloutDialog({ open, onOpenChange, confirming, onConfirm }: { + open: boolean; + onOpenChange: (open: boolean) => void; + confirming: boolean; + onConfirm: (reason: string) => Promise; +}) { + const [reason, setReason] = useState(''); + const trimmed = reason.trim(); + return ( + + + +
+ + setReason(event.target.value)} + placeholder="waiting for the maintenance window" + className="font-mono text-xs" + maxLength={280} + data-testid="gitops-pause-reason" + /> +
+

+ Nodes keep running what they already deployed. A pause stops the rollout from advancing, nothing more. +

+ + onOpenChange(false)} disabled={confirming}> + Cancel + + )} + primary={( + + )} + /> + + ); +} + +type RollbackScopeKind = RolloutRollbackScope['kind']; + +function RollbackRolloutDialog({ + open, + onOpenChange, + applicationId, + candidates, + targets, + nodeLabel, + onChanged, +}: { + open: boolean; + onOpenChange: (open: boolean) => void; + applicationId: string; + candidates: RollbackCandidate[]; + targets: readonly GitOpsTargetProjection[]; + nodeLabel?: (nodeId: number) => string; + onChanged: () => void; +}) { + const [generationId, setGenerationId] = useState(candidates[0]?.generationId ?? ''); + const [scopeKind, setScopeKind] = useState('all_changed'); + const [targetNodeId, setTargetNodeId] = useState(targets[0]?.nodeId ?? null); + const [confirming, setConfirming] = useState(false); + const label = (nodeId: number): string => nodeLabel?.(nodeId) ?? `node ${nodeId}`; + + const activeTargets = targets.filter(target => !target.tombstoned); + const failedTargets = activeTargets.filter(targetLooksFailed); + const selectedGenerationId = candidates.some(candidate => candidate.generationId === generationId) + ? generationId + : (candidates[0]?.generationId ?? ''); + // Targets can arrive after the dialog mounted, so the selected node follows + // the current list rather than freezing the first empty value. + const selectedTargetNodeId = activeTargets.some(target => target.nodeId === targetNodeId) + ? targetNodeId + : (activeTargets[0]?.nodeId ?? null); + + const scopeOptions = [ + { value: 'all_changed' as const, label: 'All changed targets' }, + ...(failedTargets.length > 0 ? [{ value: 'failed' as const, label: 'Failed targets' }] : []), + { value: 'target' as const, label: 'One target' }, + ]; + + async function handleConfirm(): Promise { + if (selectedGenerationId.length === 0) return; + if (scopeKind === 'target' && selectedTargetNodeId === null) return; + const scope: RolloutRollbackScope = scopeKind === 'target' + ? { kind: 'target', nodeId: selectedTargetNodeId as number } + : { kind: scopeKind }; + setConfirming(true); + try { + const result = await rollbackGitOpsRollout(applicationId, { generationId: selectedGenerationId, scope }); + const restored = result.results.filter(entry => entry.status === 'restored').length; + const failed = result.results.filter(entry => entry.status === 'failed'); + if (result.ok) { + toast.success(`Rolled back ${restored} target${restored === 1 ? '' : 's'}`); + } else { + const firstError = failed[0]?.error; + toast.warning( + firstError + ? `Rollback failed on ${failed.length} of ${result.results.length} targets: ${firstError}` + : `Rollback failed on ${failed.length} of ${result.results.length} targets`, + ); + } + onOpenChange(false); + onChanged(); + } catch (error) { + toast.error(errorMessage(error, 'Failed to roll back the rollout')); + } finally { + setConfirming(false); + } + } + + return ( + + + +
+
+ +
+ ({ + value: candidate.generationId, + label: candidate.nodeIds.length > 0 + ? `${shortId(candidate.generationId)} · ${candidate.nodeIds.map(label).join(', ')}` + : shortId(candidate.generationId), + }))} + value={selectedGenerationId} + onValueChange={setGenerationId} + placeholder="Select a generation" + /> +
+
+
+ + + {scopeKind === 'target' && ( +
+ ({ value: String(target.nodeId), label: label(target.nodeId) }))} + value={selectedTargetNodeId === null ? '' : String(selectedTargetNodeId)} + onValueChange={(next) => setTargetNodeId(Number(next))} + placeholder="Select a target" + /> +
+ )} +

+ {scopeKind === 'failed' + ? `${failedTargets.length} target${failedTargets.length === 1 ? '' : 's'} recorded as failed, unreachable, or stale.` + : scopeKind === 'all_changed' + ? `${activeTargets.length} target${activeTargets.length === 1 ? '' : 's'} currently in the rollout.` + : 'One target recovers from its own captured recovery point.'} +

+
+

+ Rollback restores the authored project captured on each node. It does not restore application data. A locally + built image is rebuilt from its authored inputs unless the node's recovery point still holds the previous + image; the per-target result reports what actually happened. Only the generation captured immediately before + each node's latest rollout can be restored there, so an older selection is reported as a failed target. +

+
+ + onOpenChange(false)} disabled={confirming}> + Cancel + + )} + primary={( + + )} + /> + + ); +} diff --git a/frontend/src/components/gitops/application/GitOpsApplicationView.tsx b/frontend/src/components/gitops/application/GitOpsApplicationView.tsx index 52a0db4f..90d5612d 100644 --- a/frontend/src/components/gitops/application/GitOpsApplicationView.tsx +++ b/frontend/src/components/gitops/application/GitOpsApplicationView.tsx @@ -9,6 +9,7 @@ import { cn } from '@/lib/utils'; import { closeGitOpsApplication, owningSurfaceHandoff } from '../portfolio/portfolioNavigation'; import GitOpsApplicationDetail from './GitOpsApplicationDetail'; import GitOpsAuthorityActions from '@/components/gitops/GitOpsAuthorityActions'; +import GitOpsRolloutControls from '@/components/gitops/GitOpsRolloutControls'; import { useGitOpsApplication, type GitOpsApplicationError } from './useGitOpsApplication'; /** @@ -94,14 +95,29 @@ export function GitOpsApplicationView({ id, className, headerActions }: { +
+ + { + const target = row.targets.find(entry => entry.nodeId === nodeId); + return target?.nodeName ?? `node ${nodeId}`; + }} + /> +
)} /> diff --git a/frontend/src/lib/blueprintsApi.ts b/frontend/src/lib/blueprintsApi.ts index 605036fc..6499ec45 100644 --- a/frontend/src/lib/blueprintsApi.ts +++ b/frontend/src/lib/blueprintsApi.ts @@ -70,6 +70,8 @@ export interface BlueprintSummary extends GitOpsRevisionCarrier { deployments: BlueprintDeployment[]; statusCounts: Partial>; effectiveApproval?: EffectiveApproval; + /** Prior generations a Git-managed rollout rollback can select. */ + rollbackCandidates?: Array<{ generationId: string; rolloutGenerationId: string; createdAt: number }>; } /** diff --git a/frontend/src/lib/gitopsAuthorityApi.ts b/frontend/src/lib/gitopsAuthorityApi.ts index d3903142..b5c3a4b3 100644 --- a/frontend/src/lib/gitopsAuthorityApi.ts +++ b/frontend/src/lib/gitopsAuthorityApi.ts @@ -88,3 +88,86 @@ export async function authorizeGitOpsRollout(applicationId: string): Promise { + await postAuthorityAction( + `/gitops/applications/${encodeURIComponent(applicationId)}/rollout/pause`, + options, + 'Failed to pause the rollout', + ); +} + +/** Resume a paused rollout and continue the queue when one is authorized. */ +export async function resumeGitOpsRollout(applicationId: string): Promise { + const payload = await postAuthorityAction( + `/gitops/applications/${encodeURIComponent(applicationId)}/rollout/resume`, + {}, + 'Failed to resume the rollout', + ); + return { + dispatched: payload.dispatched === true, + note: typeof payload.note === 'string' ? payload.note : null, + }; +} + +/** Re-derive placement for the current Blueprint and open a fresh review. */ +export async function replanGitOpsRollout(applicationId: string): Promise { + await postAuthorityAction( + `/gitops/applications/${encodeURIComponent(applicationId)}/rollout/replan`, + {}, + 'Failed to replan the rollout', + ); +} + +/** Withdraw the live rollout authorization. */ +export async function supersedeGitOpsRollout(applicationId: string): Promise { + await postAuthorityAction( + `/gitops/applications/${encodeURIComponent(applicationId)}/rollout/supersede`, + {}, + 'Failed to supersede the rollout', + ); +} + +/** Restore the selected application generation on the requested target scope. */ +export async function rollbackGitOpsRollout( + applicationId: string, + confirm: { generationId: string; scope: RolloutRollbackScope }, +): Promise { + const payload = await postAuthorityAction( + `/gitops/applications/${encodeURIComponent(applicationId)}/rollout/rollback`, + confirm, + 'Failed to roll back the rollout', + ); + const rawResults = Array.isArray(payload.results) ? payload.results : []; + const results: RolloutRollbackTargetResult[] = []; + for (const entry of rawResults) { + if (typeof entry !== 'object' || entry === null) continue; + const { nodeId, status, error } = entry as { nodeId?: unknown; status?: unknown; error?: unknown }; + if (typeof nodeId !== 'number' || (status !== 'restored' && status !== 'failed')) continue; + results.push({ + nodeId, + status, + ...(typeof error === 'string' ? { error } : {}), + }); + } + return { ok: payload.ok === true, results }; +} diff --git a/frontend/src/lib/gitopsState.ts b/frontend/src/lib/gitopsState.ts index 7a835851..3c4c8341 100644 --- a/frontend/src/lib/gitopsState.ts +++ b/frontend/src/lib/gitopsState.ts @@ -392,7 +392,9 @@ export const ROLLOUT_STATE: Record = { rollout_superseded: { label: 'superseded', tone: 'neutral', - line: 'A newer rollout generation replaced this one.', + // Covers both producers: a newer generation replaced it, or an operator + // withdrew it (supersede, or a rollback that abandoned it). + line: 'This rollout generation is no longer active: an operator withdrew it or a newer generation replaced it.', icon: ArchiveX, }, target_stale: { diff --git a/frontend/src/types/gitopsPortfolio.ts b/frontend/src/types/gitopsPortfolio.ts index 09d347fc..d7ae524c 100644 --- a/frontend/src/types/gitopsPortfolio.ts +++ b/frontend/src/types/gitopsPortfolio.ts @@ -119,6 +119,17 @@ export interface GitOpsPortfolioDetailResponse { generatedAt: number; application: GitOpsPortfolioRow; projection: GitOpsRevisionProjection; + /** For a Git-managed Blueprint: the prior generations a rollout rollback can select. */ + rollbackCandidates?: GitOpsRollbackCandidate[]; + /** The bound Blueprint's enabled state, when the application has one. */ + blueprintEnabled?: boolean | null; +} + +/** A prior application generation a rollout rollback can select. */ +export interface GitOpsRollbackCandidate { + generationId: string; + rolloutGenerationId: string; + createdAt: number; } /** Filter set the list route accepts; mirrored from the backend contract. */ diff --git a/frontend/src/whats-new/entries.json b/frontend/src/whats-new/entries.json index 7d4b28a0..99c36e63 100644 --- a/frontend/src/whats-new/entries.json +++ b/frontend/src/whats-new/entries.json @@ -78,5 +78,11 @@ "title": "Decomposed approval actions", "blurb": "Accept source, approve placement, and authorize rollout as separate actions on a Git-managed Blueprint, so each authority step is reviewed and recorded on its own.", "docUrl": "https://docs.sencho.io/features/blueprint-model#accept-source-approve-placement-authorize-rollout" + }, + { + "id": "gitops-rollout-controls", + "title": "Rollout lifecycle controls", + "blurb": "Pause, resume, replan, supersede, or roll back a Blueprint rollout from the GitOps application view, with an explicit recovery scope and per-target results.", + "docUrl": "https://docs.sencho.io/features/blueprint-model#pause-resume-replan-or-supersede-a-rollout" } ]