diff --git a/backend/src/__tests__/atomic-deploy-hardening.test.ts b/backend/src/__tests__/atomic-deploy-hardening.test.ts index 1683a9e2..cf64ba96 100644 --- a/backend/src/__tests__/atomic-deploy-hardening.test.ts +++ b/backend/src/__tests__/atomic-deploy-hardening.test.ts @@ -83,7 +83,7 @@ afterAll(() => { }); beforeEach(async () => { - mockDeployStack.mockReset().mockResolvedValue({ recoveryId: null }); + mockDeployStack.mockReset().mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); mockGetBackupInfo.mockReset().mockResolvedValue({ exists: true, timestamp: Date.now() }); mockRestoreStackFiles.mockReset().mockResolvedValue(undefined); mockSnapshotStackFiles.mockReset().mockResolvedValue(async () => {}); @@ -96,7 +96,7 @@ afterEach(() => vi.restoreAllMocks()); describe('Rollback holds the stack lifecycle lock (H-1)', () => { it('blocks deploy while a rollback is in flight on the same stack', async () => { mockTier('paid'); - const gate = deferred<{ recoveryId: string | null }>(); + const gate = deferred<{ recoveryId: string | null; deployedGenerationId: string | null }>(); mockDeployStack.mockImplementationOnce(() => gate.promise); const rollback = request(app) @@ -113,14 +113,14 @@ describe('Rollback holds the stack lifecycle lock (H-1)', () => { expect(deploy.body.code).toBe('stack_op_in_progress'); expect(deploy.body.inProgress.action).toBe('rollback'); - gate.resolve({ recoveryId: null }); + gate.resolve({ recoveryId: null, deployedGenerationId: null }); const rollbackRes = await rollback; expect(rollbackRes.status).toBe(200); }); it('returns 409 when a rollback lands while a deploy is in flight', async () => { mockTier('paid'); - const gate = deferred<{ recoveryId: string | null }>(); + const gate = deferred<{ recoveryId: string | null; deployedGenerationId: string | null }>(); mockDeployStack.mockImplementationOnce(() => gate.promise); const deploy = request(app) @@ -136,13 +136,13 @@ describe('Rollback holds the stack lifecycle lock (H-1)', () => { expect(rollback.status).toBe(409); expect(rollback.body.inProgress.action).toBe('deploy'); - gate.resolve({ recoveryId: null }); + gate.resolve({ recoveryId: null, deployedGenerationId: null }); await deploy; }); it('releases the lock after a successful rollback', async () => { mockTier('paid'); - mockDeployStack.mockResolvedValue({ recoveryId: null }); + mockDeployStack.mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const first = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie); expect(first.status).toBe(200); @@ -157,7 +157,7 @@ describe('Rollback holds the stack lifecycle lock (H-1)', () => { const first = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie); expect(first.status).toBe(500); - mockDeployStack.mockResolvedValueOnce({ recoveryId: null }); + mockDeployStack.mockResolvedValueOnce({ recoveryId: null, deployedGenerationId: null }); const second = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie); expect(second.status).toBe(200); }); @@ -166,7 +166,7 @@ describe('Rollback holds the stack lifecycle lock (H-1)', () => { describe('Rollback notifications (M-2)', () => { it('dispatches a success notification when a rollback completes', async () => { mockTier('paid'); - mockDeployStack.mockResolvedValue({ recoveryId: null }); + mockDeployStack.mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const { NotificationService } = await import('../services/NotificationService'); const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); @@ -214,7 +214,7 @@ describe('Rollback returns 404 when no backup exists', () => { // The 404 is an early return inside the try; the finally must still release // the lock so the stack is not wedged at 409 afterwards. mockGetBackupInfo.mockResolvedValue({ exists: true, timestamp: Date.now() }); - mockDeployStack.mockResolvedValue({ recoveryId: null }); + mockDeployStack.mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const next = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie); expect(next.status).toBe(200); }); @@ -223,7 +223,7 @@ describe('Rollback returns 404 when no backup exists', () => { describe('Developer Mode logging matrix', () => { it('only emits rollback diagnostic logs when Developer Mode is enabled', async () => { mockTier('paid'); - mockDeployStack.mockResolvedValue({ recoveryId: null }); + mockDeployStack.mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const { DatabaseService } = await import('../services/DatabaseService'); const db = DatabaseService.getInstance(); const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); @@ -246,7 +246,7 @@ describe('Deploy safety is available on every tier', () => { it('allows rollback on community', async () => { mockTier('community'); mockGetBackupInfo.mockResolvedValue({ exists: true, timestamp: 1700000000000 }); - mockDeployStack.mockResolvedValue({ recoveryId: null }); + mockDeployStack.mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const res = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie); expect(res.status).toBe(200); expect(mockDeployStack).toHaveBeenCalled(); diff --git a/backend/src/__tests__/blueprints-compose-apply.test.ts b/backend/src/__tests__/blueprints-compose-apply.test.ts index 55f4b217..4c764ff9 100644 --- a/backend/src/__tests__/blueprints-compose-apply.test.ts +++ b/backend/src/__tests__/blueprints-compose-apply.test.ts @@ -75,7 +75,7 @@ describe('Blueprint compose apply (real filesystem)', () => { const composeContent = 'services:\n web:\n image: traefik:v3\n'; const markerContent = JSON.stringify({ blueprintId: 1, revision: 1, lastApplied: Date.now() }, null, 2); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const outcome = await BlueprintService.getInstance().applyLocalUnderLock( nodeId, @@ -120,7 +120,7 @@ describe('Blueprint compose apply (real filesystem)', () => { path.join(stackDir, 'docker-compose.yaml'), path.join(stackDir, 'docker-compose.yml'), ); - return { recoveryId: null }; + return { recoveryId: null, deployedGenerationId: null }; }); const outcome = await BlueprintService.getInstance().applyLocalUnderLock( @@ -198,7 +198,7 @@ describe('Blueprint compose apply (real filesystem)', () => { const original = 'services:\n mine:\n image: nginx:alpine\n'; await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), original); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); await expect( BlueprintService.getInstance().applyLocalUnderLock( diff --git a/backend/src/__tests__/blueprints-reconciler-gate.test.ts b/backend/src/__tests__/blueprints-reconciler-gate.test.ts index 18f41c42..da859422 100644 --- a/backend/src/__tests__/blueprints-reconciler-gate.test.ts +++ b/backend/src/__tests__/blueprints-reconciler-gate.test.ts @@ -125,6 +125,84 @@ describe('reconcileOne approval gate (real path)', () => { expect(DatabaseService.getInstance().listDeployments(bp.id)).toEqual([]); }); + /** Approve `bp` for placement on `nodeId`, then sever that placement in + * the canonical model so every auto-decision path sees it as tombstoned. */ + async function seedSeveredPlacement(bpId: number, nodeId: number): Promise { + const db = DatabaseService.getInstance().getDb(); + const bp = DatabaseService.getInstance().getBlueprint(bpId)!; + db.prepare( + `UPDATE blueprints SET approval_status = 'approved', + approved_intent_fingerprint = ?, + approved_blast_json = ? + WHERE id = ?`, + ).run( + intentFingerprint(bp), + serializeApprovedBlast([{ nodeId, outcome: 'place' as const }]), + bpId, + ); + + const { migrateInlineBlueprints } = await import('../services/gitops/migrate'); + const { GitOpsStore, emptyTargetRow } = await import('../services/gitops/store'); + migrateInlineBlueprints(); + const gitopsApp = GitOpsStore.getInstance().getLiveBlueprintApplication(bpId)!; + GitOpsStore.getInstance().upsertTarget({ + ...emptyTargetRow(gitopsApp.id, nodeId, Date.now()), + target_status: 'tombstoned', + }); + } + + function seedDeployment(bpId: number, nodeId: number, status: string, appliedRevision: number | null): void { + DatabaseService.getInstance().getDb().prepare( + `INSERT INTO blueprint_deployments (blueprint_id, node_id, status, applied_revision, last_deployed_at) + VALUES (?, ?, ?, ?, ?)`, + ).run(bpId, nodeId, status, appliedRevision, Date.now()); + } + + it('does not auto-place onto a tombstoned target', async () => { + // A withdraw (or node delete) severs the placement in the model. The + // tick must treat that as authoritative instead of resurrecting the + // workload behind the projection's back; only an explicit deploy + // re-opens the placement. + const node = seedNode(); + const bp = createBp({ nodeIds: [node.id] }); + await seedSeveredPlacement(bp.id, node.id); + + const deploySpy = vi.spyOn(BlueprintService.getInstance(), 'deployToNode').mockResolvedValue({ status: 'active' }); + await BlueprintReconciler.getInstance().reconcileOne(bp.id); + + expect(deploySpy).not.toHaveBeenCalled(); + }); + + it('does not auto-redeploy a stale revision onto a tombstoned target', async () => { + // Severance also blocks the update path: an existing deployment that + // lagged behind the blueprint must wait for an explicit deploy, never + // catch up on its own while the model says the placement is gone. + const node = seedNode(); + const bp = createBp({ nodeIds: [node.id] }); + await seedSeveredPlacement(bp.id, node.id); + seedDeployment(bp.id, node.id, 'active', bp.revision - 1); + + const deploySpy = vi.spyOn(BlueprintService.getInstance(), 'deployToNode').mockResolvedValue({ status: 'active' }); + await BlueprintReconciler.getInstance().reconcileOne(bp.id); + + expect(deploySpy).not.toHaveBeenCalled(); + }); + + it('does not redeploy a failed placement onto a tombstoned target', async () => { + // A failed run on a severed placement is evidence of the severance, not + // a retry request. Redeploying here would undo the withdraw the model + // already recorded. + const node = seedNode(); + const bp = createBp({ nodeIds: [node.id] }); + await seedSeveredPlacement(bp.id, node.id); + seedDeployment(bp.id, node.id, 'failed', bp.revision); + + const deploySpy = vi.spyOn(BlueprintService.getInstance(), 'deployToNode').mockResolvedValue({ status: 'active' }); + await BlueprintReconciler.getInstance().reconcileOne(bp.id); + + expect(deploySpy).not.toHaveBeenCalled(); + }); + it('does not mutate when approval_status is approved but blast JSON is malformed', async () => { const node = seedNode(); const bp = createBp({ nodeIds: [node.id] }); diff --git a/backend/src/__tests__/blueprints.test.ts b/backend/src/__tests__/blueprints.test.ts index 8eadc069..a4ce5cdc 100644 --- a/backend/src/__tests__/blueprints.test.ts +++ b/backend/src/__tests__/blueprints.test.ts @@ -410,7 +410,7 @@ describe('BlueprintService per-stack lock', () => { vi.spyOn(FileSystemService.prototype, 'createStack').mockResolvedValue(undefined); const writeSpy = vi.spyOn(FileSystemService.prototype, 'writeStackFile').mockResolvedValue(undefined); const cleanupSpy = vi.spyOn(FileSystemService.prototype, 'removeAlternateRootComposeFiles').mockResolvedValue(undefined); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const outcome = await BlueprintService.getInstance().deployToNode(bp, node); diff --git a/backend/src/__tests__/community-deploy-policy-route.test.ts b/backend/src/__tests__/community-deploy-policy-route.test.ts index 039bd2f1..988b908f 100644 --- a/backend/src/__tests__/community-deploy-policy-route.test.ts +++ b/backend/src/__tests__/community-deploy-policy-route.test.ts @@ -49,7 +49,7 @@ beforeAll(async () => { const { ComposeService } = await import('../services/ComposeService'); listImagesSpy = vi.spyOn(ComposeService.prototype, 'listStackImages').mockResolvedValue(['nginx:bad']); - deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); + deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const TrivyService = (await import('../services/TrivyService')).default; const trivy = TrivyService.getInstance(); diff --git a/backend/src/__tests__/compose-service.test.ts b/backend/src/__tests__/compose-service.test.ts index ea2bba1e..e9748212 100644 --- a/backend/src/__tests__/compose-service.test.ts +++ b/backend/src/__tests__/compose-service.test.ts @@ -740,7 +740,7 @@ describe('ComposeService - deployStack', () => { const promise = ComposeService.getInstance(1).deployStack('my-stack'); await vi.advanceTimersByTimeAsync(3100); - await expect(promise).resolves.toEqual({ recoveryId: null }); + await expect(promise).resolves.toEqual({ recoveryId: null, deployedGenerationId: null }); expect(mockGetLegacyOrphanContainersByStack).toHaveBeenCalledWith('my-stack'); }); @@ -844,7 +844,7 @@ describe('ComposeService - deployStack', () => { await vi.advanceTimersByTimeAsync(3100); const result = await promise; - expect(result).toEqual({ recoveryId: 'recovery-1' }); + expect(result).toEqual({ recoveryId: 'recovery-1', deployedGenerationId: null }); expect(mockCaptureCandidate).toHaveBeenCalledWith(expect.objectContaining({ stackName: 'my-stack', operationKind: 'deployment', @@ -1221,7 +1221,7 @@ describe('ComposeService - updateStack prune-on-update', () => { // The update already succeeded before the prune ran, so a prune failure // must neither reject nor trigger the atomic restore. - await expect(promise).resolves.toEqual({ recoveryId: 'recovery-1' }); + await expect(promise).resolves.toEqual({ recoveryId: 'recovery-1', deployedGenerationId: null }); expect(mockRestoreStackFiles).not.toHaveBeenCalled(); }); @@ -1235,7 +1235,7 @@ describe('ComposeService - updateStack prune-on-update', () => { const promise = svc.updateStack('my-stack'); await vi.advanceTimersByTimeAsync(3100); - await expect(promise).resolves.toEqual({ recoveryId: 'recovery-1' }); + await expect(promise).resolves.toEqual({ recoveryId: 'recovery-1', deployedGenerationId: null }); }); }); diff --git a/backend/src/__tests__/drift-route.test.ts b/backend/src/__tests__/drift-route.test.ts index f1d531d9..32441ba5 100644 --- a/backend/src/__tests__/drift-route.test.ts +++ b/backend/src/__tests__/drift-route.test.ts @@ -3,23 +3,26 @@ * read-only report is reachable on the Community tier (no tier gate). Deep diff * behaviour is covered by drift-detection.test.ts. */ -import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'; +import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest'; import fs from 'fs'; import path from 'path'; import request from 'supertest'; import jwt from 'jsonwebtoken'; import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb'; +import { directApplicationFixture } from './helpers/gitopsFixtures'; import DockerController from '../services/DockerController'; let tmpDir: string; let app: import('express').Express; let authHeader: string; let LicenseService: typeof import('../services/LicenseService').LicenseService; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; beforeAll(async () => { tmpDir = await setupTestDb(); ({ app } = await import('../index')); ({ LicenseService } = await import('../services/LicenseService')); + ({ DatabaseService } = await import('../services/DatabaseService')); const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' }); authHeader = `Bearer ${token}`; }); @@ -63,3 +66,197 @@ describe('GET /api/stacks/:stackName/drift', () => { fs.rmSync(stackDir, { recursive: true, force: true }); }); }); + +describe('drift payload carries the GitOps revision', () => { + const STACK = 'driftgitopstest'; + + // Cleanup belongs here, not at the end of each test body. A failing + // assertion would otherwise leak a blueprint, a deployment, and an + // application into the next test, which reuses this stack name and + // asserts not_applicable: one real failure would become two, and the + // second would point at innocent code. + afterEach(() => { + vi.restoreAllMocks(); + fs.rmSync(path.join(process.env.COMPOSE_DIR as string, STACK), { recursive: true, force: true }); + const db = DatabaseService.getInstance().getDb(); + for (const table of ['gitops_applications', 'blueprint_deployments', 'blueprints']) { + db.prepare(`DELETE FROM ${table}`).run(); + } + }); + + function defaultNodeId(): number { + const id = DatabaseService.getInstance().getNodes().find(n => n.is_default)?.id; + if (id === undefined) throw new Error('the test database has no default node'); + return id; + } + + /** A Blueprint named after the stack, which is what makes the directory its work. */ + function seedBlueprint(nodeId: number): import('../services/DatabaseService').Blueprint { + return DatabaseService.getInstance().createBlueprint({ + name: STACK, + description: null, + compose_content: 'services:\n web:\n image: nginx:1.27\n', + selector: { type: 'nodes', ids: [nodeId] }, + drift_mode: 'suggest', + classification: 'stateless', + classification_reasons: [], + enabled: true, + created_by: 'admin', + }); + } + + async function activateBlueprintApplication(blueprintId: number, applicationId: string): Promise { + const { GitOpsTransitions } = await import('../services/gitops/transitions'); + const { blankInlineApplication } = await import('../services/gitops/blueprintProducers'); + GitOpsTransitions.getInstance().activateInlineBlueprint({ + application: blankInlineApplication(applicationId, blueprintId, Date.now()), + envelope: { operationId: `op-${applicationId}`, actor: 'tester', trigger: 'manual', at: Date.now() }, + }); + } + + function stubDockerBoundary(): void { + vi.spyOn(DockerController, 'getInstance').mockReturnValue({ + getDependencySnapshot: vi.fn().mockResolvedValue({ containers: [], networks: [], volumes: [] }), + } as unknown as DockerController); + } + + function makeStack(): void { + const stackDir = path.join(process.env.COMPOSE_DIR as string, STACK); + fs.mkdirSync(stackDir, { recursive: true }); + fs.writeFileSync(path.join(stackDir, 'compose.yaml'), 'services:\n web:\n image: nginx:1.27\n'); + } + + it('adds gitopsRevision to the GET without disturbing the ledger fields', async () => { + makeStack(); + stubDockerBoundary(); + + const res = await request(app).get(`/api/stacks/${STACK}/drift`).set('Authorization', authHeader); + expect(res.status).toBe(200); + // A stack with no Git source has no application, so the uniform + // not-applicable shape is what a reader gets rather than a missing key. + expect(res.body.gitopsRevision).toMatchObject({ schemaVersion: 1, targetMode: 'not_applicable' }); + expect(res.body.gitopsRevision.drift).toEqual([]); + // The ledger surface is untouched: this field is additive, not a rewrite. + expect(res.body).toMatchObject({ stack: STACK }); + expect(Array.isArray(res.body.findings)).toBe(true); + expect(Array.isArray(res.body.ledger)).toBe(true); + expect(res.body.temporal).toBeDefined(); + }); + + it('resolves the Blueprint that owns the stack directory, not just Direct Git', async () => { + // A Blueprint application is stored with stack_name NULL, so no lookup by + // stack name reaches it, yet the reconciler materializes the Blueprint as a + // stack directory of that name. Without the deployment bridge the Drift tab + // reports not_applicable for a stack GitOps is actively managing, while the + // Blueprint page reports a live application for the very same thing. + const nodeId = defaultNodeId(); + const blueprint = seedBlueprint(nodeId); + // last_deployed_at is what proves this Blueprint actually wrote the + // directory, which is the predicate the bridge requires. + DatabaseService.getInstance().upsertDeployment({ + blueprint_id: blueprint.id, + node_id: nodeId, + status: 'active', + applied_revision: 1, + last_deployed_at: Date.now(), + }); + await activateBlueprintApplication(blueprint.id, 'app-bp-drift'); + + makeStack(); + stubDockerBoundary(); + const res = await request(app).get(`/api/stacks/${STACK}/drift`).set('Authorization', authHeader); + expect(res.status).toBe(200); + expect(res.body.gitopsRevision).toMatchObject({ + applicationId: 'app-bp-drift', + targetMode: 'inline_blueprint', + blueprintId: blueprint.id, + }); + }); + + it('refuses to claim a stack the Blueprint could not deploy onto', async () => { + // name_conflict is written precisely when a stack of that name already + // exists on the node and Sencho does not own it. A deployment row exists, + // so a present-row check would treat it as ownership and hand the unrelated + // stack's operator this Blueprint's repository, ref, and SHA pointers: the + // exact collision the deployment check is supposed to rule out. + const nodeId = defaultNodeId(); + const blueprint = seedBlueprint(nodeId); + DatabaseService.getInstance().upsertDeployment({ blueprint_id: blueprint.id, node_id: nodeId, status: 'name_conflict' }); + await activateBlueprintApplication(blueprint.id, 'app-bp-conflict'); + + makeStack(); + stubDockerBoundary(); + const res = await request(app).get(`/api/stacks/${STACK}/drift`).set('Authorization', authHeader); + expect(res.status).toBe(200); + expect(res.body.gitopsRevision).toMatchObject({ targetMode: 'not_applicable', applicationId: null }); + }); + + it('says a proven Blueprint owner has no application, instead of answering with another one', async () => { + // The hazard this pins: a stack that once had Direct Git and was detached, + // whose directory a Blueprint later took over, whose application row is + // then lost. Falling through the resolution chain would report the old + // Direct application's repository, ref, and SHA as this directory's state, + // confidently and wrongly. + const nodeId = defaultNodeId(); + const { GitOpsTransitions } = await import('../services/gitops/transitions'); + const tx = GitOpsTransitions.getInstance(); + + const stale = directApplicationFixture('app-stale-direct', STACK); + tx.activateDirect({ + application: stale, + nodeId, + envelope: { operationId: 'op-stale', actor: 'tester', trigger: 'manual', at: Date.now() }, + }); + tx.applicationTombstoned(stale.id, 'detached', { + operationId: 'op-stale-2', actor: 'tester', trigger: 'manual', at: Date.now(), + }); + + const blueprint = seedBlueprint(nodeId); + // Ownership proven by the deployment row, but no application row exists. + DatabaseService.getInstance().upsertDeployment({ + blueprint_id: blueprint.id, + node_id: nodeId, + status: 'active', + applied_revision: 1, + last_deployed_at: Date.now(), + }); + + makeStack(); + stubDockerBoundary(); + const res = await request(app).get(`/api/stacks/${STACK}/drift`).set('Authorization', authHeader); + expect(res.status).toBe(200); + expect(res.body.gitopsRevision).toMatchObject({ targetMode: 'not_applicable', applicationId: null }); + // Not the plain sentinel: the fault is named, and the detached Direct + // application is nowhere in the answer. + expect(res.body.gitopsRevision.limitations).toEqual([ + expect.objectContaining({ code: 'blueprint_application_missing' }), + ]); + expect(JSON.stringify(res.body.gitopsRevision)).not.toContain('app-stale-direct'); + }); + + it('refuses to claim a stack the Blueprint has never deployed', async () => { + // A pending or first-deploy-failed row has nothing of ours on the node + // either, so last_deployed_at is what proves the directory is the + // Blueprint's work. + const nodeId = defaultNodeId(); + const blueprint = seedBlueprint(nodeId); + DatabaseService.getInstance().upsertDeployment({ blueprint_id: blueprint.id, node_id: nodeId, status: 'pending' }); + await activateBlueprintApplication(blueprint.id, 'app-bp-pending'); + + makeStack(); + stubDockerBoundary(); + const res = await request(app).get(`/api/stacks/${STACK}/drift`).set('Authorization', authHeader); + expect(res.status).toBe(200); + expect(res.body.gitopsRevision).toMatchObject({ targetMode: 'not_applicable', applicationId: null }); + }); + + it('adds the same gitopsRevision to the re-check', async () => { + makeStack(); + stubDockerBoundary(); + + const res = await request(app).post(`/api/stacks/${STACK}/drift/recheck`).set('Authorization', authHeader); + expect(res.status).toBe(200); + expect(res.body.gitopsRevision).toMatchObject({ schemaVersion: 1, targetMode: 'not_applicable' }); + expect(Array.isArray(res.body.ledger)).toBe(true); + }); +}); diff --git a/backend/src/__tests__/fleet-snapshot-routes.test.ts b/backend/src/__tests__/fleet-snapshot-routes.test.ts index f44ac149..08324347 100644 --- a/backend/src/__tests__/fleet-snapshot-routes.test.ts +++ b/backend/src/__tests__/fleet-snapshot-routes.test.ts @@ -298,7 +298,7 @@ describe('Single-stack snapshot restore (behavior lock)', () => { fs.writeFileSync(composePath('corrupt-web'), beforeCompose); fs.writeFileSync(envPath('corrupt-web'), beforeEnv); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const res = await request(app) .post(`/api/fleet/snapshots/${id}/restore`) .set('Cookie', adminCookie) @@ -329,7 +329,7 @@ describe('Single-stack snapshot restore (behavior lock)', () => { fs.writeFileSync(composePath('mixed-web'), beforeCompose); fs.writeFileSync(envPath('mixed-web'), beforeEnv); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const res = await request(app) .post(`/api/fleet/snapshots/${id}/restore`) .set('Cookie', adminCookie) @@ -355,7 +355,7 @@ describe('Single-stack snapshot restore (behavior lock)', () => { const beforeCompose = 'services:\n keep: {}\n'; fs.writeFileSync(composePath('delim-web'), beforeCompose); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const res = await request(app) .post(`/api/fleet/snapshots/${id}/restore`) .set('Cookie', adminCookie) @@ -396,7 +396,7 @@ describe('Single-stack snapshot restore (behavior lock)', () => { it('redeploys after restore when requested', async () => { vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const db = DatabaseService.getInstance(); const id = db.createSnapshot('restore-redeploy', 'admin', 1, 1, '[]', '[]'); db.insertSnapshotFiles(id, [ @@ -758,7 +758,7 @@ describe('Restore-all', () => { it('isolates corrupt decrypt stacks before any mutation with notes and redeploy requested', async () => { vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const db = DatabaseService.getInstance(); const id = db.createSnapshot('restore-all-corrupt', 'admin', 1, 2, '[]', '[]'); const good = CryptoService.getInstance().encrypt('services:\n app: {}\n'); @@ -800,7 +800,7 @@ describe('Restore-all', () => { it('isolates delimiter-byte corruption before restore-all mutation', async () => { vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const db = DatabaseService.getInstance(); const id = db.createSnapshot('restore-all-delim', 'admin', 1, 2, '[]', '[]'); const good = CryptoService.getInstance().encrypt('services:\n app: {}\n'); @@ -835,7 +835,7 @@ describe('Restore-all', () => { it('redeploys each restored stack when requested', async () => { vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const db = DatabaseService.getInstance(); const id = db.createSnapshot('restore-all-redeploy', 'admin', 1, 1, '[]', '[]'); db.insertSnapshotFiles(id, [ @@ -856,7 +856,7 @@ describe('Restore-all', () => { }); it('records a policy-blocked redeploy as a per-stack failure and still restores the rest', async () => { - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); vi.spyOn(policyGate, 'assertPolicyGateAllows').mockImplementation(async (stackName: string) => { if (stackName === 'blocked-web') throw new Error('Policy "block-criticals" blocked deploy: 1 image(s) exceed high'); }); diff --git a/backend/src/__tests__/git-source-apply-recovery.test.ts b/backend/src/__tests__/git-source-apply-recovery.test.ts index 13fbd1c5..3f080b7b 100644 --- a/backend/src/__tests__/git-source-apply-recovery.test.ts +++ b/backend/src/__tests__/git-source-apply-recovery.test.ts @@ -129,6 +129,14 @@ vi.mock('../services/DatabaseService', () => ({ setGitSourceLastPlan: mockSetGitSourceLastPlan, addNotificationHistory: mockAddNotificationHistory, getStackProjectEnvFiles: vi.fn().mockReturnValue([]), + // The apply path now asks whether this stack has a GitOps application. + // These fixtures predate the revision-state model, so the lookup finds + // nothing and every GitOps producer stays a no-op, which is exactly the + // behavior an install with pre-existing Git stacks gets. + getDb: () => ({ + prepare: () => ({ get: () => undefined, all: () => [], run: () => ({ changes: 0 }) }), + transaction: (fn: () => unknown) => () => fn(), + }), }), }, })); @@ -277,7 +285,7 @@ describe('git-source apply recovery (R1)', () => { it('refuses to promote when recovery capture fails', async () => { mockCaptureCandidate.mockRejectedValue(new Error('Exact authored-project rollback coverage is unavailable')); - mockDeployStack.mockResolvedValue({ recoveryId: null }); + mockDeployStack.mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const { GitSourceService, GitSourceError } = await import('../services/GitSourceService'); const svc = GitSourceService.getInstance(); diff --git a/backend/src/__tests__/git-source-routes.test.ts b/backend/src/__tests__/git-source-routes.test.ts index ccbf4363..572b7672 100644 --- a/backend/src/__tests__/git-source-routes.test.ts +++ b/backend/src/__tests__/git-source-routes.test.ts @@ -1,9 +1,10 @@ /** * Route-layer tests for the git-source API. * - * Covers input-validation and guard behavior that lives in the Express - * handlers (not in GitSourceService), specifically: - * - HTTPS-only repo URL enforcement + * Covers input-validation and guard behavior reachable through the Express + * handlers (the URL rules themselves live in services/gitops/repoIdentity.ts, + * not in GitSourceService), specifically: + * - HTTPS-only repo URL enforcement, including userinfo/query/fragment rejection * - Max-length caps on repo_url / branch / compose_path / env_path / token * - Stack-existence 404 guard on PUT * - 400 on invalid stack names @@ -20,6 +21,70 @@ import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './he import { DatabaseService } from '../services/DatabaseService'; import { ComposeService } from '../services/ComposeService'; import { GitSourceService, GitSourceError } from '../services/GitSourceService'; +import { GitOpsStore } from '../services/gitops/store'; +import { GitOpsTransitions } from '../services/gitops/transitions'; +import { insertHistory } from '../services/gitops/history'; +import type { GitOpsApplicationRow } from '../services/gitops/types'; + +/** A minimal live Direct application row for GitOps read-path fixtures. */ +function directApplicationFixture(id: string, stackName: string): GitOpsApplicationRow { + return { + id, + lifecycle_key: `direct:${stackName}`, + lifecycle_status: 'active', + target_mode: 'direct', + stack_name: stackName, + blueprint_id: null, + configured_repo_url: 'https://github.com/example/repo.git', + repo_identity_json: '{"host":"github.com","pathname":"/example/repo.git"}', + configured_ref: 'main', + compose_paths_json: '["compose.yaml"]', + context_dir: null, + sync_env: 0, + env_path: null, + materialization_fingerprint: 'a'.repeat(64), + desired_commit_sha: null, + fetched_commit_sha: null, + candidate_generation_id: null, + accepted_generation_id: null, + candidate_plan_blocked: 0, + review_required: 0, + artifact_set_id: null, + latest_artifact_set_id: null, + intent_revision_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + rollout_authorization_ref: null, + legacy_combined_approval_ref: null, + preflight_fingerprint: null, + latest_operation_id: null, + active_operation_id: null, + active_operation_stage: null, + active_operation_at: null, + active_generation_id: null, + pause_at: null, + pause_reason: null, + partial_json: null, + failure_stage: null, + failure_class: null, + failure_at: null, + retry_at: null, + retry_count: 0, + suspended_at: null, + recovery_ref: null, + recovery_phase: null, + interruption_stage: null, + interruption_at: null, + interruption_operation_id: null, + interruption_generation_id: null, + evidence_fresh_at: null, + evidence_limitations_json: null, + created_at: 1, + updated_at: 1, + }; +} // ── Hoisted mocks (must come before importing the app) ───────────────── @@ -93,6 +158,27 @@ describe('PUT /api/stacks/:stackName/git-source — URL validation', () => { expect(res.body.error).toMatch(/HTTPS/i); }); + it('rejects repo URLs with userinfo, query, or fragment', async () => { + const cases = [ + { repo_url: 'https://user:pass@github.com/example/repo.git', error: /userinfo/i }, + { repo_url: 'https://github.com/example/repo.git?token=1', error: /query/i }, + { repo_url: 'https://github.com/example/repo.git#head', error: /fragment/i }, + ]; + for (const c of cases) { + const res = await request(app) + .put('/api/stacks/existing-stack/git-source') + .set('Authorization', `Bearer ${adminToken()}`) + .send({ + repo_url: c.repo_url, + branch: 'main', + compose_path: 'compose.yaml', + auth_type: 'none', + }); + expect(res.status).toBe(400); + expect(res.body.error).toMatch(c.error); + } + }); + it('rejects missing repo_url with 400', async () => { const res = await request(app) .put('/api/stacks/existing-stack/git-source') @@ -107,6 +193,28 @@ describe('PUT /api/stacks/:stackName/git-source — URL validation', () => { }); }); +describe('POST /api/git-sources/browse: URL validation', () => { + it('rejects non-HTTPS, userinfo, query, and fragment URLs before cloning', async () => { + const listRepoTree = vi.spyOn(GitSourceService.getInstance(), 'listRepoTree'); + const cases = [ + { repo_url: 'http://github.com/example/repo.git', error: /HTTPS/i }, + { repo_url: 'https://user:pass@github.com/example/repo.git', error: /userinfo/i }, + { repo_url: 'https://github.com/example/repo.git?token=1', error: /query/i }, + { repo_url: 'https://github.com/example/repo.git#head', error: /fragment/i }, + ]; + for (const c of cases) { + const res = await request(app) + .post('/api/git-sources/browse') + .set('Authorization', `Bearer ${adminToken()}`) + .send({ repo_url: c.repo_url, branch: 'main', auth_type: 'none' }); + expect(res.status).toBe(400); + expect(res.body.error).toMatch(c.error); + } + expect(listRepoTree).not.toHaveBeenCalled(); + listRepoTree.mockRestore(); + }); +}); + describe('PUT /api/stacks/:stackName/git-source — max-length caps', () => { const baseBody = { branch: 'main', @@ -276,7 +384,15 @@ describe('GET /api/stacks/:stackName/git-source', () => { .get('/api/stacks/unlinked-stack/git-source') .set('Authorization', `Bearer ${adminToken()}`); expect(res.status).toBe(200); - expect(res.body).toEqual({ linked: false }); + expect(res.body.linked).toBe(false); + // The stack is real but carries no Git source, so it has no GitOps + // application to project and the directory is still on disk. + expect(res.body.stackResourcePresent).toBe(true); + expect(res.body.gitopsRevision).toMatchObject({ + schemaVersion: 1, + targetMode: 'not_applicable', + applicationId: null, + }); }); it('returns 404 when the stack does not exist on the active node', async () => { @@ -479,6 +595,15 @@ describe('POST /api/stacks/from-git', () => { expect(res.body.error).toMatch(/HTTPS/i); }); + it('rejects repo URLs with userinfo, query, or fragment', async () => { + const res = await request(app) + .post('/api/stacks/from-git') + .set('Authorization', `Bearer ${adminToken()}`) + .send({ ...validBody, repo_url: 'https://github.com/example/repo.git?token=1' }); + expect(res.status).toBe(400); + expect(res.body.error).toMatch(/query/i); + }); + it('rejects oversized repo_url with 400', async () => { const res = await request(app) .post('/api/stacks/from-git') @@ -1213,3 +1338,371 @@ describe('POST /api/stacks/:stackName/git-source/pull permissions and actor', () } }); }); + +describe('GitOps additive fields and history routes', () => { + let viewerCookie: string; + let auditorCookie: string; + + async function loginAs(username: string, role: 'viewer' | 'auditor'): Promise { + const bcrypt = (await import('bcrypt')).default; + const password = `${username}-pass`; + DatabaseService.getInstance().addUser({ + username, + password_hash: await bcrypt.hash(password, 1), + role, + }); + const login = await request(app).post('/api/auth/login').send({ username, password }); + const cookies = login.headers['set-cookie'] as string | string[]; + return Array.isArray(cookies) ? cookies[0] : cookies; + } + + beforeAll(async () => { + viewerCookie = await loginAs('gitops-viewer', 'viewer'); + auditorCookie = await loginAs('gitops-auditor', 'auditor'); + }); + + function makeStackDir(stackName: string): void { + const composeDir = process.env.COMPOSE_DIR!; + fs.mkdirSync(path.join(composeDir, stackName), { recursive: true }); + fs.writeFileSync(path.join(composeDir, stackName, 'compose.yaml'), 'services:\n x:\n image: nginx\n'); + } + + /** Bring a real Direct application into the store, which also writes its first history row. */ + function activateApplication( + id: string, + stackName: string, + lifecycleStatus: GitOpsApplicationRow['lifecycle_status'] = 'active', + ): void { + const application: GitOpsApplicationRow = { + ...directApplicationFixture(id, stackName), + lifecycle_status: lifecycleStatus, + }; + GitOpsTransitions.getInstance().activateDirect({ + application, + nodeId: 1, + envelope: { operationId: `op-${id}`, actor: 'tester', trigger: 'manual', at: Date.now() }, + }); + } + + /** Append one more history row for an existing application. */ + function recordFetch(applicationId: string, stackName: string, operationId: string, sha: string): void { + const application = GitOpsStore.getInstance().getApplication(applicationId) + ?? directApplicationFixture(applicationId, stackName); + insertHistory(DatabaseService.getInstance().getDb(), { + application, + nodeId: 1, + dedupeTarget: 'app', + operationId, + stage: 'fetched', + outcome: 'committed', + trigger: 'manual', + actor: 'tester', + before: { desiredCommitSha: null }, + after: { desiredCommitSha: sha }, + commitSha: sha, + at: Date.now(), + }); + } + + it('carries gitopsRevision and stackResourcePresent on each git-source row', async () => { + makeStackDir('additive-stack'); + seedGitSource('additive-stack'); + const res = await request(app) + .get('/api/git-sources') + .set('Authorization', `Bearer ${adminToken()}`); + expect(res.status).toBe(200); + const row = res.body.find((r: { stack_name: string }) => r.stack_name === 'additive-stack'); + expect(row).toBeDefined(); + expect(row.stackResourcePresent).toBe(true); + expect(row.gitopsRevision.schemaVersion).toBe(1); + }); + + it('withholds a row with no GitOps application from a non-admin', async () => { + makeStackDir('unmodelled-stack'); + seedGitSource('unmodelled-stack'); + // A viewer holds global stack:read, but a source we cannot tie to a + // live application has no lifecycle to prove, so it stays with Admin. + const res = await request(app) + .get('/api/git-sources') + .set('Cookie', viewerCookie); + expect(res.status).toBe(200); + expect(res.body.map((r: { stack_name: string }) => r.stack_name)).not.toContain('unmodelled-stack'); + }); + + it('projects a live application, not just the not-applicable shape', async () => { + makeStackDir('live-app-stack'); + seedGitSource('live-app-stack'); + activateApplication('app-live-route', 'live-app-stack'); + const res = await request(app) + .get('/api/git-sources') + .set('Authorization', `Bearer ${adminToken()}`); + const row = res.body.find((r: { stack_name: string }) => r.stack_name === 'live-app-stack'); + expect(row.gitopsRevision).toMatchObject({ + schemaVersion: 1, + targetMode: 'direct', + applicationId: 'app-live-route', + lifecycleStatus: 'active', + }); + expect(row.gitopsRevision.facets).not.toBeNull(); + }); + + it('shows a modelled row to a non-admin holding stack read', async () => { + // The deny case alone would pass if the route dropped every row for a + // non-admin, so the allow case is what proves the classifier runs. + makeStackDir('viewer-visible-stack'); + seedGitSource('viewer-visible-stack'); + activateApplication('app-viewer-visible', 'viewer-visible-stack'); + const res = await request(app) + .get('/api/git-sources') + .set('Cookie', viewerCookie); + expect(res.status).toBe(200); + expect(res.body.map((r: { stack_name: string }) => r.stack_name)).toContain('viewer-visible-stack'); + }); + + it('filters cross-stack history per row for a non-admin', async () => { + makeStackDir('viewer-hist-stack'); + activateApplication('app-viewer-hist', 'viewer-hist-stack'); + // No directory, so this application's rows are unprovable and Admin-only. + activateApplication('app-hidden-hist', 'absent-hist-stack'); + + const asAdmin = await request(app) + .get('/api/git-sources/history?limit=100') + .set('Authorization', `Bearer ${adminToken()}`); + const adminStacks = asAdmin.body.items.map((i: { stackName: string }) => i.stackName); + expect(adminStacks).toContain('viewer-hist-stack'); + expect(adminStacks).toContain('absent-hist-stack'); + + const asViewer = await request(app) + .get('/api/git-sources/history?limit=100') + .set('Cookie', viewerCookie); + const viewerStacks = asViewer.body.items.map((i: { stackName: string }) => i.stackName); + expect(viewerStacks).toContain('viewer-hist-stack'); + expect(viewerStacks).not.toContain('absent-hist-stack'); + }); + + it('shows an auditor the history entries a viewer cannot prove', async () => { + // 'absent-hist-stack' has no directory, so its entries cannot be tied + // to a readable stack. They are still an audit record, so the audit + // permission reaches them where a plain stack grant does not. + const asAuditor = await request(app) + .get('/api/git-sources/history?limit=100') + .set('Cookie', auditorCookie); + expect(asAuditor.status).toBe(200); + const auditorStacks = asAuditor.body.items.map((i: { stackName: string }) => i.stackName); + expect(auditorStacks).toContain('absent-hist-stack'); + expect(auditorStacks).toContain('viewer-hist-stack'); + }); + + it('does not let the audit permission reach Git configuration', async () => { + // The source list is live configuration, not a record of events, so an + // auditor sees no more of it than any other non-admin. + makeStackDir('auditor-config-stack'); + seedGitSource('auditor-config-stack'); + const res = await request(app) + .get('/api/git-sources') + .set('Cookie', auditorCookie); + expect(res.status).toBe(200); + // Seeded with no GitOps application, so it stays Admin-only. + expect(res.body.map((r: { stack_name: string }) => r.stack_name)).not.toContain('auditor-config-stack'); + }); + + it('advances the cursor past rows the caller may not read', async () => { + // The viewer cannot read the absent-stack rows seeded above. Paging + // must still move forward, or a narrowly scoped caller re-reads the + // same rejected window for ever. + const first = await request(app) + .get('/api/git-sources/history?limit=1') + .set('Cookie', viewerCookie); + expect(first.status).toBe(200); + expect(first.body.nextCursor).not.toBeNull(); + + const second = await request(app) + .get(`/api/git-sources/history?limit=1&cursor=${encodeURIComponent(first.body.nextCursor)}`) + .set('Cookie', viewerCookie); + expect(second.status).toBe(200); + const firstIds = first.body.items.map((i: { id: string }) => i.id); + const secondIds = second.body.items.map((i: { id: string }) => i.id); + expect(secondIds.filter((id: string) => firstIds.includes(id))).toEqual([]); + }); + + it('hands back a cursor when a page fills and none when the window is spent', async () => { + makeStackDir('paging-stack'); + activateApplication('app-paging', 'paging-stack'); + recordFetch('app-paging', 'paging-stack', 'op-page-1', 'aaa1111'); + recordFetch('app-paging', 'paging-stack', 'op-page-2', 'bbb2222'); + + const full = await request(app) + .get('/api/stacks/paging-stack/git-source/history?limit=2') + .set('Authorization', `Bearer ${adminToken()}`); + expect(full.body.items).toHaveLength(2); + expect(full.body.nextCursor).not.toBeNull(); + + const rest = await request(app) + .get(`/api/stacks/paging-stack/git-source/history?limit=2&cursor=${encodeURIComponent(full.body.nextCursor)}`) + .set('Authorization', `Bearer ${adminToken()}`); + expect(rest.body.items).toHaveLength(1); + expect(rest.body.nextCursor).toBeNull(); + }); + + it('keeps a creating stack own history readable through the per-stack route', async () => { + // The row classifier sends `creating` to Admin. The per-stack route + // authorizes its collection by name instead, which is the whole reason + // that distinction exists. + makeStackDir('creating-stack'); + activateApplication('app-creating', 'creating-stack', 'creating'); + + recordFetch('app-creating', 'creating-stack', 'op-creating-1', 'creat111'); + const perStack = await request(app) + .get('/api/stacks/creating-stack/git-source/history') + .set('Cookie', viewerCookie); + expect(perStack.status).toBe(200); + // Asserted by identity, not by count. A non-empty page would also be + // satisfied by an exemption that had widened to rows it should not + // cover, which is the failure this route's scope exists to prevent. + expect(perStack.body.items.map((i: { applicationId: string }) => i.applicationId)) + .toContain('app-creating'); + expect(perStack.body.items.map((i: { commitSha: string | null }) => i.commitSha)) + .toContain('creat111'); + + const crossStack = await request(app) + .get('/api/git-sources/history?limit=100') + .set('Cookie', viewerCookie); + const stacks = crossStack.body.items.map((i: { stackName: string }) => i.stackName); + expect(stacks).not.toContain('creating-stack'); + }); + + it('does not expose a predecessor application through a reused stack name', async () => { + // A stack name outlives the applications that hold it. A grant on the + // one holding it now says nothing about the repository, actors or + // commits of the one that held it before, so those rows stay behind + // the audit permission on this route exactly as they do cross-stack. + makeStackDir('reused-name'); + activateApplication('app-reused-old', 'reused-name'); + recordFetch('app-reused-old', 'reused-name', 'op-reused-old', 'old11111'); + GitOpsTransitions.getInstance().applicationTombstoned('app-reused-old', 'deleted', { + operationId: 'op-reused-old', actor: 'tester', trigger: 'manual', at: Date.now(), + }); + activateApplication('app-reused-new', 'reused-name'); + recordFetch('app-reused-new', 'reused-name', 'op-reused-new', 'new22222'); + + const viewer = await request(app) + .get('/api/stacks/reused-name/git-source/history?limit=100') + .set('Cookie', viewerCookie); + expect(viewer.status).toBe(200); + const viewerShas = viewer.body.items.map((i: { commitSha: string | null }) => i.commitSha); + expect(viewerShas).toContain('new22222'); + expect(viewerShas).not.toContain('old11111'); + + // The audit trail is not lost, only moved behind the permission that + // exists for reading it. + const auditor = await request(app) + .get('/api/stacks/reused-name/git-source/history?limit=100') + .set('Cookie', auditorCookie); + expect(auditor.status).toBe(200); + const auditorShas = auditor.body.items.map((i: { commitSha: string | null }) => i.commitSha); + expect(auditorShas).toContain('old11111'); + expect(auditorShas).toContain('new22222'); + }); + + it('moves a detached application behind system:audit even with no successor', async () => { + // Detach leaves the files on disk, which once justified reading its + // trail on a stack grant. A grant covers whatever occupies the name + // today, and nothing in these tables can prove the detached + // application still does: some successors hide from every lookup this + // route could run, so detach joins `deleted` as an audit-only + // predecessor. + makeStackDir('detached-kept'); + activateApplication('app-detached-kept', 'detached-kept'); + recordFetch('app-detached-kept', 'detached-kept', 'op-detached-kept', 'kept1111'); + GitOpsTransitions.getInstance().applicationTombstoned('app-detached-kept', 'detached', { + operationId: 'op-detached-kept', actor: 'tester', trigger: 'manual', at: Date.now(), + }); + + const viewer = await request(app) + .get('/api/stacks/detached-kept/git-source/history?limit=100') + .set('Cookie', viewerCookie); + expect(viewer.status).toBe(200); + const shas = viewer.body.items.map((i: { commitSha: string | null }) => i.commitSha); + expect(shas).not.toContain('kept1111'); + + // Moved behind the audit permission, not lost. + const auditor = await request(app) + .get('/api/stacks/detached-kept/git-source/history?limit=100') + .set('Cookie', auditorCookie); + expect(auditor.status).toBe(200); + const auditorShas = auditor.body.items.map((i: { commitSha: string | null }) => i.commitSha); + expect(auditorShas).toContain('kept1111'); + }); + + it('keeps a detached predecessor behind system:audit once a successor takes the name', async () => { + // The successor makes the reuse visible, but the answer does not depend + // on detecting it: a detached trail is audit-only on its own. This pins + // that a successor neither restores nor widens what the stack grant + // reaches. + makeStackDir('reused-detached'); + activateApplication('app-detached-old', 'reused-detached'); + recordFetch('app-detached-old', 'reused-detached', 'op-detached-old', 'det11111'); + GitOpsTransitions.getInstance().applicationTombstoned('app-detached-old', 'detached', { + operationId: 'op-detached-old', actor: 'tester', trigger: 'manual', at: Date.now(), + }); + activateApplication('app-detached-new', 'reused-detached'); + recordFetch('app-detached-new', 'reused-detached', 'op-detached-new', 'det22222'); + + const viewer = await request(app) + .get('/api/stacks/reused-detached/git-source/history?limit=100') + .set('Cookie', viewerCookie); + expect(viewer.status).toBe(200); + const shas = viewer.body.items.map((i: { commitSha: string | null }) => i.commitSha); + expect(shas).toContain('det22222'); + expect(shas).not.toContain('det11111'); + + // Moved behind the audit permission, not lost. + const auditor = await request(app) + .get('/api/stacks/reused-detached/git-source/history?limit=100') + .set('Cookie', auditorCookie); + expect(auditor.status).toBe(200); + const auditorShas = auditor.body.items.map((i: { commitSha: string | null }) => i.commitSha); + expect(auditorShas).toContain('det11111'); + expect(auditorShas).toContain('det22222'); + }); + + it('rejects a malformed cursor instead of silently restarting', async () => { + const res = await request(app) + .get('/api/git-sources/history?cursor=123.not-a-uuid') + .set('Authorization', `Bearer ${adminToken()}`); + expect(res.status).toBe(400); + expect(res.body.error).toMatch(/cursor/i); + }); + + it('rejects a recognized filter carrying an unusable value', async () => { + const outcome = await request(app) + .get('/api/git-sources/history?outcome=success') + .set('Authorization', `Bearer ${adminToken()}`); + expect(outcome.status).toBe(400); + expect(outcome.body.error).toMatch(/outcome/i); + + const nodeId = await request(app) + .get('/api/git-sources/history?nodeId=abc') + .set('Authorization', `Bearer ${adminToken()}`); + expect(nodeId.status).toBe(400); + expect(nodeId.body.error).toMatch(/nodeId/i); + }); + + it('rejects an invalid stack name on the per-stack history route', async () => { + const res = await request(app) + .get('/api/stacks/..%2Fetc/git-source/history') + .set('Authorization', `Bearer ${adminToken()}`); + expect(res.status).toBe(400); + expect(res.body.error).toMatch(/stack name/i); + }); + + it('returns an empty page for a stack with no recorded history', async () => { + makeStackDir('quiet-stack'); + const res = await request(app) + .get('/api/stacks/quiet-stack/git-source/history') + .set('Authorization', `Bearer ${adminToken()}`); + expect(res.status).toBe(200); + expect(res.body.items).toEqual([]); + expect(res.body.nextCursor).toBeNull(); + }); +}); diff --git a/backend/src/__tests__/git-source-service.test.ts b/backend/src/__tests__/git-source-service.test.ts index d3cf162c..e3959567 100644 --- a/backend/src/__tests__/git-source-service.test.ts +++ b/backend/src/__tests__/git-source-service.test.ts @@ -17,6 +17,14 @@ import fs from 'fs'; import path from 'path'; import { describe, it, expect, vi, beforeAll, afterAll, beforeEach } from 'vitest'; import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { GitOpsStore } from '../services/gitops/store'; +import { GitOpsTransitions } from '../services/gitops/transitions'; +import { + buildGenerationRow, + directSourceIdentity, + newGitOpsId, + type DirectSourceConfig, +} from '../services/gitops/directApplication'; // ── Hoisted mocks ────────────────────────────────────────────────────── @@ -849,6 +857,98 @@ describe('GitSourceService pending lifecycle', () => { expect(db.getGitSource('pending-stack')?.pending_commit_sha).toBeNull(); }); + it('dismissPending clears the canonical candidate and records a dismissed history row', async () => { + mockSuccessfulClone(); + const svc = GitSourceService.getInstance(); + const db = DatabaseService.getInstance(); + const stackName = 'dismiss-canonical'; + await svc.upsert({ + stackName, + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + db.setGitSourcePending(stackName, 'sha-xxx', 'services: {}', null); + const { appId, generationId } = seedDirectCandidate(stackName); + expect(GitOpsStore.getInstance().getApplication(appId)?.candidate_generation_id).toBe(generationId); + + svc.dismissPending(stackName, 'operator-1'); + + const app = GitOpsStore.getInstance().getApplication(appId)!; + expect(app.candidate_generation_id).toBeNull(); + expect(app.candidate_plan_blocked).toBe(0); + expect(app.review_required).toBe(0); + expect(db.getGitSource(stackName)?.pending_commit_sha).toBeNull(); + const stages = (db.getDb().prepare( + 'SELECT stage, outcome FROM gitops_history WHERE application_id = ? ORDER BY id', + ).all(appId) as Array<{ stage: string; outcome: string }>).map((r) => `${r.stage}:${r.outcome}`); + expect(stages).toContain('dismissed:skipped'); + }); + + it('dismissPending refuses while an operation is in flight and mutates nothing', async () => { + mockSuccessfulClone(); + const svc = GitSourceService.getInstance(); + const db = DatabaseService.getInstance(); + const stackName = 'dismiss-in-flight'; + await svc.upsert({ + stackName, + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + db.setGitSourcePending(stackName, 'sha-yyy', 'services: {}', null); + const { appId, generationId } = seedDirectCandidate(stackName); + GitOpsTransitions.getInstance().fetchStarted(appId, testEnvelope()); + + let caught: unknown; + try { + svc.dismissPending(stackName, 'operator-1'); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(GitSourceError); + if (!(caught instanceof GitSourceError)) throw new Error('expected GitSourceError'); + expect(caught.code).toBe('OPERATION_IN_FLIGHT'); + // The refusal is the outcome: neither the model nor the legacy columns move. + expect(GitOpsStore.getInstance().getApplication(appId)?.candidate_generation_id).toBe(generationId); + expect(db.getGitSource(stackName)?.pending_commit_sha).toBe('sha-yyy'); + }); + + it('dismissPending stays a legacy-only no-op without a canonical application', async () => { + mockSuccessfulClone(); + const svc = GitSourceService.getInstance(); + const db = DatabaseService.getInstance(); + const stackName = 'dismiss-legacy-only'; + await svc.upsert({ + stackName, + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + db.setGitSourcePending(stackName, 'sha-zzz', 'services: {}', null); + + expect(() => svc.dismissPending(stackName, 'operator-1')).not.toThrow(); + expect(db.getGitSource(stackName)?.pending_commit_sha).toBeNull(); + }); + it('clearGitSourceAppliedRevision clears pending plan columns', async () => { mockSuccessfulClone(); const svc = GitSourceService.getInstance(); @@ -1206,6 +1306,144 @@ describe('GitSourceService.pull', () => { const svc = GitSourceService.getInstance(); await expect(svc.pull('does-not-exist')).rejects.toMatchObject({ code: 'GIT_ERROR' }); }); + + function generationCount(stackName: string): number { + const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName)!; + return (DatabaseService.getInstance().getDb() + .prepare('SELECT COUNT(*) AS n FROM gitops_generations WHERE application_id = ?') + .get(app.id) as { n: number }).n; + } + + async function createFromGit(stackName: string, sha: string, autoApplyOnWebhook = false): Promise { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n web:\n image: nginx\n', sha }); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + try { + await svc.createStackFromGit({ + stackName, + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + token: null, + autoApplyOnWebhook, + autoDeployOnApply: false, + }); + } finally { + validateSpy.mockRestore(); + } + } + + it('an up-to-date pull against an accepted commit opens a fresh staging generation', async () => { + // Deliberate counterpart to the dedupe below: once the candidate was + // accepted, nothing is staged, and staging again is a new dispatch + // cycle that apply needs as its acceptance target. + const svc = GitSourceService.getInstance(); + await createFromGit('pull-after-apply', '1111111111111111111111111111111111111111'); + const base = generationCount('pull-after-apply'); + + await svc.pull('pull-after-apply'); + expect(generationCount('pull-after-apply')).toBe(base + 1); + const app = GitOpsStore.getInstance().getLiveDirectApplication('pull-after-apply')!; + expect(app.candidate_generation_id).toBeTruthy(); + expect(DatabaseService.getInstance().getGitSource('pull-after-apply')?.pending_commit_sha).toBeTruthy(); + await cleanupStackDir('pull-after-apply'); + }); + + it('repeat pulls of an unapplied update keep one candidate', async () => { + const svc = GitSourceService.getInstance(); + await createFromGit('pull-repeat', '2222222222222222222222222222222222222222'); + const base = generationCount('pull-repeat'); + + const updatedSha = '3333333333333333333333333333333333333333'; + mockSuccessfulClone({ + compose: 'services:\n web:\n image: nginx:1.29\n', + sha: updatedSha, + }); + await svc.pull('pull-repeat'); + const stagedId = GitOpsStore.getInstance().getLiveDirectApplication('pull-repeat')!.candidate_generation_id; + expect(stagedId).toBeTruthy(); + expect(generationCount('pull-repeat')).toBe(base + 1); + + mockSuccessfulClone({ + compose: 'services:\n web:\n image: nginx:1.29\n', + sha: updatedSha, + }); + await svc.pull('pull-repeat'); + expect(generationCount('pull-repeat')).toBe(base + 1); + expect(GitOpsStore.getInstance().getLiveDirectApplication('pull-repeat')!.candidate_generation_id).toBe(stagedId); + expect(DatabaseService.getInstance().getGitSource('pull-repeat')?.pending_commit_sha).toBe(updatedSha); + await cleanupStackDir('pull-repeat'); + }); + + it('a pull whose source fingerprint drifted from the staged candidate mints anew', async () => { + const svc = GitSourceService.getInstance(); + await createFromGit('pull-fp-drift', '4444444444444444444444444444444444444444'); + const base = generationCount('pull-fp-drift'); + const updatedSha = '5555555555555555555555555555555555555555'; + mockSuccessfulClone({ + compose: 'services:\n web:\n image: nginx:1.29\n', + sha: updatedSha, + }); + await svc.pull('pull-fp-drift'); + const stagedId = GitOpsStore.getInstance().getLiveDirectApplication('pull-fp-drift')!.candidate_generation_id; + expect(stagedId).toBeTruthy(); + + // Simulates a standing candidate produced under different source + // wiring than the configuration in effect now. Commit and plan + // verdict are unchanged, but the fingerprint term alone must defeat + // equivalence so the candidate never misrepresents what a pull stages. + DatabaseService.getInstance().getDb() + .prepare('UPDATE gitops_generations SET materialization_fingerprint = ? WHERE id = ?') + .run('drifted-fingerprint', stagedId); + + mockSuccessfulClone({ + compose: 'services:\n web:\n image: nginx:1.29\n', + sha: updatedSha, + }); + await svc.pull('pull-fp-drift'); + expect(generationCount('pull-fp-drift')).toBe(base + 2); + expect(GitOpsStore.getInstance().getLiveDirectApplication('pull-fp-drift')!.candidate_generation_id).not.toBe(stagedId); + await cleanupStackDir('pull-fp-drift'); + }); + + it('a pull whose plan verdict differs from the staged candidate mints anew', async () => { + const svc = GitSourceService.getInstance(); + await createFromGit('pull-verdict-flip', '6666666666666666666666666666666666666666'); + const base = generationCount('pull-verdict-flip'); + const updatedSha = '7777777777777777777777777777777777777777'; + mockSuccessfulClone({ + compose: 'services:\n web:\n image: nginx:1.29\n', + sha: updatedSha, + }); + await svc.pull('pull-verdict-flip'); + const stagedId = GitOpsStore.getInstance().getLiveDirectApplication('pull-verdict-flip')!.candidate_generation_id; + expect(stagedId).toBeTruthy(); + const seeded = DatabaseService.getInstance().getDb() + .prepare('SELECT plan_blocked FROM gitops_generations WHERE id = ?') + .get(stagedId) as { plan_blocked: number }; + expect(seeded.plan_blocked).toBe(0); + + // The plan is re-evaluated on every pull and can flip without a new + // commit, for example when stack policy changes between pulls. + // Simulating a candidate staged under the other verdict proves the + // verdict term defeats equivalence on its own. + DatabaseService.getInstance().getDb() + .prepare('UPDATE gitops_generations SET plan_blocked = 1 WHERE id = ?') + .run(stagedId); + + mockSuccessfulClone({ + compose: 'services:\n web:\n image: nginx:1.29\n', + sha: updatedSha, + }); + await svc.pull('pull-verdict-flip'); + expect(generationCount('pull-verdict-flip')).toBe(base + 2); + expect(GitOpsStore.getInstance().getLiveDirectApplication('pull-verdict-flip')!.candidate_generation_id).not.toBe(stagedId); + await cleanupStackDir('pull-verdict-flip'); + }); }); describe('GitSourceService.createStackFromGit', () => { @@ -1537,7 +1775,7 @@ describe('GitSourceService.apply', () => { const { ComposeService } = await import('../services/ComposeService'); const { HealthGateService } = await import('../services/HealthGateService'); const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-git'); const nodeId = DatabaseService.getInstance().getDefaultNode()!.id!; @@ -1548,7 +1786,7 @@ describe('GitSourceService.apply', () => { source: 'git_apply', actor: 'system:git-source', }); - expect(beginSpy).toHaveBeenCalledWith(nodeId, 'apply-deploy-gate', 'deploy', 'system:git-source'); + expect(beginSpy).toHaveBeenCalledWith(nodeId, 'apply-deploy-gate', 'deploy', 'system:git-source', { deployedGenerationId: null }); expect(mockRecoveryLinkGateOrRetain).toHaveBeenCalledWith('rec-test-1', 'gate-git'); } finally { validateSpy.mockRestore(); @@ -1648,7 +1886,7 @@ describe('GitSourceService.apply', () => { const TrivyService = (await import('../services/TrivyService')).default; const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); const listImagesSpy = vi.spyOn(ComposeService.prototype, 'listStackImages').mockResolvedValue(['nginx:bad']); - const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null }); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const trivy = TrivyService.getInstance(); const trivyAvailableSpy = vi.spyOn(trivy, 'isTrivyAvailable').mockReturnValue(true); const scanSpy = vi.spyOn(trivy, 'scanImagePreflight').mockResolvedValue({ @@ -2646,3 +2884,50 @@ describe('GitSourceService classified plan fingerprint', () => { } }); }); + +// ── Canonical dismissal fixtures ─────────────────────────────────────── + +function testEnvelope(): { operationId: string; actor: string; trigger: string; at: number } { + return { operationId: newGitOpsId(), actor: 'test', trigger: 'test', at: Date.now() }; +} + +/** + * Mint an unblocked candidate for `stackName`, the same shape a pull produces, + * without driving a real fetch. Reuses the live application the preceding + * `svc.upsert` created; the identity is re-derived from the same configuration + * so the generation fingerprint matches the application's. + */ +function seedDirectCandidate(stackName: string): { appId: string; generationId: string } { + const at = Date.now(); + const config: DirectSourceConfig = { + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + }; + const identity = directSourceIdentity(config); + const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName); + if (!app) throw new Error(`no live direct application for ${stackName}`); + const appId = app.id; + const generationId = newGitOpsId(); + GitOpsStore.getInstance().insertGeneration(buildGenerationRow({ + id: generationId, + applicationId: appId, + commitSha: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + identity, + configuredRef: 'main', + candidateRelPath: 'generations/cand', + appliedRelPath: 'applied/1', + manifestVersion: 1, + expectedInvocation: null, + changePlanFingerprint: 'fp-seed', + operationId: newGitOpsId(), + trigger: 'test', + actor: 'test', + at, + })); + GitOpsTransitions.getInstance().candidateReady(appId, generationId, false, testEnvelope()); + return { appId, generationId }; +} diff --git a/backend/src/__tests__/gitops-additive-json.test.ts b/backend/src/__tests__/gitops-additive-json.test.ts new file mode 100644 index 00000000..ce050855 --- /dev/null +++ b/backend/src/__tests__/gitops-additive-json.test.ts @@ -0,0 +1,393 @@ +/** + * Exact API shapes for the additive GitOps revision fields on the Blueprint, + * node-label, and node surfaces. + * + * Two things are being defended here. The first is that the fields are + * genuinely additive: the routes keep their status codes, the two DELETEs stay + * 204 with no body, and the pre-existing keys are untouched. The second is that + * `gitopsRevisions` reports only what a mutation actually moved. A label or a + * cordon that no selector reacts to must answer with an empty list rather than + * every Blueprint in the fleet, because a consumer reading that list as "these + * changed" would otherwise invalidate the whole catalog over an edit nobody can + * observe. + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; +import request from 'supertest'; +import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb'; +import { directApplicationFixture } from './helpers/gitopsFixtures'; + +let tmpDir: string; +let app: import('express').Express; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; +let LicenseService: typeof import('../services/LicenseService').LicenseService; +let adminCookie: string; +let counter = 0; + +/** A fresh non-default node row, so a cordon or delete never touches the default node. */ +function seedNode(): number { + counter += 1; + const result = DatabaseService.getInstance().getDb().prepare( + `INSERT INTO nodes (name, type, mode, compose_dir, is_default, status, created_at) + VALUES (?, 'local', 'proxy', '/tmp/compose', 0, 'online', ?)`, + ).run(`additive-node-${counter}`, Date.now()); + return result.lastInsertRowid as number; +} + +/** A Blueprint inserted straight into the table, so no GitOps application exists for it. */ +function seedUnmodelledBlueprint() { + counter += 1; + return DatabaseService.getInstance().createBlueprint({ + name: `additive-unmodelled-${counter}`, + description: null, + compose_content: 'services:\n app:\n image: nginx\n', + selector: { type: 'nodes', ids: [] }, + drift_mode: 'suggest', + classification: 'stateless', + classification_reasons: [], + enabled: true, + created_by: 'admin', + }); +} + +/** Create through the route, which is the path that activates an application. */ +async function createBlueprint(selector: { type: string; ids?: number[]; all?: string[]; any?: string[] }) { + counter += 1; + const res = await request(app) + .post('/api/blueprints') + .set('Cookie', adminCookie) + .send({ + name: `additive-bp-${counter}`, + compose_content: 'services:\n app:\n image: nginx\n', + selector, + }); + expect(res.status).toBe(201); + return res; +} + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ DatabaseService } = await import('../services/DatabaseService')); + ({ LicenseService } = await import('../services/LicenseService')); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); + ({ app } = await import('../index')); + adminCookie = await loginAsTestAdmin(app); +}); + +afterAll(() => cleanupTestDb(tmpDir)); + +beforeEach(() => { + vi.restoreAllMocks(); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); + const db = DatabaseService.getInstance().getDb(); + for (const table of [ + 'blueprint_deployments', 'gitops_history', 'gitops_target_current', 'gitops_rollout_candidates', + 'gitops_intent_revisions', 'gitops_applications', 'blueprints', 'node_labels', + ]) { + db.prepare(`DELETE FROM ${table}`).run(); + } + // The default node is the one every stack route resolves against, so only + // the seeded ones go. + db.prepare('DELETE FROM nodes WHERE is_default = 0').run(); +}); + +describe('Blueprint routes carry gitopsRevision', () => { + it('projects the live application the create activated, and reports it identically on list and detail', async () => { + const created = await createBlueprint({ type: 'nodes', ids: [] }); + expect(created.body.gitopsRevision).toMatchObject({ + schemaVersion: 1, + targetMode: 'inline_blueprint', + lifecycleStatus: 'active', + blueprintId: created.body.id, + }); + const applicationId = created.body.gitopsRevision.applicationId; + expect(typeof applicationId).toBe('string'); + + // The same application id has to come back from every surface, or two + // views of one Blueprint would disagree about which application is live. + const detail = await request(app).get(`/api/blueprints/${created.body.id}`).set('Cookie', adminCookie); + expect(detail.status).toBe(200); + expect(detail.body.gitopsRevision.applicationId).toBe(applicationId); + + const list = await request(app).get('/api/blueprints').set('Cookie', adminCookie); + expect(list.status).toBe(200); + const row = list.body.find((b: { id: number }) => b.id === created.body.id); + expect(row.gitopsRevision.applicationId).toBe(applicationId); + }); + + it('gives a Blueprint with no application the uniform not-applicable shape', async () => { + const bp = seedUnmodelledBlueprint(); + const detail = await request(app).get(`/api/blueprints/${bp.id}`).set('Cookie', adminCookie); + expect(detail.status).toBe(200); + // Not an omitted key and not a throw: the catalog needs one shape across + // rows whether or not migration has brought a Blueprint into the model. + expect(detail.body.gitopsRevision).toMatchObject({ + schemaVersion: 1, + targetMode: 'not_applicable', + applicationId: null, + facets: null, + }); + }); + + it('carries gitopsRevision on update and on pin, and leaves the existing keys alone', async () => { + const nodeId = seedNode(); + const created = await createBlueprint({ type: 'nodes', ids: [nodeId] }); + + const updated = await request(app) + .put(`/api/blueprints/${created.body.id}`) + .set('Cookie', adminCookie) + .send({ description: 'revised' }); + expect(updated.status).toBe(200); + expect(updated.body.description).toBe('revised'); + expect(updated.body.id).toBe(created.body.id); + expect(updated.body.gitopsRevision.applicationId).toBe(created.body.gitopsRevision.applicationId); + + const pinned = await request(app) + .put(`/api/blueprints/${created.body.id}/pin`) + .set('Cookie', adminCookie) + .send({ nodeId }); + expect(pinned.status).toBe(200); + expect(pinned.body.pinned_node_id).toBe(nodeId); + expect(pinned.body.gitopsRevision.applicationId).toBe(created.body.gitopsRevision.applicationId); + }); + + it('keeps DELETE at 204 with no body', async () => { + const created = await createBlueprint({ type: 'nodes', ids: [] }); + const res = await request(app).delete(`/api/blueprints/${created.body.id}`).set('Cookie', adminCookie); + expect(res.status).toBe(204); + expect(res.body).toEqual({}); + expect(res.text).toBeFalsy(); + }); +}); + +describe('Projection resolution reaches every application that owns a surface', () => { + it('retires a deleted Blueprint to the plain not-applicable shape', async () => { + // Driven through the real delete route rather than a hand-made + // tombstone. Blueprint retirement writes `deleted`, never `detached`, + // which is why no detached-Blueprint lookup exists: one would index and + // query a state the product cannot produce. Asserting it here keeps + // that fact tied to the path that decides it. + const created = await createBlueprint({ type: 'nodes', ids: [] }); + const applicationId = created.body.gitopsRevision.applicationId; + const del = await request(app).delete(`/api/blueprints/${created.body.id}`).set('Cookie', adminCookie); + expect(del.status).toBe(204); + + const store = (await import('../services/gitops/store')).GitOpsStore.getInstance(); + expect(store.getApplication(applicationId)?.lifecycle_status).toBe('deleted'); + expect(store.getLiveBlueprintApplication(created.body.id)).toBeUndefined(); + + const { projectBlueprintRevision } = await import('../helpers/gitopsResponse'); + expect(projectBlueprintRevision(created.body.id)).toMatchObject({ targetMode: 'not_applicable' }); + }); + + it('reports a detached Direct source as not_live, which nothing could reach before', async () => { + const store = (await import('../services/gitops/store')).GitOpsStore.getInstance(); + const tx = (await import('../services/gitops/transitions')).GitOpsTransitions.getInstance(); + const application = directApplicationFixture('app-direct-detach', 'detached-direct-stack'); + const nodeId = seedNode(); + tx.activateDirect({ + application, + nodeId, + envelope: { operationId: 'op-direct-detach', actor: 'tester', trigger: 'manual', at: Date.now() }, + }); + tx.applicationTombstoned(application.id, 'detached', { + operationId: 'op-direct-detach-2', actor: 'tester', trigger: 'manual', at: Date.now(), + }); + expect(store.getLiveDirectApplication('detached-direct-stack')).toBeUndefined(); + expect(store.getDetachedDirectApplication('detached-direct-stack')?.id).toBe(application.id); + + // The tombstone keeps repository, ref, and SHA pointers as frozen facts + // so the projection can still say what was there. Before this lookup + // existed, the source deriver's not_live branch had no way to be + // reached and a deliberate detach read as "never had Git". + const { projectManagedStackRevision, projectStackRevision } = await import('../helpers/gitopsResponse'); + const projection = projectManagedStackRevision('detached-direct-stack', nodeId); + expect(projection).toMatchObject({ applicationId: application.id, lifecycleStatus: 'detached' }); + if (projection.targetMode === 'not_applicable') throw new Error('expected an application'); + expect(projection.facets.source).toMatchObject({ status: 'not_live', lifecycleStatus: 'detached' }); + + // The Git-source resolver stays live-only, so it cannot feed a detached + // lifecycle to the row classifier that decides who may read the row. + expect(projectStackRevision('detached-direct-stack')).toMatchObject({ targetMode: 'not_applicable' }); + }); + + it('does not resurrect a deleted application for a stack name that gets reused', async () => { + const tx = (await import('../services/gitops/transitions')).GitOpsTransitions.getInstance(); + const application = directApplicationFixture('app-reuse', 'reused-name-stack'); + tx.activateDirect({ + application, + nodeId: seedNode(), + envelope: { operationId: 'op-reuse', actor: 'tester', trigger: 'manual', at: Date.now() }, + }); + tx.applicationTombstoned(application.id, 'deleted', { + operationId: 'op-reuse-2', actor: 'tester', trigger: 'manual', at: Date.now(), + }); + + // Deletion means the stack is gone, so a directory of that name now is + // a different stack. Reporting the old repository and SHA against it + // would disclose one stack's Git identity through another's name. + const { projectManagedStackRevision } = await import('../helpers/gitopsResponse'); + expect(projectManagedStackRevision('reused-name-stack', 1)).toMatchObject({ targetMode: 'not_applicable' }); + }); + + it('says why when the application it resolved has gone missing', async () => { + const created = await createBlueprint({ type: 'nodes', ids: [] }); + const store = (await import('../services/gitops/store')).GitOpsStore.getInstance(); + const live = store.getLiveBlueprintApplication(created.body.id); + // Resolve the row, then delete it before the projection re-reads it by + // id. That is the window the two non-transactional reads leave open. + vi.spyOn(store, 'getLiveBlueprintApplication').mockImplementation((id: number) => { + DatabaseService.getInstance().getDb() + .prepare('DELETE FROM gitops_applications WHERE blueprint_id = ?').run(id); + return live; + }); + + const detail = await request(app).get(`/api/blueprints/${created.body.id}`).set('Cookie', adminCookie); + expect(detail.status).toBe(200); + expect(detail.body.gitopsRevision.targetMode).toBe('not_applicable'); + // The distinguishing fact: an unmodelled Blueprint carries no limitation. + expect(detail.body.gitopsRevision.limitations).toEqual([ + expect.objectContaining({ code: 'application_row_missing' }), + ]); + }); + + it('leaves an unmodelled Blueprint with no limitation, so the two stay distinguishable', async () => { + const bp = seedUnmodelledBlueprint(); + const detail = await request(app).get(`/api/blueprints/${bp.id}`).set('Cookie', adminCookie); + expect(detail.body.gitopsRevision.limitations).toEqual([]); + }); +}); + +describe('Node-label routes report only the Blueprints a label moved', () => { + it('carries gitopsRevisions for a Blueprint whose selector reacts to the label', async () => { + const nodeId = seedNode(); + const created = await createBlueprint({ type: 'labels', all: ['edge'] }); + + const res = await request(app) + .post(`/api/node-labels/${nodeId}`) + .set('Cookie', adminCookie) + .send({ label: 'edge' }); + expect(res.status).toBe(201); + expect(res.body).toMatchObject({ nodeId, label: 'edge' }); + expect(res.body.gitopsRevisions).toHaveLength(1); + expect(res.body.gitopsRevisions[0]).toMatchObject({ + blueprintId: created.body.id, + applicationId: created.body.gitopsRevision.applicationId, + }); + }); + + it('reports an empty list for a label no selector mentions', async () => { + const nodeId = seedNode(); + await createBlueprint({ type: 'nodes', ids: [] }); + + const res = await request(app) + .post(`/api/node-labels/${nodeId}`) + .set('Cookie', adminCookie) + .send({ label: 'unrelated' }); + expect(res.status).toBe(201); + expect(res.body.gitopsRevisions).toEqual([]); + }); + + it('keeps DELETE at 204 with no body', async () => { + const nodeId = seedNode(); + await request(app) + .post(`/api/node-labels/${nodeId}`) + .set('Cookie', adminCookie) + .send({ label: 'edge' }); + + const res = await request(app) + .delete(`/api/node-labels/${nodeId}/edge`) + .set('Cookie', adminCookie); + expect(res.status).toBe(204); + expect(res.body).toEqual({}); + expect(res.text).toBeFalsy(); + }); +}); + +describe('Node routes carry gitopsRevisions', () => { + it('carries the field on cordon, empty because a cordon revises no intent', async () => { + const nodeId = seedNode(); + await createBlueprint({ type: 'nodes', ids: [nodeId] }); + + const res = await request(app).post(`/api/nodes/${nodeId}/cordon`).set('Cookie', adminCookie).send({}); + expect(res.status).toBe(200); + expect(res.body.cordoned).toBe(true); + // Deliberately empty, and asserted so the reason is not lost. A cordon + // suppresses new placements; it does not change what a Blueprint asks + // for, and `listDesiredNodes` reports what is asked for. The set is + // therefore identical either side of the write, so nothing is revised + // and nothing is reported. The field is still present, so a consumer + // reads one shape across every mutation. + expect(res.body.gitopsRevisions).toEqual([]); + }); + + it('carries the field on uncordon', async () => { + const nodeId = seedNode(); + await createBlueprint({ type: 'nodes', ids: [nodeId] }); + await request(app).post(`/api/nodes/${nodeId}/cordon`).set('Cookie', adminCookie).send({}); + + const res = await request(app).post(`/api/nodes/${nodeId}/uncordon`).set('Cookie', adminCookie).send({}); + expect(res.status).toBe(200); + expect(res.body.cordoned).toBe(false); + expect(res.body.gitopsRevisions).toEqual([]); + }); + + it('orders revisions by blueprintId ascending when a mutation moves several', async () => { + const nodeId = seedNode(); + const first = await createBlueprint({ type: 'labels', all: ['fleet'] }); + const second = await createBlueprint({ type: 'labels', all: ['fleet'] }); + + const res = await request(app) + .post(`/api/node-labels/${nodeId}`) + .set('Cookie', adminCookie) + .send({ label: 'fleet' }); + expect(res.status).toBe(201); + // Ordering is the contract, not the order the producer happened to visit + // the Blueprints in, which is a Map iteration order. + const ids = res.body.gitopsRevisions.map((r: { blueprintId: number }) => r.blueprintId); + expect(ids).toEqual([first.body.id, second.body.id].sort((a, b) => a - b)); + }); + + it('reports the Blueprints that lost a target when a node is deleted', async () => { + const nodeId = seedNode(); + const bp = await createBlueprint({ type: 'nodes', ids: [nodeId] }); + const applicationId = bp.body.gitopsRevision.applicationId; + // A target has to exist on the node for the deletion to retire one. The + // route reads the owners before the tombstone, which is the only moment + // the link from target back to Blueprint still exists. + DatabaseService.getInstance().getDb().prepare( + `INSERT INTO gitops_target_current (application_id, node_id, target_status, updated_at) + VALUES (?, ?, 'active', ?)`, + ).run(applicationId, nodeId, Date.now()); + + const res = await request(app).delete(`/api/nodes/${nodeId}`).set('Cookie', adminCookie); + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + expect(res.body.gitopsRevisions.map((r: { blueprintId: number }) => r.blueprintId)).toEqual([bp.body.id]); + }); + + it('still reports a node deletion as successful when the revision projection fails', async () => { + const nodeId = seedNode(); + await createBlueprint({ type: 'nodes', ids: [nodeId] }); + // The write commits before the decoration is built. If a projection + // fault escaped, the operator would be told a hard delete failed and + // would retry it, and the retry answers "Node not found": two wrong + // answers about an operation that actually succeeded. + const store = (await import('../services/gitops/store')).GitOpsStore.getInstance(); + vi.spyOn(store, 'getLiveBlueprintApplication').mockImplementation(() => { + throw new Error('projection exploded'); + }); + + const res = await request(app).delete(`/api/nodes/${nodeId}`).set('Cookie', adminCookie); + expect(res.status).toBe(200); + expect(res.body).toMatchObject({ success: true, gitopsRevisions: [] }); + // And the node really is gone, so the success it reported was true. + expect(DatabaseService.getInstance().getNode(nodeId)).toBeUndefined(); + }); + + it('reports an empty list when a deleted node held no Blueprint target', async () => { + const nodeId = seedNode(); + const res = await request(app).delete(`/api/nodes/${nodeId}`).set('Cookie', adminCookie); + expect(res.status).toBe(200); + expect(res.body).toMatchObject({ success: true, gitopsRevisions: [] }); + }); +}); diff --git a/backend/src/__tests__/gitops-approvals.test.ts b/backend/src/__tests__/gitops-approvals.test.ts new file mode 100644 index 00000000..ece53e8a --- /dev/null +++ b/backend/src/__tests__/gitops-approvals.test.ts @@ -0,0 +1,353 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { GitOpsStore } from '../services/gitops/store'; +import { encodeGitOpsApprovedTargetEffectJson, encodeGitOpsRequiredTargetsJson } from '../services/gitops/json'; +import type { + GitOpsApplicationRow, + GitOpsApprovalRow, + GitOpsGenerationRow, + GitOpsIntentRevisionRow, + GitOpsRolloutCandidateRow, +} from '../services/gitops/types'; + +describe('gitops approvals', () => { + let tmpDir: string; + + beforeAll(async () => { + tmpDir = await setupTestDb(); + GitOpsStore.resetForTests(); + const store = GitOpsStore.getInstance(); + store.insertApplication(directApp('app-a', 'stack-a')); + store.insertGeneration(generation('gen-a', 'app-a')); + store.insertGeneration(generation('gen-b', 'app-a')); + }); + + afterAll(() => { + cleanupTestDb(tmpDir); + }); + + it('rejects exact kind/authority mismatches at the CHECK floor', async () => { + const { DatabaseService } = await import('../services/DatabaseService'); + const db = DatabaseService.getInstance().getDb(); + const insert = db.prepare( + `INSERT INTO gitops_approvals ( + id, kind, authority, authoritative, application_id, generation_id, intent_revision_id, + artifact_set_id, rollout_candidate_id, rollout_generation_id, source_acceptance_ref, + placement_approval_ref, required_targets_json, preflight_fingerprint, fingerprint, + blast_json, policy_provenance_json, actor, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ); + expect(() => insert.run( + 'bad-src-legacy', 'source_acceptance', 'legacy_combined', 1, 'app-a', 'gen-a', + null, null, null, null, null, null, null, null, null, null, null, 'tester', 1, + )).toThrow(); + expect(() => insert.run( + 'bad-src-auth0', 'source_acceptance', 'operator', 0, 'app-a', 'gen-a', + null, null, null, null, null, null, null, null, null, null, null, 'tester', 1, + )).toThrow(); + expect(() => insert.run( + 'bad-legacy-auth1', 'legacy_combined', 'legacy_combined', 1, 'app-a', null, + null, null, null, null, null, null, null, null, null, null, null, 'tester', 1, + )).toThrow(); + expect(() => insert.run( + 'bad-legacy-op', 'legacy_combined', 'operator', 0, 'app-a', null, + null, null, null, null, null, null, null, null, null, null, null, 'tester', 1, + )).toThrow(); + }); + + it('resolves source acceptance only for the expected generation', () => { + const store = GitOpsStore.getInstance(); + store.insertApproval(sourceAcceptance('acc-a', 'app-a', 'gen-a')); + store.insertApproval(sourceAcceptance('acc-b', 'app-a', 'gen-b')); + expect(store.resolveApprovalRef('acc-a', { + kind: 'source_acceptance', + applicationId: 'app-a', + generationId: 'gen-a', + })?.id).toBe('acc-a'); + expect(store.resolveApprovalRef('acc-a', { + kind: 'source_acceptance', + applicationId: 'app-a', + generationId: 'gen-b', + })).toBeNull(); + expect(store.newestSourceAcceptanceId('app-a', 'gen-a')).toBe('acc-a'); + }); + + it('validates placement effects against required nodes without set equality', () => { + const store = GitOpsStore.getInstance(); + store.insertIntentRevision(intent('intent-1', 'app-a')); + store.insertApproval(placement('place-subset', 'app-a', 'intent-1', [ + { nodeId: 2, outcome: 'place' }, + ])); + store.insertApproval(placement('place-empty', 'app-a', 'intent-1', [])); + store.insertApproval(placement('place-remove-extra', 'app-a', 'intent-1', [ + { nodeId: 3, outcome: 'remove' }, + ])); + const required = [1, 2]; + expect(store.resolveApprovalRef('place-subset', { + kind: 'placement_approval', + applicationId: 'app-a', + intentRevisionId: 'intent-1', + requiredNodeIds: required, + })?.id).toBe('place-subset'); + expect(store.resolveApprovalRef('place-empty', { + kind: 'placement_approval', + applicationId: 'app-a', + intentRevisionId: 'intent-1', + requiredNodeIds: required, + })?.id).toBe('place-empty'); + expect(store.resolveApprovalRef('place-remove-extra', { + kind: 'placement_approval', + applicationId: 'app-a', + intentRevisionId: 'intent-1', + requiredNodeIds: required, + })?.id).toBe('place-remove-extra'); + store.insertApproval(placement('place-bad-required', 'app-a', 'intent-1', [ + { nodeId: 9, outcome: 'place' }, + ])); + expect(store.resolveApprovalRef('place-bad-required', { + kind: 'placement_approval', + applicationId: 'app-a', + intentRevisionId: 'intent-1', + requiredNodeIds: required, + })).toBeNull(); + store.insertApproval(placement('remove-required', 'app-a', 'intent-1', [ + { nodeId: 1, outcome: 'remove' }, + ])); + expect(store.resolveApprovalRef('remove-required', { + kind: 'placement_approval', + applicationId: 'app-a', + intentRevisionId: 'intent-1', + requiredNodeIds: required, + })).toBeNull(); + }); + + it('refuses to persist an approval whose evidence JSON cannot be decoded', () => { + const store = GitOpsStore.getInstance(); + store.insertApplication(directApp('app-badjson', 'badjson-web')); + store.insertGeneration(generation('gen-badjson', 'app-badjson')); + store.insertIntentRevision(intent('intent-badjson', 'app-badjson')); + expect(() => store.insertApproval({ + ...placement('appr-badblast', 'app-badjson', 'intent-badjson', []), + generation_id: null, + blast_json: '[{"nodeId":2,"outcome":"place"},{"nodeId":1,"outcome":"remove"}]', + })).toThrow(); + expect(() => store.insertApproval({ + ...placement('appr-badtargets', 'app-badjson', 'intent-badjson', []), + generation_id: null, + required_targets_json: '{"nodeIds":[2,1]}', + })).toThrow(); + expect(store.getApproval('appr-badblast')).toBeUndefined(); + expect(store.getApproval('appr-badtargets')).toBeUndefined(); + }); + + it('does not treat a CHECK-valid row as proof when expected identity differs', () => { + const store = GitOpsStore.getInstance(); + store.insertIntentRevision(intent('intent-2', 'app-a')); + store.insertRolloutCandidate(candidate('cand-1', 'app-a', 'intent-2', 'gen-a')); + store.insertApproval(sourceAcceptance('acc-bind-a', 'app-a', 'gen-a')); + store.insertApproval(placement('place-bind', 'app-a', 'intent-2', [ + { nodeId: 1, outcome: 'place' }, + ])); + const fingerprint = 'ab'.repeat(32); + store.insertApproval({ + id: 'rollout-1', + kind: 'rollout_authorization', + authority: 'operator', + authoritative: 1, + application_id: 'app-a', + generation_id: 'gen-a', + intent_revision_id: 'intent-2', + artifact_set_id: 'art-missing', + rollout_candidate_id: 'cand-1', + rollout_generation_id: null, + source_acceptance_ref: 'acc-bind-a', + placement_approval_ref: 'place-bind', + required_targets_json: encodeGitOpsRequiredTargetsJson([1]), + preflight_fingerprint: fingerprint, + fingerprint: null, + blast_json: null, + policy_provenance_json: null, + actor: 'tester', + created_at: 10, + }); + expect(store.resolveApprovalRef('rollout-1', { + kind: 'rollout_authorization', + applicationId: 'app-a', + binding: { + rolloutCandidateId: 'cand-1', + acceptedGenerationId: 'gen-b', + artifactSetId: 'art-missing', + intentRevisionId: 'intent-2', + requiredNodeIds: [1], + sourceAcceptanceRef: 'acc-bind-a', + placementApprovalRef: 'place-bind', + preflightFingerprint: fingerprint, + }, + })).toBeNull(); + }); +}); + +function sourceAcceptance(id: string, applicationId: string, generationId: string): GitOpsApprovalRow { + return { + id, + kind: 'source_acceptance', + authority: 'operator', + authoritative: 1, + application_id: applicationId, + generation_id: generationId, + intent_revision_id: null, + artifact_set_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + required_targets_json: null, + preflight_fingerprint: null, + fingerprint: null, + blast_json: null, + policy_provenance_json: null, + actor: 'tester', + created_at: 1, + }; +} + +function placement( + id: string, + applicationId: string, + intentRevisionId: string, + effect: Array<{ nodeId: number; outcome: 'place' | 'remove' }>, +): GitOpsApprovalRow { + return { + ...sourceAcceptance(id, applicationId, 'gen-a'), + kind: 'placement_approval', + generation_id: null, + intent_revision_id: intentRevisionId, + blast_json: encodeGitOpsApprovedTargetEffectJson(effect), + }; +} + +function intent(id: string, applicationId: string): GitOpsIntentRevisionRow { + return { + id, + application_id: applicationId, + blueprint_id: 1, + compose_content_sha256: 'c'.repeat(64), + blueprint_revision: 1, + deploy_stack_name: 'web', + selector_json: '{}', + pinned_node_id: null, + cordon_implications_json: '[]', + rollout_strategy_json: '{}', + runtime_drift_policy: null, + stateful_policy_json: null, + health_failure_rollback_policy_json: null, + operation_id: 'op-intent', + actor: 'tester', + created_at: 1, + }; +} + +function candidate( + id: string, + applicationId: string, + intentRevisionId: string, + acceptedGenerationId: string, +): GitOpsRolloutCandidateRow { + return { + id, + application_id: applicationId, + intent_revision_id: intentRevisionId, + compose_content_sha256: 'c'.repeat(64), + accepted_generation_id: acceptedGenerationId, + artifact_set_id: null, + required_targets_json: encodeGitOpsRequiredTargetsJson([1]), + authoritative: 0, + provenance: 'legacy_inline', + operation_id: 'op-cand', + created_at: 1, + }; +} + +function directApp(id: string, stackName: string): GitOpsApplicationRow { + return { + id, + lifecycle_key: `direct:${stackName}`, + lifecycle_status: 'active', + target_mode: 'direct', + stack_name: stackName, + blueprint_id: null, + configured_repo_url: 'https://github.com/org/repo.git', + repo_identity_json: '{"host":"github.com","pathname":"/org/repo.git"}', + configured_ref: 'main', + compose_paths_json: '["compose.yml"]', + context_dir: null, + sync_env: 0, + env_path: null, + materialization_fingerprint: 'a'.repeat(64), + desired_commit_sha: null, + fetched_commit_sha: null, + candidate_generation_id: null, + accepted_generation_id: null, + candidate_plan_blocked: 0, + review_required: 0, + artifact_set_id: null, + latest_artifact_set_id: null, + intent_revision_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + rollout_authorization_ref: null, + legacy_combined_approval_ref: null, + preflight_fingerprint: null, + latest_operation_id: null, + active_operation_id: null, + active_operation_stage: null, + active_operation_at: null, + active_generation_id: null, + pause_at: null, + pause_reason: null, + partial_json: null, + failure_stage: null, + failure_class: null, + failure_at: null, + retry_at: null, + retry_count: 0, + suspended_at: null, + recovery_ref: null, + recovery_phase: null, + interruption_stage: null, + interruption_at: null, + interruption_operation_id: null, + interruption_generation_id: null, + evidence_fresh_at: null, + evidence_limitations_json: null, + created_at: 1, + updated_at: 1, + }; +} + +function generation(id: string, applicationId: string): GitOpsGenerationRow { + return { + id, + application_id: applicationId, + commit_sha: id, + repo_url: 'https://github.com/org/repo.git', + configured_ref: 'main', + repo_identity_json: '{"host":"github.com","pathname":"/org/repo.git"}', + manifest_version: 0, + candidate_dir: `generations/candidate-${id}`, + applied_dir: `generations/applied-${id}-0`, + expected_invocation_json: '{"composeFileOrder":[],"projectName":null,"projectDirectory":null,"envFileOrder":[]}', + materialization_fingerprint: 'a'.repeat(64), + validation_ok: 1, + plan_blocked: 0, + change_plan_fingerprint: null, + operation_id: `op-${id}`, + trigger: 'manual', + actor: 'tester', + previous_generation_id: null, + redacted_limitations_json: '[]', + created_at: 1, + }; +} diff --git a/backend/src/__tests__/gitops-blueprint-deployment-causes.test.ts b/backend/src/__tests__/gitops-blueprint-deployment-causes.test.ts new file mode 100644 index 00000000..1cbd9541 --- /dev/null +++ b/backend/src/__tests__/gitops-blueprint-deployment-causes.test.ts @@ -0,0 +1,272 @@ +/** + * Blueprint deployment writes, recorded by cause. + * + * The cause has to be carried rather than inferred, because several land on the + * same deployment status. A deploy that failed and a withdraw that failed both + * read `failed`, and they mean opposite things about whether the deployment is + * still on the node. + */ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { DatabaseService, type Blueprint } from '../services/DatabaseService'; +import { GitOpsStore } from '../services/gitops/store'; +import { GitOpsTransitions } from '../services/gitops/transitions'; +import { commitBlueprintCreate, commitBlueprintUpdate } from '../services/gitops/blueprintProducers'; +import { + commitBlueprintDeploymentCause, + commitBlueprintDeploymentRemoved, +} from '../services/gitops/blueprintDeploymentProducers'; + +const NODE = 1; +const NEXT_COMPOSE = 'services:\n web:\n image: nginx:1.29\n'; + +describe('gitops blueprint deployment causes', () => { + let tmpDir: string; + + beforeAll(async () => { + tmpDir = await setupTestDb(); + GitOpsStore.resetForTests(); + GitOpsTransitions.resetForTests(); + }); + + afterAll(() => { + cleanupTestDb(tmpDir); + }); + + it('creates the target on the first deploy and records what was requested', () => { + const store = GitOpsStore.getInstance(); + const blueprint = create('dc-first'); + const app = store.getLiveBlueprintApplication(blueprint.id)!; + // A Blueprint application has no targets until something is sent somewhere. + expect(store.getTarget(app.id, NODE)).toBeUndefined(); + + deploying(blueprint); + + const target = store.getTarget(app.id, NODE)!; + expect(target.active_operation_stage).toBe('blueprint_deploy_started'); + expect(target.active_intent_revision_id).toBe(app.intent_revision_id); + }); + + it('acknowledges the intent the node was actually sent', () => { + const store = GitOpsStore.getInstance(); + const blueprint = create('dc-ack'); + const app = store.getLiveBlueprintApplication(blueprint.id)!; + deploying(blueprint); + + commitBlueprintDeploymentCause('deploy_ack', blueprint.id, NODE, { + status: 'active', last_checked_at: Date.now(), + }, 'tester'); + + const target = store.getTarget(app.id, NODE)!; + expect(target.intent_revision_id).toBe(app.intent_revision_id); + expect(target.active_operation_stage).toBeNull(); + }); + + it('records nothing when an observation repeats the state it already reported', () => { + const store = GitOpsStore.getInstance(); + const db = DatabaseService.getInstance(); + const blueprint = create('dc-repeat'); + const app = store.getLiveBlueprintApplication(blueprint.id)!; + deploying(blueprint); + commitBlueprintDeploymentCause('drift_observed', blueprint.id, NODE, { + status: 'drifted', last_checked_at: Date.now(), drift_summary: 'moved', + }, 'tester'); + const before = historyCount(db, app.id); + + // A reconciler tick re-asserting a state it already reported must not + // append a second event describing the same fact. + commitBlueprintDeploymentCause('drift_observed', blueprint.id, NODE, { + status: 'drifted', last_checked_at: Date.now(), drift_summary: 'moved again', + }, 'tester'); + expect(historyCount(db, app.id)).toBe(before); + }); + + it('supersedes a stuck deploy rather than answering the request it replaced', () => { + const store = GitOpsStore.getInstance(); + const blueprint = create('dc-stuck'); + const app = store.getLiveBlueprintApplication(blueprint.id)!; + deploying(blueprint); + const stale = store.getTarget(app.id, NODE)!.active_intent_revision_id; + + // The Blueprint changed while the row sat at `deploying`, so a redeploy is + // asking for something new. The status does not move, and gating the start + // on that would let the later acknowledgement answer the stale request. + commitBlueprintUpdate( + blueprint.id, { compose_content: NEXT_COMPOSE }, 'tester', () => [NODE], + ); + const revised = store.getLiveBlueprintApplication(blueprint.id)!.intent_revision_id; + expect(revised).not.toBe(stale); + + deploying(blueprint); + expect(store.getTarget(app.id, NODE)!.active_intent_revision_id).toBe(revised); + + commitBlueprintDeploymentCause('deploy_ack', blueprint.id, NODE, { + status: 'active', last_checked_at: Date.now(), + }, 'tester'); + // Converged on what was actually asked for last, not on the superseded one. + expect(store.getTarget(app.id, NODE)!.intent_revision_id).toBe(revised); + }); + + it('keeps a failed deploy distinct from a failed withdraw', () => { + const store = GitOpsStore.getInstance(); + const failed = create('dc-deploy-fail'); + deploying(failed); + commitBlueprintDeploymentCause('deploy_fail', failed.id, NODE, { + status: 'failed', last_checked_at: Date.now(), last_error: 'boom', + }, 'tester'); + const deployTarget = store.getTarget(store.getLiveBlueprintApplication(failed.id)!.id, NODE)!; + expect(deployTarget.failure_stage).toBe('blueprint_deploy'); + expect(deployTarget.target_status).toBe('active'); + + const withdrawn = create('dc-withdraw-fail'); + deploying(withdrawn); + commitBlueprintDeploymentCause('deploy_ack', withdrawn.id, NODE, { + status: 'active', last_checked_at: Date.now(), + }, 'tester'); + commitBlueprintDeploymentCause('withdraw_start', withdrawn.id, NODE, { + status: 'withdrawing', last_checked_at: Date.now(), + }, 'tester'); + commitBlueprintDeploymentCause('withdraw_fail', withdrawn.id, NODE, { + status: 'failed', last_checked_at: Date.now(), last_error: 'boom', + }, 'tester'); + + const withdrawTarget = store.getTarget(store.getLiveBlueprintApplication(withdrawn.id)!.id, NODE)!; + // Same deployment status, opposite meaning: the deployment is still there. + expect(withdrawTarget.failure_stage).toBe('blueprint_withdraw'); + expect(withdrawTarget.target_status).toBe('active'); + }); + + it('classifies a name conflict as its own failure', () => { + const store = GitOpsStore.getInstance(); + const blueprint = create('dc-conflict'); + deploying(blueprint); + commitBlueprintDeploymentCause('name_conflict', blueprint.id, NODE, { + status: 'name_conflict', last_checked_at: Date.now(), last_error: 'taken', + }, 'tester'); + + const target = store.getTarget(store.getLiveBlueprintApplication(blueprint.id)!.id, NODE)!; + expect(target.failure_class).toBe('name_conflict'); + }); + + it('tombstones the target when the deployment row is removed', () => { + const store = GitOpsStore.getInstance(); + const blueprint = create('dc-removed'); + deploying(blueprint); + commitBlueprintDeploymentCause('deploy_ack', blueprint.id, NODE, { + status: 'active', last_checked_at: Date.now(), + }, 'tester'); + commitBlueprintDeploymentCause('withdraw_start', blueprint.id, NODE, { + status: 'withdrawing', last_checked_at: Date.now(), + }, 'tester'); + + commitBlueprintDeploymentRemoved(blueprint.id, NODE, 'tester'); + + const target = store.getTarget(store.getLiveBlueprintApplication(blueprint.id)!.id, NODE)!; + expect(target.target_status).toBe('tombstoned'); + }); + + it('re-opens a severed target when a deploy starts again', () => { + // After a withdraw the reconciler must not redeploy onto the severed + // placement unrecorded, while an explicit deploy must land in the model + // instead of being refused and ignored. + const store = GitOpsStore.getInstance(); + const blueprint = create('dc-revive'); + deploying(blueprint); + commitBlueprintDeploymentCause('deploy_ack', blueprint.id, NODE, { + status: 'active', last_checked_at: Date.now(), + }, 'tester'); + commitBlueprintDeploymentCause('withdraw_start', blueprint.id, NODE, { + status: 'withdrawing', last_checked_at: Date.now(), + }, 'tester'); + commitBlueprintDeploymentRemoved(blueprint.id, NODE, 'tester'); + + const appId = store.getLiveBlueprintApplication(blueprint.id)!.id; + expect(store.getTarget(appId, NODE)!.target_status).toBe('tombstoned'); + + deploying(blueprint); + + const revived = store.getTarget(appId, NODE)!; + expect(revived.target_status).toBe('active'); + expect(revived.active_operation_stage).toBe('blueprint_deploy_started'); + }); + + it('observes without acknowledging anything', () => { + const store = GitOpsStore.getInstance(); + const blueprint = create('dc-observe'); + deploying(blueprint); + + commitBlueprintDeploymentCause('drift_observed', blueprint.id, NODE, { + status: 'drifted', last_checked_at: Date.now(), drift_summary: 'image moved', + }, 'tester'); + + const target = store.getTarget(store.getLiveBlueprintApplication(blueprint.id)!.id, NODE)!; + expect(target.latest_stage).toBe('blueprint_drifted'); + // An observation says what was seen, never what was agreed. + expect(target.intent_revision_id).toBeNull(); + }); + + it('records a stateful first placement, which happens before any deploy', () => { + const store = GitOpsStore.getInstance(); + const blueprint = create('dc-first-placement'); + const app = store.getLiveBlueprintApplication(blueprint.id)!; + // No deploy has run, so there is no target: this is the case that used to + // drop the observation and leave the hold unrecorded. + expect(store.getTarget(app.id, NODE)).toBeUndefined(); + + commitBlueprintDeploymentCause('await_state_review', blueprint.id, NODE, { + status: 'pending_state_review', last_checked_at: Date.now(), + }, 'tester'); + + const target = store.getTarget(app.id, NODE)!; + expect(target.latest_stage).toBe('blueprint_state_review'); + // First contact only. Nothing has been sent, applied or agreed. + expect(target.intent_revision_id).toBeNull(); + expect(target.desired_generation_id).toBeNull(); + expect(target.active_operation_stage).toBeNull(); + // Unset rather than reachable: a node that has only been asked to hold + // something has not been contacted, and claiming reachability here would + // make the rollout facet answer for a node nobody has spoken to. + expect(target.connectivity).toBeNull(); + }); + + it('still drops an observation for a node with no target and no placement', () => { + // Only the first-placement hold creates a target. A drift or evict report + // for a node nothing was ever sent to describes a deployment this model + // does not have, so it stays dropped. + const store = GitOpsStore.getInstance(); + const blueprint = create('dc-observe-no-target'); + const app = store.getLiveBlueprintApplication(blueprint.id)!; + + commitBlueprintDeploymentCause('drift_observed', blueprint.id, NODE, { + status: 'drifted', last_checked_at: Date.now(), drift_summary: 'image moved', + }, 'tester'); + + expect(store.getTarget(app.id, NODE)).toBeUndefined(); + }); +}); + +function historyCount(db: DatabaseService, applicationId: string): number { + return (db.getDb() + .prepare('SELECT COUNT(*) AS n FROM gitops_history WHERE application_id = ?') + .get(applicationId) as { n: number }).n; +} + +function deploying(blueprint: Blueprint): void { + commitBlueprintDeploymentCause('deploy_start', blueprint.id, NODE, { + status: 'deploying', last_checked_at: Date.now(), + }, 'tester'); +} + +function create(name: string): Blueprint { + return commitBlueprintCreate({ + name, + description: null, + compose_content: 'services:\n web:\n image: nginx:1.27\n', + selector: { type: 'nodes', ids: [NODE] }, + drift_mode: 'suggest', + classification: 'stateless', + classification_reasons: [], + enabled: true, + created_by: 'tester', + }, () => [NODE]); +} diff --git a/backend/src/__tests__/gitops-blueprint-producers.test.ts b/backend/src/__tests__/gitops-blueprint-producers.test.ts new file mode 100644 index 00000000..e5c7eaf9 --- /dev/null +++ b/backend/src/__tests__/gitops-blueprint-producers.test.ts @@ -0,0 +1,338 @@ +/** + * Blueprint source-mutation producers. + * + * These are the seam between the Blueprint routes and the revision state, and + * the question they exist to answer is when an edit invalidates what the fleet + * already acknowledged. Renaming or re-selecting does; rewording a description + * does not, and minting an intent for the latter would make every node's + * acknowledgement read as stale over a change no node can observe. + */ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { DatabaseService, type Blueprint } from '../services/DatabaseService'; +import { GitOpsStore } from '../services/gitops/store'; +import { GitOpsTransitions } from '../services/gitops/transitions'; +import { + classifyBlueprintChange, + commitBlueprintCreate, + commitBlueprintDelete, + commitBlueprintPin, + commitBlueprintUpdate, +} from '../services/gitops/blueprintProducers'; + +const DESIRED = [1]; +const desiredNodeIdsFor = (): number[] => DESIRED; + +describe('gitops blueprint producers', () => { + let tmpDir: string; + + beforeAll(async () => { + tmpDir = await setupTestDb(); + GitOpsStore.resetForTests(); + GitOpsTransitions.resetForTests(); + }); + + afterAll(() => { + cleanupTestDb(tmpDir); + }); + + it('creates the Blueprint, its application, and the first intent and candidate together', () => { + const store = GitOpsStore.getInstance(); + const blueprint = create('bp-create'); + + const app = store.getLiveBlueprintApplication(blueprint.id)!; + expect(app.target_mode).toBe('inline_blueprint'); + expect(app.lifecycle_status).toBe('active'); + // No Git identity: an Inline Blueprint has no generations to point at. + expect(app.stack_name).toBeNull(); + expect(app.configured_repo_url).toBeNull(); + + const intent = store.getIntentRevision(app.intent_revision_id!)!; + expect(intent.blueprint_id).toBe(blueprint.id); + expect(intent.deploy_stack_name).toBe('bp-create'); + + const candidate = store.getRolloutCandidate(app.rollout_candidate_id!)!; + expect(candidate.intent_revision_id).toBe(intent.id); + expect(JSON.parse(candidate.required_targets_json)).toEqual({ nodeIds: DESIRED }); + + // History starts where the application does. Beginning at the first intent + // would describe an application nothing records coming into existence. + const stages = DatabaseService.getInstance().getDb().prepare( + 'SELECT stage FROM gitops_history WHERE application_id = ? ORDER BY rowid ASC', + ).all(app.id) as Array<{ stage: string }>; + expect(stages.map(row => row.stage)) + .toEqual(['application_activated', 'intent_revised', 'rollout_candidate_opened']); + + // No targets until something is deployed somewhere. + expect(store.listTargets(app.id)).toEqual([]); + }); + + it('refuses a second live application for the same Blueprint', () => { + const store = GitOpsStore.getInstance(); + const blueprint = create('bp-single'); + const app = store.getLiveBlueprintApplication(blueprint.id)!; + const tx = GitOpsTransitions.getInstance(); + + expect(() => tx.activateInlineBlueprint({ + application: { ...app, id: 'second-app' }, + envelope: { operationId: 'op-dup', actor: 'tester', trigger: 'manual', at: Date.now() }, + })).toThrow(/already exists/); + }); + + it('mints nothing when an edit changes no value', () => { + const store = GitOpsStore.getInstance(); + const blueprint = create('bp-noop'); + const app = store.getLiveBlueprintApplication(blueprint.id)!; + + const result = commitBlueprintUpdate( + blueprint.id, { name: 'bp-noop', drift_mode: blueprint.drift_mode }, 'tester', desiredNodeIdsFor, + ); + + expect(result.change).toBe('none'); + const after = store.getLiveBlueprintApplication(blueprint.id)!; + expect(after.intent_revision_id).toBe(app.intent_revision_id); + expect(after.rollout_candidate_id).toBe(app.rollout_candidate_id); + }); + + it('leaves the acknowledged intent alone when only the description changes', () => { + const store = GitOpsStore.getInstance(); + const blueprint = create('bp-meta'); + const app = store.getLiveBlueprintApplication(blueprint.id)!; + + const result = commitBlueprintUpdate( + blueprint.id, { description: 'now with a longer explanation' }, 'tester', desiredNodeIdsFor, + ); + + expect(result.change).toBe('metadata_only'); + expect(result.blueprint?.description).toBe('now with a longer explanation'); + // The source row moved and the intent did not: no node's acknowledgement + // became stale because someone reworded the description. + const after = store.getLiveBlueprintApplication(blueprint.id)!; + expect(after.intent_revision_id).toBe(app.intent_revision_id); + }); + + it('mints a new intent and candidate when the deployed content changes', () => { + const store = GitOpsStore.getInstance(); + const blueprint = create('bp-op'); + const app = store.getLiveBlueprintApplication(blueprint.id)!; + + const result = commitBlueprintUpdate( + blueprint.id, { compose_content: 'services:\n web:\n image: nginx:1.29\n' }, 'tester', desiredNodeIdsFor, + ); + + expect(result.change).toBe('operational'); + const after = store.getLiveBlueprintApplication(blueprint.id)!; + expect(after.intent_revision_id).not.toBe(app.intent_revision_id); + expect(after.rollout_candidate_id).not.toBe(app.rollout_candidate_id); + const intent = store.getIntentRevision(after.intent_revision_id!)!; + expect(intent.compose_content_sha256).not.toBe( + store.getIntentRevision(app.intent_revision_id!)!.compose_content_sha256, + ); + }); + + it('treats a pin as a placement change, and re-pinning the same node as nothing', () => { + const store = GitOpsStore.getInstance(); + const blueprint = create('bp-pin'); + const app = store.getLiveBlueprintApplication(blueprint.id)!; + + const pinned = commitBlueprintPin(blueprint.id, 1, 'tester', desiredNodeIdsFor); + expect(pinned.changed).toBe(true); + const afterPin = store.getLiveBlueprintApplication(blueprint.id)!; + expect(afterPin.intent_revision_id).not.toBe(app.intent_revision_id); + expect(store.getRolloutCandidate(afterPin.rollout_candidate_id!)?.provenance).toBe('roster_change'); + + const again = commitBlueprintPin(blueprint.id, 1, 'tester', desiredNodeIdsFor); + expect(again.changed).toBe(false); + expect(store.getLiveBlueprintApplication(blueprint.id)?.intent_revision_id) + .toBe(afterPin.intent_revision_id); + }); + + it('records the required set in a canonical order so a reorder is not a change', () => { + const store = GitOpsStore.getInstance(); + const blueprint = create('bp-order'); + commitBlueprintUpdate(blueprint.id, { name: 'bp-order-2' }, 'tester', () => [3, 1, 2]); + + const app = store.getLiveBlueprintApplication(blueprint.id)!; + expect(JSON.parse(store.getRolloutCandidate(app.rollout_candidate_id!)!.required_targets_json)) + .toEqual({ nodeIds: [1, 2, 3] }); + }); + + it('retires the application when the Blueprint is deleted', () => { + const store = GitOpsStore.getInstance(); + const blueprint = create('bp-delete'); + const app = store.getLiveBlueprintApplication(blueprint.id)!; + + expect(commitBlueprintDelete(blueprint.id, 'tester')).toBe(true); + + // The live slot has to be released, or the Blueprint name cannot be used + // again while a record of a deleted one still claims it. + expect(store.getLiveBlueprintApplication(blueprint.id)).toBeUndefined(); + expect(store.getApplication(app.id)?.lifecycle_status).toBe('deleted'); + }); + + it('does not bump the revision or void approval when only the description changed', () => { + const db = DatabaseService.getInstance(); + const store = GitOpsStore.getInstance(); + const blueprint = create('bp-full-save'); + const app = store.getLiveBlueprintApplication(blueprint.id)!; + const intentBefore = store.getIntentRevision(app.intent_revision_id!)!; + + // What the editor actually sends: every field, every save. The source layer + // decides what to invalidate from which keys are present, so submitting an + // unchanged compose body used to advance the revision past the one the + // current intent describes, and clear the approval, while this layer + // classified it as metadata and minted nothing. + const result = commitBlueprintUpdate(blueprint.id, { + name: blueprint.name, + description: 'reworded', + compose_content: blueprint.compose_content, + selector: blueprint.selector, + drift_mode: blueprint.drift_mode, + enabled: blueprint.enabled, + bumpRevision: true, + }, 'tester', desiredNodeIdsFor); + + expect(result.change).toBe('metadata_only'); + const after = db.getBlueprint(blueprint.id)!; + expect(after.description).toBe('reworded'); + expect(after.revision).toBe(blueprint.revision); + expect(after.approval_status).toBe(blueprint.approval_status); + // The intent still describes the revision that is actually stored. + expect(store.getIntentRevision(app.intent_revision_id!)!.blueprint_revision).toBe(after.revision); + expect(store.getLiveBlueprintApplication(blueprint.id)!.intent_revision_id).toBe(intentBefore.id); + }); + + it('treats a reordered selector naming the same nodes as no change', () => { + const store = GitOpsStore.getInstance(); + const blueprint = create('bp-reorder'); + const app = store.getLiveBlueprintApplication(blueprint.id)!; + + const result = commitBlueprintUpdate( + blueprint.id, { selector: { type: 'nodes', ids: [1] } }, 'tester', desiredNodeIdsFor, + ); + + expect(result.change).toBe('none'); + expect(store.getLiveBlueprintApplication(blueprint.id)!.intent_revision_id).toBe(app.intent_revision_id); + }); + + it('rolls the Blueprint back when recording it fails', () => { + const db = DatabaseService.getInstance(); + const applicationsBefore = (db.getDb() + .prepare("SELECT COUNT(*) AS n FROM gitops_applications WHERE target_mode = 'inline_blueprint'") + .get() as { n: number }).n; + + // The source write and its record commit together or not at all, so a + // Blueprint can never exist with nothing describing what it means. + expect(() => commitBlueprintCreate({ + name: 'bp-rollback', + description: null, + compose_content: 'services:\n web:\n image: nginx:1.27\n', + selector: { type: 'nodes', ids: [1] }, + drift_mode: 'suggest', + classification: 'stateless', + classification_reasons: [], + enabled: true, + created_by: 'tester', + }, () => { throw new Error('placement lookup failed'); })).toThrow(/placement lookup failed/); + + expect(db.getBlueprintByName('bp-rollback')).toBeUndefined(); + // And no orphan application survived the rolled-back source write. + expect(db.getDb() + .prepare("SELECT COUNT(*) AS n FROM gitops_applications WHERE target_mode = 'inline_blueprint'") + .get() as { n: number }).toEqual({ n: applicationsBefore }); + }); + + it('leaves every other Blueprint untouched when one is edited', () => { + const store = GitOpsStore.getInstance(); + const other = create('bp-bystander'); + const edited = create('bp-edited'); + const otherBefore = store.getLiveBlueprintApplication(other.id)!; + + commitBlueprintUpdate(edited.id, { name: 'bp-edited-2' }, 'tester', desiredNodeIdsFor); + + const otherAfter = store.getLiveBlueprintApplication(other.id)!; + expect(otherAfter.intent_revision_id).toBe(otherBefore.intent_revision_id); + expect(otherAfter.rollout_candidate_id).toBe(otherBefore.rollout_candidate_id); + expect(store.getIntentRevision(otherAfter.intent_revision_id!)!.deploy_stack_name).toBe('bp-bystander'); + }); + + it('classifies each field without touching the database', () => { + const before = { + name: 'a', description: 'd', compose_content: 'c', selector: { type: 'nodes', ids: [1] }, + drift_mode: 'suggest', enabled: true, classification: 'stateless', classification_reasons: [], + } as unknown as Blueprint; + + expect(classifyBlueprintChange(before, {})).toBe('none'); + expect(classifyBlueprintChange(before, { name: 'a' })).toBe('none'); + expect(classifyBlueprintChange(before, { description: 'd' })).toBe('none'); + expect(classifyBlueprintChange(before, { description: 'other' })).toBe('metadata_only'); + expect(classifyBlueprintChange(before, { name: 'b' })).toBe('operational'); + expect(classifyBlueprintChange(before, { enabled: false })).toBe('operational'); + // Selector equality is by value: the same set written again is not a change. + expect(classifyBlueprintChange(before, { selector: { type: 'nodes', ids: [1] } })).toBe('none'); + expect(classifyBlueprintChange(before, { selector: { type: 'nodes', ids: [2] } })).toBe('operational'); + // An operational change alongside a metadata one is still operational. + expect(classifyBlueprintChange(before, { name: 'b', description: 'other' })).toBe('operational'); + }); + + describe('an application that is not yet active', () => { + // The live-slot lookup answers with `active` or `creating`, because its + // other callers ask whether the slot is taken. The transitions that mint + // intents accept only `active` and reject anything else by throwing, inside + // the caller's own transaction, so a `creating` row reaching one would fail + // the operator's edit and roll the Blueprint write back with it. + // + // The state is written directly here because no production path creates a + // Blueprint-mode application in `creating`: they all go through + // `blankInlineApplication`, which hardcodes `active`. These cases pin a + // deliberately defensive guard rather than a reachable behaviour, and are + // marked as such so nobody later reads them as live coverage. The Git + // backed `blueprint` mode is what makes the state producible. + const toCreating = (blueprintId: number): void => { + DatabaseService.getInstance().getDb() + .prepare('UPDATE gitops_applications SET lifecycle_status = ? WHERE blueprint_id = ?') + .run('creating', blueprintId); + }; + + it('lets an edit through without minting an intent', () => { + const store = GitOpsStore.getInstance(); + const blueprint = create('bp-creating-update'); + const app = store.getLiveBlueprintApplication(blueprint.id)!; + const intentBefore = app.intent_revision_id; + toCreating(blueprint.id); + + const result = commitBlueprintUpdate(blueprint.id, { name: 'bp-creating-renamed' }, 'tester', desiredNodeIdsFor); + + expect(result.blueprint?.name).toBe('bp-creating-renamed'); + expect(store.getLiveBlueprintApplication(blueprint.id)!.intent_revision_id).toBe(intentBefore); + }); + + it('lets a pin through without minting an intent', () => { + const store = GitOpsStore.getInstance(); + const blueprint = create('bp-creating-pin'); + const app = store.getLiveBlueprintApplication(blueprint.id)!; + const intentBefore = app.intent_revision_id; + toCreating(blueprint.id); + + const result = commitBlueprintPin(blueprint.id, 1, 'tester', desiredNodeIdsFor); + + expect(result.changed).toBe(true); + expect(result.blueprint?.pinned_node_id).toBe(1); + expect(store.getLiveBlueprintApplication(blueprint.id)!.intent_revision_id).toBe(intentBefore); + }); + }); +}); + +function create(name: string): Blueprint { + return commitBlueprintCreate({ + name, + description: null, + compose_content: 'services:\n web:\n image: nginx:1.27\n', + selector: { type: 'nodes', ids: [1] }, + drift_mode: 'suggest', + classification: 'stateless', + classification_reasons: [], + enabled: true, + created_by: 'tester', + }, desiredNodeIdsFor); +} diff --git a/backend/src/__tests__/gitops-blueprint-transitions.test.ts b/backend/src/__tests__/gitops-blueprint-transitions.test.ts new file mode 100644 index 00000000..1f84b695 --- /dev/null +++ b/backend/src/__tests__/gitops-blueprint-transitions.test.ts @@ -0,0 +1,628 @@ +/** + * Blueprint source and deployment transitions. + * + * These have no production caller yet; the Blueprint routes and the reconciler + * are wired to them in the same step. They are tested directly so the shape a + * caller must satisfy is pinned here rather than inferred from the deriver. + * + * The rule they share is that a terminal event has to name the request it is + * answering. A node that acknowledges a superseded intent has not converged on + * anything anyone asked for, and recording it as an acknowledgement is how a + * fleet comes to report agreement it does not have. + */ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { GitOpsStore, emptyTargetRow } from '../services/gitops/store'; +import { GitOpsTransitions, type EventEnvelope } from '../services/gitops/transitions'; +import { projectApplication } from '../services/gitops/derive'; +import type { + GitOpsApplicationRow, + GitOpsIntentRevisionRow, + GitOpsRolloutCandidateRow, +} from '../services/gitops/types'; + +describe('gitops blueprint transitions', () => { + let tmpDir: string; + + beforeAll(async () => { + tmpDir = await setupTestDb(); + GitOpsStore.resetForTests(); + GitOpsTransitions.resetForTests(); + }); + + afterAll(() => { + cleanupTestDb(tmpDir); + }); + + it('mints an intent and opens a candidate against it', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedInline('app-intent', 101); + + tx.intentRevised({ + applicationId: 'app-intent', + intent: intent('int-1', 'app-intent', 101), + envelope: env('op-int-1'), + }); + expect(store.getApplication('app-intent')?.intent_revision_id).toBe('int-1'); + + tx.rolloutCandidateOpened({ + applicationId: 'app-intent', + candidate: candidate('cand-1', 'app-intent', 'int-1'), + envelope: env('op-cand-1'), + }); + const app = store.getApplication('app-intent')!; + expect(app.rollout_candidate_id).toBe('cand-1'); + // Candidate-time facts only: nothing here claims anything was authorized. + const row = store.getRolloutCandidate('cand-1')!; + expect(row.intent_revision_id).toBe('int-1'); + expect(row.accepted_generation_id).toBeNull(); + }); + + it('refuses a candidate that does not name the current intent', () => { + const tx = GitOpsTransitions.getInstance(); + seedInline('app-stale-cand', 102); + tx.intentRevised({ + applicationId: 'app-stale-cand', + intent: intent('int-2', 'app-stale-cand', 102), + envelope: env('op-int-2'), + }); + + expect(() => tx.rolloutCandidateOpened({ + applicationId: 'app-stale-cand', + candidate: { ...candidate('cand-2', 'app-stale-cand', 'int-nonexistent') }, + envelope: env('op-cand-2'), + })).toThrow(/current intent/); + }); + + it('records a deploy, then accepts the ack that names it', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedInline('app-ack', 103, 1); + tx.intentRevised({ applicationId: 'app-ack', intent: intent('int-3', 'app-ack', 103), envelope: env('op-int-3') }); + + tx.blueprintDeployStarted({ + applicationId: 'app-ack', + nodeId: 1, + intentRevisionId: 'int-3', + rolloutCandidateId: null, + envelope: env('op-dep-3'), + }); + let target = store.getTarget('app-ack', 1)!; + expect(target.active_operation_stage).toBe('blueprint_deploy_started'); + expect(target.active_intent_revision_id).toBe('int-3'); + // Nothing is acknowledged yet: the request is in flight, not converged. + expect(target.intent_revision_id).toBeNull(); + + tx.blueprintAckRecorded({ + applicationId: 'app-ack', + nodeId: 1, + intentRevisionId: 'int-3', + rolloutCandidateId: null, + legacyAppliedRevision: 7, + envelope: env('op-ack-3'), + }); + target = store.getTarget('app-ack', 1)!; + expect(target.intent_revision_id).toBe('int-3'); + expect(target.active_operation_stage).toBeNull(); + expect(target.legacy_applied_revision).toBe(7); + // A Blueprint target has no Git generation to point at. + expect(target.desired_generation_id).toBeNull(); + }); + + it('ignores an acknowledgement for an intent the target was never asked to run', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedInline('app-super', 104, 1); + tx.intentRevised({ applicationId: 'app-super', intent: intent('int-4', 'app-super', 104), envelope: env('op-int-4') }); + tx.blueprintDeployStarted({ + applicationId: 'app-super', + nodeId: 1, + intentRevisionId: 'int-4', + rolloutCandidateId: null, + envelope: env('op-dep-4'), + }); + + // A newer intent superseded the one this node is running. + tx.intentRevised({ applicationId: 'app-super', intent: intent('int-5', 'app-super', 104), envelope: env('op-int-5') }); + + expect(() => tx.blueprintAckRecorded({ + applicationId: 'app-super', + nodeId: 1, + intentRevisionId: 'int-5', + rolloutCandidateId: null, + legacyAppliedRevision: null, + envelope: env('op-ack-5'), + })).toThrow(/was not asked to run/); + expect(store.getTarget('app-super', 1)?.intent_revision_id).toBeNull(); + }); + + it('clears a deploy failure only when the next deploy is acknowledged', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedInline('app-fail', 105, 1); + tx.intentRevised({ applicationId: 'app-fail', intent: intent('int-6', 'app-fail', 105), envelope: env('op-int-6') }); + tx.blueprintDeployStarted({ + applicationId: 'app-fail', + nodeId: 1, + intentRevisionId: 'int-6', + rolloutCandidateId: null, + envelope: env('op-dep-6'), + }); + tx.blueprintDeployFailed({ + applicationId: 'app-fail', + nodeId: 1, + failureClass: 'name_conflict', + envelope: env('op-dep-6'), + }); + + let target = store.getTarget('app-fail', 1)!; + expect(target.failure_stage).toBe('blueprint_deploy'); + expect(target.failure_class).toBe('name_conflict'); + expect(target.active_operation_stage).toBeNull(); + // A failure does not acknowledge anything. + expect(target.intent_revision_id).toBeNull(); + + tx.blueprintDeployStarted({ + applicationId: 'app-fail', + nodeId: 1, + intentRevisionId: 'int-6', + rolloutCandidateId: null, + envelope: env('op-dep-6b'), + }); + tx.blueprintAckRecorded({ + applicationId: 'app-fail', + nodeId: 1, + intentRevisionId: 'int-6', + rolloutCandidateId: null, + legacyAppliedRevision: null, + envelope: env('op-ack-6b'), + }); + target = store.getTarget('app-fail', 1)!; + expect(target.failure_stage).toBeNull(); + expect(target.intent_revision_id).toBe('int-6'); + }); + + it('withdraws against the intent being removed, not a later one', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedInline('app-wd', 106, 1); + tx.intentRevised({ applicationId: 'app-wd', intent: intent('int-7', 'app-wd', 106), envelope: env('op-int-7') }); + tx.blueprintWithdrawStarted({ + applicationId: 'app-wd', + nodeId: 1, + intentRevisionId: 'int-7', + envelope: env('op-wd-7'), + }); + + expect(() => tx.blueprintWithdrawn({ + applicationId: 'app-wd', + nodeId: 1, + intentRevisionId: 'int-other', + envelope: env('op-wd-7'), + })).toThrow(/was not asked to run/); + + tx.blueprintWithdrawn({ + applicationId: 'app-wd', + nodeId: 1, + intentRevisionId: 'int-7', + envelope: env('op-wd-7'), + }); + const target = store.getTarget('app-wd', 1)!; + expect(target.target_status).toBe('tombstoned'); + expect(target.active_operation_stage).toBeNull(); + }); + + it('re-opens a severed placement when a deploy starts again', () => { + // Withdrawal is terminal for the placement, not for the node. A later + // explicit deploy re-activates the target and records the revival in the + // same event, so the projection and the workload cannot disagree about + // whether this node runs the Blueprint. + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedInline('app-revive', 220, 1); + tx.intentRevised({ + applicationId: 'app-revive', + intent: intent('int-rev', 'app-revive', 220), + envelope: env('op-rev-int'), + }); + tx.blueprintDeployStarted({ + applicationId: 'app-revive', nodeId: 1, intentRevisionId: 'int-rev', + rolloutCandidateId: null, envelope: env('op-rev-d1'), + }); + tx.blueprintAckRecorded({ + applicationId: 'app-revive', nodeId: 1, intentRevisionId: 'int-rev', + rolloutCandidateId: null, legacyAppliedRevision: null, envelope: env('op-rev-a1'), + }); + tx.blueprintWithdrawStarted({ + applicationId: 'app-revive', nodeId: 1, intentRevisionId: 'int-rev', + envelope: env('op-rev-w1'), + }); + tx.blueprintWithdrawn({ + applicationId: 'app-revive', nodeId: 1, intentRevisionId: 'int-rev', + envelope: env('op-rev-w2'), + }); + expect(store.getTarget('app-revive', 1)?.target_status).toBe('tombstoned'); + + tx.blueprintDeployStarted({ + applicationId: 'app-revive', nodeId: 1, intentRevisionId: 'int-rev', + rolloutCandidateId: null, envelope: env('op-rev-d2'), + }); + + const revived = store.getTarget('app-revive', 1)!; + expect(revived.target_status).toBe('active'); + expect(revived.active_operation_stage).toBe('blueprint_deploy_started'); + // The acknowledged intent survives severance; only a fresh ack rewrites it. + expect(revived.intent_revision_id).toBe('int-rev'); + }); + + it('keeps a failed withdraw distinct from a failed deploy', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedInline('app-wdf', 107, 1); + tx.intentRevised({ applicationId: 'app-wdf', intent: intent('int-8', 'app-wdf', 107), envelope: env('op-int-8') }); + tx.blueprintWithdrawStarted({ + applicationId: 'app-wdf', + nodeId: 1, + intentRevisionId: 'int-8', + envelope: env('op-wd-8'), + }); + tx.blueprintWithdrawFailed({ + applicationId: 'app-wdf', + nodeId: 1, + failureClass: 'post_mutation', + envelope: env('op-wd-8'), + }); + + const target = store.getTarget('app-wdf', 1)!; + expect(target.failure_stage).toBe('blueprint_withdraw'); + // Still active: a withdraw that failed has not removed the deployment. + expect(target.target_status).toBe('active'); + }); + + it('releases the request identity with the operation, not just the stage', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedInline('app-ident', 120, 1); + tx.intentRevised({ applicationId: 'app-ident', intent: intent('int-20', 'app-ident', 120), envelope: env('op-int-20') }); + tx.blueprintDeployStarted({ + applicationId: 'app-ident', + nodeId: 1, + intentRevisionId: 'int-20', + rolloutCandidateId: null, + envelope: env('op-dep-20'), + }); + tx.blueprintDeployFailed({ + applicationId: 'app-ident', + nodeId: 1, + failureClass: 'pre_mutation', + envelope: env('op-dep-20'), + }); + + // Identity has to go with the stage. Left behind, a later start that only + // sets a stage would make the superseded intent read as live again, and a + // duplicate ack for it would then be accepted. + const target = store.getTarget('app-ident', 1)!; + expect(target.active_operation_stage).toBeNull(); + expect(target.active_intent_revision_id).toBeNull(); + expect(target.active_rollout_candidate_id).toBeNull(); + + expect(() => tx.blueprintAckRecorded({ + applicationId: 'app-ident', + nodeId: 1, + intentRevisionId: 'int-20', + rolloutCandidateId: null, + legacyAppliedRevision: null, + envelope: env('op-ack-20'), + })).toThrow(/was not asked to run/); + }); + + it('acknowledges an interrupted deploy, and retires the interruption with it', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedInline('app-int', 121, 1); + tx.intentRevised({ applicationId: 'app-int', intent: intent('int-21', 'app-int', 121), envelope: env('op-int-21') }); + tx.blueprintDeployStarted({ + applicationId: 'app-int', + nodeId: 1, + intentRevisionId: 'int-21', + rolloutCandidateId: 'cand-21', + envelope: env('op-dep-21'), + }); + tx.interruptActiveOperations('app-int', env('op-boot-21')); + expect(store.getTarget('app-int', 1)?.interruption_stage).toBe('blueprint_deploy_started'); + + // An ack that arrives after a restart still names a request this target was + // genuinely given, so it is accepted. + tx.blueprintAckRecorded({ + applicationId: 'app-int', + nodeId: 1, + intentRevisionId: 'int-21', + rolloutCandidateId: 'cand-21', + legacyAppliedRevision: null, + envelope: env('op-ack-21'), + }); + + const target = store.getTarget('app-int', 1)!; + expect(target.intent_revision_id).toBe('int-21'); + expect(target.rollout_candidate_id).toBe('cand-21'); + // Retired, or it would keep matching and let a third ack regress the + // pointer after two later deploys had succeeded. + expect(target.interruption_stage).toBeNull(); + expect(target.interruption_intent_revision_id).toBeNull(); + expect(target.interruption_rollout_candidate_id).toBeNull(); + }); + + it('refuses an acknowledgement that pairs the deployed intent with another candidate', () => { + const tx = GitOpsTransitions.getInstance(); + seedInline('app-pair', 122, 1); + tx.intentRevised({ applicationId: 'app-pair', intent: intent('int-22', 'app-pair', 122), envelope: env('op-int-22') }); + tx.blueprintDeployStarted({ + applicationId: 'app-pair', + nodeId: 1, + intentRevisionId: 'int-22', + rolloutCandidateId: 'cand-22', + envelope: env('op-dep-22'), + }); + + expect(() => tx.blueprintAckRecorded({ + applicationId: 'app-pair', + nodeId: 1, + intentRevisionId: 'int-22', + rolloutCandidateId: 'cand-other', + legacyAppliedRevision: null, + envelope: env('op-ack-22'), + })).toThrow(/not the one deployed/); + }); + + it('will not settle a deploy out of a withdraw, or the reverse', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedInline('app-cross', 123, 1); + tx.intentRevised({ applicationId: 'app-cross', intent: intent('int-23', 'app-cross', 123), envelope: env('op-int-23') }); + tx.blueprintWithdrawStarted({ + applicationId: 'app-cross', + nodeId: 1, + intentRevisionId: 'int-23', + envelope: env('op-wd-23'), + }); + + // Same intent, but it names the deploy this target is not running. Taking + // it would claim the deployment is live while it is being torn down. + expect(() => tx.blueprintAckRecorded({ + applicationId: 'app-cross', + nodeId: 1, + intentRevisionId: 'int-23', + rolloutCandidateId: null, + legacyAppliedRevision: null, + envelope: env('op-ack-23'), + })).toThrow(/was not asked to run/); + expect(store.getTarget('app-cross', 1)?.target_status).toBe('active'); + expect(store.getTarget('app-cross', 1)?.active_operation_stage).toBe('blueprint_withdraw_started'); + }); + + it('refuses a start that would displace an unrelated operation', () => { + const tx = GitOpsTransitions.getInstance(); + seedInline('app-conflict', 124, 1); + tx.intentRevised({ applicationId: 'app-conflict', intent: intent('int-24', 'app-conflict', 124), envelope: env('op-int-24') }); + tx.blueprintDeployStarted({ + applicationId: 'app-conflict', + nodeId: 1, + intentRevisionId: 'int-24', + rolloutCandidateId: null, + envelope: env('op-dep-24'), + }); + + // Overwriting would leave the displaced operation with no terminal event + // and no history saying it was abandoned. + expect(() => tx.blueprintWithdrawStarted({ + applicationId: 'app-conflict', + nodeId: 1, + intentRevisionId: 'int-24', + envelope: env('op-wd-24-other'), + })).toThrow(/conflicting target operation/); + }); + + it('records an observation without acknowledging or minting anything', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedInline('app-obs', 108, 1); + tx.intentRevised({ applicationId: 'app-obs', intent: intent('int-9', 'app-obs', 108), envelope: env('op-int-9') }); + const before = store.getApplication('app-obs')!; + + for (const stage of ['blueprint_state_review', 'blueprint_evict_blocked', 'blueprint_drifted', 'blueprint_correcting'] as const) { + tx.blueprintObservation({ applicationId: 'app-obs', nodeId: 1, stage, envelope: env(`op-obs-${stage}`) }); + } + + const after = store.getApplication('app-obs')!; + expect(after.intent_revision_id).toBe(before.intent_revision_id); + expect(after.rollout_candidate_id).toBe(before.rollout_candidate_id); + expect(store.getTarget('app-obs', 1)?.intent_revision_id).toBeNull(); + }); + + it('projects every observation stage as its runtime status', () => { + // Recording an observation nothing reads would leave a deployed Blueprint + // reporting itself as never applied, which is what the pointers alone say. + const tx = GitOpsTransitions.getInstance(); + const expected = { + blueprint_state_review: 'pending_state_review', + blueprint_evict_blocked: 'evict_blocked', + blueprint_drifted: 'drifted', + blueprint_correcting: 'correcting', + } as const; + + // One live application per Blueprint, so each case needs its own id. + Object.entries(expected).forEach(([stage, status], index) => { + const applicationId = `app-proj-${stage}`; + seedInline(applicationId, 200 + index, 1); + tx.blueprintObservation({ + applicationId, + nodeId: 1, + stage: stage as keyof typeof expected, + envelope: env(`op-proj-${stage}`), + }); + + expect(runtimeStatusOf(applicationId), stage).toBe(status); + }); + }); + + it('stops projecting an observation once something else happens to the target', () => { + // The observation is what was seen last, not a state the target is stuck + // in. A deploy after it has to win, or a corrected stack reads as drifting + // for ever. A deploy start rather than a tombstone, so the runtime + // assertion is load-bearing: the tombstone check sits above the observation + // branch and would hold whatever `latest_stage` said. + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedInline('app-superseded', 210, 1); + tx.intentRevised({ + applicationId: 'app-superseded', + intent: intent('int-sup', 'app-superseded', 210), + envelope: env('op-sup-int'), + }); + tx.blueprintObservation({ + applicationId: 'app-superseded', nodeId: 1, stage: 'blueprint_drifted', envelope: env('op-sup-obs'), + }); + expect(runtimeStatusOf('app-superseded')).toBe('drifted'); + + tx.blueprintDeployStarted({ + applicationId: 'app-superseded', + nodeId: 1, + intentRevisionId: 'int-sup', + rolloutCandidateId: null, + envelope: env('op-sup-deploy'), + }); + + expect(store.getTarget('app-superseded', 1)?.latest_stage).toBe('blueprint_deploy_started'); + expect(runtimeStatusOf('app-superseded')).not.toBe('drifted'); + }); + + it('does not let an observation mask a failure this node actually hit', () => { + // The ordering claim in the deriver, asserted at its upper boundary. A + // failed mutation describes what this node did; an observation describes + // what was seen about it. Reporting the observation instead would hide a + // deploy that broke the running workload. + const tx = GitOpsTransitions.getInstance(); + seedInline('app-failfirst', 211, 1); + tx.deployFailed('app-failfirst', 1, 'post_mutation', env('op-fail')); + tx.blueprintObservation({ + applicationId: 'app-failfirst', nodeId: 1, stage: 'blueprint_drifted', envelope: env('op-fail-obs'), + }); + + expect(runtimeStatusOf('app-failfirst')).toBe('failed_after_mutation'); + }); +}); + +function runtimeStatusOf(applicationId: string): string | undefined { + const projection = projectApplication(applicationId, false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + return projection.targets[0]?.runtime.status; +} + +function seedInline(applicationId: string, blueprintId: number, nodeId?: number): void { + const store = GitOpsStore.getInstance(); + store.insertApplication(inlineApp(applicationId, blueprintId)); + if (nodeId !== undefined) { + store.upsertTarget(emptyTargetRow(applicationId, nodeId, 1)); + } +} + +function env(operationId: string): EventEnvelope { + return { operationId, actor: 'tester', trigger: 'manual', at: Date.now() }; +} + +function intent(id: string, applicationId: string, blueprintId: number): GitOpsIntentRevisionRow { + return { + id, + application_id: applicationId, + blueprint_id: blueprintId, + compose_content_sha256: 'c'.repeat(64), + blueprint_revision: 1, + deploy_stack_name: 'bp-stack', + selector_json: '{"nodeIds":[1]}', + pinned_node_id: null, + cordon_implications_json: '{}', + rollout_strategy_json: '{}', + runtime_drift_policy: null, + stateful_policy_json: null, + health_failure_rollback_policy_json: null, + operation_id: `op-${id}`, + actor: 'tester', + created_at: 1, + }; +} + +function candidate(id: string, applicationId: string, intentRevisionId: string): GitOpsRolloutCandidateRow { + return { + id, + application_id: applicationId, + intent_revision_id: intentRevisionId, + compose_content_sha256: 'c'.repeat(64), + accepted_generation_id: null, + artifact_set_id: null, + required_targets_json: '{"nodeIds":[1]}', + authoritative: 1, + provenance: 'intent_change', + operation_id: `op-${id}`, + created_at: 1, + }; +} + +function inlineApp(id: string, blueprintId: number): GitOpsApplicationRow { + return { + id, + lifecycle_key: `blueprint:${blueprintId}`, + lifecycle_status: 'active', + target_mode: 'inline_blueprint', + stack_name: null, + blueprint_id: blueprintId, + configured_repo_url: null, + repo_identity_json: null, + configured_ref: null, + compose_paths_json: null, + context_dir: null, + sync_env: 0, + env_path: null, + materialization_fingerprint: null, + desired_commit_sha: null, + fetched_commit_sha: null, + candidate_generation_id: null, + accepted_generation_id: null, + candidate_plan_blocked: 0, + review_required: 0, + artifact_set_id: null, + latest_artifact_set_id: null, + intent_revision_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + rollout_authorization_ref: null, + legacy_combined_approval_ref: null, + preflight_fingerprint: null, + latest_operation_id: null, + active_operation_id: null, + active_operation_stage: null, + active_operation_at: null, + active_generation_id: null, + pause_at: null, + pause_reason: null, + partial_json: null, + failure_stage: null, + failure_class: null, + failure_at: null, + retry_at: null, + retry_count: 0, + suspended_at: null, + recovery_ref: null, + recovery_phase: null, + interruption_stage: null, + interruption_at: null, + interruption_operation_id: null, + interruption_generation_id: null, + evidence_fresh_at: null, + evidence_limitations_json: null, + created_at: 1, + updated_at: 1, + }; +} diff --git a/backend/src/__tests__/gitops-create-recovery.test.ts b/backend/src/__tests__/gitops-create-recovery.test.ts new file mode 100644 index 00000000..5a385887 --- /dev/null +++ b/backend/src/__tests__/gitops-create-recovery.test.ts @@ -0,0 +1,443 @@ +/** + * Boot-time settlement of creates that a previous process left in flight. + * + * Each case seeds the durable state a crash would have left at one phase, runs + * the recovery the startup sweep runs, and asserts the outcome: finish the + * create only when its project is already on disk, tear it down only after its + * files are gone, and never touch a source row that outlived the application. + */ +import fs from 'fs'; +import path from 'path'; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { DatabaseService } from '../services/DatabaseService'; +import { GitOpsStore } from '../services/gitops/store'; +import { GitOpsTransitions, type EventEnvelope } from '../services/gitops/transitions'; +import { assertCreatesSettled, resolveInterruptedCreates } from '../services/gitops/createRecovery'; +import { candidateRelPathForSha, CREATE_STAGING_MARKER_FILENAME } from '../services/gitops/createStagingMarker'; +import { stackManagedRoot } from '../services/gitops/directApplication'; +import type { + GitOpsApplicationRow, + GitOpsCreateCheckpointRow, + GitOpsGenerationRow, +} from '../services/gitops/types'; + +const SHA = 'feed1234'; + +describe('gitops interrupted create recovery', () => { + let tmpDir: string; + + beforeAll(async () => { + tmpDir = await setupTestDb(); + GitOpsStore.resetForTests(); + GitOpsTransitions.resetForTests(); + }); + + afterAll(() => { + cleanupTestDb(tmpDir); + }); + + beforeEach(() => { + const db = DatabaseService.getInstance().getDb(); + db.prepare('DELETE FROM gitops_create_checkpoints').run(); + db.prepare('DELETE FROM gitops_history').run(); + db.prepare('DELETE FROM gitops_target_current').run(); + db.prepare('DELETE FROM gitops_generations').run(); + db.prepare('DELETE FROM gitops_applications').run(); + }); + + it('tears down a create that stopped before the stack existed', async () => { + const store = GitOpsStore.getInstance(); + seedCreate('app-pre', 'pre-stack-web', 'pre_stack'); + const managedRoot = stackManagedRoot('pre-stack-web'); + fs.mkdirSync(path.join(managedRoot, candidateRelPathForSha(SHA)), { recursive: true }); + + const settled = await resolveInterruptedCreates(); + + expect(settled).toEqual([ + { stackName: 'pre-stack-web', applicationId: 'app-pre', outcome: 'tombstoned' }, + ]); + expect(store.getApplication('app-pre')?.lifecycle_status).toBe('deleted'); + expect(store.getCreateCheckpoint('app-pre')).toBeUndefined(); + expect(store.getLiveDirectApplication('pre-stack-web')).toBeUndefined(); + expect(fs.existsSync(managedRoot)).toBe(false); + }); + + it('leaves a stack directory alone when the create never recorded making it', async () => { + // pre_stack is durable proof that createStack had not returned, so a + // directory present now may be the operator's own. Deleting it is the one + // mistake recovery cannot take back. + seedCreate('app-notours', 'notours-web', 'pre_stack'); + const composeDir = process.env.COMPOSE_DIR!; + const stackDir = path.join(composeDir, 'notours-web'); + fs.mkdirSync(stackDir, { recursive: true }); + fs.writeFileSync(path.join(stackDir, 'compose.yaml'), 'services: {}\n'); + + const settled = await resolveInterruptedCreates(); + + expect(settled[0].outcome).toBe('tombstoned'); + expect(fs.existsSync(path.join(stackDir, 'compose.yaml'))).toBe(true); + }); + + it('removes the stack directory a crashed create had already made', async () => { + seedCreate('app-mid', 'mid-web', 'stack_created'); + const composeDir = process.env.COMPOSE_DIR!; + fs.mkdirSync(path.join(composeDir, 'mid-web'), { recursive: true }); + fs.writeFileSync(path.join(composeDir, 'mid-web', 'compose.yaml'), 'services: {}\n'); + + const settled = await resolveInterruptedCreates(); + + expect(settled[0].outcome).toBe('tombstoned'); + expect(fs.existsSync(path.join(composeDir, 'mid-web'))).toBe(false); + }); + + it('preserves a managed root the create did not create', async () => { + seedCreate('app-shared', 'shared-web', 'pre_stack', { createdManagedRoot: 0 }); + const managedRoot = stackManagedRoot('shared-web'); + const sentinel = path.join(managedRoot, 'generations', 'applied-earlier'); + fs.mkdirSync(sentinel, { recursive: true }); + fs.mkdirSync(path.join(managedRoot, candidateRelPathForSha(SHA)), { recursive: true }); + + await resolveInterruptedCreates(); + + expect(fs.existsSync(sentinel)).toBe(true); + expect(fs.existsSync(path.join(managedRoot, candidateRelPathForSha(SHA)))).toBe(false); + }); + + it('finishes a create whose manifest was already committed on disk', async () => { + const store = GitOpsStore.getInstance(); + const db = DatabaseService.getInstance(); + seedCreate('app-finish', 'finish-web', 'manifest_committed'); + const composeDir = process.env.COMPOSE_DIR!; + fs.mkdirSync(path.join(composeDir, 'finish-web'), { recursive: true }); + + const settled = await resolveInterruptedCreates(); + + expect(settled[0].outcome).toBe('completed'); + const app = store.getApplication('app-finish')!; + expect(app.lifecycle_status).toBe('active'); + expect(app.accepted_generation_id).toBe('gen-app-finish'); + expect(app.source_acceptance_ref).not.toBeNull(); + expect(store.getTarget('app-finish', 1)?.applied_generation_id).toBe('gen-app-finish'); + expect(db.getGitSource('finish-web')?.last_applied_commit_sha).toBe(SHA); + expect(store.getCreateCheckpoint('app-finish')).toBeUndefined(); + }); + + it('clears the checkpoint of a create that already reached its boundary', async () => { + const store = GitOpsStore.getInstance(); + seedCreate('app-done', 'done-web', 'pointers_committed'); + DatabaseService.getInstance().getDb().prepare( + "UPDATE gitops_applications SET lifecycle_status = 'active' WHERE id = 'app-done'", + ).run(); + + const settled = await resolveInterruptedCreates(); + + expect(settled[0].outcome).toBe('checkpoint_cleared'); + expect(store.getApplication('app-done')?.lifecycle_status).toBe('active'); + expect(store.getCreateCheckpoint('app-done')).toBeUndefined(); + }); + + it('tombstones a creating application with no checkpoint and keeps its source row', async () => { + const store = GitOpsStore.getInstance(); + const db = DatabaseService.getInstance(); + seedCreate('app-orphan', 'orphan-web', 'pre_stack'); + store.deleteCreateCheckpoint('app-orphan'); + db.upsertGitSource({ + stack_name: 'orphan-web', + repo_url: 'https://github.com/org/repo.git', + branch: 'main', + compose_path: 'compose.yml', + compose_paths: ['compose.yml'], + context_dir: null, + sync_env: false, + env_path: null, + auth_type: 'none', + encrypted_token: null, + auto_apply_on_webhook: false, + auto_deploy_on_apply: false, + last_applied_commit_sha: SHA, + last_applied_content_hash: null, + pending_commit_sha: null, + pending_compose_content: null, + pending_env_content: null, + pending_fetched_at: null, + last_debounce_at: null, + }); + + const settled = await resolveInterruptedCreates(); + + expect(settled).toEqual([ + { stackName: 'orphan-web', applicationId: 'app-orphan', outcome: 'source_preserved' }, + ]); + expect(store.getApplication('app-orphan')?.lifecycle_status).toBe('deleted'); + expect(db.getGitSource('orphan-web')).toBeTruthy(); + expect(store.getLiveDirectApplication('orphan-web')).toBeUndefined(); + }); + + it('retains a create whose files could not be removed, and refuses to start on it', async () => { + // A cleanup that cannot finish must not be recorded as a clean failure: the + // application stays `creating`, so nothing downstream may treat the stack + // name as free. The failure is forced through the containment guard, which + // is a real refusal rather than a stubbed one. + const store = GitOpsStore.getInstance(); + seedCreate('app-stuck', 'stuck-web', 'stack_created', { createdManagedRoot: 0 }); + store.updateCreateCheckpoint('app-stuck', { generationId: 'gen-app-stuck' }, Date.now()); + const managedRoot = stackManagedRoot('stuck-web'); + // Outside the managed area, which is what makes the guard refuse. + const external = path.join(process.env.DATA_DIR!, 'external-stuck'); + fs.mkdirSync(external, { recursive: true }); + fs.mkdirSync(managedRoot, { recursive: true }); + fs.symlinkSync(external, path.join(managedRoot, 'generations'), 'junction'); + + const settled = await resolveInterruptedCreates(); + + expect(settled).toEqual([ + { stackName: 'stuck-web', applicationId: 'app-stuck', outcome: 'retained' }, + ]); + // Still creating, and its checkpoint survives so the next boot retries. + expect(store.getApplication('app-stuck')?.lifecycle_status).toBe('creating'); + expect(store.getCreateCheckpoint('app-stuck')).toBeDefined(); + expect(fs.existsSync(external)).toBe(true); + + // What startup does with that outcome: stop, before any mutation service + // starts or HTTP binds. + expect(() => assertCreatesSettled(settled)).toThrow(/stuck-web/); + }); + + it('reports a settled create whose marker survived, and still starts', async () => { + // The counterpart to the test above, driven through the real code rather + // than a hand-built outcome list. Ownership is decided here, so a marker + // file that could not be deleted decides nothing and must not stop a boot. + // The marker path is made a directory so the unlink fails while everything + // else about the create is already settled. + const store = GitOpsStore.getInstance(); + seedCreate('app-marker', 'marker-web', 'pointers_committed'); + DatabaseService.getInstance().getDb().prepare( + "UPDATE gitops_applications SET lifecycle_status = 'active' WHERE id = 'app-marker'", + ).run(); + fs.mkdirSync(path.join(stackManagedRoot('marker-web'), CREATE_STAGING_MARKER_FILENAME), { recursive: true }); + + const settled = await resolveInterruptedCreates(); + + expect(settled).toEqual([ + { stackName: 'marker-web', applicationId: 'app-marker', outcome: 'marker_retained' }, + ]); + // The checkpoint is what makes the next boot retry the marker. Dropping it + // would leave a claim on the name with nothing left to clear it. + expect(store.getCreateCheckpoint('app-marker')).toBeDefined(); + expect(() => assertCreatesSettled(settled)).not.toThrow(); + }); + + it('clears the marker before the checkpoint for a create that is no longer creating', async () => { + // An application tombstoned on some other path leaves a stale checkpoint + // behind. It settles like any other finished create, and it has to clear the + // marker on the way out: dropping the checkpoint first would leave a claim + // on the stack name with nothing left to retry it, and every later create + // for that name would be refused by a marker nothing could remove. + const store = GitOpsStore.getInstance(); + seedCreate('app-gone', 'gone-web', 'stack_created', { createdManagedRoot: 0 }); + DatabaseService.getInstance().getDb().prepare( + "UPDATE gitops_applications SET lifecycle_status = 'deleted' WHERE id = 'app-gone'", + ).run(); + // A directory at the marker path, so the unlink fails the way a permission + // error would and the ordering becomes observable. + fs.mkdirSync(path.join(stackManagedRoot('gone-web'), CREATE_STAGING_MARKER_FILENAME), { recursive: true }); + + const settled = await resolveInterruptedCreates(); + + expect(settled).toEqual([ + { stackName: 'gone-web', applicationId: 'app-gone', outcome: 'marker_retained' }, + ]); + expect(store.getCreateCheckpoint('app-gone')).toBeDefined(); + expect(() => assertCreatesSettled(settled)).not.toThrow(); + }); + + it('drops the checkpoint for a create that is no longer creating once its marker is clear', async () => { + // The same route with nothing blocking the marker: this is the ordinary + // outcome, and it must still end with the checkpoint gone. + const store = GitOpsStore.getInstance(); + seedCreate('app-gone-ok', 'gone-ok-web', 'stack_created', { createdManagedRoot: 0 }); + DatabaseService.getInstance().getDb().prepare( + "UPDATE gitops_applications SET lifecycle_status = 'deleted' WHERE id = 'app-gone-ok'", + ).run(); + + const settled = await resolveInterruptedCreates(); + + expect(settled[0].outcome).toBe('checkpoint_cleared'); + expect(store.getCreateCheckpoint('app-gone-ok')).toBeUndefined(); + }); + + it('reports a marker left by a torn-down create without blocking the boot', async () => { + // The teardown path reaches the same condition by a different route. Its + // staged directories are gone, so nothing deployable survives and the + // create is effectively torn down; only the marker is stuck. Treating that + // as unresolved would make one failed unlink cost an operator their + // instance, which is the opposite of the settled path's answer. + const store = GitOpsStore.getInstance(); + seedCreate('app-tearmark', 'tearmark-web', 'stack_created', { createdManagedRoot: 0 }); + fs.mkdirSync(path.join(stackManagedRoot('tearmark-web'), CREATE_STAGING_MARKER_FILENAME), { recursive: true }); + + const settled = await resolveInterruptedCreates(); + + expect(settled[0].outcome).toBe('marker_retained'); + expect(store.getCreateCheckpoint('app-tearmark')).toBeDefined(); + expect(() => assertCreatesSettled(settled)).not.toThrow(); + }); + + it('settles a create when the managed area is not on disk at all', async () => { + // A database restored without its data directory, or a volume that failed + // to mount. Nothing under the area exists, so there is nothing to remove + // and the create tears down normally. Reporting this as unresolved would, + // with the boot gate, stop the instance starting on every boot over a + // directory that is merely absent. + const previous = process.env.DATA_DIR; + process.env.DATA_DIR = path.join(tmpDir, 'data-without-managed-area'); + try { + seedCreate('app-noarea', 'noarea-web', 'stack_created', { createdManagedRoot: 0 }); + const settled = await resolveInterruptedCreates(); + expect(settled[0].outcome).toBe('tombstoned'); + expect(() => assertCreatesSettled(settled)).not.toThrow(); + } finally { + process.env.DATA_DIR = previous; + } + }); + + it('is idempotent across repeated boots', async () => { + seedCreate('app-replay', 'replay-web', 'pre_stack'); + const first = await resolveInterruptedCreates(); + const second = await resolveInterruptedCreates(); + expect(first[0].outcome).toBe('tombstoned'); + expect(second).toEqual([]); + }); +}); + +function seedCreate( + applicationId: string, + stackName: string, + phase: GitOpsCreateCheckpointRow['phase'], + options: { createdManagedRoot?: number } = {}, +): void { + const store = GitOpsStore.getInstance(); + const generationId = `gen-${applicationId}`; + GitOpsTransitions.getInstance().activateCreateFromGit({ + application: creatingApp(applicationId, stackName), + nodeId: 1, + commitSha: SHA, + generation: gen(generationId, applicationId), + checkpoint: { + application_id: applicationId, + stack_name: stackName, + phase: 'pre_stack', + generation_id: null, + operation_id: `op-${applicationId}`, + repo_url: 'https://github.com/org/repo.git', + branch: 'main', + compose_path: 'compose.yml', + compose_paths_json: '["compose.yml"]', + context_dir: null, + sync_env: 0, + env_path: null, + auth_type: 'none', + encrypted_token: null, + auto_apply_on_webhook: 0, + auto_deploy_on_apply: 0, + commit_sha: SHA, + applied_spec_json: null, + created_managed_root: options.createdManagedRoot ?? 1, + created_at: 1, + updated_at: 1, + }, + envelope: envelope(`op-${applicationId}`), + }); + if (phase !== 'pre_stack') { + store.updateCreateCheckpoint(applicationId, { phase }, Date.now()); + } +} + +function envelope(operationId: string): EventEnvelope { + return { operationId, actor: 'tester', trigger: 'manual', at: Date.now() }; +} + +function creatingApp(id: string, stackName: string): GitOpsApplicationRow { + return { + id, + lifecycle_key: `direct:${stackName}`, + lifecycle_status: 'creating', + target_mode: 'direct', + stack_name: stackName, + blueprint_id: null, + configured_repo_url: 'https://github.com/org/repo.git', + repo_identity_json: '{"host":"github.com","pathname":"/org/repo.git"}', + configured_ref: 'main', + compose_paths_json: '["compose.yml"]', + context_dir: null, + sync_env: 0, + env_path: null, + materialization_fingerprint: 'a'.repeat(64), + desired_commit_sha: null, + fetched_commit_sha: null, + candidate_generation_id: null, + accepted_generation_id: null, + candidate_plan_blocked: 0, + review_required: 0, + artifact_set_id: null, + latest_artifact_set_id: null, + intent_revision_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + rollout_authorization_ref: null, + legacy_combined_approval_ref: null, + preflight_fingerprint: null, + latest_operation_id: null, + active_operation_id: null, + active_operation_stage: null, + active_operation_at: null, + active_generation_id: null, + pause_at: null, + pause_reason: null, + partial_json: null, + failure_stage: null, + failure_class: null, + failure_at: null, + retry_at: null, + retry_count: 0, + suspended_at: null, + recovery_ref: null, + recovery_phase: null, + interruption_stage: null, + interruption_at: null, + interruption_operation_id: null, + interruption_generation_id: null, + evidence_fresh_at: null, + evidence_limitations_json: null, + created_at: 1, + updated_at: 1, + }; +} + +function gen(id: string, applicationId: string): GitOpsGenerationRow { + return { + id, + application_id: applicationId, + commit_sha: SHA, + repo_url: 'https://github.com/org/repo.git', + configured_ref: 'main', + repo_identity_json: '{"host":"github.com","pathname":"/org/repo.git"}', + manifest_version: 1, + candidate_dir: candidateRelPathForSha(SHA), + applied_dir: `generations/applied-${SHA}-1`, + expected_invocation_json: '{"composeFileOrder":[],"projectName":null,"projectDirectory":null,"envFileOrder":[]}', + materialization_fingerprint: 'a'.repeat(64), + validation_ok: 1, + plan_blocked: 0, + change_plan_fingerprint: null, + operation_id: `op-${id}`, + trigger: 'manual', + actor: 'tester', + previous_generation_id: null, + redacted_limitations_json: '[]', + created_at: 1, + }; +} diff --git a/backend/src/__tests__/gitops-create.test.ts b/backend/src/__tests__/gitops-create.test.ts new file mode 100644 index 00000000..44de101c --- /dev/null +++ b/backend/src/__tests__/gitops-create.test.ts @@ -0,0 +1,746 @@ +/** + * Create-from-Git durability: the activation transaction, the teardown of a + * create that never reached `applied`, and the staging marker plus + * operation-owned cleanup that together decide what a crashed create is + * allowed to delete. + */ +import fs from 'fs'; +import fsPromises from 'fs/promises'; +import path from 'path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { DatabaseService } from '../services/DatabaseService'; +import { GitOpsStore } from '../services/gitops/store'; +import { GitOpsTransitions, type EventEnvelope } from '../services/gitops/transitions'; +import { + appliedRelPathFor, + candidateRelPathForSha, + deleteStagingMarker, + CREATE_STAGING_MARKER_FILENAME, + readStagingMarker, + stagingMarkerPath, + writeStagingMarker, + CreateStagingMarkerError, +} from '../services/gitops/createStagingMarker'; +import { cleanupUnclaimedManagedRoot, removeOperationOwnedPaths } from '../services/gitops/createCleanup'; +import { GENERATIONS_DIR, MANAGED_ROOT_NAME, managedAreaBase } from '../services/gitops/managedPaths'; +import type { + GitOpsApplicationRow, + GitOpsCreateCheckpointRow, + GitOpsGenerationRow, +} from '../services/gitops/types'; + +const SHA = 'a1b2c3d4'; + +describe('gitops create-from-git', () => { + let tmpDir: string; + + beforeAll(async () => { + tmpDir = await setupTestDb(); + GitOpsStore.resetForTests(); + GitOpsTransitions.resetForTests(); + }); + + afterAll(() => { + cleanupTestDb(tmpDir); + }); + + it('commits application, fetch, generation, checkpoint, and candidate in one transaction', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + const result = tx.activateCreateFromGit({ + application: creatingApp('app-create', 'create-web'), + nodeId: 1, + commitSha: SHA, + generation: gen('gen-create', 'app-create'), + checkpoint: checkpoint('app-create', 'create-web'), + envelope: envelope('op-create'), + }); + + const app = store.getApplication('app-create')!; + expect(app.lifecycle_status).toBe('creating'); + expect(app.desired_commit_sha).toBe(SHA); + expect(app.fetched_commit_sha).toBe(SHA); + expect(app.candidate_generation_id).toBe('gen-create'); + expect(app.accepted_generation_id).toBeNull(); + expect(app.source_acceptance_ref).toBeNull(); + + const target = store.getTarget('app-create', 1)!; + expect(target.candidate_generation_id).toBe('gen-create'); + expect(target.desired_generation_id).toBeNull(); + expect(target.applied_generation_id).toBeNull(); + + expect(store.getCreateCheckpoint('app-create')?.generation_id).toBe('gen-create'); + expect(store.getCreateCheckpoint('app-create')?.phase).toBe('pre_stack'); + + expect(result.historyIds).toHaveLength(3); + const stages = DatabaseService.getInstance().getDb().prepare( + 'SELECT stage FROM gitops_history WHERE application_id = ? ORDER BY rowid ASC', + ).all('app-create') as Array<{ stage: string }>; + expect(stages.map((row) => row.stage)).toEqual(['application_activated', 'fetched', 'candidate_ready']); + }); + + it('refuses to persist a create whose candidate is blocked or stale', () => { + const tx = GitOpsTransitions.getInstance(); + expect(() => tx.activateCreateFromGit({ + application: creatingApp('app-blocked', 'blocked-web'), + nodeId: 1, + commitSha: SHA, + generation: { ...gen('gen-blocked', 'app-blocked'), plan_blocked: 1 }, + checkpoint: checkpoint('app-blocked', 'blocked-web'), + envelope: envelope('op-blocked'), + })).toThrow(/invalid or blocked candidate/); + + expect(() => tx.activateCreateFromGit({ + application: creatingApp('app-stale', 'stale-web'), + nodeId: 1, + commitSha: SHA, + generation: { ...gen('gen-stale', 'app-stale'), materialization_fingerprint: 'b'.repeat(64) }, + checkpoint: checkpoint('app-stale', 'stale-web'), + envelope: envelope('op-stale'), + })).toThrow(/fingerprint/); + + expect(GitOpsStore.getInstance().getApplication('app-blocked')).toBeUndefined(); + expect(GitOpsStore.getInstance().getApplication('app-stale')).toBeUndefined(); + }); + + it('activates the application only at applied, which is the success boundary', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateCreateFromGit({ + application: creatingApp('app-boundary', 'boundary-web'), + nodeId: 1, + commitSha: SHA, + generation: gen('gen-boundary', 'app-boundary'), + checkpoint: checkpoint('app-boundary', 'boundary-web'), + envelope: envelope('op-boundary'), + }); + expect(store.getApplication('app-boundary')?.lifecycle_status).toBe('creating'); + + tx.applied({ + applicationId: 'app-boundary', + generationId: 'gen-boundary', + artifactSetId: 'art-boundary', + sourceAcceptanceId: 'acc-boundary', + authority: 'operator', + envelope: envelope('op-boundary-applied'), + activateCreating: true, + }); + + const app = store.getApplication('app-boundary')!; + expect(app.lifecycle_status).toBe('active'); + expect(app.accepted_generation_id).toBe('gen-boundary'); + expect(store.getTarget('app-boundary', 1)?.applied_generation_id).toBe('gen-boundary'); + + // After the success boundary the create can no longer be torn down. + expect(() => tx.createFailed('app-boundary', 'post_boundary', envelope('op-boundary-fail'))) + .toThrow(/requires a creating application/); + expect(store.getApplication('app-boundary')?.lifecycle_status).toBe('active'); + }); + + it('tombstones a failed create, drops its checkpoint, and frees the stack name', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateCreateFromGit({ + application: creatingApp('app-fail', 'fail-web'), + nodeId: 1, + commitSha: SHA, + generation: gen('gen-fail', 'app-fail'), + checkpoint: checkpoint('app-fail', 'fail-web'), + envelope: envelope('op-fail'), + }); + tx.createFailed('app-fail', 'validation', envelope('op-fail')); + + const app = store.getApplication('app-fail')!; + expect(app.lifecycle_status).toBe('deleted'); + expect(app.failure_stage).toBe('create'); + expect(app.failure_class).toBe('validation'); + expect(store.getCreateCheckpoint('app-fail')).toBeUndefined(); + expect(store.getTarget('app-fail', 1)?.target_status).toBe('tombstoned'); + expect(store.getLiveDirectApplication('fail-web')).toBeUndefined(); + + // Retry is a brand new application id against the now-free stack name. + tx.activateCreateFromGit({ + application: creatingApp('app-fail-retry', 'fail-web'), + nodeId: 1, + commitSha: SHA, + generation: gen('gen-fail-retry', 'app-fail-retry'), + checkpoint: checkpoint('app-fail-retry', 'fail-web'), + envelope: envelope('op-fail-retry'), + }); + expect(store.getLiveDirectApplication('fail-web')?.id).toBe('app-fail-retry'); + }); +}); + +describe('gitops create staging marker', () => { + let root: string; + let dataDir: string; + let priorDataDir: string | undefined; + + beforeAll(() => { + // A managed root only ever lives inside the managed area, and the marker + // helpers enforce that at every filesystem call, so the fixture has to be a + // real managed area rather than a bare temp directory. + priorDataDir = process.env.DATA_DIR; + dataDir = fs.mkdtempSync(path.join(process.env.TEMP || '/tmp', 'sencho-marker-')); + process.env.DATA_DIR = dataDir; + root = managedAreaBase(); + fs.mkdirSync(root, { recursive: true }); + }); + + afterAll(() => { + if (priorDataDir === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = priorDataDir; + if (dataDir) fs.rmSync(dataDir, { recursive: true, force: true }); + }); + + function areaFor(name: string): string { + return path.join(root, name); + } + + it('derives generation paths without depending on import order', () => { + // These were briefly built from a constant imported across a module cycle, + // which evaluated as undefined and produced `undefined/candidate-`: + // a path that passes containment, names nothing, and makes cleanup a no-op. + expect(candidateRelPathForSha('abc123')).toBe('generations/candidate-abc123'); + expect(appliedRelPathFor('abc123', 2)).toBe('generations/applied-abc123-2'); + }); + + it('round-trips a valid marker and refuses a foreign live marker', async () => { + const area = areaFor('round-trip'); + await writeStagingMarker(area, { + schemaVersion: 1, + operationId: 'op-1', + rootPreexisted: true, + candidateRelPath: candidateRelPathForSha(SHA), + createdAt: 1, + }); + const read = await readStagingMarker(area); + expect(read.state).toBe('valid'); + if (read.state !== 'valid') throw new Error('expected a valid marker'); + expect(read.marker.operationId).toBe('op-1'); + expect(read.marker.candidateRelPath).toBe(`generations/candidate-${SHA}`); + + // Same operation may rewrite its own marker; a different one may not. + await writeStagingMarker(area, { ...read.marker, createdAt: 2 }); + await expect(writeStagingMarker(area, { ...read.marker, operationId: 'op-2' })) + .rejects.toBeInstanceOf(CreateStagingMarkerError); + + await deleteStagingMarker(area); + expect((await readStagingMarker(area)).state).toBe('missing'); + }); + + it('treats every unsafe candidate path as corrupt', async () => { + const cases: Array<[string, unknown]> = [ + ['absolute', path.resolve(root, 'elsewhere')], + ['dotdot', '../escape'], + ['empty', ''], + ['wrong prefix', 'applied/candidate-abc'], + ['escape', 'generations/candidate-../../../etc'], + ['null', null], + ]; + for (const [label, candidateRelPath] of cases) { + const area = areaFor(`corrupt-${label.replace(/\s/g, '-')}`); + await fsPromises.mkdir(area, { recursive: true }); + await fsPromises.writeFile( + stagingMarkerPath(area), + JSON.stringify({ schemaVersion: 1, operationId: 'op-x', rootPreexisted: true, candidateRelPath, createdAt: 1 }), + 'utf8', + ); + const read = await readStagingMarker(area); + expect(read.state, `${label} should be corrupt`).toBe('corrupt'); + } + }); + + it('rejects a marker with a bad schema version or missing fields', async () => { + const area = areaFor('bad-shape'); + await fsPromises.mkdir(area, { recursive: true }); + await fsPromises.writeFile(stagingMarkerPath(area), '{"schemaVersion":2}', 'utf8'); + expect((await readStagingMarker(area)).state).toBe('corrupt'); + await fsPromises.writeFile(stagingMarkerPath(area), 'not json', 'utf8'); + expect((await readStagingMarker(area)).state).toBe('corrupt'); + }); + + it('refuses to claim an area whose marker cannot be read', async () => { + // A marker that exists but will not parse is still someone's claim. + // Writing over it would hand this operation deletion authority over what + // the last one staged. + const area = areaFor('unreadable-claim'); + await fsPromises.mkdir(area, { recursive: true }); + await fsPromises.writeFile(stagingMarkerPath(area), 'not json', 'utf8'); + await expect(writeStagingMarker(area, { + schemaVersion: 1, + operationId: 'op-new', + rootPreexisted: true, + candidateRelPath: candidateRelPathForSha(SHA), + createdAt: 1, + })).rejects.toThrow(/unreadable staging marker/); + }); + + it('refuses every marker operation on a root outside the managed area', async () => { + // The stack name reaches this root without being validated here, so each + // call checks containment itself. Without these the checks are deletable + // and nothing notices. + const outside = path.join(dataDir, 'not-the-managed-area', 'web'); + await fsPromises.mkdir(outside, { recursive: true }); + + const read = await readStagingMarker(outside); + expect(read.state).toBe('corrupt'); + if (read.state !== 'corrupt') throw new Error('expected a corrupt result'); + expect(read.reason).toMatch(/managed area/); + + await expect(writeStagingMarker(outside, { + schemaVersion: 1, + operationId: 'op-outside', + rootPreexisted: true, + candidateRelPath: candidateRelPathForSha(SHA), + createdAt: 1, + })).rejects.toBeInstanceOf(CreateStagingMarkerError); + + await expect(deleteStagingMarker(outside)).rejects.toBeInstanceOf(CreateStagingMarkerError); + }); +}); + +describe('gitops create cleanup', () => { + let root: string; + let dataDir: string; + let priorDataDir: string | undefined; + + beforeAll(() => { + // Same as the marker describe: cleanup refuses to touch anything outside + // the managed area, so the fixture areas have to live inside one. + priorDataDir = process.env.DATA_DIR; + dataDir = fs.mkdtempSync(path.join(process.env.TEMP || '/tmp', 'sencho-cleanup-')); + process.env.DATA_DIR = dataDir; + root = managedAreaBase(); + fs.mkdirSync(root, { recursive: true }); + }); + + afterAll(() => { + if (priorDataDir === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = priorDataDir; + if (dataDir) fs.rmSync(dataDir, { recursive: true, force: true }); + }); + + async function seedArea(name: string): Promise<{ area: string; candidateRel: string; sentinel: string }> { + const area = path.join(root, name); + const candidateRel = candidateRelPathForSha(SHA); + await fsPromises.mkdir(path.join(area, candidateRel), { recursive: true }); + const sentinel = path.join(area, 'generations', 'applied-old'); + await fsPromises.mkdir(sentinel, { recursive: true }); + return { area, candidateRel, sentinel }; + } + + it('removes only the staged candidate when the managed root pre-existed', async () => { + const { area, candidateRel, sentinel } = await seedArea('preexisting'); + await removeOperationOwnedPaths({ stackManagedRoot: area, candidateRelPath: candidateRel, ownsManagedRoot: false }); + expect(fs.existsSync(path.join(area, candidateRel))).toBe(false); + expect(fs.existsSync(sentinel)).toBe(true); + expect(fs.existsSync(area)).toBe(true); + }); + + it('removes the whole root only when the operation created it', async () => { + const { area, candidateRel } = await seedArea('owned'); + await removeOperationOwnedPaths({ stackManagedRoot: area, candidateRelPath: candidateRel, ownsManagedRoot: true }); + expect(fs.existsSync(area)).toBe(false); + }); + + it('refuses to remove a managed root outside the managed area', async () => { + // The guard that keeps a recursive removal inside the area it is meant to + // clean. Without a test it is deletable and nothing notices. + const outside = path.join(dataDir, 'not-the-managed-area', 'web'); + fs.mkdirSync(outside, { recursive: true }); + await expect(removeOperationOwnedPaths({ + stackManagedRoot: outside, + candidateRelPath: null, + ownsManagedRoot: true, + })).rejects.toThrow(/outside the managed area/); + expect(fs.existsSync(outside)).toBe(true); + + // The reaper reports rather than throws, so it answers `preserved`. + expect(await cleanupUnclaimedManagedRoot(outside, { + operationId: 'op-outside', + rootPreexisted: false, + candidateRelPath: candidateRelPathForSha(SHA), + })).toBe('preserved'); + expect(fs.existsSync(outside)).toBe(true); + }); + + it('refuses to remove a path outside the managed root', async () => { + const { area } = await seedArea('escape-guard'); + await expect(removeOperationOwnedPaths({ + stackManagedRoot: area, + candidateRelPath: '../../outside', + ownsManagedRoot: false, + })).rejects.toThrow(/outside the managed root/); + }); + + /** + * A directory link that lands outside the managed area. + * + * `junction` is what Windows can create without elevation, and Node ignores + * the type argument everywhere else, so one call covers both platforms. + */ + async function linkOutside(linkPath: string, name: string, victimRelPath?: string): Promise { + const external = path.join(dataDir, 'external', name); + await fsPromises.mkdir(external, { recursive: true }); + await fsPromises.writeFile(path.join(external, 'keepme.txt'), 'not ours', 'utf8'); + // The path the escaping delete would actually resolve to. Without content + // at exactly that path the removal is a no-op even unguarded, and the + // survival assertion would pass against the unfixed code too. + if (victimRelPath) { + const victim = path.join(external, victimRelPath); + await fsPromises.mkdir(victim, { recursive: true }); + await fsPromises.writeFile(path.join(victim, 'victim.txt'), 'would have been deleted', 'utf8'); + } + await fsPromises.mkdir(path.dirname(linkPath), { recursive: true }); + await fsPromises.symlink(external, linkPath, 'junction'); + return external; + } + + it('refuses to remove a path whose parent links out of the managed area', async () => { + // The lexical checks all pass here: `/generations/candidate-*` reads + // as contained no matter what `generations` points at. Containment has to + // be proven against the real filesystem, because the recursive delete is + // what follows the link. + const area = path.join(root, 'junction-parent'); + const candidateRel = candidateRelPathForSha(SHA); + const external = await linkOutside(path.join(area, 'generations'), 'parent-escape', `candidate-${SHA}`); + + await expect(removeOperationOwnedPaths({ + stackManagedRoot: area, + candidateRelPath: candidateRel, + ownsManagedRoot: false, + })).rejects.toThrow(/links outside its managed location/); + // The path the escaping delete would have resolved to, not just a bystander + // file: this is the data an unguarded removal destroys. + expect(fs.existsSync(path.join(external, `candidate-${SHA}`, 'victim.txt'))).toBe(true); + expect(fs.existsSync(path.join(external, 'keepme.txt'))).toBe(true); + }); + + it('refuses to remove a managed root that is itself a link out of the area', async () => { + const area = path.join(root, 'junction-root'); + const external = await linkOutside(area, 'root-escape'); + + await expect(removeOperationOwnedPaths({ + stackManagedRoot: area, + candidateRelPath: null, + ownsManagedRoot: true, + })).rejects.toThrow(/links outside its managed location/); + expect(fs.existsSync(path.join(external, 'keepme.txt'))).toBe(true); + // The link itself survives too. Unlinking it is the damage an unguarded + // delete does here, and it is the operator's own relocation pointer. + expect(fs.existsSync(area)).toBe(true); + + // The boot sweep reaches the same root by a different route and must reach + // the same answer, reporting rather than throwing as it does everywhere. + expect(await cleanupUnclaimedManagedRoot(area, { + operationId: 'op-junction', + rootPreexisted: false, + candidateRelPath: candidateRelPathForSha(SHA), + })).toBe('preserved'); + expect(fs.existsSync(path.join(external, 'keepme.txt'))).toBe(true); + expect(fs.existsSync(area)).toBe(true); + }); + + /** + * A directory link that stays *inside* the managed area but lands in another + * stack's subtree. + * + * The area-membership check cannot see this: the link's target is a real path + * under the managed area, so "is this inside the area" answers yes while the + * delete walks into a generation that belongs to someone else. Containment has + * to be proven against the path's own place in the area, not the area itself. + */ + it('refuses to remove a candidate reached through a junction into a sibling stack', async () => { + const victim = await seedArea('sibling-victim'); + const attacker = path.join(root, 'sibling-attacker'); + await fsPromises.mkdir(attacker, { recursive: true }); + await fsPromises.symlink( + path.join(victim.area, GENERATIONS_DIR), + path.join(attacker, GENERATIONS_DIR), + 'junction', + ); + + await expect(removeOperationOwnedPaths({ + stackManagedRoot: attacker, + candidateRelPath: candidateRelPathForSha(SHA), + ownsManagedRoot: false, + })).rejects.toThrow(/links outside its managed location/); + // The other stack's staged generation, which an area-only guard removes. + expect(fs.existsSync(path.join(victim.area, victim.candidateRel))).toBe(true); + expect(fs.existsSync(victim.sentinel)).toBe(true); + }); + + it('refuses to remove a managed root that junctions into another node subtree', async () => { + // Mirrors the production layout `//`, because the + // node segment is the one an area-only guard also fails to pin. + const victimRoot = path.join(root, 'node-2', 'shared-name'); + const victimGeneration = path.join(victimRoot, candidateRelPathForSha(SHA)); + await fsPromises.mkdir(victimGeneration, { recursive: true }); + + const attackerRoot = path.join(root, 'node-1', 'shared-name'); + await fsPromises.mkdir(path.dirname(attackerRoot), { recursive: true }); + await fsPromises.symlink(victimRoot, attackerRoot, 'junction'); + + await expect(removeOperationOwnedPaths({ + stackManagedRoot: attackerRoot, + candidateRelPath: null, + ownsManagedRoot: true, + })).rejects.toThrow(/links outside its managed location/); + expect(fs.existsSync(victimGeneration)).toBe(true); + + // The boot sweep reaches the same root by another route and must agree. + expect(await cleanupUnclaimedManagedRoot(attackerRoot, { + operationId: 'op-sibling-node', + rootPreexisted: false, + candidateRelPath: candidateRelPathForSha(SHA), + })).toBe('preserved'); + expect(fs.existsSync(victimGeneration)).toBe(true); + }); + + it('refuses to write or delete a staging marker through a junction into a sibling stack', async () => { + // The write sink needs the same rule as the delete: a marker written into + // another stack's root would hand this operation deletion authority there, + // and would overwrite the claim that stack is relying on. + const victim = path.join(root, 'sibling-marker-victim'); + await fsPromises.mkdir(victim, { recursive: true }); + const attacker = path.join(root, 'sibling-marker-attacker'); + await fsPromises.symlink(victim, attacker, 'junction'); + + // No marker at the victim yet, so the write reaches the containment check + // rather than being turned back by the "someone already owns this" guard. + await expect(writeStagingMarker(attacker, { + schemaVersion: 1, + operationId: 'op-sibling-marker', + rootPreexisted: false, + candidateRelPath: candidateRelPathForSha(SHA), + createdAt: 1, + })).rejects.toThrow(/links outside its managed location/); + expect(fs.existsSync(path.join(victim, CREATE_STAGING_MARKER_FILENAME))).toBe(false); + + // Now the victim holds its own claim, and the delete must not clear it: + // that claim is what stops a second create racing this stack. + await fsPromises.writeFile(path.join(victim, CREATE_STAGING_MARKER_FILENAME), 'theirs', 'utf8'); + await expect(deleteStagingMarker(attacker)).rejects.toThrow(/links outside its managed location/); + expect(fs.readFileSync(path.join(victim, CREATE_STAGING_MARKER_FILENAME), 'utf8')).toBe('theirs'); + }); + + it('refuses to write a staging marker through a link out of the area', async () => { + // The write sink gets the same barrier as the delete. Without it a marker + // could be written through a link and then refused by the hardened delete, + // wedging the stack name behind a claim nothing could clear. + const area = path.join(root, 'junction-write'); + const external = await linkOutside(area, 'write-escape'); + + await expect(writeStagingMarker(area, { + schemaVersion: 1, + operationId: 'op-write-escape', + rootPreexisted: false, + candidateRelPath: candidateRelPathForSha(SHA), + createdAt: 1, + })).rejects.toThrow(/links outside its managed location/); + expect(fs.existsSync(path.join(external, CREATE_STAGING_MARKER_FILENAME))).toBe(false); + }); + + it('refuses to delete a staging marker through a link out of the area', async () => { + // Reached only through the real-path barrier: the marker path is lexically + // inside the area, so every string check passes. + const area = path.join(root, 'junction-marker'); + const external = await linkOutside(area, 'marker-escape'); + await fsPromises.writeFile(path.join(external, CREATE_STAGING_MARKER_FILENAME), '{}', 'utf8'); + + await expect(deleteStagingMarker(area)).rejects.toThrow(/links outside its managed location/); + expect(fs.existsSync(path.join(external, CREATE_STAGING_MARKER_FILENAME))).toBe(true); + }); + + it('treats a managed area that does not exist as nothing to remove', async () => { + // A database restored without its data directory, or a volume that failed + // to mount. Every path under the area is absent, so a forced remove is a + // no-op. Refusing here instead would make an absent directory look like a + // link escape and, with the boot gate, stop the instance starting at all. + const missingData = path.join(dataDir, 'no-area-here'); + const previous = process.env.DATA_DIR; + process.env.DATA_DIR = missingData; + try { + const area = path.join(managedAreaBase(), 'ghost-stack'); + await expect(removeOperationOwnedPaths({ + stackManagedRoot: area, + candidateRelPath: candidateRelPathForSha(SHA), + ownsManagedRoot: false, + })).resolves.toBe('cleared'); + } finally { + process.env.DATA_DIR = previous; + } + }); + + it('still cleans up when the managed area itself is relocated onto a link', async () => { + // The counterpart to the two tests above: an operator who points the data + // directory at another volume has moved the whole area rather than escaped + // it, and cleanup must keep working for them. + const relocatedData = path.join(dataDir, 'relocated-data'); + const storage = path.join(dataDir, 'other-volume'); + await fsPromises.mkdir(relocatedData, { recursive: true }); + await fsPromises.mkdir(storage, { recursive: true }); + await fsPromises.symlink(storage, path.join(relocatedData, MANAGED_ROOT_NAME), 'junction'); + + const previous = process.env.DATA_DIR; + process.env.DATA_DIR = relocatedData; + try { + const area = path.join(managedAreaBase(), 'relocated-stack'); + const candidateRel = candidateRelPathForSha(SHA); + await fsPromises.mkdir(path.join(area, candidateRel), { recursive: true }); + + await removeOperationOwnedPaths({ stackManagedRoot: area, candidateRelPath: candidateRel, ownsManagedRoot: false }); + expect(fs.existsSync(path.join(area, candidateRel))).toBe(false); + } finally { + process.env.DATA_DIR = previous; + } + }); + + it('preserves an unclaimed root whose marker is missing or corrupt', async () => { + const { area, sentinel } = await seedArea('unclaimed'); + expect(await cleanupUnclaimedManagedRoot(area, null)).toBe('preserved'); + expect(fs.existsSync(sentinel)).toBe(true); + + expect(await cleanupUnclaimedManagedRoot(area, { + operationId: 'op-x', + rootPreexisted: true, + candidateRelPath: '../escape', + })).toBe('preserved'); + expect(fs.existsSync(sentinel)).toBe(true); + }); + + it('applies operation-owned cleanup for an unclaimed root with a valid marker', async () => { + const preexisting = await seedArea('unclaimed-preexisting'); + expect(await cleanupUnclaimedManagedRoot(preexisting.area, { + operationId: 'op-x', + rootPreexisted: true, + candidateRelPath: preexisting.candidateRel, + })).toBe('removed_candidate'); + expect(fs.existsSync(path.join(preexisting.area, preexisting.candidateRel))).toBe(false); + expect(fs.existsSync(preexisting.sentinel)).toBe(true); + + const owned = await seedArea('unclaimed-owned'); + expect(await cleanupUnclaimedManagedRoot(owned.area, { + operationId: 'op-x', + rootPreexisted: false, + candidateRelPath: owned.candidateRel, + })).toBe('removed_root'); + expect(fs.existsSync(owned.area)).toBe(false); + }); +}); + +function envelope(operationId: string): EventEnvelope { + return { operationId, actor: 'tester', trigger: 'manual', at: Date.now() }; +} + +function checkpoint(applicationId: string, stackName: string): GitOpsCreateCheckpointRow { + return { + application_id: applicationId, + stack_name: stackName, + phase: 'pre_stack', + generation_id: null, + operation_id: `op-${applicationId}`, + repo_url: 'https://github.com/org/repo.git', + branch: 'main', + compose_path: 'compose.yml', + compose_paths_json: '["compose.yml"]', + context_dir: null, + sync_env: 0, + env_path: null, + auth_type: 'none', + encrypted_token: null, + auto_apply_on_webhook: 0, + auto_deploy_on_apply: 0, + commit_sha: SHA, + applied_spec_json: null, + created_managed_root: 1, + created_at: 1, + updated_at: 1, + }; +} + +function creatingApp(id: string, stackName: string): GitOpsApplicationRow { + return { + id, + lifecycle_key: `direct:${stackName}`, + lifecycle_status: 'creating', + target_mode: 'direct', + stack_name: stackName, + blueprint_id: null, + configured_repo_url: 'https://github.com/org/repo.git', + repo_identity_json: '{"host":"github.com","pathname":"/org/repo.git"}', + configured_ref: 'main', + compose_paths_json: '["compose.yml"]', + context_dir: null, + sync_env: 0, + env_path: null, + materialization_fingerprint: 'a'.repeat(64), + desired_commit_sha: null, + fetched_commit_sha: null, + candidate_generation_id: null, + accepted_generation_id: null, + candidate_plan_blocked: 0, + review_required: 0, + artifact_set_id: null, + latest_artifact_set_id: null, + intent_revision_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + rollout_authorization_ref: null, + legacy_combined_approval_ref: null, + preflight_fingerprint: null, + latest_operation_id: null, + active_operation_id: null, + active_operation_stage: null, + active_operation_at: null, + active_generation_id: null, + pause_at: null, + pause_reason: null, + partial_json: null, + failure_stage: null, + failure_class: null, + failure_at: null, + retry_at: null, + retry_count: 0, + suspended_at: null, + recovery_ref: null, + recovery_phase: null, + interruption_stage: null, + interruption_at: null, + interruption_operation_id: null, + interruption_generation_id: null, + evidence_fresh_at: null, + evidence_limitations_json: null, + created_at: 1, + updated_at: 1, + }; +} + +function gen(id: string, applicationId: string): GitOpsGenerationRow { + return { + id, + application_id: applicationId, + commit_sha: SHA, + repo_url: 'https://github.com/org/repo.git', + configured_ref: 'main', + repo_identity_json: '{"host":"github.com","pathname":"/org/repo.git"}', + manifest_version: 0, + candidate_dir: candidateRelPathForSha(SHA), + applied_dir: `generations/applied-${id}-0`, + expected_invocation_json: '{"composeFileOrder":[],"projectName":null,"projectDirectory":null,"envFileOrder":[]}', + materialization_fingerprint: 'a'.repeat(64), + validation_ok: 1, + plan_blocked: 0, + change_plan_fingerprint: null, + operation_id: `op-${id}`, + trigger: 'manual', + actor: 'tester', + previous_generation_id: null, + redacted_limitations_json: '[]', + created_at: 1, + }; +} diff --git a/backend/src/__tests__/gitops-deferred.test.ts b/backend/src/__tests__/gitops-deferred.test.ts new file mode 100644 index 00000000..72c3a46b --- /dev/null +++ b/backend/src/__tests__/gitops-deferred.test.ts @@ -0,0 +1,359 @@ +/** + * Deferred-state events: retry, suspend, pause, and partial rollout. + * + * These have no production writer by design; later tickets emit them. They are + * implemented and tested now so the deriver has no branch a writer cannot + * reach, and so the shape a future producer must satisfy is pinned rather than + * inferred from the deriver. + * + * The rule they all share is that none of them is a statement about health. A + * suspended source, a paused rollout, and a partial rollout each leave every + * success pointer exactly where it was. + */ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { GitOpsStore } from '../services/gitops/store'; +import { GitOpsTransitions, type EventEnvelope } from '../services/gitops/transitions'; +import { projectApplication } from '../services/gitops/derive'; +import type { GitOpsApplicationRow, GitOpsGenerationRow } from '../services/gitops/types'; + +describe('gitops deferred state', () => { + let tmpDir: string; + + beforeAll(async () => { + tmpDir = await setupTestDb(); + GitOpsStore.resetForTests(); + GitOpsTransitions.resetForTests(); + }); + + afterAll(() => { + cleanupTestDb(tmpDir); + }); + + it('schedules a retry without hiding the failure that caused it', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedApplied('app-retry', 'retry-web'); + tx.fetchStarted('app-retry', env('op-retry-f')); + tx.fetchFailed('app-retry', env('op-retry-f')); + + tx.sourceRetryScheduled('app-retry', 5_000, 2, env('op-retry-s')); + + const app = store.getApplication('app-retry')!; + expect(app.retry_at).toBe(5_000); + expect(app.retry_count).toBe(2); + // A retry is a plan, not a resolution: a stack that keeps failing must not + // read as merely busy. + expect(app.failure_stage).toBe('fetch'); + expect(projectOf('app-retry').facets.source.status).toBe('source_failed'); + + // Starting the retry clears the schedule and keeps the count. + tx.fetchStarted('app-retry', env('op-retry-f2')); + expect(store.getApplication('app-retry')?.retry_at).toBeNull(); + expect(store.getApplication('app-retry')?.retry_count).toBe(2); + }); + + it('suspends a source without forgetting what it had accepted', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedApplied('app-susp', 'susp-web'); + const accepted = store.getApplication('app-susp')!.accepted_generation_id; + + tx.sourceSuspended('app-susp', 'operator paused sync', env('op-susp')); + + const app = store.getApplication('app-susp')!; + expect(app.suspended_at).not.toBeNull(); + expect(app.accepted_generation_id).toBe(accepted); + expect(projectOf('app-susp').facets.source.status).toBe('source_suspended'); + // A suspended source refuses new work rather than queueing it. + expect(() => tx.fetchStarted('app-susp', env('op-susp-f'))).toThrow(/suspended/); + + tx.sourceUnsuspended('app-susp', env('op-unsusp')); + expect(store.getApplication('app-susp')?.suspended_at).toBeNull(); + expect(projectOf('app-susp').facets.source.status).toBe('application_generation_accepted'); + }); + + it('interrupts an operation in flight when the source is suspended', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedApplied('app-susp2', 'susp2-web'); + tx.fetchStarted('app-susp2', env('op-susp2-f')); + + tx.sourceSuspended('app-susp2', 'operator paused sync', env('op-susp2')); + + const app = store.getApplication('app-susp2')!; + // Abandoning the operation without recording it would leave the source + // reporting a fetch in flight that nothing will ever finish. + expect(app.active_operation_stage).toBeNull(); + expect(app.interruption_stage).toBe('fetch_started'); + expect(app.suspended_at).not.toBeNull(); + }); + + it('pauses a rollout without claiming anything about health', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedApplied('app-pause', 'pause-web'); + tx.deployStarted('app-pause', 1, 'gen-app-pause', env('op-pause-d')); + tx.deployBound('app-pause', 1, 'gen-app-pause', env('op-pause-d')); + + tx.rolloutPaused('app-pause', 1, 'awaiting approval', env('op-pause')); + + const target = store.getTarget('app-pause', 1)!; + expect(target.pause_at).not.toBeNull(); + // What was deployed is still deployed. + expect(target.deployed_generation_id).toBe('gen-app-pause'); + expect(projectOf('app-pause').targets[0]?.runtime.status).toBe('paused'); + + tx.rolloutUnpaused('app-pause', 1, env('op-unpause')); + expect(store.getTarget('app-pause', 1)?.pause_at).toBeNull(); + expect(projectOf('app-pause').targets[0]?.runtime.status).not.toBe('paused'); + }); + + it('records a partial rollout without inventing a deployed pointer', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedApplied('app-partial', 'partial-web'); + + tx.partiallyRolledOut('app-partial', 1, '{"reached":[1],"pending":[2]}', env('op-partial')); + + const target = store.getTarget('app-partial', 1)!; + expect(target.partial_json).toBe('{"reached":[1],"pending":[2]}'); + expect(target.deployed_generation_id).toBeNull(); + expect(projectOf('app-partial').targets[0]?.runtime.status).toBe('partially_rolled_out'); + + tx.partialCleared('app-partial', 1, env('op-partial-clear')); + expect(store.getTarget('app-partial', 1)?.partial_json).toBeNull(); + }); + + it('refuses partial state that is not decodable', () => { + const tx = GitOpsTransitions.getInstance(); + seedApplied('app-partial-bad', 'partial-bad-web'); + expect(() => tx.partiallyRolledOut('app-partial-bad', 1, 'not json', env('op-partial-bad'))) + .toThrow(); + }); + + it('reports a rollback in flight on both the application and the target', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedApplied('app-rb-start', 'rb-start-web'); + const generationId = 'gen-app-rb-start'; + + tx.rollbackInProgress({ + applicationId: 'app-rb-start', + nodeId: 1, + recoveryRef: 'rb-1', + recoveryGenerationId: generationId, + envelope: env('op-rb-start'), + }); + + const target = store.getTarget('app-rb-start', 1)!; + expect(target.recovery_phase).toBe('restoring'); + expect(target.recovery_ref).toBe('rb-1'); + expect(target.recovery_generation_id).toBe(generationId); + // Written to both, because a target-only write left the source facet + // reporting whatever the source last did instead of the rollback. + expect(store.getApplication('app-rb-start')?.recovery_phase).toBe('restoring'); + expect(projectOf('app-rb-start').facets.rollout.status).toBe('rollback_in_progress'); + }); + + it('persists the failure class a partial rollback was given', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedApplied('app-rb-partial', 'rb-partial-web'); + const applied = store.getTarget('app-rb-partial', 1)!.applied_generation_id; + + tx.rollbackPartialFailed({ + applicationId: 'app-rb-partial', + nodeId: 1, + recoveryRef: 'rb-2', + failureClass: 'partial', + envelope: env('op-rb-partial'), + }); + + const target = store.getTarget('app-rb-partial', 1)!; + expect(target.recovery_phase).toBe('failed'); + expect(target.failure_stage).toBe('recovery'); + // Reported verbatim: the deriver reads these columns rather than inventing + // a class, and `partial` is the one this alias adds over a recovery. + expect(target.failure_class).toBe('partial'); + // A failed rollback moves no success pointer. + expect(target.applied_generation_id).toBe(applied); + expect(target.healthy_generation_id).toBeNull(); + }); + + it('completes a rollback only against a generation it can prove', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedApplied('app-rb-done', 'rb-done-web'); + const generationId = 'gen-app-rb-done'; + + // Nothing bound yet, so there is no generation to complete against. + expect(() => tx.rollbackCompleted({ + applicationId: 'app-rb-done', + nodeId: 1, + recoveryRef: 'rb-3', + capturedArtifactSetId: null, + capturedSourceAcceptanceRef: null, + envelope: env('op-rb-done-early'), + })).toThrow(/bound recovery generation/); + + tx.rollbackInProgress({ + applicationId: 'app-rb-done', + nodeId: 1, + recoveryRef: 'rb-3', + recoveryGenerationId: generationId, + envelope: env('op-rb-done-start'), + }); + tx.rollbackCompleted({ + applicationId: 'app-rb-done', + nodeId: 1, + recoveryRef: 'rb-3', + capturedArtifactSetId: null, + capturedSourceAcceptanceRef: null, + envelope: env('op-rb-done'), + }); + + const target = store.getTarget('app-rb-done', 1)!; + expect(target.recovery_phase).toBe('complete'); + expect(target.desired_generation_id).toBe(generationId); + expect(target.applied_generation_id).toBe(generationId); + // The workload is back but nothing has observed it yet. + expect(target.healthy_generation_id).toBeNull(); + }); + + it('refuses to complete a rollback onto another application generation', () => { + const tx = GitOpsTransitions.getInstance(); + seedApplied('app-rb-foreign', 'rb-foreign-web'); + seedApplied('app-rb-owner', 'rb-owner-web'); + + tx.rollbackInProgress({ + applicationId: 'app-rb-foreign', + nodeId: 1, + recoveryRef: 'rb-4', + recoveryGenerationId: 'gen-app-rb-owner', + envelope: env('op-rb-foreign-start'), + }); + + expect(() => tx.rollbackCompleted({ + applicationId: 'app-rb-foreign', + nodeId: 1, + recoveryRef: 'rb-4', + capturedArtifactSetId: null, + capturedSourceAcceptanceRef: null, + envelope: env('op-rb-foreign'), + })).toThrow(/does not own/); + }); +}); + +function projectOf(applicationId: string) { + const projection = projectApplication(applicationId, true); + if (projection.targetMode === 'not_applicable') throw new Error('expected an application'); + return projection; +} + +function seedApplied(applicationId: string, stackName: string): void { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + const generationId = `gen-${applicationId}`; + tx.activateDirect({ application: app(applicationId, stackName), nodeId: 1, envelope: env(`op-act-${applicationId}`) }); + store.insertGeneration(gen(generationId, applicationId)); + tx.fetchStarted(applicationId, env(`op-f-${applicationId}`)); + tx.fetched(applicationId, 'abc123', env(`op-f-${applicationId}`)); + tx.candidateReady(applicationId, generationId, false, env(`op-c-${applicationId}`)); + tx.applied({ + applicationId, + generationId, + artifactSetId: `art-${applicationId}`, + sourceAcceptanceId: `acc-${applicationId}`, + authority: 'operator', + envelope: env(`op-a-${applicationId}`), + }); +} + +function env(operationId: string): EventEnvelope { + return { operationId, actor: 'tester', trigger: 'manual', at: Date.now() }; +} + +function app(id: string, stackName: string): GitOpsApplicationRow { + return { + id, + lifecycle_key: `direct:${stackName}`, + lifecycle_status: 'active', + target_mode: 'direct', + stack_name: stackName, + blueprint_id: null, + configured_repo_url: 'https://github.com/org/repo.git', + repo_identity_json: '{"host":"github.com","pathname":"/org/repo.git"}', + configured_ref: 'main', + compose_paths_json: '["compose.yml"]', + context_dir: null, + sync_env: 0, + env_path: null, + materialization_fingerprint: 'a'.repeat(64), + desired_commit_sha: null, + fetched_commit_sha: null, + candidate_generation_id: null, + accepted_generation_id: null, + candidate_plan_blocked: 0, + review_required: 0, + artifact_set_id: null, + latest_artifact_set_id: null, + intent_revision_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + rollout_authorization_ref: null, + legacy_combined_approval_ref: null, + preflight_fingerprint: null, + latest_operation_id: null, + active_operation_id: null, + active_operation_stage: null, + active_operation_at: null, + active_generation_id: null, + pause_at: null, + pause_reason: null, + partial_json: null, + failure_stage: null, + failure_class: null, + failure_at: null, + retry_at: null, + retry_count: 0, + suspended_at: null, + recovery_ref: null, + recovery_phase: null, + interruption_stage: null, + interruption_at: null, + interruption_operation_id: null, + interruption_generation_id: null, + evidence_fresh_at: null, + evidence_limitations_json: null, + created_at: 1, + updated_at: 1, + }; +} + +function gen(id: string, applicationId: string): GitOpsGenerationRow { + return { + id, + application_id: applicationId, + commit_sha: 'abc123', + repo_url: 'https://github.com/org/repo.git', + configured_ref: 'main', + repo_identity_json: '{"host":"github.com","pathname":"/org/repo.git"}', + manifest_version: 0, + candidate_dir: `generations/candidate-${id}`, + applied_dir: `generations/applied-${id}-0`, + expected_invocation_json: '{"composeFileOrder":[],"projectName":null,"projectDirectory":null,"envFileOrder":[]}', + materialization_fingerprint: 'a'.repeat(64), + validation_ok: 1, + plan_blocked: 0, + change_plan_fingerprint: null, + operation_id: `op-${id}`, + trigger: 'manual', + actor: 'tester', + previous_generation_id: null, + redacted_limitations_json: '[]', + created_at: 1, + }; +} diff --git a/backend/src/__tests__/gitops-derive.test.ts b/backend/src/__tests__/gitops-derive.test.ts new file mode 100644 index 00000000..c2a7e8e2 --- /dev/null +++ b/backend/src/__tests__/gitops-derive.test.ts @@ -0,0 +1,1012 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { FACET_EVIDENCE_SOURCE } from '../services/gitops/types'; +import { GitOpsStore, emptyTargetRow } from '../services/gitops/store'; +import { GitOpsTransitions, type EventEnvelope } from '../services/gitops/transitions'; +import { projectApplication } from '../services/gitops/derive'; +import type { GitOpsApplicationRow, GitOpsGenerationRow } from '../services/gitops/types'; + +describe('gitops derivation', () => { + let tmpDir: string; + + beforeAll(async () => { + tmpDir = await setupTestDb(); + GitOpsStore.resetForTests(); + GitOpsTransitions.resetForTests(); + }); + + afterAll(() => { + cleanupTestDb(tmpDir); + }); + + it('registers every facet status exactly once', () => { + expect(FACET_EVIDENCE_SOURCE.source.applying).toBe('current'); + expect(FACET_EVIDENCE_SOURCE.rollout.completion_unknown).toBe('current_or_future'); + expect(FACET_EVIDENCE_SOURCE.source.source_superseded).toBe('future'); + expect(FACET_EVIDENCE_SOURCE.runtime.rollout_artifact_drift).toBe('future'); + expect(FACET_EVIDENCE_SOURCE.lkg.none).toBe('current'); + }); + + it('projects applying with no fetch/apply/dismiss actions', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-apply-facet', 'facet-web'), nodeId: 1, envelope: env('op-act') }); + store.insertGeneration(gen('gen-facet', 'app-apply-facet')); + tx.fetchStarted('app-apply-facet', env('op-f')); + tx.fetched('app-apply-facet', 'abc123', env('op-f')); + tx.candidateReady('app-apply-facet', 'gen-facet', false, env('op-c')); + tx.applyStarted('app-apply-facet', 'gen-facet', env('op-a')); + const projection = projectApplication('app-apply-facet', false); + expect(projection.targetMode).toBe('direct'); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.facets.source.status).toBe('applying'); + expect(projection.availableActions).toEqual(['none']); + expect(projection.targets[0]?.desiredGenerationId).toBeNull(); + expect(projection.targets[0]?.candidateGenerationId).toBe('gen-facet'); + }); + + it('projects a freshly activated target as never applied and offers no deploy', () => { + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-idle', 'idle-web'), nodeId: 1, envelope: env('op-act-idle') }); + const projection = projectApplication('app-idle', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.targets[0]?.runtime.status).toBe('never_applied'); + expect(projection.availableActions).not.toContain('deploy'); + expect(projection.availableActions).toContain('fetch'); + }); + + it('keeps a never-applied target out of deploy actions when health gating is disabled', () => { + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-idle-nohealth', 'idle-nohealth-web'), nodeId: 1, envelope: env('op-act-idle-2') }); + const projection = projectApplication('app-idle-nohealth', true); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.targets[0]?.runtime.status).toBe('never_applied'); + expect(projection.availableActions).not.toContain('deploy'); + }); + + it('projects accepted application and applied-not-deployed after apply', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-done', 'done-web'), nodeId: 1, envelope: env('op-act-2') }); + store.insertGeneration(gen('gen-done', 'app-done')); + tx.fetchStarted('app-done', env('op-f2')); + tx.fetched('app-done', 'abc123', env('op-f2')); + tx.candidateReady('app-done', 'gen-done', false, env('op-c2')); + tx.applied({ + applicationId: 'app-done', + generationId: 'gen-done', + artifactSetId: 'art-done', + sourceAcceptanceId: 'acc-done', + authority: 'operator', + envelope: env('op-a2'), + }); + const projection = projectApplication('app-done', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.facets.source.status).toBe('application_generation_accepted'); + expect(projection.facets.artifact.status).toBe('artifact_resolution_pending'); + expect(projection.facets.placement.status).toBe('unbound_direct'); + expect(projection.targets[0]?.runtime.status).toBe('applied_not_deployed'); + expect(projection.targets[0]?.lkg.status).toBe('none'); + expect(projection.availableActions).toContain('deploy'); + }); + + it('keeps a stale deployment deploy-pending instead of synced and healthy', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-stale-deploy', 'stale-deploy-web'), nodeId: 1, envelope: env('op-stale') }); + store.insertGeneration(gen('gen-a-stale', 'app-stale-deploy')); + store.insertGeneration(gen('gen-b-stale', 'app-stale-deploy')); + // Generation A is deployed and healthy; generation B is applied and + // desired, with automatic deployment off so nothing moves it. + const target = { + ...emptyTargetRow('app-stale-deploy', 1, 1), + desired_generation_id: 'gen-b-stale', + applied_generation_id: 'gen-b-stale', + deployed_generation_id: 'gen-a-stale', + healthy_generation_id: 'gen-a-stale', + }; + store.upsertTarget(target); + + let projection = projectApplication('app-stale-deploy', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.targets[0]?.runtime.status).toBe('applied_not_deployed'); + expect(projection.targets[0]?.health.status).toBe('pending'); + expect(projection.availableActions).toContain('deploy'); + // The mismatch is a confirmed drift item, not only a facet status: the + // canonical drift list must not contradict what the runtime facet says. + expect(projection.drift).toHaveLength(1); + expect(projection.drift[0]).toEqual({ + class: 'runtime', + expected: { kind: 'generation', id: 'gen-b-stale' }, + observed: { kind: 'generation', id: 'gen-a-stale' }, + freshnessAt: null, + owner: 'ComposeService', + reason: 'the target is running a different generation than the one it was asked to run', + configuredPolicy: null, + affectedTargets: [{ nodeId: 1, stackName: 'stale-deploy-web' }], + action: 'deploy', + }); + + // Re-derived from the store rows rather than any carried-over state, so a + // restart reads the same answer, item included. + expect(GitOpsStore.getInstance().getTarget('app-stale-deploy', 1)?.deployed_generation_id).toBe('gen-a-stale'); + projection = projectApplication('app-stale-deploy', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.targets[0]?.runtime.status).toBe('applied_not_deployed'); + expect(projection.drift).toHaveLength(1); + + // Once the deploy lands the target awaits its own health run instead of + // inheriting generation A's green verdict, and the mismatch item clears: + // desired and deployed now agree, so there is nothing left to report. + store.upsertTarget({ ...target, deployed_generation_id: 'gen-b-stale' }); + projection = projectApplication('app-stale-deploy', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.targets[0]?.runtime.status).toBe('fully_deployed_health_pending'); + expect(projection.targets[0]?.health.status).toBe('pending'); + expect(projection.drift).toHaveLength(0); + + // A passing run recorded against the desired generation answers for it + // even while a different generation is deployed. No producer reaches this + // combination today; the pin keeps any tightening of the comparison a + // conscious decision rather than an accident. + store.upsertTarget({ ...target, healthy_generation_id: 'gen-b-stale' }); + projection = projectApplication('app-stale-deploy', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.targets[0]?.runtime.status).toBe('applied_not_deployed'); + expect(projection.targets[0]?.health.status).toBe('passed'); + }); + + it('keeps the generation-mismatch drift item after a failed redeploy', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-fail-drift', 'fail-drift-web'), nodeId: 1, envelope: env('op-fd-act') }); + store.insertGeneration(gen('gen-fd-a', 'app-fail-drift')); + store.insertGeneration(gen('gen-fd-b', 'app-fail-drift')); + // Generation A ships and binds, then B is applied as the new desired + // state while A keeps serving. + tx.fetchStarted('app-fail-drift', env('op-fd-f1')); + tx.fetched('app-fail-drift', 'abc123', env('op-fd-f1')); + tx.candidateReady('app-fail-drift', 'gen-fd-a', false, env('op-fd-c1')); + tx.applied({ + applicationId: 'app-fail-drift', + generationId: 'gen-fd-a', + artifactSetId: 'art-fd-a', + sourceAcceptanceId: 'acc-fd-a', + authority: 'operator', + envelope: env('op-fd-a1'), + }); + tx.deployStarted('app-fail-drift', 1, 'gen-fd-a', env('op-fd-d1')); + tx.deployBound('app-fail-drift', 1, 'gen-fd-a', env('op-fd-d1')); + tx.fetchStarted('app-fail-drift', env('op-fd-f2')); + tx.fetched('app-fail-drift', 'def456', env('op-fd-f2')); + tx.candidateReady('app-fail-drift', 'gen-fd-b', false, env('op-fd-c2')); + tx.applied({ + applicationId: 'app-fail-drift', + generationId: 'gen-fd-b', + artifactSetId: 'art-fd-b', + sourceAcceptanceId: 'acc-fd-b', + authority: 'operator', + envelope: env('op-fd-a2'), + }); + + // Sanity: the clean mismatch reports one item offering deploy. + let projection = projectApplication('app-fail-drift', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.targets[0]?.runtime.status).toBe('applied_not_deployed'); + expect(projection.drift).toHaveLength(1); + expect(projection.drift[0].action).toBe('deploy'); + + // Mid-deploy the divergence is factual while nothing can offer deploying + // again: one item, action none, gone the moment B binds. + tx.deployStarted('app-fail-drift', 1, 'gen-fd-b', env('op-fd-d2')); + projection = projectApplication('app-fail-drift', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.targets[0]?.runtime.status).toBe('deploying'); + expect(projection.drift).toHaveLength(1); + expect(projection.drift[0].action).toBe('none'); + + // The deploy of B fails before mutating anything; A keeps serving and the + // deployed pointer stays on it. The runtime facet now shows the failure, + // but the mismatch between what was asked for and what is running did not + // go anywhere, so the drift item must survive the presentation change. + tx.deployStarted('app-fail-drift', 1, 'gen-fd-b', env('op-fd-d2')); + tx.deployFailed('app-fail-drift', 1, 'pre_mutation', env('op-fd-d2')); + projection = projectApplication('app-fail-drift', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(store.getTarget('app-fail-drift', 1)?.deployed_generation_id).toBe('gen-fd-a'); + expect(projection.targets[0]?.runtime.status).toBe('failed_previous_workload_intact'); + // No target reads applied_not_deployed here, so availableActions withholds + // deploy and the item must agree instead of advertising an absent action. + expect(projection.availableActions).not.toContain('deploy'); + expect(projection.drift).toHaveLength(1); + expect(projection.drift[0]).toEqual({ + class: 'runtime', + expected: { kind: 'generation', id: 'gen-fd-b' }, + observed: { kind: 'generation', id: 'gen-fd-a' }, + freshnessAt: null, + owner: 'ComposeService', + reason: 'the target is running a different generation than the one it was asked to run', + configuredPolicy: null, + affectedTargets: [{ nodeId: 1, stackName: 'fail-drift-web' }], + action: 'none', + }); + + // Re-derived from the same rows, the report is stable. + projection = projectApplication('app-fail-drift', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.drift).toHaveLength(1); + expect(projection.drift[0].expected).toEqual({ kind: 'generation', id: 'gen-fd-b' }); + expect(projection.drift[0].action).toBe('none'); + + // The artifact observation describing the workload that is being replaced + // stays suppressed while the generation question stands. + // Version 2 because the apply already seeded an unresolved v1 row for + // this generation and the table is unique per generation and version. + store.insertArtifactSet({ + id: 'art-fd-b-expected', + generation_id: 'gen-fd-b', + evidence_version: 2, + authoritative: 0, + qualification: 'exact', + evidence_json: JSON.stringify({ kind: 'exact', identity: 'sha256:wanted' }), + created_at: 1, + }); + const failed = store.getTarget('app-fail-drift', 1)!; + store.upsertTarget({ + ...failed, + expected_artifact_set_id: 'art-fd-b-expected', + observed_artifact_identity_json: JSON.stringify({ kind: 'exact', identity: 'sha256:serving', observedAt: 7 }), + }); + projection = projectApplication('app-fail-drift', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.drift).toHaveLength(1); + expect(projection.drift[0].expected.kind).toBe('generation'); + + // The post-mutation variant reports the same way. Even when Compose was + // handed off, the deployed pointer stays on the old generation until a + // successful bind proves the new one, so the report stays anchored to + // whatever is actually serving. + tx.deployStarted('app-fail-drift', 1, 'gen-fd-b', env('op-fd-d4')); + tx.deployFailed('app-fail-drift', 1, 'post_mutation', env('op-fd-d4')); + projection = projectApplication('app-fail-drift', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.targets[0]?.runtime.status).toBe('failed_after_mutation'); + expect(projection.drift).toHaveLength(1); + expect(projection.drift[0].observed).toEqual({ kind: 'generation', id: 'gen-fd-a' }); + expect(projection.drift[0].action).toBe('none'); + + // Binding B clears the item along with the failure. The artifact probe + // from the suppression check goes with it, so the converged target is + // judged on pointers and health alone. + tx.deployStarted('app-fail-drift', 1, 'gen-fd-b', env('op-fd-d3')); + tx.deployBound('app-fail-drift', 1, 'gen-fd-b', env('op-fd-d3')); + store.upsertTarget({ + ...store.getTarget('app-fail-drift', 1)!, + expected_artifact_set_id: null, + observed_artifact_identity_json: null, + }); + projection = projectApplication('app-fail-drift', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.targets[0]?.runtime.status).toBe('fully_deployed_health_pending'); + expect(projection.drift).toHaveLength(0); + }); + + it('does not report generation mismatch on a retired target', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-tomb-drift', 'tomb-drift-web'), nodeId: 1, envelope: env('op-td-act') }); + store.insertGeneration(gen('gen-td-a', 'app-tomb-drift')); + store.insertGeneration(gen('gen-td-b', 'app-tomb-drift')); + // Retirement clears failure and LKG state but leaves the pointers alone, + // so a target retired mid-pending-deploy keeps divergent pointers. No + // transition can rebind it afterwards, so the mismatch must stay silent + // instead of becoming an item nothing could ever clear. + store.upsertTarget({ + ...emptyTargetRow('app-tomb-drift', 1, 1), + target_status: 'tombstoned', + desired_generation_id: 'gen-td-b', + applied_generation_id: 'gen-td-b', + deployed_generation_id: 'gen-td-a', + }); + + const projection = projectApplication('app-tomb-drift', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.targets[0]?.runtime.status).toBe('tombstoned'); + expect(projection.drift).toHaveLength(0); + }); + + it('keeps a failed sibling out of another target\'s deploy action', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-sib', 'sib-web'), nodeId: 1, envelope: env('op-sib') }); + store.insertGeneration(gen('gen-a-sib', 'app-sib')); + store.insertGeneration(gen('gen-b-sib', 'app-sib')); + // Node 1 diverges cleanly; node 2 carries the same divergence but sits in + // a failed state. The deploy question is per target: node 2 must not be + // told to deploy because node 1 legally can. + store.upsertTarget({ + ...emptyTargetRow('app-sib', 1, 1), + desired_generation_id: 'gen-b-sib', + applied_generation_id: 'gen-b-sib', + deployed_generation_id: 'gen-a-sib', + healthy_generation_id: 'gen-a-sib', + }); + store.upsertTarget({ + ...emptyTargetRow('app-sib', 2, 2), + desired_generation_id: 'gen-b-sib', + applied_generation_id: 'gen-b-sib', + deployed_generation_id: 'gen-a-sib', + failure_stage: 'deploy', + failure_class: 'pre_mutation', + }); + // Node 3 carries the same divergence under an operator pause: a paused + // target cannot act, so its item stays none like the failed one. + store.upsertTarget({ + ...emptyTargetRow('app-sib', 3, 3), + desired_generation_id: 'gen-b-sib', + applied_generation_id: 'gen-b-sib', + deployed_generation_id: 'gen-a-sib', + pause_at: 1, + pause_reason: 'operator', + }); + + const projection = projectApplication('app-sib', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.availableActions).toContain('deploy'); + expect(projection.drift).toHaveLength(3); + expect(projection.drift[0].affectedTargets[0]?.nodeId).toBe(1); + expect(projection.drift[0].action).toBe('deploy'); + expect(projection.drift[1].affectedTargets[0]?.nodeId).toBe(2); + expect(projection.drift[1].action).toBe('none'); + expect(projection.drift[2].affectedTargets[0]?.nodeId).toBe(3); + expect(projection.drift[2].action).toBe('none'); + }); + + it('never advertises Direct deployment for a Blueprint-mode application', () => { + const store = GitOpsStore.getInstance(); + store.insertGeneration(gen('gen-bp-wanted', 'app-bp-deploy')); + store.insertGeneration(gen('gen-bp-serving', 'app-bp-deploy')); + store.insertApplication(rawApp('app-bp-deploy', { + target_mode: 'inline_blueprint', + blueprint_id: 9, + lifecycle_key: 'blueprint:9', + stack_name: null, + configured_repo_url: null, + repo_identity_json: null, + configured_ref: null, + })); + // A divergent Blueprint target reads applied_not_deployed like any other, + // but Direct deployment is not a legal move for this mode: only an + // identity-matched interruption retry ever deploys here. + store.upsertTarget({ + ...emptyTargetRow('app-bp-deploy', 1, 1), + desired_generation_id: 'gen-bp-wanted', + applied_generation_id: 'gen-bp-wanted', + deployed_generation_id: 'gen-bp-serving', + }); + + const projection = projectApplication('app-bp-deploy', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.availableActions).not.toContain('deploy'); + expect(projection.drift).toHaveLength(1); + expect(projection.drift[0].action).toBe('none'); + }); + + it('retries an interrupted Blueprint deploy only while the recorded identities still match', () => { + const store = GitOpsStore.getInstance(); + const bpApp = (id: string, blueprintId: number, overrides: Partial) => + rawApp(id, { + target_mode: 'inline_blueprint', + blueprint_id: blueprintId, + lifecycle_key: `blueprint:${blueprintId}`, + stack_name: null, + configured_repo_url: null, + repo_identity_json: null, + configured_ref: null, + ...overrides, + }); + const divergentTarget = (appId: string) => ({ + ...emptyTargetRow(appId, 1, 1), + desired_generation_id: `wanted-${appId}`, + applied_generation_id: `wanted-${appId}`, + deployed_generation_id: `serving-${appId}`, + interruption_stage: 'blueprint_deploy_started' as const, + interruption_at: 1, + }); + + // Inline reality today: no rollout candidate producer has run, so the + // application and the recorded crash carry no candidate id at all. The + // absent pair matches, leaving the intent revision as the live identity + // that decides the retry. + store.insertGeneration(gen('wanted-app-bp-r-vac', 'app-bp-r-vac')); + store.insertGeneration(gen('serving-app-bp-r-vac', 'app-bp-r-vac')); + store.insertApplication(bpApp('app-bp-r-vac', 12, { intent_revision_id: 'ir-v' })); + store.upsertTarget({ + ...divergentTarget('app-bp-r-vac'), + interruption_intent_revision_id: 'ir-v', + }); + + let projection = projectApplication('app-bp-r-vac', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + const vacRuntime = projection.targets[0]?.runtime; + if (!vacRuntime || vacRuntime.status !== 'completion_unknown') throw new Error('expected completion_unknown'); + expect(vacRuntime.interruptedStage).toBe('blueprint_deploy_started'); + expect(projection.availableActions).toContain('deploy'); + + // With a candidate in play, both recorded identities must equal what the + // application requires for the repeat to stay legal. + store.insertGeneration(gen('wanted-app-bp-r-match', 'app-bp-r-match')); + store.insertGeneration(gen('serving-app-bp-r-match', 'app-bp-r-match')); + store.insertApplication(bpApp('app-bp-r-match', 13, { intent_revision_id: 'ir-m', rollout_candidate_id: 'rc-m' })); + store.upsertTarget({ + ...divergentTarget('app-bp-r-match'), + interruption_intent_revision_id: 'ir-m', + interruption_rollout_candidate_id: 'rc-m', + }); + projection = projectApplication('app-bp-r-match', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.availableActions).toContain('deploy'); + + // Candidate-only mismatch: the intent still matches but the recorded + // rollout candidate was superseded. Both persisted identities are + // contractually significant, so either one drifting alone suppresses + // the retry. + store.upsertTarget({ + ...store.getTarget('app-bp-r-match', 1)!, + interruption_rollout_candidate_id: 'rc-superseded', + }); + projection = projectApplication('app-bp-r-match', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.availableActions).not.toContain('deploy'); + expect(projection.drift[0]?.action).toBe('none'); + + // A superseded intent revision means the recorded operation names an + // identity nobody requires anymore, so the retry disappears even though + // the divergence itself still reports. + store.upsertTarget({ + ...store.getTarget('app-bp-r-match', 1)!, + interruption_intent_revision_id: 'ir-superseded', + interruption_rollout_candidate_id: 'rc-m', + }); + projection = projectApplication('app-bp-r-match', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.targets[0]?.runtime.status).toBe('completion_unknown'); + expect(projection.availableActions).not.toContain('deploy'); + expect(projection.drift).toHaveLength(1); + expect(projection.drift[0].action).toBe('none'); + }); + + it('retries an interrupted Direct deploy only while the interrupted generation still matches', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-int-dep', 'int-dep-web'), nodeId: 1, envelope: env('op-id-act') }); + store.insertGeneration(gen('gen-a-id', 'app-int-dep')); + // Cycle B fetches a different commit; carrying that sha keeps the fixture + // in the accepted state instead of a reconcile-required one. + store.insertGeneration({ ...gen('gen-b-id', 'app-int-dep'), commit_sha: 'def456' }); + store.insertGeneration(gen('gen-c-id', 'app-int-dep')); + tx.fetchStarted('app-int-dep', env('op-id-f1')); + tx.fetched('app-int-dep', 'abc123', env('op-id-f1')); + tx.candidateReady('app-int-dep', 'gen-a-id', false, env('op-id-c1')); + tx.applied({ + applicationId: 'app-int-dep', + generationId: 'gen-a-id', + artifactSetId: 'art-id-a', + sourceAcceptanceId: 'acc-id-a', + authority: 'operator', + envelope: env('op-id-a1'), + }); + tx.deployStarted('app-int-dep', 1, 'gen-a-id', env('op-id-d1')); + tx.deployBound('app-int-dep', 1, 'gen-a-id', env('op-id-d1')); + tx.fetchStarted('app-int-dep', env('op-id-f2')); + tx.fetched('app-int-dep', 'def456', env('op-id-f2')); + tx.candidateReady('app-int-dep', 'gen-b-id', false, env('op-id-c2')); + tx.applied({ + applicationId: 'app-int-dep', + generationId: 'gen-b-id', + artifactSetId: 'art-id-b', + sourceAcceptanceId: 'acc-id-b', + authority: 'operator', + envelope: env('op-id-a2'), + }); + // Crash mid-deploy: the interruption records the generation that was + // being deployed, and a retry is legal while that still matches what the + // target wants applied. + tx.deployStarted('app-int-dep', 1, 'gen-b-id', env('op-id-d2')); + tx.interruptActiveOperations('app-int-dep', env('op-id-x')); + + let projection = projectApplication('app-int-dep', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + const runtime = projection.targets[0]?.runtime; + if (!runtime || runtime.status !== 'completion_unknown') throw new Error('expected completion_unknown'); + expect(runtime.interruptedStage).toBe('deploy_started'); + expect(projection.availableActions).toContain('deploy'); + + // Once the target's applied and desired identities move on, the recorded + // interruption names a generation nobody wants anymore, so the retry + // disappears even though the divergence itself still reports. + const interrupted = store.getTarget('app-int-dep', 1)!; + store.upsertTarget({ + ...interrupted, + desired_generation_id: 'gen-c-id', + applied_generation_id: 'gen-c-id', + // The old generation's artifact expectations cannot follow the new + // desired id; clearing them keeps the interruption the only reported + // divergence here. + expected_artifact_set_id: null, + latest_artifact_set_id: null, + }); + projection = projectApplication('app-int-dep', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.targets[0]?.runtime.status).toBe('completion_unknown'); + expect(projection.drift).toHaveLength(1); + expect(projection.availableActions).not.toContain('deploy'); + expect(projection.drift[0].action).toBe('none'); + }); + + it('offers apply after an interrupted apply only when every apply precondition holds', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + const interruptedApp = (id: string, overrides: Partial = {}) => + rawApp(id, { + stack_name: `${id}-web`, + interruption_stage: 'apply_started', + interruption_at: 1, + interruption_operation_id: `op-${id}`, + interruption_generation_id: `gen-${id}`, + candidate_generation_id: `gen-${id}`, + ...overrides, + }); + + // Transition-legal positive: the recorded generation exists under this + // application with an unchanged materialization fingerprint, remains the + // current candidate, and neither suspension nor blockage intervenes, so + // finishing the apply is exactly what applyStarted would accept. + store.insertGeneration(gen('gen-app-int-ap-match', 'app-int-ap-match')); + store.insertApplication(interruptedApp('app-int-ap-match')); + let projection = projectApplication('app-int-ap-match', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + if (projection.facets.source.status !== 'source_unknown') throw new Error('expected source_unknown'); + expect(projection.facets.source.interruptedStage).toBe('apply_started'); + expect(projection.availableActions).toContain('apply'); + // The recommendation is only as good as the transition it names, so the + // projected action is executed rather than trusted: this must not throw. + tx.applyStarted('app-int-ap-match', 'gen-app-int-ap-match', env('op-ap-resume')); + + // Missing row: the recorded generation is gone, so applyStarted would + // refuse and the recommendation must fail closed. + store.insertApplication(interruptedApp('app-int-ap-missing')); + projection = projectApplication('app-int-ap-missing', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.availableActions).not.toContain('apply'); + + // Foreign owner: the candidate row exists but belongs to another + // application, which applyStarted refuses just the same. + store.insertGeneration(gen('gen-app-int-ap-foreign', 'app-not-the-owner')); + store.insertApplication(interruptedApp('app-int-ap-foreign')); + projection = projectApplication('app-int-ap-foreign', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.availableActions).not.toContain('apply'); + + // Fingerprint mismatch: finishing the recorded apply would use bytes built + // from different configuration. Defensive today, since shipped producers + // clear the candidate when configuration changes; the gate mirrors the + // transition's refusal either way. + store.insertGeneration({ + ...gen('gen-app-int-ap-fp', 'app-int-ap-fp'), + materialization_fingerprint: 'b'.repeat(64), + }); + store.insertApplication(interruptedApp('app-int-ap-fp')); + projection = projectApplication('app-int-ap-fp', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.availableActions).not.toContain('apply'); + + // Suspended: identities line up, but the source was suspended after the + // crash and applyStarted refuses suspended sources outright. + store.insertGeneration(gen('gen-app-int-ap-susp', 'app-int-ap-susp')); + store.insertApplication(interruptedApp('app-int-ap-susp', { suspended_at: 1 })); + projection = projectApplication('app-int-ap-susp', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.availableActions).not.toContain('apply'); + + // Stale: the candidate moved on after the crash, so the recorded apply + // can no longer prove what it was applying and apply must not be offered. + store.insertGeneration(gen('gen-ap-new', 'app-int-ap-stale')); + store.insertApplication(interruptedApp('app-int-ap-stale', { + interruption_generation_id: 'gen-ap-old', + candidate_generation_id: 'gen-ap-new', + })); + projection = projectApplication('app-int-ap-stale', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.facets.source.status).toBe('source_unknown'); + expect(projection.availableActions).not.toContain('apply'); + + // Blocked: the recorded apply still names the current candidate, but a + // later classification blocked that candidate, so finishing it is refused + // even though every identity still lines up. + store.insertGeneration(gen('gen-ap-b', 'app-int-ap-block')); + store.insertApplication(interruptedApp('app-int-ap-block', { candidate_plan_blocked: 1 })); + projection = projectApplication('app-int-ap-block', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.availableActions).toEqual(['dismiss']); + }); + + it('offers ordinary apply only when the candidate generation is present, owned, and current', () => { + const store = GitOpsStore.getInstance(); + // Valid: an owned, fingerprint-matched candidate reads ready and offers + // apply exactly as the transition would accept it. + store.insertGeneration(gen('gen-cr-valid', 'app-cr-valid')); + store.insertApplication(rawApp('app-cr-valid', { stack_name: 'cr-valid-web', candidate_generation_id: 'gen-cr-valid' })); + let projection = projectApplication('app-cr-valid', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.facets.source.status).toBe('candidate_ready'); + expect(projection.availableActions).toContain('apply'); + + // Missing: the candidate names a generation that does not exist, so + // ready would recommend an apply the transition refuses; the source must + // fail closed to reconcile-required instead, naming what was lost, and + // fetch stays on offer as the way out. + store.insertApplication(rawApp('app-cr-missing', { stack_name: 'cr-missing-web', candidate_generation_id: 'gen-cr-gone' })); + projection = projectApplication('app-cr-missing', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.facets.source.status).toBe('source_reconcile_required'); + expect(projection.limitations.map((item) => item.code)).toContain('candidate_generation_invalid'); + expect(projection.limitations.some((item) => item.evidence === 'gen-cr-gone')).toBe(true); + expect(projection.availableActions).not.toContain('apply'); + expect(projection.availableActions).toContain('fetch'); + + // Foreign: the candidate row exists but belongs to another application, + // which applyStarted refuses just as surely as a missing one. + store.insertGeneration(gen('gen-cr-foreign', 'app-not-the-owner')); + store.insertApplication(rawApp('app-cr-foreign', { stack_name: 'cr-foreign-web', candidate_generation_id: 'gen-cr-foreign' })); + projection = projectApplication('app-cr-foreign', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.facets.source.status).toBe('source_reconcile_required'); + expect(projection.availableActions).not.toContain('apply'); + + // Fingerprint mismatch: the generation's recorded fingerprint no longer + // equals the application's current one. No shipped producer leaves a + // candidate pointer across a configuration change today, so this pins + // the derivation's fail-safe side of that refusal. + store.insertGeneration({ + ...gen('gen-cr-stalefp', 'app-cr-fp'), + materialization_fingerprint: 'b'.repeat(64), + }); + store.insertApplication(rawApp('app-cr-fp', { stack_name: 'cr-fp-web', candidate_generation_id: 'gen-cr-stalefp' })); + projection = projectApplication('app-cr-fp', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.facets.source.status).toBe('source_reconcile_required'); + expect(projection.availableActions).not.toContain('apply'); + }); + + it('reports an accepted generation only when its evidence is present, owned, and current', () => { + const store = GitOpsStore.getInstance(); + const acceptedApp = (id: string, overrides: Partial = {}) => + rawApp(id, { + stack_name: `${id}-web`, + accepted_generation_id: `gen-${id}`, + desired_commit_sha: 'abc123', + ...overrides, + }); + + // Valid: the accepted row exists under this application with the + // materialization fingerprint it was built from and the commit the + // configuration asks for, so success is the honest answer. + store.insertGeneration(gen('gen-app-acc-valid', 'app-acc-valid')); + store.insertApplication(acceptedApp('app-acc-valid')); + let projection = projectApplication('app-acc-valid', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.facets.source.status).toBe('application_generation_accepted'); + expect(projection.availableActions).toEqual(['none']); + + // Missing: the accepted pointer names a generation that is gone, so + // neither the fingerprint nor the sha comparison can run and success + // would be claimed without any evidence behind it. + store.insertApplication(acceptedApp('app-acc-missing')); + projection = projectApplication('app-acc-missing', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.facets.source.status).toBe('source_reconcile_required'); + expect(projection.limitations.map((item) => item.code)).toContain('accepted_generation_invalid'); + expect(projection.limitations.some((item) => item.evidence === 'gen-app-acc-missing')).toBe(true); + expect(projection.availableActions).toContain('fetch'); + + // Foreign: the row exists but belongs to another application, which is + // the same refusal with the same recovery path. + store.insertGeneration(gen('gen-app-acc-foreign', 'app-not-the-owner')); + store.insertApplication(acceptedApp('app-acc-foreign')); + projection = projectApplication('app-acc-foreign', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.facets.source.status).toBe('source_reconcile_required'); + expect(projection.limitations.map((item) => item.code)).toContain('accepted_generation_invalid'); + expect(projection.availableActions).toContain('fetch'); + + // Fingerprint mismatch: the accepted row is present and owned but its + // materialization fingerprint differs from the application's current one. + store.insertGeneration({ + ...gen('gen-app-acc-fp', 'app-acc-fp'), + materialization_fingerprint: 'b'.repeat(64), + }); + store.insertApplication(acceptedApp('app-acc-fp')); + projection = projectApplication('app-acc-fp', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.facets.source.status).toBe('source_reconcile_required'); + expect(projection.availableActions).toContain('fetch'); + + // Sha mismatch: built from the right configuration but not the commit the + // configuration currently names. + store.insertGeneration({ + ...gen('gen-app-acc-sha', 'app-acc-sha'), + commit_sha: 'def456', + }); + store.insertApplication(acceptedApp('app-acc-sha', { desired_commit_sha: '789abc' })); + projection = projectApplication('app-acc-sha', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.facets.source.status).toBe('source_reconcile_required'); + expect(projection.availableActions).toContain('fetch'); + }); + + it('limits fetch to live Direct applications', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + // Direct control: a never-reconciled stack is offered fetch. + tx.activateDirect({ application: app('app-fetch-direct', 'fetch-direct-web'), nodeId: 1, envelope: env('op-fd') }); + const direct = projectApplication('app-fetch-direct', false); + if (direct.targetMode === 'not_applicable') throw new Error('expected application'); + expect(direct.availableActions).toContain('fetch'); + + // A Git-backed Blueprint application with the same unreconciled source + // state gets no fetch: the revision-state action rules reserve fetch for + // Direct applications, and Blueprint source integration ships later. + store.insertApplication(rawApp('app-fetch-bp', { + target_mode: 'blueprint', + blueprint_id: 21, + lifecycle_key: 'blueprint:21', + stack_name: null, + })); + const bp = projectApplication('app-fetch-bp', false); + if (bp.targetMode === 'not_applicable') throw new Error('expected application'); + expect(bp.availableActions).not.toContain('fetch'); + }); + + it('offers approve_legacy while Inline placement review is pending', () => { + const store = GitOpsStore.getInstance(); + store.insertApplication(rawApp('app-bp-legacy', { + target_mode: 'inline_blueprint', + blueprint_id: 11, + lifecycle_key: 'blueprint:11', + stack_name: null, + configured_repo_url: null, + repo_identity_json: null, + configured_ref: null, + intent_revision_id: 'ir-11', + legacy_combined_approval_ref: 'legacy-combined-11', + })); + + const projection = projectApplication('app-bp-legacy', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.facets.placement.status).toBe('placement_review_pending'); + expect(projection.availableActions).toEqual(['approve_legacy']); + }); + + it('still judges a target with no desired id against its deployed pointer', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-null-desired', 'null-desired-web'), nodeId: 1, envelope: env('op-null') }); + store.insertGeneration(gen('gen-a-null', 'app-null-desired')); + // Recovered and legacy rows can carry pointers with no desired id. The + // deployed pointer stays their only basis to judge. + store.upsertTarget({ + ...emptyTargetRow('app-null-desired', 1, 1), + applied_generation_id: 'gen-a-null', + deployed_generation_id: 'gen-a-null', + healthy_generation_id: 'gen-a-null', + }); + + const projection = projectApplication('app-null-desired', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.targets[0]?.runtime.status).toBe('synced_and_healthy'); + expect(projection.targets[0]?.health.status).toBe('passed'); + }); + + it('emits the runtime drift item when a comparable observation disagrees', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-drift-item', 'drift-item-web'), nodeId: 1, envelope: env('op-drift') }); + store.insertGeneration(gen('gen-drift', 'app-drift-item')); + store.insertArtifactSet({ + id: 'art-expected-drift', + generation_id: 'gen-drift', + evidence_version: 1, + authoritative: 0, + qualification: 'exact', + evidence_json: JSON.stringify({ kind: 'exact', identity: 'sha256:wanted' }), + created_at: 1, + }); + store.upsertTarget({ + ...emptyTargetRow('app-drift-item', 1, 1), + desired_generation_id: 'gen-drift', + applied_generation_id: 'gen-drift', + deployed_generation_id: 'gen-drift', + expected_artifact_set_id: 'art-expected-drift', + observed_artifact_identity_json: JSON.stringify({ kind: 'exact', identity: 'sha256:serving', observedAt: 42 }), + }); + + let projection = projectApplication('app-drift-item', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.targets[0]?.runtime.status).toBe('runtime_artifact_drift'); + expect(projection.drift).toHaveLength(1); + expect(projection.drift[0]).toEqual({ + class: 'runtime', + expected: { kind: 'artifact_set', id: 'art-expected-drift', qualification: 'exact', evidenceVersion: 1 }, + observed: { kind: 'runtime_artifact', identity: 'sha256:serving', observedAt: 42 }, + freshnessAt: 42, + owner: 'observed_artifact_identity', + reason: 'the running workload reports an artifact identity other than the expected artifact set', + configuredPolicy: null, + affectedTargets: [{ nodeId: 1, stackName: 'drift-item-web' }], + action: 'none', + }); + + // Equal comparable identities are not drift: the item disappears and the + // chain continues to health instead of parking in verification pending. + store.upsertTarget({ + ...emptyTargetRow('app-drift-item', 1, 1), + desired_generation_id: 'gen-drift', + applied_generation_id: 'gen-drift', + deployed_generation_id: 'gen-drift', + expected_artifact_set_id: 'art-expected-drift', + observed_artifact_identity_json: JSON.stringify({ kind: 'qualified', identity: 'sha256:wanted', observedAt: 43 }), + }); + projection = projectApplication('app-drift-item', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.drift).toHaveLength(0); + + // An observation that is not comparable never becomes a confirmed item. + store.upsertTarget({ + ...emptyTargetRow('app-drift-item', 1, 1), + desired_generation_id: 'gen-drift', + applied_generation_id: 'gen-drift', + deployed_generation_id: 'gen-drift', + expected_artifact_set_id: 'art-expected-drift', + observed_artifact_identity_json: JSON.stringify({ kind: 'stale', identity: 'sha256:serving', observedAt: 44 }), + }); + projection = projectApplication('app-drift-item', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.targets[0]?.runtime.status).toBe('artifact_verification_pending'); + expect(projection.drift).toHaveLength(0); + + // Ordering pin: a stale deployment outranks artifact verification. With + // the desired generation applied but an older one deployed, the deploy + // question comes first, so the mismatch item is emitted while the artifact + // observation describing the workload about to be replaced is not. + store.insertGeneration(gen('gen-b-drift', 'app-drift-item')); + store.upsertTarget({ + ...emptyTargetRow('app-drift-item', 1, 1), + desired_generation_id: 'gen-drift', + applied_generation_id: 'gen-drift', + deployed_generation_id: 'gen-b-drift', + healthy_generation_id: 'gen-b-drift', + expected_artifact_set_id: 'art-expected-drift', + observed_artifact_identity_json: JSON.stringify({ kind: 'exact', identity: 'sha256:serving', observedAt: 45 }), + }); + projection = projectApplication('app-drift-item', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.targets[0]?.runtime.status).toBe('applied_not_deployed'); + expect(projection.drift).toHaveLength(1); + expect(projection.drift[0]).toEqual({ + class: 'runtime', + expected: { kind: 'generation', id: 'gen-drift' }, + observed: { kind: 'generation', id: 'gen-b-drift' }, + // Pointer-to-pointer comparison carries no observation timestamp. + freshnessAt: null, + owner: 'ComposeService', + reason: 'the target is running a different generation than the one it was asked to run', + configuredPolicy: null, + affectedTargets: [{ nodeId: 1, stackName: 'drift-item-web' }], + action: 'deploy', + }); + + // An application-level gate withholds the action without removing the + // fact: a fetch in flight makes availableActions none, so the item must + // say none too rather than contradicting the payload it travels in. + tx.fetchStarted('app-drift-item', env('op-f-drift')); + projection = projectApplication('app-drift-item', false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + expect(projection.availableActions).toEqual(['none']); + expect(projection.drift).toHaveLength(1); + expect(projection.drift[0].action).toBe('none'); + }); +}); + +function env(operationId: string): EventEnvelope { + return { operationId, actor: 'tester', trigger: 'manual', at: 1 }; +} + +function app(id: string, stackName: string): GitOpsApplicationRow { + return { + id, + lifecycle_key: `direct:${stackName}`, + lifecycle_status: 'active', + target_mode: 'direct', + stack_name: stackName, + blueprint_id: null, + configured_repo_url: 'https://github.com/org/repo.git', + repo_identity_json: '{"host":"github.com","pathname":"/org/repo.git"}', + configured_ref: 'main', + compose_paths_json: '["compose.yml"]', + context_dir: null, + sync_env: 0, + env_path: null, + materialization_fingerprint: 'a'.repeat(64), + desired_commit_sha: null, + fetched_commit_sha: null, + candidate_generation_id: null, + accepted_generation_id: null, + candidate_plan_blocked: 0, + review_required: 0, + artifact_set_id: null, + latest_artifact_set_id: null, + intent_revision_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + rollout_authorization_ref: null, + legacy_combined_approval_ref: null, + preflight_fingerprint: null, + latest_operation_id: null, + active_operation_id: null, + active_operation_stage: null, + active_operation_at: null, + active_generation_id: null, + pause_at: null, + pause_reason: null, + partial_json: null, + failure_stage: null, + failure_class: null, + failure_at: null, + retry_at: null, + retry_count: 0, + suspended_at: null, + recovery_ref: null, + recovery_phase: null, + interruption_stage: null, + interruption_at: null, + interruption_operation_id: null, + interruption_generation_id: null, + evidence_fresh_at: null, + evidence_limitations_json: null, + created_at: 1, + updated_at: 1, + }; +} + +/** Seeds application rows directly (the Direct fixture with overrides) rather than driving the transitions that would produce these modes and states. */ +function rawApp(id: string, overrides: Partial): GitOpsApplicationRow { + return { ...app(id, 'raw-fixture-stack'), ...overrides }; +} + +function gen(id: string, applicationId: string): GitOpsGenerationRow { + return { + id, + application_id: applicationId, + commit_sha: 'abc123', + repo_url: 'https://github.com/org/repo.git', + configured_ref: 'main', + repo_identity_json: '{"host":"github.com","pathname":"/org/repo.git"}', + manifest_version: 0, + candidate_dir: `generations/candidate-${id}`, + applied_dir: `generations/applied-${id}-0`, + expected_invocation_json: '{"composeFileOrder":[],"projectName":null,"projectDirectory":null,"envFileOrder":[]}', + materialization_fingerprint: 'a'.repeat(64), + validation_ok: 1, + plan_blocked: 0, + change_plan_fingerprint: null, + operation_id: `op-${id}`, + trigger: 'manual', + actor: 'tester', + previous_generation_id: null, + redacted_limitations_json: '[]', + created_at: 1, + }; +} diff --git a/backend/src/__tests__/gitops-direct-producers.test.ts b/backend/src/__tests__/gitops-direct-producers.test.ts new file mode 100644 index 00000000..4e162a2f --- /dev/null +++ b/backend/src/__tests__/gitops-direct-producers.test.ts @@ -0,0 +1,652 @@ +/** + * End-to-end coverage for the Direct Git producers. + * + * Only the two boundaries the host owns are stubbed: `git.clone` writes a real + * project into the clone directory and `git.log` returns a commit, and the + * compose commands that need a running daemon report an exit code each test + * chooses. Compose commands that only parse files, `config` above all, still + * shell out for real, so the Compose CLI is a genuine prerequisite here even + * though the daemon is not. Everything after that runs for real, so fetch, + * candidate materialization, change-plan classification, the apply, the + * deploy, and the detach all drive the GitOps state model the way they do in + * production. + * + * This exists because the producer wiring is the seam between the operational + * Git path and the revision state, and a mismatch there type-checks and passes + * transition-level tests. + */ +import { EventEmitter } from 'events'; +import fsPromises from 'fs/promises'; +import path from 'path'; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; + +const { mockGitClone, mockGitLog, compose } = vi.hoisted(() => ({ + mockGitClone: vi.fn(), + mockGitLog: vi.fn(), + /** Exit code the next daemon-dependent compose command reports. */ + compose: { exitCode: 1 }, +})); + +vi.mock('isomorphic-git', () => { + const api = { clone: mockGitClone, log: mockGitLog }; + return { default: api, clone: mockGitClone, log: mockGitLog }; +}); +vi.mock('isomorphic-git/http/node', () => ({ default: {} })); + +/** + * Compose verbs that need a running daemon and are issued through `spawn`. + * + * `ps` is deliberately absent: it is issued through `execFile`, which this mock + * does not replace, so listing it would advertise coverage that is not there. + */ +const DAEMON_COMPOSE_VERBS = new Set(['up', 'down', 'pull', 'build', 'start', 'stop', 'restart']); +const COMPOSE_FLAGS_WITH_VALUE = new Set([ + '-f', '--file', '-p', '--project-name', '--env-file', '--project-directory', +]); + +/** + * The verb in a `docker compose …` argv, skipping global flags and their + * values so a stack or file named after a verb cannot be mistaken for one. + */ +function composeVerbOf(args: readonly string[]): string | null { + if (args[0] !== 'compose') return null; + for (let i = 1; i < args.length; i++) { + const token = args[i]; + if (COMPOSE_FLAGS_WITH_VALUE.has(token)) { + i++; + continue; + } + if (token.startsWith('-')) continue; + return token; + } + return null; +} + +function fakeComposeChild(): EventEmitter { + const child = new EventEmitter() as EventEmitter & { + stdout: EventEmitter; + stderr: EventEmitter; + kill: () => boolean; + }; + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + child.kill = () => true; + // The caller attaches its listeners synchronously after spawn returns, so the + // exit cannot be announced until the current turn finishes. + setImmediate(() => child.emit('close', compose.exitCode)); + return child; +} + +/** + * Only the compose commands that need a daemon are answered here; `config` and + * everything else still runs for real. + * + * That split is the whole point. A deploy failing is otherwise a fact about the + * host rather than about the adapter: a workstation with the CLI but no daemon + * parses compose files happily and fails `up`, while a CI runner succeeds at + * both, so any test that reads "the deploy failed" from the environment says + * something different in the two places. + */ +vi.mock('child_process', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + spawn: (command: string, args: readonly string[], options?: unknown) => { + if (command === 'docker' && DAEMON_COMPOSE_VERBS.has(composeVerbOf(args) ?? '')) { + return fakeComposeChild(); + } + return (actual.spawn as unknown as (...a: unknown[]) => unknown)(command, args, options); + }, + }; +}); + +// Rollback capture talks to Docker, which is not available here. Stubbing it +// keeps the apply on its success path so the GitOps wiring is what the test +// actually exercises. +vi.mock('../services/StackUpdateRecoveryService', () => ({ + StackUpdateRecoveryService: { + getInstance: () => ({ + captureCandidate: vi.fn(async () => ({ id: 'rec-producers-1' })), + abandon: vi.fn(async () => true), + markAcquired: vi.fn(() => true), + handoff: vi.fn(() => true), + markReconciling: vi.fn(() => true), + markImmediateVerified: vi.fn(() => true), + get: vi.fn(() => ({ id: 'rec-producers-1', is_current: 1 })), + linkGateOrRetain: vi.fn(), + compensateWithCandidate: vi.fn(async () => true), + start: vi.fn(), + }), + }, +})); + +const REPO = 'https://github.com/example/project.git'; +const COMPOSE = 'services:\n web:\n image: nginx:1.27\n'; +const COMPOSE_V2 = 'services:\n web:\n image: nginx:1.28\n'; +const COMPOSE_PROD = 'services:\n web:\n restart: always\n'; + +let tmpDir: string; +let GitSourceService: typeof import('../services/GitSourceService').GitSourceService; +let GitOpsStore: typeof import('../services/gitops/store').GitOpsStore; +let GitOpsTransitions: typeof import('../services/gitops/transitions').GitOpsTransitions; +let projectApplication: typeof import('../services/gitops/derive').projectApplication; + +/** Make the next clone produce a project containing this compose content. */ +function stageRepo(content: string, sha: string, extraFiles: Record = {}): void { + mockGitClone.mockImplementation(async ({ dir }: { dir: string }) => { + await fsPromises.mkdir(dir, { recursive: true }); + await fsPromises.writeFile(path.join(dir, 'compose.yaml'), content, 'utf8'); + for (const [name, body] of Object.entries(extraFiles)) { + await fsPromises.writeFile(path.join(dir, name), body, 'utf8'); + } + }); + mockGitLog.mockResolvedValue([{ oid: sha }]); +} + +function projectOf(applicationId: string) { + const projection = projectApplication(applicationId, true); + if (projection.targetMode === 'not_applicable') throw new Error('expected an application'); + return projection; +} + +describe('Direct Git producers drive the revision state', () => { + beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ GitSourceService } = await import('../services/GitSourceService')); + ({ GitOpsStore } = await import('../services/gitops/store')); + ({ GitOpsTransitions } = await import('../services/gitops/transitions')); + ({ projectApplication } = await import('../services/gitops/derive')); + GitOpsStore.resetForTests(); + GitOpsTransitions.resetForTests(); + }); + + afterAll(() => { + cleanupTestDb(tmpDir); + }); + + beforeEach(() => { + mockGitClone.mockReset(); + mockGitLog.mockReset(); + compose.exitCode = 1; + }); + + // Prototype spies a test installs are restored here rather than in a per-test + // finally, so a failing test cannot leak one into the next. Only spies are + // touched, so the hoisted git mocks keep the reset above. + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('creates, fetches, applies, and detaches a Git stack through the state model', async () => { + const svc = GitSourceService.getInstance(); + const store = GitOpsStore.getInstance(); + const stackName = 'producers-web'; + + // ── create ──────────────────────────────────────────────────────────── + stageRepo(COMPOSE, 'aaaaaaa1'); + await svc.createStackFromGit({ + stackName, + repoUrl: REPO, + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + token: null, + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + + const app = store.getLiveDirectApplication(stackName); + expect(app).toBeTruthy(); + if (!app) throw new Error('expected an application'); + expect(app.lifecycle_status).toBe('active'); + expect(app.accepted_generation_id).not.toBeNull(); + expect(app.desired_commit_sha).toBe('aaaaaaa1'); + // The create records a secret-free identity, never the operational URL. + expect(app.configured_repo_url).toBe('https://github.com/example/project.git'); + // The success boundary cleared its own checkpoint. + expect(store.getCreateCheckpoint(app.id)).toBeUndefined(); + + const afterCreate = projectOf(app.id); + expect(afterCreate.facets.source.status).toBe('application_generation_accepted'); + expect(afterCreate.targets[0]?.runtime.status).toBe('applied_not_deployed'); + + // ── fetch a newer commit ────────────────────────────────────────────── + stageRepo(COMPOSE_V2, 'bbbbbbb2'); + await svc.pull(stackName, { actor: 'tester' }); + + const afterPull = store.getApplication(app.id)!; + expect(afterPull.desired_commit_sha).toBe('bbbbbbb2'); + expect(afterPull.fetched_commit_sha).toBe('bbbbbbb2'); + // A fetch advances the resolved commit and offers a candidate, but the + // accepted generation does not move until the apply. + expect(afterPull.accepted_generation_id).toBe(app.accepted_generation_id); + expect(afterPull.candidate_generation_id).not.toBeNull(); + expect(afterPull.candidate_generation_id).not.toBe(app.accepted_generation_id); + + const candidateId = afterPull.candidate_generation_id!; + const candidate = store.getGeneration(candidateId)!; + expect(candidate.commit_sha).toBe('bbbbbbb2'); + expect(candidate.application_id).toBe(app.id); + expect(candidate.materialization_fingerprint).toBe(afterPull.materialization_fingerprint); + expect(store.getTarget(app.id, 1)?.candidate_generation_id).toBe(candidateId); + expect(projectOf(app.id).availableActions).toContain('apply'); + + // ── apply ───────────────────────────────────────────────────────────── + await svc.apply(stackName, 'bbbbbbb2', { requirePlanFingerprint: false, deploy: false, actor: 'tester' }); + + const afterApply = store.getApplication(app.id)!; + expect(afterApply.accepted_generation_id).toBe(candidateId); + expect(afterApply.candidate_generation_id).toBeNull(); + expect(afterApply.active_operation_stage).toBeNull(); + expect(afterApply.source_acceptance_ref).not.toBeNull(); + + const target = store.getTarget(app.id, 1)!; + expect(target.desired_generation_id).toBe(candidateId); + expect(target.applied_generation_id).toBe(candidateId); + expect(target.candidate_generation_id).toBeNull(); + + // The acceptance is provable against the exact generation it authorized. + expect(store.resolveApprovalRef(afterApply.source_acceptance_ref!, { + kind: 'source_acceptance', + applicationId: app.id, + generationId: candidateId, + })).toBeTruthy(); + expect(store.resolveApprovalRef(afterApply.source_acceptance_ref!, { + kind: 'source_acceptance', + applicationId: app.id, + generationId: app.accepted_generation_id!, + })).toBeNull(); + + // ── detach ──────────────────────────────────────────────────────────── + await svc.detach(stackName); + + expect(store.getLiveDirectApplication(stackName)).toBeUndefined(); + const tombstoned = store.getApplication(app.id)!; + expect(tombstoned.lifecycle_status).toBe('detached'); + // Configured identity and resolved commit survive as frozen facts. + expect(tombstoned.configured_repo_url).toBe('https://github.com/example/project.git'); + expect(tombstoned.desired_commit_sha).toBe('bbbbbbb2'); + expect(store.getTarget(app.id, 1)?.target_status).toBe('tombstoned'); + expect(projectOf(app.id).facets.source.status).toBe('not_live'); + }); + + it('binds the deployed generation through the Compose adapter', async () => { + const { ComposeService } = await import('../services/ComposeService'); + const { default: DockerController } = await import('../services/DockerController'); + const svc = GitSourceService.getInstance(); + const store = GitOpsStore.getInstance(); + const stackName = 'producers-deploy'; + + stageRepo(COMPOSE, '11111111'); + await svc.createStackFromGit({ + stackName, + repoUrl: REPO, + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + token: null, + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const app = store.getLiveDirectApplication(stackName)!; + const applied = store.getTarget(app.id, 1)!.applied_generation_id; + expect(applied).not.toBeNull(); + expect(store.getTarget(app.id, 1)?.deployed_generation_id).toBeNull(); + + // The post-deploy probe asks the daemon what came up. It is not what this + // test is about, and `getInstance` hands back a fresh controller each call, + // so the stubs go on the prototype. `afterEach` restores them, so a failing + // test cannot leak one into the next. + const composeSvc = ComposeService.getInstance(1); + vi.spyOn(DockerController.prototype, 'getLegacyOrphanContainersByStack') + .mockResolvedValue([]); + vi.spyOn(DockerController.prototype, 'getDocker') + .mockReturnValue({ listContainers: async () => [] } as unknown as ReturnType< + typeof DockerController.prototype.getDocker + >); + + // ── the compose command fails ────────────────────────────────────── + compose.exitCode = 1; + await expect(composeSvc.deployStack(stackName)).rejects.toThrow(); + + const failed = store.getTarget(app.id, 1)!; + expect(failed.deployed_generation_id).toBeNull(); + expect(failed.failure_stage).toBe('deploy'); + // Classified conservatively: once the compose command has been handed + // off, we cannot prove the workload was untouched, and claiming it was + // intact would be the more dangerous error. + expect(failed.failure_class).toBe('post_mutation'); + expect(projectOf(app.id).targets[0]?.runtime.status).toBe('failed_after_mutation'); + expect(failed.applied_generation_id).toBe(applied); + + // ── the compose command succeeds ─────────────────────────────────── + compose.exitCode = 0; + const result = await composeSvc.deployStack(stackName); + + // The adapter reports the generation it bound, which is what lets the + // caller start health against that exact generation rather than against + // whatever is applied by the time health runs. + expect(result.deployedGenerationId).toBe(applied); + + const bound = store.getTarget(app.id, 1)!; + expect(bound.deployed_generation_id).toBe(applied); + expect(bound.applied_generation_id).toBe(applied); + // A bound deploy clears the earlier failure: the target is no longer in + // the state the operator was asked to act on. + expect(bound.failure_stage).toBeNull(); + expect(bound.failure_class).toBeNull(); + expect(projectOf(app.id).targets[0]?.runtime.status).not.toBe('failed_after_mutation'); + }); + + it('reports the deployed generation from an update so health can bind to it', async () => { + const { ComposeService } = await import('../services/ComposeService'); + const { StackUpdateOrchestrator } = await import('../services/StackUpdateOrchestrator'); + const svc = GitSourceService.getInstance(); + const store = GitOpsStore.getInstance(); + const stackName = 'producers-update'; + + stageRepo(COMPOSE, '22222222'); + await svc.createStackFromGit({ + stackName, + repoUrl: REPO, + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + token: null, + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const app = store.getLiveDirectApplication(stackName)!; + const applied = store.getTarget(app.id, 1)!.applied_generation_id; + + // The image pull fails, so the update dies during preparation, before the + // recreate Compose is handed anything. The deploy operation is opened only + // at that recreate, so nothing is recorded: an update that never touched + // the workload must not leave a deploy failure behind for the deriver to + // report. + compose.exitCode = 1; + await expect(ComposeService.getInstance(1).updateStack(stackName)).rejects.toThrow(); + const target = store.getTarget(app.id, 1)!; + expect(target.deployed_generation_id).toBeNull(); + expect(target.failure_stage).toBeNull(); + expect(target.active_operation_stage).toBeNull(); + expect(target.applied_generation_id).toBe(applied); + expect(projectOf(app.id).targets[0]?.runtime.status).toBe('applied_not_deployed'); + + // The same holds through the orchestrator, which is what the update callers + // actually use and which carries the binding on to beginStack. + await expect(StackUpdateOrchestrator.getInstance().execute( + { nodeId: 1, stackName, target: { scope: 'stack' }, trigger: 'manual', actor: 'tester' }, + { atomic: false, terminalWs: null }, + )).rejects.toThrow(); + expect(store.getTarget(app.id, 1)?.failure_stage).toBeNull(); + }); + + it('records a failed fetch without moving any pointer', async () => { + const svc = GitSourceService.getInstance(); + const store = GitOpsStore.getInstance(); + const stackName = 'producers-fail'; + + stageRepo(COMPOSE, 'ccccccc3'); + await svc.createStackFromGit({ + stackName, + repoUrl: REPO, + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + token: null, + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const app = store.getLiveDirectApplication(stackName)!; + const acceptedBefore = app.accepted_generation_id; + + mockGitClone.mockRejectedValue(new Error('could not resolve host')); + await expect(svc.pull(stackName, { actor: 'tester' })).rejects.toThrow(); + + const afterFailure = store.getApplication(app.id)!; + expect(afterFailure.failure_stage).toBe('fetch'); + expect(afterFailure.active_operation_stage).toBeNull(); + expect(afterFailure.accepted_generation_id).toBe(acceptedBefore); + expect(afterFailure.candidate_generation_id).toBeNull(); + + const projection = projectOf(app.id); + expect(projection.facets.source.status).toBe('source_failed'); + expect(projection.availableActions).toContain('fetch'); + + // A later successful fetch clears the failure. + stageRepo(COMPOSE_V2, 'ddddddd4'); + await svc.pull(stackName, { actor: 'tester' }); + expect(store.getApplication(app.id)?.failure_stage).toBeNull(); + }); + + it('brings a newly linked stack into the model and invalidates its candidate on a material edit', async () => { + const svc = GitSourceService.getInstance(); + const store = GitOpsStore.getInstance(); + const stackName = 'producers-link'; + + const composeDir = process.env.COMPOSE_DIR!; + await fsPromises.mkdir(path.join(composeDir, stackName), { recursive: true }); + await fsPromises.writeFile(path.join(composeDir, stackName, 'compose.yaml'), COMPOSE, 'utf8'); + + stageRepo(COMPOSE, '33333333'); + await svc.upsert({ + stackName, + repoUrl: REPO, + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + token: null, + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + + const app = store.getLiveDirectApplication(stackName); + expect(app).toBeTruthy(); + if (!app) throw new Error('expected an application'); + // Linked, not fetched: nothing is desired or accepted until a pull runs. + expect(app.lifecycle_status).toBe('active'); + expect(app.desired_commit_sha).toBeNull(); + expect(app.accepted_generation_id).toBeNull(); + let projection = projectOf(app.id); + expect(projection.facets.source.status).toBe('never_reconciled'); + expect(projection.availableActions).toContain('fetch'); + + // A pull produces a candidate against the current configuration. + stageRepo(COMPOSE_V2, '44444444'); + await svc.pull(stackName, { actor: 'tester' }); + const candidateId = store.getApplication(app.id)!.candidate_generation_id; + expect(candidateId).not.toBeNull(); + + // A credential-only edit changes nothing material, so the candidate stands. + stageRepo(COMPOSE_V2, '44444444'); + await svc.upsert({ + stackName, + repoUrl: REPO, + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + token: null, + autoApplyOnWebhook: true, + autoDeployOnApply: false, + }); + expect(store.getApplication(app.id)?.candidate_generation_id).toBe(candidateId); + + // Changing the compose file set does invalidate it: that candidate was + // built from a different set and can no longer be applied. + stageRepo(COMPOSE_V2, '44444444', { 'compose.prod.yaml': COMPOSE_PROD }); + await svc.upsert({ + stackName, + repoUrl: REPO, + branch: 'main', + composePaths: ['compose.yaml', 'compose.prod.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + token: null, + autoApplyOnWebhook: true, + autoDeployOnApply: false, + }); + const afterEdit = store.getApplication(app.id)!; + expect(afterEdit.candidate_generation_id).toBeNull(); + expect(afterEdit.desired_commit_sha).toBeNull(); + expect(store.getTarget(app.id, 1)?.candidate_generation_id).toBeNull(); + projection = projectOf(app.id); + expect(projection.availableActions).toContain('fetch'); + expect(projection.availableActions).not.toContain('apply'); + }); + + it('retires the application when the stack itself is deleted', async () => { + const { DeployedStackDeletionService } = await import('../services/DeployedStackDeletionService'); + const svc = GitSourceService.getInstance(); + const store = GitOpsStore.getInstance(); + const stackName = 'producers-delete'; + + stageRepo(COMPOSE, '55555555'); + await svc.createStackFromGit({ + stackName, + repoUrl: REPO, + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + token: null, + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const app = store.getLiveDirectApplication(stackName)!; + + await DeployedStackDeletionService.getInstance().deleteDeployedStack({ + nodeId: 1, + stackName, + pruneVolumes: false, + actor: 'tester', + }); + + // A deleted stack must not leave a live application behind: it would keep + // claiming the name and block re-creating it. + expect(store.getLiveDirectApplication(stackName)).toBeUndefined(); + expect(store.getApplication(app.id)?.lifecycle_status).toBe('deleted'); + expect(store.getTarget(app.id, 1)?.target_status).toBe('tombstoned'); + }); + + it('closes the operation when a terminal transition is rejected', async () => { + const svc = GitSourceService.getInstance(); + const store = GitOpsStore.getInstance(); + const stackName = 'producers-reject'; + + stageRepo(COMPOSE, '66666666'); + await svc.createStackFromGit({ + stackName, + repoUrl: REPO, + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + token: null, + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const app = store.getLiveDirectApplication(stackName)!; + + // Reject the transition that closes a fetch. Recording must not fail the + // pull, but it must not leave the operation open either: a fetch that never + // terminates blocks every later pull from being recorded at all. + const tx = GitOpsTransitions.getInstance(); + const realFetched = tx.fetched.bind(tx); + tx.fetched = () => { throw new Error('rejected for test'); }; + try { + stageRepo(COMPOSE_V2, '77777777'); + await svc.pull(stackName, { actor: 'tester' }); + } finally { + tx.fetched = realFetched; + } + + const afterReject = store.getApplication(app.id)!; + expect(afterReject.active_operation_stage).toBeNull(); + expect(afterReject.failure_stage).toBe('fetch'); + // The projection reports an error the operator can act on, not a spinner. + const projection = projectOf(app.id); + expect(projection.facets.source.status).toBe('source_failed'); + expect(projection.availableActions).toContain('fetch'); + + // And the next pull records normally, rather than being locked out. + stageRepo(COMPOSE_V2, '88888888'); + await svc.pull(stackName, { actor: 'tester' }); + const recovered = store.getApplication(app.id)!; + expect(recovered.fetched_commit_sha).toBe('88888888'); + expect(recovered.failure_stage).toBeNull(); + expect(recovered.active_operation_stage).toBeNull(); + }); + + it('leaves a stack with no GitOps application untouched', async () => { + const svc = GitSourceService.getInstance(); + const store = GitOpsStore.getInstance(); + const stackName = 'producers-legacy'; + + // A Git stack exactly as an install carries it across an upgrade: the + // source row was written before this model existed, so there is no + // application and nothing has migrated it yet. Seeded directly, because + // linking through the service now creates one. + const composeDir = process.env.COMPOSE_DIR!; + await fsPromises.mkdir(path.join(composeDir, stackName), { recursive: true }); + await fsPromises.writeFile(path.join(composeDir, stackName, 'compose.yaml'), COMPOSE, 'utf8'); + (await import('../services/DatabaseService')).DatabaseService.getInstance().upsertGitSource({ + stack_name: stackName, + repo_url: REPO, + branch: 'main', + compose_path: 'compose.yaml', + compose_paths: ['compose.yaml'], + context_dir: null, + sync_env: false, + env_path: null, + auth_type: 'none', + encrypted_token: null, + auto_apply_on_webhook: false, + auto_deploy_on_apply: false, + last_applied_commit_sha: 'eeeeeee5', + last_applied_content_hash: null, + pending_commit_sha: null, + pending_compose_content: null, + pending_env_content: null, + pending_fetched_at: null, + last_debounce_at: null, + }); + expect(store.getLiveDirectApplication(stackName)).toBeUndefined(); + + stageRepo(COMPOSE_V2, 'fffffff6'); + await svc.pull(stackName, { actor: 'tester' }); + + // The pull succeeded operationally and wrote no GitOps rows. + expect(store.getLiveDirectApplication(stackName)).toBeUndefined(); + const historyRows = (await import('../services/DatabaseService')).DatabaseService + .getInstance().getDb() + .prepare('SELECT COUNT(*) AS n FROM gitops_history WHERE stack_name = ?') + .get(stackName) as { n: number }; + expect(historyRows.n).toBe(0); + }); +}); diff --git a/backend/src/__tests__/gitops-history-read.test.ts b/backend/src/__tests__/gitops-history-read.test.ts new file mode 100644 index 00000000..bcf56f36 --- /dev/null +++ b/backend/src/__tests__/gitops-history-read.test.ts @@ -0,0 +1,550 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { DatabaseService } from '../services/DatabaseService'; +import { GitOpsStore } from '../services/gitops/store'; +import { GitOpsTransitions } from '../services/gitops/transitions'; +import { + HISTORY_DEFAULT_LIMIT, + HISTORY_MAX_LIMIT, + HISTORY_SCAN_CAP, + decodeHistoryCursor, + encodeHistoryCursor, + insertHistory, + queryHistoryRows, + toHistoryItem, +} from '../services/gitops/history'; +import { parseHistoryFilters, parseLimit } from '../helpers/gitopsHistoryPage'; +import { + classifyHistoryRow, + classifySourceRow, + normalizeStackResourcePresent, +} from '../services/gitops/readAuth'; +import type { GitOpsApplicationRow, GitOpsHistoryRow } from '../services/gitops/types'; + +describe('gitops history read layer', () => { + let tmpDir: string; + + beforeAll(async () => { + tmpDir = await setupTestDb(); + GitOpsStore.resetForTests(); + GitOpsTransitions.resetForTests(); + seedHistory(); + }); + + afterAll(() => { + cleanupTestDb(tmpDir); + }); + + describe('cursor', () => { + const SAMPLE_ID = '0b3f4a1c-8d2e-4c7b-9f10-5a6b7c8d9e0f'; + + it('round-trips a created_at and id pair', () => { + const encoded = encodeHistoryCursor({ createdAt: 1700, id: SAMPLE_ID }); + expect(decodeHistoryCursor(encoded)).toEqual({ createdAt: 1700, id: SAMPLE_ID }); + }); + + it('rejects malformed cursors rather than guessing a position', () => { + expect(decodeHistoryCursor('')).toBeNull(); + expect(decodeHistoryCursor('nodot')).toBeNull(); + expect(decodeHistoryCursor(`.${SAMPLE_ID}`)).toBeNull(); + expect(decodeHistoryCursor('123.')).toBeNull(); + expect(decodeHistoryCursor(`notanumber.${SAMPLE_ID}`)).toBeNull(); + expect(decodeHistoryCursor(`-5.${SAMPLE_ID}`)).toBeNull(); + }); + + it('rejects an id that is not a real row id', () => { + // A truncated cursor is the likely case, and an unvalidated id would not + // fail: it would shift the page boundary and quietly drop or repeat rows. + expect(decodeHistoryCursor(`1700.${SAMPLE_ID.slice(0, 12)}`)).toBeNull(); + expect(decodeHistoryCursor('1700.not-a-uuid')).toBeNull(); + expect(decodeHistoryCursor(`1700.${SAMPLE_ID.toUpperCase()}`)).toBeNull(); + }); + + it('accepts the cursor it emits for a stored row', () => { + const row = query({ commitSha: 'sha-b' })[0] as GitOpsHistoryRow; + const encoded = encodeHistoryCursor({ createdAt: row.created_at, id: row.id }); + expect(decodeHistoryCursor(encoded)).toEqual({ createdAt: row.created_at, id: row.id }); + }); + }); + + describe('queryHistoryRows', () => { + it('returns newest first', () => { + const rows = query({ stackName: 'history-web' }); + expect(rows.map(r => r.commit_sha)).toEqual(['sha-c', 'sha-b', 'sha-a', null]); + }); + + // Every filter is asserted to reach the right column. A wrong column name + // is not a subtly wrong page, it is a "no such column" throw at query + // time, so an untested filter is a 500 waiting behind any link carrying it. + it.each([ + ['applicationId', { applicationId: 'app-history' }, 'sha-b'], + ['stackName', { stackName: 'history-web' }, 'sha-b'], + ['repoIdentity', { repoIdentity: 'https://github.com/org/repo.git' }, 'sha-b'], + ['configuredRef', { configuredRef: 'main' }, 'sha-b'], + ['commitSha', { commitSha: 'sha-b' }, 'sha-b'], + ['nodeId', { nodeId: 7 }, 'sha-b'], + ['trigger', { trigger: 'webhook' }, 'sha-b'], + ['actor', { actor: 'operator-2' }, 'sha-b'], + ['outcome', { outcome: 'failed' as const }, 'sha-c'], + ['rolloutCandidateId', { rolloutCandidateId: 'cand-1' }, 'sha-c'], + ])('routes the %s filter to its own column', (_name, filters, expectedSha) => { + const rows = query(filters); + expect(rows.length).toBeGreaterThan(0); + expect(rows.map(r => r.commit_sha)).toContain(expectedSha); + }); + + it('accepts the identity filters that match nothing in this fixture', () => { + // Exercises the remaining column names so a typo still throws here. + expect(query({ generationId: 'gen-absent' })).toHaveLength(0); + expect(query({ artifactSetId: 'artifact-absent' })).toHaveLength(0); + expect(query({ blueprintId: 4242 })).toHaveLength(0); + expect(query({ rolloutGenerationId: 'rollout-gen-absent' })).toHaveLength(0); + }); + + it('narrows node rows while keeping application-level rows', () => { + // A proxied hub view filters by its own node id, but activation and + // similar stages carry no node. Dropping them under `node_id = ?` + // would make the history read as if the application never came into + // being. + const byNode = query({ nodeId: 7 }); + expect(byNode.map(r => r.node_id)).toEqual([7, null]); + expect(byNode[0]?.commit_sha).toBe('sha-b'); + expect(query({ stackName: 'no-such-stack' })).toHaveLength(0); + }); + + it('never matches a rollout candidate id as a rollout generation id', () => { + // The candidate is a proposal; a generation is a dispatch that ran. + // Answering the generation filter from the candidate column would report + // a rollout that never happened. + expect(query({ rolloutCandidateId: 'cand-1' })).toHaveLength(1); + expect(query({ rolloutGenerationId: 'cand-1' })).toHaveLength(0); + }); + + it('pages past the cursor without repeating a row', () => { + const first = query({ stackName: 'history-web' }, null, 2); + expect(first).toHaveLength(2); + const last = first[1] as GitOpsHistoryRow; + const second = query({ stackName: 'history-web' }, { createdAt: last.created_at, id: last.id }, 2); + expect(second).toHaveLength(2); + expect(second[0]?.id).not.toBe(last.id); + expect(second[0]?.commit_sha).toBe('sha-a'); + }); + + it('separates rows sharing one millisecond by id', () => { + const sameMs = query({ commitSha: 'sha-tie' }); + expect(sameMs).toHaveLength(2); + const [newer, older] = sameMs as [GitOpsHistoryRow, GitOpsHistoryRow]; + expect(newer.id > older.id).toBe(true); + const after = query({ commitSha: 'sha-tie' }, { createdAt: newer.created_at, id: newer.id }, 10); + expect(after.map(r => r.id)).toEqual([older.id]); + }); + }); + + describe('toHistoryItem', () => { + it('exposes the producer delta as before and after', () => { + const row = query({ commitSha: 'sha-b' })[0] as GitOpsHistoryRow; + const item = toHistoryItem(row); + expect(item.before).toEqual({ desiredCommitSha: null }); + expect(item.after).toEqual({ desiredCommitSha: 'sha-b' }); + expect(item.limitations).toEqual([]); + expect(item.stage).toBe('fetched'); + }); + + it('keeps identity and stage when the recorded delta cannot be read', () => { + const row = query({ commitSha: 'sha-a' })[0] as GitOpsHistoryRow; + const corrupt: GitOpsHistoryRow = { ...row, after_json: '{not json' }; + const item = toHistoryItem(corrupt); + expect(item.after).toBeNull(); + expect(item.stage).toBe(row.stage); + expect(item.applicationId).toBe(row.application_id); + expect(item.limitations).toEqual([ + { + code: 'history_json_invalid', + message: 'Recorded change detail for this entry could not be read.', + evidence: { before: false, after: true }, + }, + ]); + }); + + it('treats a non-object payload as unreadable', () => { + const row = query({ commitSha: 'sha-a' })[0] as GitOpsHistoryRow; + const item = toHistoryItem({ ...row, before_json: '"a string"' }); + expect(item.before).toBeNull(); + expect(item.limitations[0]?.code).toBe('history_json_invalid'); + }); + + it('maps every identity column to its own field', () => { + // The four approval refs are same-typed and same-shaped, so a swap + // between them is invisible without asserting each one individually. + const row = query({ commitSha: 'sha-b' })[0] as GitOpsHistoryRow; + const populated: GitOpsHistoryRow = { + ...row, + generation_id: 'gen-x', + artifact_set_id: 'art-x', + intent_revision_id: 'intent-x', + rollout_candidate_id: 'cand-x', + rollout_generation_id: 'rgen-x', + source_acceptance_ref: 'ref-source', + placement_approval_ref: 'ref-placement', + rollout_authorization_ref: 'ref-rollout', + legacy_combined_approval_ref: 'ref-legacy', + blueprint_id: 11, + }; + const item = toHistoryItem(populated); + expect(item).toMatchObject({ + id: row.id, + createdAt: row.created_at, + applicationId: row.application_id, + targetMode: row.target_mode, + stackName: row.stack_name, + repoIdentity: 'https://github.com/org/repo.git', + configuredRef: 'main', + blueprintId: 11, + nodeId: row.node_id, + commitSha: row.commit_sha, + generationId: 'gen-x', + artifactSetId: 'art-x', + intentRevisionId: 'intent-x', + rolloutCandidateId: 'cand-x', + rolloutGenerationId: 'rgen-x', + operationId: row.operation_id, + stage: row.stage, + outcome: row.outcome, + trigger: row.trigger, + actor: row.actor, + approvals: { + sourceAcceptanceRef: 'ref-source', + placementApprovalRef: 'ref-placement', + rolloutAuthorizationRef: 'ref-rollout', + legacyCombinedApprovalRef: 'ref-legacy', + }, + }); + }); + }); + + describe('stackResourcePresent validation', () => { + it('accepts only a real boolean true', () => { + expect(normalizeStackResourcePresent(true)).toBe(true); + expect(normalizeStackResourcePresent(false)).toBe(false); + expect(normalizeStackResourcePresent('true')).toBe(false); + expect(normalizeStackResourcePresent(1)).toBe(false); + expect(normalizeStackResourcePresent(null)).toBe(false); + expect(normalizeStackResourcePresent(undefined)).toBe(false); + }); + }); + + describe('source-row classifier', () => { + const revision = (lifecycleStatus: unknown): Record => ({ + schemaVersion: 1, + targetMode: 'direct', + lifecycleStatus, + }); + + it('authorizes a live present stack by stack read', () => { + expect(classifySourceRow({ + stackName: 'web', + gitopsRevision: revision('active'), + stackResourcePresent: true, + })).toEqual({ kind: 'stack_read', stackName: 'web' }); + }); + + it('authorizes a detached stack whose resource is present', () => { + expect(classifySourceRow({ + stackName: 'web', + gitopsRevision: revision('detached'), + stackResourcePresent: true, + })).toEqual({ kind: 'stack_read', stackName: 'web' }); + }); + + it('falls back to Admin for every unprovable row', () => { + const admin = { kind: 'admin' }; + expect(classifySourceRow({ stackName: '', gitopsRevision: revision('active'), stackResourcePresent: true })).toEqual(admin); + expect(classifySourceRow({ stackName: null, gitopsRevision: revision('active'), stackResourcePresent: true })).toEqual(admin); + expect(classifySourceRow({ stackName: 'web', gitopsRevision: null, stackResourcePresent: true })).toEqual(admin); + expect(classifySourceRow({ stackName: 'web', gitopsRevision: 'nope', stackResourcePresent: true })).toEqual(admin); + expect(classifySourceRow({ stackName: 'web', gitopsRevision: revision(undefined), stackResourcePresent: true })).toEqual(admin); + expect(classifySourceRow({ stackName: 'web', gitopsRevision: revision('deleted'), stackResourcePresent: true })).toEqual(admin); + expect(classifySourceRow({ stackName: 'web', gitopsRevision: revision('creating'), stackResourcePresent: true })).toEqual(admin); + expect(classifySourceRow({ stackName: 'web', gitopsRevision: revision('active'), stackResourcePresent: false })).toEqual(admin); + expect(classifySourceRow({ stackName: 'web', gitopsRevision: revision('active'), stackResourcePresent: 'yes' })).toEqual(admin); + }); + + it('sends a stack with no GitOps application to Admin', () => { + // The not_applicable projection carries no lifecycleStatus at all. + expect(classifySourceRow({ + stackName: 'web', + gitopsRevision: { schemaVersion: 1, targetMode: 'not_applicable', applicationId: null }, + stackResourcePresent: true, + })).toEqual({ kind: 'admin' }); + }); + }); + + describe('history-row classifier', () => { + it('authorizes from the application lifecycle, not the recorded delta', () => { + expect(classifyHistoryRow({ + stackName: 'web', + applicationLifecycleStatus: 'active', + stackResourcePresent: true, + })).toEqual({ kind: 'stack_read', stackName: 'web' }); + }); + + it('falls back to the audit audience for every unprovable row', () => { + // History entries are an audit trail, so an entry nobody can tie to a + // readable stack goes to whoever audits rather than to Admin alone. + const audit = { kind: 'audit' }; + const held = { stackResourcePresent: true }; + expect(classifyHistoryRow({ stackName: null, applicationLifecycleStatus: 'active', ...held })).toEqual(audit); + expect(classifyHistoryRow({ stackName: '', applicationLifecycleStatus: 'active', ...held })).toEqual(audit); + expect(classifyHistoryRow({ stackName: 'web', applicationLifecycleStatus: undefined, ...held })).toEqual(audit); + expect(classifyHistoryRow({ stackName: 'web', applicationLifecycleStatus: 'deleted', ...held })).toEqual(audit); + expect(classifyHistoryRow({ stackName: 'web', applicationLifecycleStatus: 'creating', ...held })).toEqual(audit); + expect(classifyHistoryRow({ + stackName: 'web', + applicationLifecycleStatus: 'active', + stackResourcePresent: false, + })).toEqual(audit); + }); + + it('sends every detached predecessor to the audit audience', () => { + // Detach leaves the files on disk, but a stack grant covers whatever + // occupies the name today, and nothing in these tables can prove the + // detached application still does: a Blueprint successor records the + // name off-row (`deploy_stack_name`) and a plain Compose stack recreated + // at the name leaves no trace at all. An allowance that holds only for + // the successors this classifier happens to see is worse than none, so + // detach joins `deleted` and `creating`. + const detached = { stackName: 'web', applicationLifecycleStatus: 'detached', stackResourcePresent: true }; + expect(classifyHistoryRow({ ...detached })).toEqual({ kind: 'audit' }); + }); + + it('keeps source rows on Admin rather than the audit audience', () => { + // Git configuration is not a record of events, so an auditing mandate + // does not reach it. + expect(classifySourceRow({ + stackName: 'web', + gitopsRevision: { schemaVersion: 1, targetMode: 'direct', lifecycleStatus: 'deleted' }, + stackResourcePresent: true, + })).toEqual({ kind: 'admin' }); + }); + }); + + it('honours the scan cap as the query bound', () => { + // Asserts the cap is actually applied to the read, not merely declared. + expect(query({}, null, HISTORY_SCAN_CAP).length).toBeLessThanOrEqual(HISTORY_SCAN_CAP); + expect(query({}, null, 1)).toHaveLength(1); + }); + + describe('filter and limit parsing', () => { + it('reads every supported filter off the query string', () => { + const parsed = parseHistoryFilters({ + applicationId: 'app-1', + repoIdentity: 'https://github.com/org/repo.git', + configuredRef: 'main', + commitSha: 'sha-1', + generationId: 'gen-1', + artifactSetId: 'art-1', + blueprintId: '9', + rolloutCandidateId: 'cand-1', + rolloutGenerationId: 'rgen-1', + nodeId: '3', + trigger: 'manual', + actor: 'operator', + outcome: 'failed', + }); + if (!parsed.ok) throw new Error(parsed.message); + expect(parsed.filters).toEqual({ + applicationId: 'app-1', + repoIdentity: 'https://github.com/org/repo.git', + configuredRef: 'main', + commitSha: 'sha-1', + generationId: 'gen-1', + artifactSetId: 'art-1', + blueprintId: 9, + rolloutCandidateId: 'cand-1', + rolloutGenerationId: 'rgen-1', + nodeId: 3, + trigger: 'manual', + actor: 'operator', + outcome: 'failed', + }); + }); + + it('never takes stackName from the caller', () => { + const parsed = parseHistoryFilters({ stackName: 'somebody-elses-stack' }); + if (!parsed.ok) throw new Error(parsed.message); + expect(parsed.filters.stackName).toBeUndefined(); + }); + + it('rejects a recognized filter with an unusable value', () => { + expect(parseHistoryFilters({ outcome: 'success' })).toEqual({ + ok: false, + message: expect.stringContaining('outcome'), + }); + expect(parseHistoryFilters({ nodeId: 'abc' })).toEqual({ + ok: false, + message: expect.stringContaining('nodeId'), + }); + expect(parseHistoryFilters({ blueprintId: '1.5' })).toEqual({ + ok: false, + message: expect.stringContaining('blueprintId'), + }); + }); + + it('clamps the page size and falls back for nonsense', () => { + expect(parseLimit('5000')).toBe(HISTORY_MAX_LIMIT); + expect(parseLimit('10')).toBe(10); + expect(parseLimit(undefined)).toBe(HISTORY_DEFAULT_LIMIT); + expect(parseLimit('0')).toBe(HISTORY_DEFAULT_LIMIT); + expect(parseLimit('-1')).toBe(HISTORY_DEFAULT_LIMIT); + expect(parseLimit('abc')).toBe(HISTORY_DEFAULT_LIMIT); + }); + }); +}); + +function query( + filters: Parameters[1], + cursor: Parameters[2] = null, + limit = 50, +): GitOpsHistoryRow[] { + return queryHistoryRows(DatabaseService.getInstance().getDb(), filters, cursor, limit); +} + +function seedHistory(): void { + const db = DatabaseService.getInstance().getDb(); + const base = application(); + // The activation event belongs to the application, not to a node, so it + // carries no node id. This is the row shape every application-level stage + // writes and the one a `node_id = ?` filter silently drops. + insertHistory(db, { + application: base, + nodeId: null, + dedupeTarget: 'app', + operationId: 'op-app-level', + stage: 'application_activated', + outcome: 'committed', + trigger: 'manual', + actor: 'operator-1', + before: { lifecycleStatus: null }, + after: { lifecycleStatus: 'active', targetMode: 'direct' }, + at: 500, + }); + insertHistory(db, { + application: base, + nodeId: 1, + dedupeTarget: 'app', + operationId: 'op-a', + stage: 'fetched', + outcome: 'committed', + trigger: 'manual', + actor: 'operator-1', + before: { desiredCommitSha: null }, + after: { desiredCommitSha: 'sha-a' }, + commitSha: 'sha-a', + at: 1000, + }); + insertHistory(db, { + application: base, + nodeId: 7, + dedupeTarget: 'app', + operationId: 'op-b', + stage: 'fetched', + outcome: 'committed', + trigger: 'webhook', + actor: 'operator-2', + before: { desiredCommitSha: null }, + after: { desiredCommitSha: 'sha-b' }, + commitSha: 'sha-b', + at: 2000, + }); + insertHistory(db, { + application: { ...base, rollout_candidate_id: 'cand-1' }, + nodeId: 1, + dedupeTarget: 'app', + operationId: 'op-c', + stage: 'apply_failed', + outcome: 'failed', + trigger: 'manual', + actor: 'operator-1', + before: {}, + after: { failureClass: 'validation' }, + commitSha: 'sha-c', + rolloutCandidateId: 'cand-1', + at: 3000, + }); + // Two rows inside one millisecond, which a real transaction produces. + for (const operationId of ['op-tie-1', 'op-tie-2']) { + insertHistory(db, { + application: { ...base, stack_name: 'tie-web', lifecycle_key: 'direct:tie-web' }, + nodeId: 1, + dedupeTarget: 'app', + operationId, + stage: 'fetched', + outcome: 'committed', + trigger: 'manual', + actor: 'operator-1', + before: {}, + after: {}, + commitSha: 'sha-tie', + at: 4000, + }); + } +} + +function application(): GitOpsApplicationRow { + return { + id: 'app-history', + lifecycle_key: 'direct:history-web', + lifecycle_status: 'active', + target_mode: 'direct', + stack_name: 'history-web', + blueprint_id: null, + configured_repo_url: 'https://github.com/org/repo.git', + repo_identity_json: '{"host":"github.com","pathname":"/org/repo.git"}', + configured_ref: 'main', + compose_paths_json: '["compose.yml"]', + context_dir: null, + sync_env: 0, + env_path: null, + materialization_fingerprint: 'a'.repeat(64), + desired_commit_sha: null, + fetched_commit_sha: null, + candidate_generation_id: null, + accepted_generation_id: null, + candidate_plan_blocked: 0, + review_required: 0, + artifact_set_id: null, + latest_artifact_set_id: null, + intent_revision_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + rollout_authorization_ref: null, + legacy_combined_approval_ref: null, + preflight_fingerprint: null, + latest_operation_id: null, + active_operation_id: null, + active_operation_stage: null, + active_operation_at: null, + active_generation_id: null, + pause_at: null, + pause_reason: null, + partial_json: null, + failure_stage: null, + failure_class: null, + failure_at: null, + retry_at: null, + retry_count: 0, + suspended_at: null, + recovery_ref: null, + recovery_phase: null, + interruption_stage: null, + interruption_at: null, + interruption_operation_id: null, + interruption_generation_id: null, + evidence_fresh_at: null, + evidence_limitations_json: null, + created_at: 1, + updated_at: 1, + }; +} diff --git a/backend/src/__tests__/gitops-identity-proxy.test.ts b/backend/src/__tests__/gitops-identity-proxy.test.ts new file mode 100644 index 00000000..267ad384 --- /dev/null +++ b/backend/src/__tests__/gitops-identity-proxy.test.ts @@ -0,0 +1,636 @@ +import { describe, expect, it } from 'vitest'; +import { IncomingMessage } from 'http'; +import { Socket } from 'net'; +import zlib from 'zlib'; +import { + IDENTITY_PROXY_MAX_BYTES, + handleIdentityResponse, + isGitOpsHistoryRoute, + isGitOpsIdentityJsonRoute, + prepareIdentityQuery, + rewriteIdentityPayload, + stripConditionalRequestHeaders, + filterIdentityCollection, + filterRemoteIdentityPayload, + type IdentityResponseSink, + type IdentityTerminalKind, +} from '../proxy/gitopsIdentityProxy'; + +describe('gitops identity proxy', () => { + describe('route matching', () => { + it('intercepts the identity GETs', () => { + for (const path of [ + '/git-sources', + '/git-sources/history', + '/stacks/web/git-source', + '/stacks/web/git-source/history', + '/stacks/web/drift', + ]) { + expect(isGitOpsIdentityJsonRoute(path, 'GET')).toBe(true); + } + }); + + it('intercepts the drift re-check, the one mutation that answers with a revision', () => { + // The GET beside it is rewritten, and both return the same projection + // object. Leaving the re-check on the streaming hop would make one object + // carry the hub's node numbering or the remote's depending only on how it + // was asked for. + expect(isGitOpsIdentityJsonRoute('/stacks/web/drift/recheck', 'POST')).toBe(true); + expect(isGitOpsIdentityJsonRoute('/stacks/web/drift/recheck', 'GET')).toBe(false); + expect(isGitOpsIdentityJsonRoute('/stacks/web/drift', 'POST')).toBe(false); + }); + + it('leaves streaming and unrelated routes to the streaming hop', () => { + // Buffering any of these to rewrite identities they do not carry would + // break streaming or cap a legitimately large response. + for (const path of [ + '/stacks/web/logs', + '/containers/abc/logs', + '/stacks/web/files/download', + '/git-sources/browse', + '/stacks/web/git-source/manifest', + '/blueprints', + ]) { + expect(isGitOpsIdentityJsonRoute(path, 'GET')).toBe(false); + } + }); + + it('never intercepts a mutation of a git-source route', () => { + for (const method of ['POST', 'PUT', 'DELETE', 'PATCH']) { + expect(isGitOpsIdentityJsonRoute('/stacks/web/git-source', method)).toBe(false); + expect(isGitOpsIdentityJsonRoute('/git-sources', method)).toBe(false); + } + }); + + it('recognizes only the history pair as history', () => { + expect(isGitOpsHistoryRoute('/git-sources/history')).toBe(true); + expect(isGitOpsHistoryRoute('/stacks/web/git-source/history')).toBe(true); + expect(isGitOpsHistoryRoute('/git-sources')).toBe(false); + expect(isGitOpsHistoryRoute('/stacks/web/git-source')).toBe(false); + }); + }); + + describe('outbound query', () => { + const prep = (qs: string, path: string, hubNodeId: number | undefined) => + prepareIdentityQuery(new URLSearchParams(qs), path, hubNodeId); + + it('always strips a caller-supplied local-target flag', () => { + // Only the hub may tell a remote to filter to its own node. + const result = prep('gitopsLocalTarget=1', '/git-sources', 3); + if (result.kind !== 'forward') throw new Error('expected forward'); + expect(result.search.get('gitopsLocalTarget')).toBeNull(); + }); + + it('strips nodeId on the non-history routes', () => { + const result = prep('nodeId=3', '/git-sources', 3); + if (result.kind !== 'forward') throw new Error('expected forward'); + expect(result.search.get('nodeId')).toBeNull(); + expect(result.search.get('gitopsLocalTarget')).toBeNull(); + }); + + it('translates a matching node into the local-target flag on history', () => { + const result = prep('nodeId=3&limit=10', '/git-sources/history', 3); + if (result.kind !== 'forward') throw new Error('expected forward'); + expect(result.search.get('nodeId')).toBeNull(); + expect(result.search.get('gitopsLocalTarget')).toBe('1'); + expect(result.search.get('limit')).toBe('10'); + }); + + it('refuses history for a node this hop cannot answer for', () => { + // Forwarding would make the remote answer about itself, which reads as an + // answer to a question nobody asked. + expect(prep('nodeId=7', '/git-sources/history', 3).kind).toBe('refuse'); + expect(prep('nodeId=7', '/stacks/web/git-source/history', 3).kind).toBe('refuse'); + expect(prep('nodeId=3', '/git-sources/history', undefined).kind).toBe('refuse'); + }); + + it('narrows to the proxied node even when history names none', () => { + // A request routed to this node is a question about this node. Without + // the filter the remote answers with rows from all of its own nodes, and + // the hub stamps a single id across every row it rewrites, so those rows + // would come back claiming to belong to a node they do not. + const result = prep('limit=5', '/git-sources/history', 3); + if (result.kind !== 'forward') throw new Error('expected forward'); + expect(result.search.get('gitopsLocalTarget')).toBe('1'); + expect(result.search.get('limit')).toBe('5'); + }); + + it('leaves the non-history routes without a node filter', () => { + // Git sources are per-instance rather than per-node, so there is nothing + // to narrow. + const result = prep('', '/git-sources', 3); + if (result.kind !== 'forward') throw new Error('expected forward'); + expect(result.search.get('gitopsLocalTarget')).toBeNull(); + }); + + it('strips a forged local-target even when it refuses', () => { + expect(prep('nodeId=7&gitopsLocalTarget=1', '/git-sources/history', 3).kind).toBe('refuse'); + }); + }); + + describe('conditional requests', () => { + it('strips every conditional request header before forwarding', () => { + const removed: string[] = []; + stripConditionalRequestHeaders({ removeHeader: (name) => removed.push(name) }); + expect(removed).toEqual(['if-none-match', 'if-modified-since', 'if-match', 'if-unmodified-since']); + }); + + it('reruns the hub filter when a caller revalidates after a permission change', async () => { + // First read: the caller may see both rows. The answer is filtered for + // them and carries no validator to cache against. + const row = (name: string): unknown => ({ + nodeId: 1, + stack_name: name, + gitopsRevision: { lifecycleStatus: 'active' }, + stackResourcePresent: true, + }); + const first = await runResponse({ + status: 200, + headers: { etag: 'W/"upstream-1"' }, + body: JSON.stringify([row('kept'), row('revoked')]), + }); + expect(first.kind).toBe('rewrite'); + expect(JSON.parse(first.body.toString())).toHaveLength(2); + expect(first.headers.etag).toBeUndefined(); + expect(first.headers['cache-control']).toBe('no-store'); + + // The revalidation attempt: a conditional request is stripped on its way + // up, so the remote cannot answer 304 and the hub must classify every + // row again under the grants in force now. + const outbound: Record = { 'if-none-match': 'W/"upstream-1"', accept: 'application/json' }; + stripConditionalRequestHeaders({ removeHeader: (name) => { delete outbound[name]; } }); + expect(outbound['if-none-match']).toBeUndefined(); + expect(outbound.accept).toBe('application/json'); + + // The fresh answer reflects the revocation: one row survives. + const second = await runResponse({ + status: 200, + body: JSON.stringify([row('kept'), row('revoked')]), + transform: (payload) => (filterRemoteIdentityPayload( + '/git-sources', + payload, + // The caller may still prove every row except the revoked stack. + (requirement) => !(requirement.kind === 'stack_read' && requirement.stackName === 'revoked'), + 1, + )), + }); + expect(second.kind).toBe('rewrite'); + expect(JSON.parse(second.body.toString())).toHaveLength(1); + }); + }); + + describe('node id rewriting', () => { + it('rewrites every enumerated position on a source row', () => { + const payload = [{ + stack_name: 'web', + nodeId: 1, + stackResourcePresent: true, + targets: [{ nodeId: 1 }, { nodeId: 2 }], + gitopsRevision: { + targets: [{ nodeId: 1 }], + drift: [{ affectedTargets: [{ nodeId: 1 }, { nodeId: null }] }], + }, + gitopsRevisions: [{ targets: [{ nodeId: 9 }] }], + }]; + rewriteIdentityPayload(payload, 42); + const row = payload[0]; + expect(row.nodeId).toBe(42); + expect(row.targets.map(t => t.nodeId)).toEqual([42, 42]); + expect(row.gitopsRevision.targets[0]?.nodeId).toBe(42); + expect(row.gitopsRevision.drift[0]?.affectedTargets[0]?.nodeId).toBe(42); + expect(row.gitopsRevisions[0]?.targets[0]?.nodeId).toBe(42); + }); + + it('preserves a null node rather than inventing a placement', () => { + const payload = [{ nodeId: null, targets: [{ nodeId: null }] }]; + rewriteIdentityPayload(payload, 42); + expect(payload[0]?.nodeId).toBeNull(); + expect(payload[0]?.targets[0]?.nodeId).toBeNull(); + }); + + it('rewrites history items including their before and after projections', () => { + const payload = { + items: [{ + nodeId: 1, + stackName: 'web', + before: { targets: [{ nodeId: 1 }] }, + after: { targets: [{ nodeId: 1 }], drift: [{ affectedTargets: [{ nodeId: 1 }] }] }, + }], + nextCursor: '100.abc', + }; + rewriteIdentityPayload(payload, 42); + const item = payload.items[0]; + expect(item?.nodeId).toBe(42); + expect(item?.before.targets[0]?.nodeId).toBe(42); + expect(item?.after.drift[0]?.affectedTargets[0]?.nodeId).toBe(42); + expect(payload.nextCursor).toBe('100.abc'); + }); + + it('rewrites the revision on a drift payload without touching the ledger', () => { + // The drift payload is a single object whose GitOps content hangs off + // `gitopsRevision`. Its own `findings` and `ledger` are the compose vs + // runtime record and carry no node identity, so they must come back byte + // for byte. + const payload = { + stack: 'web', + status: 'drifted', + findings: [{ service: 'app', kind: 'image-mismatch' }], + ledger: [{ service: 'app', kind: 'image-mismatch', detectedAt: 5 }], + gitopsRevision: { + targets: [{ nodeId: 1 }], + drift: [{ affectedTargets: [{ nodeId: 1 }] }], + }, + }; + rewriteIdentityPayload(payload, 42); + expect(payload.gitopsRevision.targets[0]?.nodeId).toBe(42); + expect(payload.gitopsRevision.drift[0]?.affectedTargets[0]?.nodeId).toBe(42); + expect(payload.findings).toEqual([{ service: 'app', kind: 'image-mismatch' }]); + expect(payload.ledger).toEqual([{ service: 'app', kind: 'image-mismatch', detectedAt: 5 }]); + expect(payload.stack).toBe('web'); + }); + + it('leaves strings and unlisted keys untouched', () => { + const payload = { + applicationId: 'app-1', + stackName: 'web', + nodeId: '1', + someOtherId: 1, + targets: [{ nodeId: 1, stackName: 'web' }], + }; + rewriteIdentityPayload(payload, 42); + expect(payload.applicationId).toBe('app-1'); + expect(payload.stackName).toBe('web'); + expect(payload.nodeId).toBe('1'); + expect(payload.someOtherId).toBe(1); + expect(payload.targets[0]?.stackName).toBe('web'); + }); + }); + + describe('collection filtering', () => { + it('drops rows in place and preserves order', () => { + const payload = [{ id: 'a' }, { id: 'b' }, { id: 'c' }]; + const filtered = filterIdentityCollection(payload, row => (row as { id: string }).id !== 'b', () => true); + expect(filtered).toEqual([{ id: 'a' }, { id: 'c' }]); + }); + + it('keeps the cursor when a page filters down to nothing', () => { + // Otherwise a caller whose grants reject a whole window concludes the + // history is empty instead of paging on. + const payload = { items: [{ id: 'a' }], nextCursor: '100.abc' }; + const filtered = filterIdentityCollection(payload, () => true, () => false); + expect(filtered).toEqual({ items: [], nextCursor: '100.abc' }); + }); + }); + + describe('hub re-authorization of remote rows', () => { + // A viewer holds global stack:read, so a row that reduces to a stack read + // survives while anything unprovable falls to Admin or audit. + const asViewer = (requirement: { kind: string }): boolean => requirement.kind === 'stack_read'; + + const sourceRow = (stackName: string, lifecycleStatus: string, present: boolean) => ({ + stack_name: stackName, + gitopsRevision: { schemaVersion: 1, targetMode: 'direct', lifecycleStatus }, + stackResourcePresent: present, + }); + + const historyItem = (stackName: string, lifecycleStatus: string | null, present: boolean) => ({ + stackName, + applicationLifecycleStatus: lifecycleStatus, + stackResourcePresent: present, + }); + + it('filters a source collection on the pre-rewrite path', () => { + // The path must be the one the hub saw before pathRewrite prefixed + // `/api`. Passing the rewritten path matches nothing and silently skips + // re-authorization on every request. + const payload = [ + sourceRow('web', 'active', true), + sourceRow('gone', 'deleted', true), + sourceRow('absent', 'active', false), + ]; + const filtered = filterRemoteIdentityPayload('/git-sources', payload, asViewer, 7); + expect(Array.isArray(filtered)).toBe(true); + expect((filtered as Array<{ stack_name: string }>).map(r => r.stack_name)).toEqual(['web']); + }); + + it('filters a history collection on the pre-rewrite path', () => { + const payload = { + items: [ + historyItem('web', 'active', true), + historyItem('creating-one', 'creating', true), + historyItem('no-app', null, true), + ], + nextCursor: '100.abc', + }; + const filtered = filterRemoteIdentityPayload('/git-sources/history', payload, asViewer, 7); + const items = (filtered as { items: Array<{ stackName: string }> }).items; + expect(items.map(i => i.stackName)).toEqual(['web']); + expect((filtered as { nextCursor: string }).nextCursor).toBe('100.abc'); + }); + + it('leaves a drift payload unfiltered', () => { + // The drift routes are per-stack, authorized by name before the hop, and + // return one object rather than a cross-stack collection. Re-filtering + // them would hide a stack's own drift from the operator who just proved + // they may read it. + const payload = { stack: 'gone', gitopsRevision: { schemaVersion: 1, targetMode: 'direct', lifecycleStatus: 'deleted' } }; + expect(filterRemoteIdentityPayload('/stacks/gone/drift', payload, asViewer, 7)).toEqual(payload); + expect(filterRemoteIdentityPayload('/stacks/gone/drift/recheck', payload, asViewer, 7)).toEqual(payload); + }); + + it('does not match the rewritten path, which is why the hop stashes the original', () => { + // Pins the defect directly: with `/api` prefixed, nothing is filtered. + const payload = [sourceRow('gone', 'deleted', true)]; + const filtered = filterRemoteIdentityPayload('/api/git-sources', payload, asViewer, 7); + expect(filtered).toEqual(payload); + }); + + it('leaves per-stack routes unfiltered, since they were authorized by name', () => { + const payload = { items: [historyItem('web', 'creating', true)] }; + const filtered = filterRemoteIdentityPayload('/stacks/web/git-source/history', payload, asViewer, 7); + expect((filtered as { items: unknown[] }).items).toHaveLength(1); + }); + }); + + describe('terminal response rules', () => { + it('rewrites a JSON 200 and reframes the body', async () => { + const result = await runResponse({ status: 200, body: JSON.stringify([{ nodeId: 1 }]) }); + expect(result.kind).toBe('rewrite'); + expect(result.statusCode).toBe(200); + expect(result.headers['content-type']).toBe('application/json; charset=utf-8'); + expect(JSON.parse(result.body.toString())).toEqual([{ nodeId: 42 }]); + expect(result.headers['content-length']).toBe(String(result.body.length)); + }); + + it('decodes a gzipped body before rewriting it', async () => { + const result = await runResponse({ + status: 200, + body: zlib.gzipSync(Buffer.from(JSON.stringify([{ nodeId: 1 }]))), + headers: { 'content-encoding': 'gzip' }, + }); + expect(result.kind).toBe('rewrite'); + expect(JSON.parse(result.body.toString())).toEqual([{ nodeId: 42 }]); + // The upstream framing described bytes that no longer exist, so it must + // be gone rather than replayed over a body of a different length. + expect(result.headers['content-encoding']).toBeUndefined(); + expect(result.headers['transfer-encoding']).toBeUndefined(); + expect(result.headers['content-length']).toBe(String(result.body.length)); + expect(result.finalizeCalls).toBe(1); + }); + + it('accumulates across chunks rather than checking one at a time', async () => { + // A per-chunk check would let an arbitrarily large body through in small + // pieces, so the cap has to be tested against a stream, not a buffer. + const result = await runResponse({ + status: 200, + body: '', + chunks: Array.from({ length: 40 }, () => Buffer.alloc(32 * 1024, 0x61)), + }); + expect(result.kind).toBe('too_large'); + expect(result.statusCode).toBe(502); + }); + + it('allows a body exactly at the ceiling', async () => { + const exact = Buffer.concat([ + Buffer.from('"'), + Buffer.alloc(IDENTITY_PROXY_MAX_BYTES - 2, 0x61), + Buffer.from('"'), + ]); + const result = await runResponse({ status: 200, body: exact }); + expect(result.kind).toBe('rewrite'); + }); + + it('passes a non-2xx body through without rewriting', async () => { + const result = await runResponse({ status: 403, body: JSON.stringify({ error: 'denied' }) }); + expect(result.kind).toBe('passthrough'); + expect(result.statusCode).toBe(403); + expect(JSON.parse(result.body.toString())).toEqual({ error: 'denied' }); + }); + + it('refuses a 200 that will not parse instead of relaying it', async () => { + // These routes answer with JSON on success, so an unparseable 200 is a + // body the hub could not read. Relaying it under the remote's success + // status would hand the client an unrewritten, unauthorized payload. + const result = await runResponse({ status: 200, body: 'not json at all' }); + expect(result.kind).toBe('parse_error'); + expect(result.statusCode).toBe(502); + expect(JSON.parse(result.body.toString()).code).toBe('gitops_proxy_unparseable'); + }); + + it('blames itself, not the remote, when its own rewrite throws', async () => { + const result = await runResponse({ + status: 200, + body: JSON.stringify([{ nodeId: 1 }]), + transform: () => { throw new Error('permission lookup failed'); }, + }); + expect(result.kind).toBe('rewrite_failed'); + // A 500, because everything in the transform runs on this instance. + expect(result.statusCode).toBe(500); + expect(JSON.parse(result.body.toString()).code).toBe('gitops_proxy_rewrite_failed'); + }); + + it('treats a stream that ends incomplete as truncation', async () => { + // Node's own truncation signal, rather than a hand-fired event: the + // message ends without `complete`, which is what a remote dying mid-body + // actually looks like. + const result = await runResponse({ status: 200, body: '[{"nodeId":1}]', endIncomplete: true }); + expect(result.kind).toBe('upstream_failed'); + expect(result.statusCode).toBe(502); + }); + + it('writes no body for 204 and answers a 304 without the upstream validators', async () => { + const noContent = await runResponse({ status: 204, body: '' }); + expect(noContent.statusCode).toBe(204); + expect(noContent.body.length).toBe(0); + expect(noContent.headers['cache-control']).toBe('no-store'); + + const notModified = await runResponse({ + status: 304, + body: '', + headers: { etag: 'W/"abc"', 'last-modified': 'Mon, 18 Aug 2026 00:00:00 GMT', 'cache-control': 'max-age=60' }, + }); + expect(notModified.statusCode).toBe(304); + expect(notModified.body.length).toBe(0); + // The upstream validators describe the remote's unfiltered + // representation, not the page this hub sends, so relaying them would + // let a cached page outlive the authorization it was filtered under. + expect(notModified.headers.etag).toBeUndefined(); + expect(notModified.headers['last-modified']).toBeUndefined(); + expect(notModified.headers['cache-control']).toBe('no-store'); + }); + + it('answers a rewritten page with no-store and no cache validators', async () => { + const result = await runResponse({ + status: 200, + body: JSON.stringify([{ nodeId: 1, stackName: 'web' }]), + headers: { + etag: 'W/"upstream-1"', + 'last-modified': 'Mon, 18 Aug 2026 00:00:00 GMT', + expires: 'Mon, 18 Aug 2026 01:00:00 GMT', + vary: 'Accept-Encoding', + 'cache-control': 'max-age=60', + }, + }); + expect(result.kind).toBe('rewrite'); + expect(result.statusCode).toBe(200); + expect(result.headers['cache-control']).toBe('no-store'); + expect(result.headers.etag).toBeUndefined(); + expect(result.headers['last-modified']).toBeUndefined(); + expect(result.headers.expires).toBeUndefined(); + expect(result.headers.vary).toBeUndefined(); + }); + + it('preserves the location on a redirect', async () => { + const result = await runResponse({ + status: 302, + body: '', + headers: { location: '/api/git-sources' }, + }); + expect(result.statusCode).toBe(302); + expect(result.headers.location).toBe('/api/git-sources'); + // Every answer this hop writes is uncacheable, redirects included. + expect(result.headers['cache-control']).toBe('no-store'); + }); + + it('refuses a body past the ceiling with its own status', async () => { + const oversized = 'x'.repeat(IDENTITY_PROXY_MAX_BYTES + 1024); + const result = await runResponse({ status: 200, body: JSON.stringify([oversized]) }); + expect(result.kind).toBe('too_large'); + // Not the upstream 200: the hub could not read the answer. + expect(result.statusCode).toBe(502); + expect(JSON.parse(result.body.toString()).code).toBe('gitops_proxy_too_large'); + expect(result.headers['cache-control']).toBe('no-store'); + }); + + it('reports an undecodable body as a decode failure', async () => { + const result = await runResponse({ + status: 200, + body: Buffer.from('this is not gzip'), + headers: { 'content-encoding': 'gzip' }, + }); + expect(result.kind).toBe('decompress_error'); + expect(result.statusCode).toBe(502); + expect(JSON.parse(result.body.toString()).code).toBe('gitops_proxy_decompress_failed'); + }); + + it('reports a truncated upstream as a failure, not an empty success', async () => { + const result = await runResponse({ status: 200, body: '[{"nodeId":1}]', abort: true }); + expect(result.kind).toBe('upstream_failed'); + expect(result.statusCode).toBe(502); + expect(JSON.parse(result.body.toString()).code).toBe('gitops_proxy_upstream_failed'); + }); + + it('finalizes once and writes nothing when the client hangs up', async () => { + const result = await runResponse({ status: 200, body: JSON.stringify([{ nodeId: 1 }]), closeEarly: true }); + expect(result.kind).toBe('downstream_close'); + expect(result.ended).toBe(false); + expect(result.finalizeCalls).toBe(1); + }); + }); +}); + +type ResponseCase = { + status: number; + body: string | Buffer; + headers?: Record; + chunks?: Buffer[]; + abort?: boolean; + endIncomplete?: boolean; + closeEarly?: boolean; + transform?: (payload: unknown) => unknown; +}; + +type ResponseResult = { + kind: IdentityTerminalKind; + statusCode: number; + headers: Record; + body: Buffer; + ended: boolean; + finalizeCalls: number; +}; + +/** Drive one upstream response through the terminal rules and capture what the client sees. */ +function runResponse(testCase: ResponseCase): Promise { + return new Promise((resolve) => { + const proxyRes = new IncomingMessage(new Socket()); + proxyRes.statusCode = testCase.status; + for (const [name, value] of Object.entries(testCase.headers ?? {})) { + proxyRes.headers[name] = value; + } + + // Pre-seeded with the framing the streaming hop would have set. Without + // this the strip assertions pass vacuously, since removing a header that + // was never present is indistinguishable from not stripping at all. + const headers: Record = { + 'content-length': '999', + 'content-encoding': 'gzip', + 'transfer-encoding': 'chunked', + }; + let ended = false; + let written = Buffer.alloc(0); + let kind: IdentityTerminalKind | undefined; + let finalizeCalls = 0; + const closeListeners: Array<() => void> = []; + + const sink: IdentityResponseSink = { + headersSent: false, + writableEnded: false, + statusCode: 0, + removeHeader: (name) => { delete headers[name]; }, + setHeader: (name, value) => { headers[name] = String(value); }, + end: (body) => { + ended = true; + sink.writableEnded = true; + if (body) written = Buffer.from(body); + finish(); + }, + on: (_event, listener) => { closeListeners.push(listener); }, + }; + + const finish = (): void => { + resolve({ + kind: kind ?? 'passthrough', + statusCode: sink.statusCode, + headers, + body: written, + ended, + finalizeCalls, + }); + }; + + handleIdentityResponse(proxyRes, sink, { + transform: testCase.transform + ?? ((payload) => { rewriteIdentityPayload(payload, 42); return payload; }), + finalizeTiming: (terminal) => { + kind = terminal; + finalizeCalls += 1; + if (terminal === 'downstream_close') setImmediate(finish); + }, + }); + + if (testCase.closeEarly) { + for (const listener of closeListeners) listener(); + return; + } + + const payload = Buffer.isBuffer(testCase.body) ? testCase.body : Buffer.from(testCase.body); + if (testCase.abort) { + proxyRes.push(payload.subarray(0, Math.max(1, payload.length - 4))); + proxyRes.emit('aborted'); + return; + } + if (testCase.endIncomplete) { + // `complete` deliberately left false, which is how Node itself reports a + // body that stopped early. + proxyRes.push(payload.subarray(0, Math.max(1, payload.length - 4))); + proxyRes.push(null); + return; + } + for (const chunk of testCase.chunks ?? []) proxyRes.push(chunk); + if (payload.length > 0) proxyRes.push(payload); + // A real HTTP response sets this once the parser has seen the whole body. + // Without it a synthetic message reports every clean end as truncated. + proxyRes.complete = true; + proxyRes.push(null); + }); +} diff --git a/backend/src/__tests__/gitops-json.test.ts b/backend/src/__tests__/gitops-json.test.ts new file mode 100644 index 00000000..18f5d6a8 --- /dev/null +++ b/backend/src/__tests__/gitops-json.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from 'vitest'; +import { + decodeArtifactEvidenceJson, + decodeGitOpsApprovedTargetEffectJson, + decodeGitOpsRequiredTargetsJson, + decodeObservedArtifactIdentity, + encodeGitOpsJson, + encodeGitOpsRequiredTargetsJson, + GitOpsJsonError, +} from '../services/gitops/json'; + +describe('gitops json codecs', () => { + it('rejects extra keys, non-integer ids, and non-canonical required targets', () => { + expect(() => decodeGitOpsRequiredTargetsJson('{"nodeIds":[1],"extra":true}')).toThrow(GitOpsJsonError); + expect(() => decodeGitOpsRequiredTargetsJson('{"nodeIds":["1"]}')).toThrow(GitOpsJsonError); + expect(() => decodeGitOpsRequiredTargetsJson('{"nodeIds":[2,1]}')).toThrow(GitOpsJsonError); + expect(() => decodeGitOpsRequiredTargetsJson('{"nodeIds":[1,1]}')).toThrow(GitOpsJsonError); + expect(decodeGitOpsRequiredTargetsJson('{"nodeIds":[1,2]}')).toEqual({ nodeIds: [1, 2] }); + expect(() => encodeGitOpsRequiredTargetsJson([2, 1])).toThrow(GitOpsJsonError); + }); + + it('decodes placement blast as a canonical action effect', () => { + expect(decodeGitOpsApprovedTargetEffectJson('[]')).toEqual([]); + expect(decodeGitOpsApprovedTargetEffectJson( + '[{"nodeId":1,"outcome":"place"},{"nodeId":3,"outcome":"remove"}]', + )).toEqual([ + { nodeId: 1, outcome: 'place' }, + { nodeId: 3, outcome: 'remove' }, + ]); + expect(() => decodeGitOpsApprovedTargetEffectJson( + '[{"nodeId":2,"outcome":"place"},{"nodeId":1,"outcome":"remove"}]', + )).toThrow(GitOpsJsonError); + expect(() => decodeGitOpsApprovedTargetEffectJson( + '[{"nodeId":1,"outcome":"place","extra":1}]', + )).toThrow(GitOpsJsonError); + }); + + it('rejects contradictory artifact evidence', () => { + expect(decodeArtifactEvidenceJson('{"kind":"unresolved"}')).toEqual({ kind: 'unresolved' }); + expect(() => decodeArtifactEvidenceJson('{"kind":"unresolved","identity":"x"}')).toThrow(GitOpsJsonError); + expect(() => decodeArtifactEvidenceJson('{"kind":"exact"}')).toThrow(GitOpsJsonError); + expect(decodeArtifactEvidenceJson('{"kind":"exact","identity":"sha256:abc"}')).toEqual({ + kind: 'exact', + identity: 'sha256:abc', + }); + }); + + it('refuses to encode a value JSON.stringify drops', () => { + // JSON.stringify returns undefined rather than throwing for these, and + // every JSON column is NOT NULL, so the encoder has to reject them itself. + expect(() => encodeGitOpsJson(undefined)).toThrow(GitOpsJsonError); + expect(() => encodeGitOpsJson(() => 'x')).toThrow(GitOpsJsonError); + expect(() => encodeGitOpsJson(Symbol('x'))).toThrow(GitOpsJsonError); + expect(encodeGitOpsJson({ a: 1 })).toBe('{"a":1}'); + }); + + it('treats null observation as unknown and rejects contradictory kinds', () => { + expect(decodeObservedArtifactIdentity(null)).toEqual({ kind: 'unknown' }); + expect(() => decodeObservedArtifactIdentity('{"kind":"missing","identity":"x"}')).toThrow(GitOpsJsonError); + expect(() => decodeObservedArtifactIdentity('{"kind":"exact","identity":"x"}')).toThrow(GitOpsJsonError); + }); +}); diff --git a/backend/src/__tests__/gitops-managed-sweep.test.ts b/backend/src/__tests__/gitops-managed-sweep.test.ts new file mode 100644 index 00000000..5e08baa6 --- /dev/null +++ b/backend/src/__tests__/gitops-managed-sweep.test.ts @@ -0,0 +1,182 @@ +/** + * What the boot sweep does with a managed area no database row claims. + * + * The three staging-marker states drive three different actions, and the + * difference between "missing" and "corrupt" is the whole rule: nothing ever + * claimed a missing-marker area, so it is an ordinary orphan and is reaped, + * while a corrupt marker is evidence of a claim we cannot read, so the area is + * preserved. See docs/internal/adrs/2026-08-16-managed-area-orphan-reaping.md. + */ +import fs from 'fs'; +import path from 'path'; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { DatabaseService } from '../services/DatabaseService'; +import { GitSourceService } from '../services/GitSourceService'; +import { GitOpsStore } from '../services/gitops/store'; +import { GitOpsTransitions } from '../services/gitops/transitions'; +import { candidateRelPathForSha, stagingMarkerPath } from '../services/gitops/createStagingMarker'; +import { stackManagedRoot } from '../services/gitops/directApplication'; +import type { GitOpsApplicationRow, GitOpsCreateCheckpointRow } from '../services/gitops/types'; + +const SHA = 'beef5678'; + +describe('managed-area orphan sweep', () => { + let tmpDir: string; + + beforeAll(async () => { + tmpDir = await setupTestDb(); + GitOpsStore.resetForTests(); + GitOpsTransitions.resetForTests(); + }); + + afterAll(() => { + cleanupTestDb(tmpDir); + }); + + beforeEach(() => { + const db = DatabaseService.getInstance().getDb(); + db.prepare('DELETE FROM gitops_create_checkpoints').run(); + db.prepare('DELETE FROM gitops_target_current').run(); + db.prepare('DELETE FROM gitops_applications').run(); + db.prepare('DELETE FROM stack_git_sources').run(); + }); + + function seedArea(stackName: string): { area: string; candidate: string; sentinel: string } { + const area = stackManagedRoot(stackName); + const candidate = path.join(area, candidateRelPathForSha(SHA)); + const sentinel = path.join(area, 'generations', 'applied-earlier'); + fs.mkdirSync(candidate, { recursive: true }); + fs.mkdirSync(sentinel, { recursive: true }); + return { area, candidate, sentinel }; + } + + it('reaps an area nothing has ever claimed', async () => { + const { area } = seedArea('orphan-none'); + + await GitSourceService.getInstance().sweepOrphans(); + + expect(fs.existsSync(area)).toBe(false); + }); + + it('preserves an area whose marker cannot be read', async () => { + const { area, sentinel } = seedArea('orphan-corrupt'); + fs.writeFileSync(stagingMarkerPath(area), '{ not json', 'utf8'); + + await GitSourceService.getInstance().sweepOrphans(); + + expect(fs.existsSync(sentinel)).toBe(true); + }); + + it('removes only the staged candidate when a valid marker claims the area', async () => { + const { area, candidate, sentinel } = seedArea('orphan-marked'); + fs.writeFileSync(stagingMarkerPath(area), JSON.stringify({ + schemaVersion: 1, + operationId: 'op-live', + rootPreexisted: true, + candidateRelPath: candidateRelPathForSha(SHA), + createdAt: Date.now(), + }), 'utf8'); + + await GitSourceService.getInstance().sweepOrphans(); + + expect(fs.existsSync(candidate)).toBe(false); + expect(fs.existsSync(sentinel)).toBe(true); + }); + + it('leaves an area claimed by an in-flight create alone', async () => { + const { area, candidate } = seedArea('orphan-inflight'); + GitOpsStore.getInstance().insertApplication(creatingApp('app-inflight', 'orphan-inflight')); + GitOpsStore.getInstance().insertCreateCheckpoint(checkpoint('app-inflight', 'orphan-inflight')); + + await GitSourceService.getInstance().sweepOrphans(); + + expect(fs.existsSync(area)).toBe(true); + expect(fs.existsSync(candidate)).toBe(true); + }); +}); + +function checkpoint(applicationId: string, stackName: string): GitOpsCreateCheckpointRow { + return { + application_id: applicationId, + stack_name: stackName, + phase: 'pre_stack', + generation_id: null, + operation_id: `op-${applicationId}`, + repo_url: 'https://github.com/org/repo.git', + branch: 'main', + compose_path: 'compose.yml', + compose_paths_json: '["compose.yml"]', + context_dir: null, + sync_env: 0, + env_path: null, + auth_type: 'none', + encrypted_token: null, + auto_apply_on_webhook: 0, + auto_deploy_on_apply: 0, + commit_sha: SHA, + applied_spec_json: null, + created_managed_root: 1, + created_at: 1, + updated_at: 1, + }; +} + +function creatingApp(id: string, stackName: string): GitOpsApplicationRow { + return { + id, + lifecycle_key: `direct:${stackName}`, + lifecycle_status: 'creating', + target_mode: 'direct', + stack_name: stackName, + blueprint_id: null, + configured_repo_url: 'https://github.com/org/repo.git', + repo_identity_json: '{"host":"github.com","pathname":"/org/repo.git"}', + configured_ref: 'main', + compose_paths_json: '["compose.yml"]', + context_dir: null, + sync_env: 0, + env_path: null, + materialization_fingerprint: 'a'.repeat(64), + desired_commit_sha: null, + fetched_commit_sha: null, + candidate_generation_id: null, + accepted_generation_id: null, + candidate_plan_blocked: 0, + review_required: 0, + artifact_set_id: null, + latest_artifact_set_id: null, + intent_revision_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + rollout_authorization_ref: null, + legacy_combined_approval_ref: null, + preflight_fingerprint: null, + latest_operation_id: null, + active_operation_id: null, + active_operation_stage: null, + active_operation_at: null, + active_generation_id: null, + pause_at: null, + pause_reason: null, + partial_json: null, + failure_stage: null, + failure_class: null, + failure_at: null, + retry_at: null, + retry_count: 0, + suspended_at: null, + recovery_ref: null, + recovery_phase: null, + interruption_stage: null, + interruption_at: null, + interruption_operation_id: null, + interruption_generation_id: null, + evidence_fresh_at: null, + evidence_limitations_json: null, + created_at: 1, + updated_at: 1, + }; +} diff --git a/backend/src/__tests__/gitops-metrics-route.test.ts b/backend/src/__tests__/gitops-metrics-route.test.ts new file mode 100644 index 00000000..e5cc2057 --- /dev/null +++ b/backend/src/__tests__/gitops-metrics-route.test.ts @@ -0,0 +1,86 @@ +/** + * Integration tests for GET /api/gitops-metrics: the Admin-only snapshot of + * in-process GitOps transition counters. + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; +import request from 'supertest'; +import bcrypt from 'bcrypt'; +import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb'; + +let tmpDir: string; +let app: import('express').Express; +let adminCookie: string; +let viewerCookie: string; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + const { DatabaseService } = await import('../services/DatabaseService'); + ({ app } = await import('../index')); + adminCookie = await loginAsTestAdmin(app); + + const viewerHash = await bcrypt.hash('viewerpass', 1); + DatabaseService.getInstance().addUser({ + username: 'gitops-metrics-viewer', + password_hash: viewerHash, + role: 'viewer', + }); + const viewerRes = await request(app) + .post('/api/auth/login') + .send({ username: 'gitops-metrics-viewer', password: 'viewerpass' }); + const cookies = viewerRes.headers['set-cookie'] as string | string[]; + viewerCookie = Array.isArray(cookies) ? cookies[0] : cookies; +}); + +afterAll(() => { + vi.restoreAllMocks(); + cleanupTestDb(tmpDir); +}); + +beforeEach(async () => { + const { GitOpsMetricsService } = await import('../services/GitOpsMetricsService'); + GitOpsMetricsService.resetForTests(); +}); + +describe('GET /api/gitops-metrics', () => { + it('returns 401 without an auth cookie', async () => { + const res = await request(app).get('/api/gitops-metrics'); + expect(res.status).toBe(401); + }); + + it('refuses a signed-in non-admin', async () => { + const res = await request(app).get('/api/gitops-metrics').set('Cookie', viewerCookie); + expect(res.status).toBe(403); + }); + + it('returns an empty list on a fresh process', async () => { + const res = await request(app).get('/api/gitops-metrics').set('Cookie', adminCookie); + expect(res.status).toBe(200); + expect(res.body).toEqual({ entries: [] }); + }); + + it('returns one entry per stage and outcome pair', async () => { + const { GitOpsMetricsService } = await import('../services/GitOpsMetricsService'); + const metrics = GitOpsMetricsService.getInstance(); + metrics.record('fetched', 'committed'); + metrics.record('fetched', 'committed'); + metrics.record('apply_failed', 'failed'); + + const res = await request(app).get('/api/gitops-metrics').set('Cookie', adminCookie); + expect(res.status).toBe(200); + expect(res.body.entries).toEqual([ + { stage: 'apply_failed', outcome: 'failed', count: 1 }, + { stage: 'fetched', outcome: 'committed', count: 2 }, + ]); + }); + + it('names no stack, node, repository or actor', async () => { + // The counters are process diagnostics, not an audit trail. Anything + // identifying would be one with no retention policy and no per-row + // authorization, which is what the history API exists to provide. + const { GitOpsMetricsService } = await import('../services/GitOpsMetricsService'); + GitOpsMetricsService.getInstance().record('deploy_started', 'committed'); + + const res = await request(app).get('/api/gitops-metrics').set('Cookie', adminCookie); + expect(Object.keys(res.body.entries[0]).sort()).toEqual(['count', 'outcome', 'stage']); + }); +}); diff --git a/backend/src/__tests__/gitops-migrate-inline-blueprint.test.ts b/backend/src/__tests__/gitops-migrate-inline-blueprint.test.ts new file mode 100644 index 00000000..2d169417 --- /dev/null +++ b/backend/src/__tests__/gitops-migrate-inline-blueprint.test.ts @@ -0,0 +1,118 @@ +/** + * Inline Blueprint migration. + * + * Migration records what a Blueprint asks for. It never records agreement: a + * Blueprint revision and a deployment's applied revision both look like + * progress, but neither proves a node is running the intent this pass just + * minted, and writing them as an acknowledgement would report convergence + * nobody verified. + */ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { DatabaseService, type Blueprint } from '../services/DatabaseService'; +import { GitOpsStore } from '../services/gitops/store'; +import { GitOpsTransitions } from '../services/gitops/transitions'; +import { migrateInlineBlueprints } from '../services/gitops/migrate'; +import { commitBlueprintCreate } from '../services/gitops/blueprintProducers'; +import { decodeGitOpsEvidenceLimitations } from '../services/gitops/json'; + +describe('gitops inline blueprint migration', () => { + let tmpDir: string; + + beforeAll(async () => { + tmpDir = await setupTestDb(); + GitOpsStore.resetForTests(); + GitOpsTransitions.resetForTests(); + }); + + afterAll(() => { + cleanupTestDb(tmpDir); + }); + + it('brings a pre-existing Blueprint in without claiming anyone agreed to it', () => { + const db = DatabaseService.getInstance(); + const store = GitOpsStore.getInstance(); + const blueprint = seedLegacy('mig-plain'); + + expect(outcomeFor(blueprint)).toBe('migrated_inline'); + + const app = store.getLiveBlueprintApplication(blueprint.id)!; + expect(app.target_mode).toBe('inline_blueprint'); + const intent = store.getIntentRevision(app.intent_revision_id!)!; + // Carried for display, never as an acknowledgement. + expect(intent.blueprint_revision).toBe(blueprint.revision); + + const candidate = store.getRolloutCandidate(app.rollout_candidate_id!)!; + expect(candidate.provenance).toBe('legacy_inline'); + // Placement is not resolved by migration. + expect(JSON.parse(candidate.required_targets_json)).toEqual({ nodeIds: [] }); + + // No target, so nothing claims a node is running this. + expect(store.listTargets(app.id)).toEqual([]); + expect(db.getBlueprint(blueprint.id)!.revision).toBe(blueprint.revision); + }); + + it('says why an unapproved Blueprint carries no authority', () => { + const store = GitOpsStore.getInstance(); + const blueprint = seedLegacy('mig-unapproved'); + migrateInlineBlueprints(); + + const app = store.getLiveBlueprintApplication(blueprint.id)!; + const limitations = decodeGitOpsEvidenceLimitations(app.evidence_limitations_json); + // Recorded rather than left blank: an absent approval and an approval that + // no longer authorizes this intent are otherwise indistinguishable. + expect(limitations.map(l => l.code)).toContain('blueprint_reapproval_required'); + }); + + it('is a no-op on replay', () => { + const store = GitOpsStore.getInstance(); + const blueprint = seedLegacy('mig-replay'); + migrateInlineBlueprints(); + const app = store.getLiveBlueprintApplication(blueprint.id)!; + + expect(outcomeFor(blueprint)).toBe('skipped_current'); + const after = store.getLiveBlueprintApplication(blueprint.id)!; + expect(after.intent_revision_id).toBe(app.intent_revision_id); + expect(after.rollout_candidate_id).toBe(app.rollout_candidate_id); + }); + + it('leaves a Blueprint the new path already described alone', () => { + const store = GitOpsStore.getInstance(); + const blueprint = commitBlueprintCreate({ + name: 'mig-live', + description: null, + compose_content: 'services:\n web:\n image: nginx:1.27\n', + selector: { type: 'nodes', ids: [1] }, + drift_mode: 'suggest', + classification: 'stateless', + classification_reasons: [], + enabled: true, + created_by: 'tester', + }, () => [1]); + const app = store.getLiveBlueprintApplication(blueprint.id)!; + + expect(outcomeFor(blueprint)).toBe('skipped_live_application'); + // Its rows were written with proof this pass does not have. + expect(store.getLiveBlueprintApplication(blueprint.id)!.intent_revision_id) + .toBe(app.intent_revision_id); + }); +}); + +function outcomeFor(blueprint: Blueprint): string { + return migrateInlineBlueprints().find(r => r.stackName === blueprint.name)!.outcome; +} + +/** A Blueprint as an install carries it across an upgrade: no GitOps rows. */ +function seedLegacy(name: string): Blueprint { + return DatabaseService.getInstance().createBlueprint({ + name, + description: null, + compose_content: `services:\n web:\n image: nginx:1.27\n# ${name}\n`, + selector: { type: 'nodes', ids: [1] }, + drift_mode: 'suggest', + classification: 'stateless', + classification_reasons: [], + enabled: true, + created_by: null, + }); +} diff --git a/backend/src/__tests__/gitops-migrate.test.ts b/backend/src/__tests__/gitops-migrate.test.ts new file mode 100644 index 00000000..6505a105 --- /dev/null +++ b/backend/src/__tests__/gitops-migrate.test.ts @@ -0,0 +1,382 @@ +/** + * Migration of Git stacks that predate the revision state model. + * + * The rule under test is that a canonical pointer is written only when the + * evidence proves that exact generation under the repository and ref configured + * now. A legacy applied commit is not that proof by itself, so the interesting + * cases are the ones where it is *not* promoted: a missing manifest, an + * unreadable one, one stamped for a repository the stack no longer points at, + * and one naming a commit the source row disagrees with. In each the commit + * survives as recorded evidence and the projection asks for a fetch instead of + * asserting something nobody verified. + */ +import fs from 'fs'; +import path from 'path'; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { DatabaseService, type StackGitSource } from '../services/DatabaseService'; +import { GitOpsStore } from '../services/gitops/store'; +import { GitOpsTransitions } from '../services/gitops/transitions'; +import { migrateDirectGitStacks, primeMigrationManifests } from '../services/gitops/migrate'; +import { directSourceIdentity, migrationDirectSourceIdentity } from '../services/gitops/directApplication'; +import { projectApplication } from '../services/gitops/derive'; + +const REPO = 'https://github.com/example/legacy.git'; +const SHA = 'legacy01'; + +type ManifestFixture = + | { manifestVersion: number; generation: { appliedDir: string }; resolvedRevision: { commitSha: string } } + | { corrupt: string } + | null; + +describe('gitops migration of pre-existing Git stacks', () => { + let tmpDir: string; + + beforeAll(async () => { + tmpDir = await setupTestDb(); + GitOpsStore.resetForTests(); + GitOpsTransitions.resetForTests(); + }); + + afterAll(() => { + cleanupTestDb(tmpDir); + }); + + beforeEach(() => { + const db = DatabaseService.getInstance().getDb(); + db.prepare('DELETE FROM gitops_migration_checkpoints').run(); + db.prepare('DELETE FROM gitops_target_current').run(); + db.prepare('DELETE FROM gitops_generations').run(); + db.prepare('DELETE FROM gitops_applications').run(); + db.prepare('DELETE FROM stack_git_sources').run(); + }); + + it('leaves a config-only stack asking for a fetch', () => { + seedStack('cfg-only', { lastApplied: null }); + primeManifests({ 'cfg-only': null }); + + expect(migrateDirectGitStacks()).toEqual([{ stackName: 'cfg-only', outcome: 'migrated_unreconciled' }]); + + const app = GitOpsStore.getInstance().getLiveDirectApplication('cfg-only')!; + expect(app.desired_commit_sha).toBeNull(); + expect(app.accepted_generation_id).toBeNull(); + expect(app.materialization_fingerprint).not.toBeNull(); + const projection = projectOf(app.id); + expect(projection.facets.source.status).toBe('never_reconciled'); + expect(projection.availableActions).toContain('fetch'); + expect(projection.limitations).toHaveLength(0); + }); + + it('accepts the applied commit only when a trusted manifest proves it', () => { + seedStack('trusted', { lastApplied: SHA }); + primeManifests({ + trusted: { + manifestVersion: 3, + generation: { appliedDir: `generations/applied-${SHA}-3` }, + resolvedRevision: { commitSha: SHA }, + }, + }); + + expect(migrateDirectGitStacks()).toEqual([{ stackName: 'trusted', outcome: 'migrated_accepted' }]); + + const store = GitOpsStore.getInstance(); + const app = store.getLiveDirectApplication('trusted')!; + expect(app.desired_commit_sha).toBe(SHA); + expect(app.fetched_commit_sha).toBe(SHA); + expect(app.accepted_generation_id).not.toBeNull(); + + const generation = store.getGeneration(app.accepted_generation_id!)!; + expect(generation.commit_sha).toBe(SHA); + // Equal fingerprints, or the accepted generation would immediately read as + // stale against the configuration that produced it. + expect(generation.materialization_fingerprint).toBe(app.materialization_fingerprint); + + const target = store.getTarget(app.id, 1)!; + expect(target.desired_generation_id).toBe(app.accepted_generation_id); + expect(target.applied_generation_id).toBe(app.accepted_generation_id); + // A manifest proves what was materialized, never what is running. + expect(target.deployed_generation_id).toBeNull(); + expect(target.healthy_generation_id).toBeNull(); + expect(target.lkg_generation_id).toBeNull(); + // Nobody approved this generation through the model. + expect(app.source_acceptance_ref).toBeNull(); + + expect(projectOf(app.id).facets.source.status).toBe('application_generation_accepted'); + }); + + it('keeps an unprovable applied commit as evidence, never as a pointer', () => { + const cases: Array<[string, ManifestFixture, string]> = [ + ['manifest-gone', null, 'manifest_absent'], + ['manifest-broken', { corrupt: 'invalid manifest shape' }, 'manifest_corrupt'], + ['manifest-foreign', { corrupt: 'identity repository mismatch' }, 'manifest_identity_invalid'], + ]; + for (const [stackName, manifest, expectedCode] of cases) { + seedStack(stackName, { lastApplied: SHA }); + primeManifests({ [stackName]: manifest }); + + migrateDirectGitStacks(); + + const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName)!; + expect(app.desired_commit_sha, stackName).toBeNull(); + expect(app.fetched_commit_sha, stackName).toBeNull(); + expect(app.accepted_generation_id, stackName).toBeNull(); + + const projection = projectOf(app.id); + expect(projection.facets.source.status, stackName).toBe('never_reconciled'); + expect(projection.limitations.map((l) => l.code), stackName).toContain(expectedCode); + // The commit is retained as the evidence behind the limitation, so an + // operator can see what the stack used to be at. + expect(projection.limitations.map((l) => l.evidence), stackName).toContain(SHA); + } + }); + + it('refuses a valid manifest that names a different commit than the source row', () => { + // The manifest validates and belongs to this stack, repository and ref, so + // every other check passes. Only the commits disagree, and that alone must + // keep the canonical pointers null: the applied directory here materializes + // MANIFEST_SHA, so accepting SHA would certify a commit whose files are not + // the ones on disk. + const manifestSha = 'manifest02'; + seedStack('commit-drift', { lastApplied: SHA }); + primeManifests({ + 'commit-drift': { + manifestVersion: 4, + generation: { appliedDir: `generations/applied-${manifestSha}-4` }, + resolvedRevision: { commitSha: manifestSha }, + }, + }); + + expect(migrateDirectGitStacks()).toEqual([{ stackName: 'commit-drift', outcome: 'migrated_unreconciled' }]); + + const app = GitOpsStore.getInstance().getLiveDirectApplication('commit-drift')!; + expect(app.desired_commit_sha).toBeNull(); + expect(app.fetched_commit_sha).toBeNull(); + expect(app.accepted_generation_id).toBeNull(); + + const target = GitOpsStore.getInstance().getTarget(app.id, 1)!; + expect(target.applied_generation_id).toBeNull(); + + const projection = projectOf(app.id); + expect(projection.facets.source.status).toBe('never_reconciled'); + expect(projection.availableActions).toContain('fetch'); + // Both commits are named, so an operator can see which two records disagree + // rather than only learning that something could not be proven. + const mismatch = projection.limitations.find((l) => l.code === 'manifest_commit_mismatch'); + expect(mismatch).toBeDefined(); + expect(mismatch!.evidence).toContain(SHA); + expect(mismatch!.evidence).toContain(manifestSha); + }); + + it('separates a manifest with no commit from one that names a conflicting commit', () => { + // A manifest adopted from an existing directory is written with an empty + // commit and state 'migrated', which the validator permits. Folding that + // into the mismatch case would tell an operator the manifest names a + // different commit while naming nothing at all. + seedStack('adopted', { lastApplied: SHA }); + primeManifests({ + adopted: { + manifestVersion: 1, + generation: { appliedDir: 'generations/applied-adopted-1' }, + resolvedRevision: { commitSha: '' }, + }, + }); + + expect(migrateDirectGitStacks()).toEqual([{ stackName: 'adopted', outcome: 'migrated_unreconciled' }]); + + const app = GitOpsStore.getInstance().getLiveDirectApplication('adopted')!; + expect(app.desired_commit_sha).toBeNull(); + expect(app.accepted_generation_id).toBeNull(); + + const codes = projectOf(app.id).limitations.map((l) => l.code); + expect(codes).toContain('manifest_commit_unresolved'); + expect(codes).not.toContain('manifest_commit_mismatch'); + }); + + it('does not let a pending pull stand in for proof', () => { + seedStack('pending-only', { lastApplied: null, pending: 'pending99' }); + primeManifests({ 'pending-only': null }); + + migrateDirectGitStacks(); + + const app = GitOpsStore.getInstance().getLiveDirectApplication('pending-only')!; + expect(app.desired_commit_sha).toBeNull(); + expect(app.candidate_generation_id).toBeNull(); + const projection = projectOf(app.id); + expect(projection.facets.source.status).toBe('never_reconciled'); + expect(projection.limitations.map((l) => l.code)).toContain('legacy_pending'); + }); + + it('is a no-op on replay and re-runs only when the configuration changes', () => { + seedStack('replay', { lastApplied: SHA }); + primeManifests({ + replay: { + manifestVersion: 1, + generation: { appliedDir: `generations/applied-${SHA}-1` }, + resolvedRevision: { commitSha: SHA }, + }, + }); + + expect(migrateDirectGitStacks()[0].outcome).toBe('migrated_accepted'); + const firstId = GitOpsStore.getInstance().getLiveDirectApplication('replay')!.id; + + expect(migrateDirectGitStacks()).toEqual([{ stackName: 'replay', outcome: 'skipped_current' }]); + expect(GitOpsStore.getInstance().getLiveDirectApplication('replay')!.id).toBe(firstId); + + // A material configuration change replays the matrix, and the existing + // application is left alone rather than being rebuilt over. + seedStack('replay', { lastApplied: SHA, composePaths: ['compose.yaml', 'compose.prod.yaml'] }); + expect(migrateDirectGitStacks()).toEqual([{ stackName: 'replay', outcome: 'skipped_live_application' }]); + expect(GitOpsStore.getInstance().getLiveDirectApplication('replay')!.id).toBe(firstId); + }); + + it('never touches a stack the new path already described', () => { + seedStack('already-modelled', { lastApplied: SHA }); + primeManifests({ 'already-modelled': null }); + migrateDirectGitStacks(); + const before = GitOpsStore.getInstance().getLiveDirectApplication('already-modelled')!; + + DatabaseService.getInstance().getDb().prepare('DELETE FROM gitops_migration_checkpoints').run(); + migrateDirectGitStacks(); + + expect(GitOpsStore.getInstance().getLiveDirectApplication('already-modelled')!.id).toBe(before.id); + }); + + it('retires a stack whose directory is gone instead of claiming its name', () => { + seedStack('vanished', { lastApplied: SHA, createDir: false }); + primeManifests({ vanished: null }); + + expect(migrateDirectGitStacks()).toEqual([ + { stackName: 'vanished', outcome: 'tombstoned_missing_stack' }, + ]); + expect(GitOpsStore.getInstance().getLiveDirectApplication('vanished')).toBeUndefined(); + }); + + it('accepts the applied commit through a trusted manifest even on a legacy URL', () => { + // The worst real-world instance of the strict-parser bug: a stack whose + // manifest proves its applied commit would have failed migration every + // boot and never entered the model at all. + const legacyUrl = `${REPO}?token=legacy-secret`; + seedStack('legacy-trusted-url', { lastApplied: SHA, repoUrl: legacyUrl }); + primeManifests({ + 'legacy-trusted-url': { + manifestVersion: 3, + generation: { appliedDir: `generations/applied-${SHA}-3` }, + resolvedRevision: { commitSha: SHA }, + }, + }); + + expect(migrateDirectGitStacks()).toEqual([{ stackName: 'legacy-trusted-url', outcome: 'migrated_accepted' }]); + + const store = GitOpsStore.getInstance(); + const app = store.getLiveDirectApplication('legacy-trusted-url')!; + expect(app.desired_commit_sha).toBe(SHA); + expect(app.accepted_generation_id).not.toBeNull(); + expect(app.configured_repo_url).toBe(REPO); + expect(DatabaseService.getInstance().getGitSource('legacy-trusted-url')?.repo_url).toBe(legacyUrl); + expect(projectOf(app.id).facets.source.status).toBe('application_generation_accepted'); + }); + + it('derives the same identity as strict ingress once the legacy decoration is stripped', () => { + const config = { + repoUrl: REPO, + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + }; + const noisy = { ...config, repoUrl: `${REPO}?token=x` }; + const lenient = migrationDirectSourceIdentity(noisy); + const strict = directSourceIdentity(config); + expect(lenient.repoUrl).toBe(strict.repoUrl); + expect(lenient.identity).toEqual(strict.identity); + // A migrated stack must be replay-recognizable against one linked fresh + // through the user path for the same repository. + expect(lenient.fingerprint).toBe(strict.fingerprint); + }); + + it('migrates a legacy operational URL that still carries a query string', () => { + const legacyUrl = `${REPO}?token=legacy-secret`; + seedStack('legacy-query-url', { lastApplied: null, repoUrl: legacyUrl }); + primeManifests({ 'legacy-query-url': null }); + + expect(migrateDirectGitStacks()).toEqual([{ stackName: 'legacy-query-url', outcome: 'migrated_unreconciled' }]); + + const app = GitOpsStore.getInstance().getLiveDirectApplication('legacy-query-url')!; + expect(app.configured_repo_url).toBe(REPO); + // The operational row keeps its query: fetch may still need it. + expect(DatabaseService.getInstance().getGitSource('legacy-query-url')?.repo_url).toBe(legacyUrl); + }); + + it('migrates a legacy URL carrying userinfo to the identity of its clean form', () => { + seedStack('legacy-userinfo-url', { lastApplied: null, repoUrl: 'https://deploy:pat@github.com/example/legacy.git' }); + seedStack('clean-url', { lastApplied: null }); + primeManifests({ 'legacy-userinfo-url': null, 'clean-url': null }); + + migrateDirectGitStacks(); + + const store = GitOpsStore.getInstance(); + const legacy = store.getLiveDirectApplication('legacy-userinfo-url')!; + const clean = store.getLiveDirectApplication('clean-url')!; + expect(legacy.configured_repo_url).toBe(REPO); + expect(legacy.repo_identity_json).toBe(clean.repo_identity_json); + // The same repository under the same configuration must produce the same + // fingerprint, or a later replay could not recognize the stack it + // already migrated. + expect(legacy.materialization_fingerprint).toBe(clean.materialization_fingerprint); + expect(DatabaseService.getInstance().getGitSource('legacy-userinfo-url')?.repo_url).toContain('deploy:pat@'); + }); +}); + +function projectOf(applicationId: string) { + const projection = projectApplication(applicationId, true); + if (projection.targetMode === 'not_applicable') throw new Error('expected an application'); + return projection; +} + +function primeManifests(fixtures: Record): void { + primeMigrationManifests((stackName) => fixtures[stackName] ?? null); +} + +function seedStack( + stackName: string, + options: { lastApplied: string | null; pending?: string; composePaths?: string[]; createDir?: boolean; repoUrl?: string }, +): void { + if (options.createDir !== false) { + const composeDir = process.env.COMPOSE_DIR!; + fs.mkdirSync(path.join(composeDir, stackName), { recursive: true }); + fs.writeFileSync(path.join(composeDir, stackName, 'compose.yaml'), 'services: {}\n'); + } + const row: Parameters[0] = { + stack_name: stackName, + repo_url: options.repoUrl ?? REPO, + branch: 'main', + compose_path: 'compose.yaml', + compose_paths: options.composePaths ?? ['compose.yaml'], + context_dir: null, + sync_env: false, + env_path: null, + auth_type: 'none', + encrypted_token: null, + auto_apply_on_webhook: false, + auto_deploy_on_apply: false, + last_applied_commit_sha: options.lastApplied, + last_applied_content_hash: null, + pending_commit_sha: options.pending ?? null, + pending_compose_content: null, + pending_env_content: null, + pending_fetched_at: null, + last_debounce_at: null, + } as StackGitSource; + DatabaseService.getInstance().upsertGitSource(row); + if (options.lastApplied) { + DatabaseService.getInstance().markGitSourceApplied(stackName, options.lastApplied, ''); + } + if (options.pending) { + // upsertGitSource does not write the pending columns on insert, so a + // legacy row carrying an unapplied pull is seeded directly. + DatabaseService.getInstance().getDb() + .prepare('UPDATE stack_git_sources SET pending_commit_sha = ? WHERE stack_name = ?') + .run(options.pending, stackName); + } +} diff --git a/backend/src/__tests__/gitops-node-placement.test.ts b/backend/src/__tests__/gitops-node-placement.test.ts new file mode 100644 index 00000000..026a5f55 --- /dev/null +++ b/backend/src/__tests__/gitops-node-placement.test.ts @@ -0,0 +1,116 @@ +/** + * Node-side placement recording. + * + * A label or a cordon is not a statement about any one Blueprint, so what + * matters is which Blueprints the change actually moved. Reacting to the event + * instead of comparing the resulting sets would invalidate every + * acknowledgement in the fleet whenever someone labelled a node nothing selects + * on. + */ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import type { Blueprint } from '../services/DatabaseService'; +import { GitOpsStore } from '../services/gitops/store'; +import { GitOpsTransitions } from '../services/gitops/transitions'; +import { commitBlueprintCreate } from '../services/gitops/blueprintProducers'; +import { + recordPlacementShift, + snapshotPlacementWith, + type PlacementSnapshot, +} from '../services/gitops/nodePlacementProducers'; + +describe('gitops node placement recording', () => { + let tmpDir: string; + + beforeAll(async () => { + tmpDir = await setupTestDb(); + GitOpsStore.resetForTests(); + GitOpsTransitions.resetForTests(); + }); + + afterAll(() => { + cleanupTestDb(tmpDir); + }); + + it('revises only the Blueprints whose desired nodes moved', () => { + const store = GitOpsStore.getInstance(); + const moved = create('np-moved'); + const still = create('np-still'); + const movedBefore = store.getLiveBlueprintApplication(moved.id)!; + const stillBefore = store.getLiveBlueprintApplication(still.id)!; + + const before: PlacementSnapshot = new Map([[moved.id, [1]], [still.id, [1]]]); + const after: PlacementSnapshot = new Map([[moved.id, [1, 2]], [still.id, [1]]]); + + expect(recordPlacementShift(before, after, 'tester', 'node_label_add')).toEqual([moved.id]); + + expect(store.getLiveBlueprintApplication(moved.id)!.intent_revision_id) + .not.toBe(movedBefore.intent_revision_id); + // The Blueprint the label did not move keeps the acknowledgement it had. + expect(store.getLiveBlueprintApplication(still.id)!.intent_revision_id) + .toBe(stillBefore.intent_revision_id); + }); + + it('records nothing when the same nodes come back in a different order', () => { + const store = GitOpsStore.getInstance(); + const blueprint = create('np-reorder'); + const app = store.getLiveBlueprintApplication(blueprint.id)!; + + const before: PlacementSnapshot = new Map([[blueprint.id, [1, 2, 3]]]); + const after: PlacementSnapshot = new Map([[blueprint.id, [3, 1, 2]]]); + + expect(recordPlacementShift(before, after, 'tester', 'node_cordon')).toEqual([]); + expect(store.getLiveBlueprintApplication(blueprint.id)!.intent_revision_id) + .toBe(app.intent_revision_id); + }); + + it('opens the revision as a roster change, not a content change', () => { + const store = GitOpsStore.getInstance(); + const blueprint = create('np-provenance'); + + recordPlacementShift( + new Map([[blueprint.id, [1]]]), + new Map([[blueprint.id, [2]]]), + 'tester', + 'node_cordon', + ); + + const app = store.getLiveBlueprintApplication(blueprint.id)!; + const candidate = store.getRolloutCandidate(app.rollout_candidate_id!)!; + expect(candidate.provenance).toBe('roster_change'); + expect(JSON.parse(candidate.required_targets_json)).toEqual({ nodeIds: [2] }); + }); + + it('leaves a Blueprint that predates the model alone', () => { + // No application, so nothing to revise. Migration brings it in; inventing a + // first intent here would claim a starting point nobody reconciled. + const before: PlacementSnapshot = new Map([[99999, [1]]]); + const after: PlacementSnapshot = new Map([[99999, [1, 2]]]); + expect(recordPlacementShift(before, after, 'tester', 'node_label_add')).toEqual([]); + }); + + it('snapshots every Blueprint it is given', () => { + const a = create('np-snap-a'); + const b = create('np-snap-b'); + const snapshot = snapshotPlacementWith( + (blueprint) => (blueprint.name === 'np-snap-a' ? [1] : [2, 3]), + [a, b], + ); + expect(snapshot.get(a.id)).toEqual([1]); + expect(snapshot.get(b.id)).toEqual([2, 3]); + }); +}); + +function create(name: string): Blueprint { + return commitBlueprintCreate({ + name, + description: null, + compose_content: 'services:\n web:\n image: nginx:1.27\n', + selector: { type: 'nodes', ids: [1] }, + drift_mode: 'suggest', + classification: 'stateless', + classification_reasons: [], + enabled: true, + created_by: 'tester', + }, () => [1]); +} diff --git a/backend/src/__tests__/gitops-publish.test.ts b/backend/src/__tests__/gitops-publish.test.ts new file mode 100644 index 00000000..2173b658 --- /dev/null +++ b/backend/src/__tests__/gitops-publish.test.ts @@ -0,0 +1,223 @@ +/** + * Announcement of committed transitions: the metric increment and the + * `state-invalidate` event that each newly inserted history row produces. + * + * The drain is deliberately exercised through the real `setImmediate` rather + * than a test-only flush. The whole reason the publisher waits for a macrotask + * is that better-sqlite3 transactions are synchronous, so a test that drained + * by hand would prove the drain works and prove nothing about when it runs. + */ +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { directApplicationFixture } from './helpers/gitopsFixtures'; +import { DatabaseService } from '../services/DatabaseService'; +import { GitOpsMetricsService } from '../services/GitOpsMetricsService'; +import { insertHistory } from '../services/gitops/history'; +import { + enqueueHistoryPublication, + resetGitOpsPublicationsForTests, + setGitOpsEventSink, + type GitOpsInvalidateEvent, +} from '../services/gitops/publish'; + +/** + * The real module, with the enqueue entry point wrapped in a spy. + * + * Needed because a replay is suppressed twice over: the insert declines to + * enqueue it, and the drain would drop it anyway since the id it carries was + * never committed. An outcome assertion therefore passes with the first + * mechanism deleted, which is exactly the false green this suite exists to + * avoid, so the call itself has to be observable. + */ +vi.mock('../services/gitops/publish', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, enqueueHistoryPublication: vi.fn(actual.enqueueHistoryPublication) }; +}); + +/** Let the publisher's own scheduling run. */ +const settle = (): Promise => new Promise((resolve) => { setImmediate(resolve); }); + +describe('gitops transition announcements', () => { + let tmpDir: string; + let events: GitOpsInvalidateEvent[]; + + beforeAll(async () => { + tmpDir = await setupTestDb(); + }); + + afterAll(() => { + cleanupTestDb(tmpDir); + }); + + afterEach(() => { + resetGitOpsPublicationsForTests(); + GitOpsMetricsService.resetForTests(); + vi.mocked(enqueueHistoryPublication).mockClear(); + }); + + const listen = (): void => { + events = []; + setGitOpsEventSink((event) => { events.push(event); }); + }; + + const db = () => DatabaseService.getInstance().getDb(); + + const write = ( + operationId: string, + stage: Parameters[1]['stage'], + outcome: Parameters[1]['outcome'] = 'committed', + overrides: Partial[1]> = {}, + ): string | null => insertHistory(db(), { + application: directApplicationFixture(`app-${operationId}`, `stack-${operationId}`), + nodeId: 3, + dedupeTarget: 'app', + operationId, + stage, + outcome, + trigger: 'manual', + actor: 'operator-1', + before: {}, + after: {}, + at: 4242, + ...overrides, + }); + + it('announces one event and one count per inserted row', async () => { + listen(); + write('op-1', 'fetch_started'); + await settle(); + + expect(events).toEqual([{ + type: 'state-invalidate', + scope: 'gitops', + action: 'fetch_started', + applicationId: 'app-op-1', + targetMode: 'direct', + stackName: 'stack-op-1', + blueprintId: null, + nodeId: 3, + ts: 4242, + }]); + expect(GitOpsMetricsService.getInstance().snapshot()).toEqual([ + { stage: 'fetch_started', outcome: 'committed', count: 1 }, + ]); + }); + + it('announces rows in the order they were inserted', async () => { + listen(); + write('op-order', 'fetch_started'); + write('op-order', 'fetched', 'committed', { dedupeTarget: 'node:3' }); + write('op-order', 'apply_failed', 'failed', { dedupeTarget: 'node:9' }); + await settle(); + + expect(events.map((e) => e.action)).toEqual(['fetch_started', 'fetched', 'apply_failed']); + }); + + it('says nothing for a transaction that rolled back', async () => { + listen(); + // The row is inserted and then discarded, which is what a transition + // throwing after its history write looks like. Announcing it would tell + // every client about a state change that never happened. + expect(() => db().transaction(() => { + write('op-rollback', 'applied'); + throw new Error('transition rejected'); + })()).toThrow('transition rejected'); + await settle(); + + expect(events).toEqual([]); + expect(GitOpsMetricsService.getInstance().snapshot()).toEqual([]); + }); + + it('does not even queue a replay of the same transition', async () => { + listen(); + expect(write('op-replay', 'applied')).not.toBeNull(); + await settle(); + expect(events).toHaveLength(1); + expect(vi.mocked(enqueueHistoryPublication)).toHaveBeenCalledTimes(1); + + // Same application, operation, stage and dedupe target: the dedupe index + // rejects it, so no row is inserted and nothing is queued. + expect(write('op-replay', 'applied')).toBeNull(); + await settle(); + + expect(vi.mocked(enqueueHistoryPublication)).toHaveBeenCalledTimes(1); + expect(events).toHaveLength(1); + expect(GitOpsMetricsService.getInstance().snapshot()).toEqual([ + { stage: 'applied', outcome: 'committed', count: 1 }, + ]); + }); + + it('counts even when no sink is installed, and says so once', async () => { + events = []; + setGitOpsEventSink(null); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + try { + write('op-nosink', 'deploy_started'); + write('op-nosink', 'deploy_bound', 'committed', { dedupeTarget: 'node:4' }); + await settle(); + + expect(events).toEqual([]); + expect(GitOpsMetricsService.getInstance().snapshot()).toEqual([ + { stage: 'deploy_bound', outcome: 'committed', count: 1 }, + { stage: 'deploy_started', outcome: 'committed', count: 1 }, + ]); + // Once for the batch, not once per row: an unwired sink is one fact, and + // a boot migration would otherwise fill the log with it. + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0][0]).toContain('no event sink installed'); + } finally { + warn.mockRestore(); + } + }); + + it('keeps announcing the batch when one broadcast throws', async () => { + const seen: string[] = []; + setGitOpsEventSink((event) => { + if (event.action === 'fetched') throw new Error('socket gone'); + seen.push(event.action); + }); + write('op-throw', 'fetch_started'); + write('op-throw', 'fetched', 'committed', { dedupeTarget: 'node:1' }); + write('op-throw', 'applied', 'committed', { dedupeTarget: 'node:2' }); + await settle(); + + expect(seen).toEqual(['fetch_started', 'applied']); + // The failed broadcast still happened as far as the model is concerned: + // the transition committed, and the count describes the transition. + expect(GitOpsMetricsService.getInstance().snapshot().map((e) => e.stage)) + .toEqual(['applied', 'fetch_started', 'fetched']); + }); +}); + +describe('GitOpsMetricsService', () => { + afterEach(() => { + GitOpsMetricsService.resetForTests(); + }); + + it('keeps one count per stage and outcome pair', () => { + const metrics = GitOpsMetricsService.getInstance(); + metrics.record('fetched', 'committed'); + metrics.record('fetched', 'committed'); + metrics.record('fetched', 'failed'); + metrics.record('applied', 'committed'); + + expect(metrics.snapshot()).toEqual([ + { stage: 'applied', outcome: 'committed', count: 1 }, + { stage: 'fetched', outcome: 'committed', count: 2 }, + { stage: 'fetched', outcome: 'failed', count: 1 }, + ]); + }); + + it('reports nothing before anything has been recorded', () => { + expect(GitOpsMetricsService.getInstance().snapshot()).toEqual([]); + }); + + it('hands out copies, so a caller cannot edit the counters', () => { + const metrics = GitOpsMetricsService.getInstance(); + metrics.record('applied', 'committed'); + const first = metrics.snapshot(); + first[0].count = 99; + + expect(metrics.snapshot()).toEqual([{ stage: 'applied', outcome: 'committed', count: 1 }]); + }); +}); diff --git a/backend/src/__tests__/gitops-recovery-capture.test.ts b/backend/src/__tests__/gitops-recovery-capture.test.ts new file mode 100644 index 00000000..707fbb41 --- /dev/null +++ b/backend/src/__tests__/gitops-recovery-capture.test.ts @@ -0,0 +1,191 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { captureGitOpsRecoveryBinding } from '../services/gitops/recoveryCapture'; +import { GitOpsStore } from '../services/gitops/store'; +import { GitOpsTransitions, type EventEnvelope } from '../services/gitops/transitions'; +import type { GitOpsApplicationRow, GitOpsGenerationRow } from '../services/gitops/types'; + +describe('gitops recovery capture', () => { + let tmpDir: string; + + beforeAll(async () => { + tmpDir = await setupTestDb(); + GitOpsStore.resetForTests(); + GitOpsTransitions.resetForTests(); + }); + + afterAll(() => { + cleanupTestDb(tmpDir); + }); + + it('returns nulls when no live Direct application exists', () => { + expect(captureGitOpsRecoveryBinding('missing-stack', 1)).toEqual({ + gitops_generation_id: null, + gitops_artifact_set_id: null, + gitops_source_acceptance_ref: null, + }); + }); + + it('captures deployed generation and generation-bound source acceptance', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-cap', 'cap-web'), nodeId: 1, envelope: env('op-act') }); + store.insertGeneration(gen('gen-cap', 'app-cap')); + tx.fetchStarted('app-cap', env('op-f')); + tx.fetched('app-cap', 'abc123', env('op-f')); + tx.candidateReady('app-cap', 'gen-cap', false, env('op-c')); + tx.applied({ + applicationId: 'app-cap', + generationId: 'gen-cap', + artifactSetId: 'art-cap', + sourceAcceptanceId: 'acc-cap', + authority: 'operator', + envelope: env('op-a'), + }); + const target = store.getTarget('app-cap', 1)!; + store.upsertTarget({ ...target, deployed_generation_id: 'gen-cap' }); + expect(captureGitOpsRecoveryBinding('cap-web', 1)).toEqual({ + gitops_generation_id: 'gen-cap', + gitops_artifact_set_id: 'art-cap', + gitops_source_acceptance_ref: 'acc-cap', + }); + }); + + it('does not capture a newer generation acceptance for an older deployed generation', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-old', 'old-web'), nodeId: 1, envelope: env('op-act-2') }); + store.insertGeneration(gen('gen-old', 'app-old')); + store.insertGeneration(gen('gen-new', 'app-old')); + tx.fetchStarted('app-old', env('op-f2')); + tx.fetched('app-old', 'abc123', env('op-f2')); + tx.candidateReady('app-old', 'gen-old', false, env('op-c2')); + tx.applied({ + applicationId: 'app-old', + generationId: 'gen-old', + artifactSetId: 'art-old', + sourceAcceptanceId: 'acc-old', + authority: 'operator', + envelope: env('op-a2'), + }); + store.insertApproval({ + id: 'acc-new', + kind: 'source_acceptance', + authority: 'operator', + authoritative: 1, + application_id: 'app-old', + generation_id: 'gen-new', + intent_revision_id: null, + artifact_set_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + required_targets_json: null, + preflight_fingerprint: null, + fingerprint: null, + blast_json: null, + policy_provenance_json: null, + actor: 'tester', + created_at: 9, + }); + const target = store.getTarget('app-old', 1)!; + store.upsertTarget({ + ...target, + deployed_generation_id: 'gen-old', + source_acceptance_ref: 'acc-new', + }); + const captured = captureGitOpsRecoveryBinding('old-web', 1); + expect(captured.gitops_generation_id).toBe('gen-old'); + expect(captured.gitops_artifact_set_id).toBe('art-old'); + expect(captured.gitops_source_acceptance_ref).toBe('acc-old'); + }); +}); + +function env(operationId: string): EventEnvelope { + return { operationId, actor: 'tester', trigger: 'manual', at: 1 }; +} + +function app(id: string, stackName: string): GitOpsApplicationRow { + return { + id, + lifecycle_key: `direct:${stackName}`, + lifecycle_status: 'active', + target_mode: 'direct', + stack_name: stackName, + blueprint_id: null, + configured_repo_url: 'https://github.com/org/repo.git', + repo_identity_json: '{"host":"github.com","pathname":"/org/repo.git"}', + configured_ref: 'main', + compose_paths_json: '["compose.yml"]', + context_dir: null, + sync_env: 0, + env_path: null, + materialization_fingerprint: 'a'.repeat(64), + desired_commit_sha: null, + fetched_commit_sha: null, + candidate_generation_id: null, + accepted_generation_id: null, + candidate_plan_blocked: 0, + review_required: 0, + artifact_set_id: null, + latest_artifact_set_id: null, + intent_revision_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + rollout_authorization_ref: null, + legacy_combined_approval_ref: null, + preflight_fingerprint: null, + latest_operation_id: null, + active_operation_id: null, + active_operation_stage: null, + active_operation_at: null, + active_generation_id: null, + pause_at: null, + pause_reason: null, + partial_json: null, + failure_stage: null, + failure_class: null, + failure_at: null, + retry_at: null, + retry_count: 0, + suspended_at: null, + recovery_ref: null, + recovery_phase: null, + interruption_stage: null, + interruption_at: null, + interruption_operation_id: null, + interruption_generation_id: null, + evidence_fresh_at: null, + evidence_limitations_json: null, + created_at: 1, + updated_at: 1, + }; +} + +function gen(id: string, applicationId: string): GitOpsGenerationRow { + return { + id, + application_id: applicationId, + commit_sha: id, + repo_url: 'https://github.com/org/repo.git', + configured_ref: 'main', + repo_identity_json: '{"host":"github.com","pathname":"/org/repo.git"}', + manifest_version: 0, + candidate_dir: `generations/candidate-${id}`, + applied_dir: `generations/applied-${id}-0`, + expected_invocation_json: '{"composeFileOrder":[],"projectName":null,"projectDirectory":null,"envFileOrder":[]}', + materialization_fingerprint: 'a'.repeat(64), + validation_ok: 1, + plan_blocked: 0, + change_plan_fingerprint: null, + operation_id: `op-${id}`, + trigger: 'manual', + actor: 'tester', + previous_generation_id: null, + redacted_limitations_json: '[]', + created_at: 1, + }; +} diff --git a/backend/src/__tests__/gitops-recovery.test.ts b/backend/src/__tests__/gitops-recovery.test.ts new file mode 100644 index 00000000..6579b982 --- /dev/null +++ b/backend/src/__tests__/gitops-recovery.test.ts @@ -0,0 +1,505 @@ +/** + * Recovery pointer rules. + * + * A restore moves a target back to an older generation, which is the one case + * where the target and its application legitimately disagree about what is + * current. These tests pin what may move with it and what may not: the + * expectation comes from what the recovery point captured, the acceptance must + * still prove the restored generation, and a last-known-good survives unless + * the generation behind it is genuinely gone. + */ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { DatabaseService } from '../services/DatabaseService'; +import { GitOpsStore } from '../services/gitops/store'; +import { GitOpsTransitions, type EventEnvelope } from '../services/gitops/transitions'; +import { projectApplication } from '../services/gitops/derive'; +import type { GitOpsApplicationRow, GitOpsGenerationRow } from '../services/gitops/types'; + +describe('gitops recovery', () => { + let tmpDir: string; + + beforeAll(async () => { + tmpDir = await setupTestDb(); + GitOpsStore.resetForTests(); + GitOpsTransitions.resetForTests(); + }); + + afterAll(() => { + cleanupTestDb(tmpDir); + }); + + it('marks the target as recovering before anything is restored', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedTwoGenerations('app-rec-start', 'rec-start-web'); + + tx.recoveryStarted({ + applicationId: 'app-rec-start', + nodeId: 1, + recoveryRef: 'rec-1', + recoveryGenerationId: 'gen-a-app-rec-start', + envelope: env('op-rec-start'), + }); + + const target = store.getTarget('app-rec-start', 1)!; + expect(target.recovery_phase).toBe('restoring'); + expect(target.recovery_ref).toBe('rec-1'); + expect(target.active_operation_stage).toBe('recovery_started'); + // Nothing has been restored, so nothing has moved. + expect(target.desired_generation_id).toBe('gen-b-app-rec-start'); + expect(projectApplication('app-rec-start', true).targets[0]?.runtime.status).toBe('recovery_required'); + }); + + it('moves the target back to the restored generation while the application stays ahead', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedTwoGenerations('app-rec-ok', 'rec-ok-web'); + const genA = 'gen-a-app-rec-ok'; + + tx.recoveryStarted({ + applicationId: 'app-rec-ok', + nodeId: 1, + recoveryRef: 'rec-ok', + recoveryGenerationId: genA, + envelope: env('op-rec-ok'), + }); + tx.recoverySucceeded({ + applicationId: 'app-rec-ok', + nodeId: 1, + recoveryRef: 'rec-ok', + recoveryGenerationId: genA, + proven: true, + gitopsBinding: 'bound', + capturedArtifactSetId: 'art-a-app-rec-ok', + capturedSourceAcceptanceRef: 'acc-a-app-rec-ok', + envelope: env('op-rec-ok'), + }); + + const target = store.getTarget('app-rec-ok', 1)!; + expect(target.desired_generation_id).toBe(genA); + expect(target.applied_generation_id).toBe(genA); + expect(target.deployed_generation_id).toBe(genA); + // The restored workload has not been observed healthy yet. + expect(target.healthy_generation_id).toBeNull(); + expect(target.recovery_phase).toBe('complete'); + // The expectation and the acceptance both describe the restored generation. + expect(target.expected_artifact_set_id).toBe('art-a-app-rec-ok'); + expect(target.source_acceptance_ref).toBe('acc-a-app-rec-ok'); + // The application is still accepted at the newer generation. + expect(store.getApplication('app-rec-ok')?.accepted_generation_id).toBe('gen-b-app-rec-ok'); + expect(store.getApplication('app-rec-ok')?.source_acceptance_ref).toBe('acc-b-app-rec-ok'); + }); + + it('refuses to bind an acceptance that authorized a different generation', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedTwoGenerations('app-rec-xacc', 'rec-xacc-web'); + + tx.recoveryStarted({ + applicationId: 'app-rec-xacc', + nodeId: 1, + recoveryRef: 'rec-xacc', + recoveryGenerationId: 'gen-a-app-rec-xacc', + envelope: env('op-rec-xacc'), + }); + tx.recoverySucceeded({ + applicationId: 'app-rec-xacc', + nodeId: 1, + recoveryRef: 'rec-xacc', + recoveryGenerationId: 'gen-a-app-rec-xacc', + proven: true, + gitopsBinding: 'bound', + capturedArtifactSetId: 'art-b-app-rec-xacc', + // The acceptance for B cannot vouch for A. + capturedSourceAcceptanceRef: 'acc-b-app-rec-xacc', + envelope: env('op-rec-xacc'), + }); + + const target = store.getTarget('app-rec-xacc', 1)!; + expect(target.source_acceptance_ref).toBeNull(); + // Nor can B's artifact set describe A. + expect(target.expected_artifact_set_id).toBeNull(); + }); + + it('leaves every pointer alone when the restore cannot be proven', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedTwoGenerations('app-rec-unproven', 'rec-unproven-web'); + const beforeTarget = store.getTarget('app-rec-unproven', 1)!; + + tx.recoveryStarted({ + applicationId: 'app-rec-unproven', + nodeId: 1, + recoveryRef: 'rec-unproven', + recoveryGenerationId: null, + envelope: env('op-rec-unproven'), + }); + tx.recoverySucceeded({ + applicationId: 'app-rec-unproven', + nodeId: 1, + recoveryRef: 'rec-unproven', + recoveryGenerationId: null, + proven: false, + gitopsBinding: 'unbound', + capturedArtifactSetId: null, + capturedSourceAcceptanceRef: null, + envelope: env('op-rec-unproven'), + }); + + const target = store.getTarget('app-rec-unproven', 1)!; + expect(target.desired_generation_id).toBe(beforeTarget.desired_generation_id); + expect(target.applied_generation_id).toBe(beforeTarget.applied_generation_id); + expect(target.healthy_generation_id).toBe(beforeTarget.healthy_generation_id); + expect(target.recovery_phase).toBe('complete'); + }); + + it('keeps a still-valid last-known-good and records why one is lost', () => { + const store = GitOpsStore.getInstance(); + const db = DatabaseService.getInstance().getDb(); + + // A last-known-good on the generation being restored survives intact. + seedTwoGenerations('app-rec-lkg', 'rec-lkg-web'); + db.prepare( + `UPDATE gitops_target_current + SET lkg_generation_id = 'gen-a-app-rec-lkg', lkg_artifact_set_id = 'art-a-app-rec-lkg' + WHERE application_id = 'app-rec-lkg'`, + ).run(); + recover('app-rec-lkg', 'gen-a-app-rec-lkg', 'art-a-app-rec-lkg', 'acc-a-app-rec-lkg'); + let target = store.getTarget('app-rec-lkg', 1)!; + expect(target.lkg_generation_id).toBe('gen-a-app-rec-lkg'); + expect(target.lkg_artifact_set_id).toBe('art-a-app-rec-lkg'); + expect(target.lkg_unavailable_at).toBeNull(); + + // A last-known-good whose generation is gone becomes explicitly + // unavailable, which is a different statement from never having had one. + seedTwoGenerations('app-rec-lkg-gone', 'rec-lkg-gone-web'); + db.prepare( + `UPDATE gitops_target_current + SET lkg_generation_id = 'gen-vanished', lkg_artifact_set_id = NULL + WHERE application_id = 'app-rec-lkg-gone'`, + ).run(); + recover('app-rec-lkg-gone', 'gen-a-app-rec-lkg-gone', 'art-a-app-rec-lkg-gone', 'acc-a-app-rec-lkg-gone'); + target = store.getTarget('app-rec-lkg-gone', 1)!; + expect(target.lkg_generation_id).toBeNull(); + expect(target.lkg_unavailable_reason).toBe('generation_missing'); + expect(projectApplication('app-rec-lkg-gone', true).targets[0]?.lkg.status).toBe('unavailable'); + }); + + it('says why it dropped a pointer it could not prove', () => { + const tx = GitOpsTransitions.getInstance(); + seedTwoGenerations('app-rec-why', 'rec-why-web'); + + tx.recoveryStarted({ + applicationId: 'app-rec-why', + nodeId: 1, + recoveryRef: 'rec-why', + recoveryGenerationId: 'gen-a-app-rec-why', + envelope: env('op-rec-why'), + }); + tx.recoverySucceeded({ + applicationId: 'app-rec-why', + nodeId: 1, + recoveryRef: 'rec-why', + recoveryGenerationId: 'gen-a-app-rec-why', + proven: true, + gitopsBinding: 'bound', + // Both captured references belong to the other generation. + capturedArtifactSetId: 'art-b-app-rec-why', + capturedSourceAcceptanceRef: 'acc-b-app-rec-why', + envelope: env('op-rec-why'), + }); + + // Without these the cleared pointers are indistinguishable from pointers + // that never existed, and the target reads healthier than it is. + const codes = projectApplication('app-rec-why', true).limitations.map((l) => l.code); + expect(codes).toContain('artifact_expectation_unprovable'); + expect(codes).toContain('source_acceptance_unprovable'); + }); + + it('flags an unproven restore so it cannot read as healthy', () => { + const tx = GitOpsTransitions.getInstance(); + seedTwoGenerations('app-rec-flag', 'rec-flag-web'); + + tx.recoveryStarted({ + applicationId: 'app-rec-flag', + nodeId: 1, + recoveryRef: 'rec-flag', + recoveryGenerationId: null, + envelope: env('op-rec-flag'), + }); + tx.recoverySucceeded({ + applicationId: 'app-rec-flag', + nodeId: 1, + recoveryRef: 'rec-flag', + recoveryGenerationId: null, + proven: false, + gitopsBinding: 'unbound', + capturedArtifactSetId: null, + capturedSourceAcceptanceRef: null, + envelope: env('op-rec-flag'), + }); + + const codes = projectApplication('app-rec-flag', true).limitations.map((l) => l.code); + expect(codes).toContain('recovery_unproven'); + }); + + it('clears a limitation once the evidence is provable again', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedTwoGenerations('app-rec-clear', 'rec-clear-web'); + const genA = 'gen-a-app-rec-clear'; + + const restore = (artifactSetId: string, acceptanceRef: string): void => { + tx.recoveryStarted({ + applicationId: 'app-rec-clear', + nodeId: 1, + recoveryRef: 'rec-clear', + recoveryGenerationId: genA, + envelope: env(`op-rec-clear-${artifactSetId}`), + }); + tx.recoverySucceeded({ + applicationId: 'app-rec-clear', + nodeId: 1, + recoveryRef: 'rec-clear', + recoveryGenerationId: genA, + proven: true, + gitopsBinding: 'bound', + capturedArtifactSetId: artifactSetId, + capturedSourceAcceptanceRef: acceptanceRef, + envelope: env(`op-rec-clear-${artifactSetId}`), + }); + }; + + restore('art-b-app-rec-clear', 'acc-b-app-rec-clear'); + expect(store.getTarget('app-rec-clear', 1)?.evidence_limitations_json).not.toBeNull(); + + restore('art-a-app-rec-clear', 'acc-a-app-rec-clear'); + // A stale limitation is worse than none: it would keep reporting doubt + // about evidence that is now proven. + expect(store.getTarget('app-rec-clear', 1)?.evidence_limitations_json).toBeNull(); + expect(projectApplication('app-rec-clear', true).limitations).toHaveLength(0); + }); + + it('opens and closes a recovery from the restore path itself', async () => { + const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService'); + const store = GitOpsStore.getInstance(); + seedTwoGenerations('app-rec-wire', 'rec-wire-web'); + const genA = 'gen-a-app-rec-wire'; + + // A recovery row bound to generation A, exactly as capture writes one. + DatabaseService.getInstance().insertStackUpdateRecoveryGeneration({ + id: 'rec-wire-1', + node_id: 1, + stack_name: 'rec-wire-web', + status: 'candidate', + phase: 'captured', + is_current: 0, + operation_kind: 'update', + content_path: null, + backup_slot_id: null, + services_json: '[]', + override_path: null, + health_gate_id: null, + gate_retain_until: null, + artifact_expires_at: null, + operation_lease_expires_at: Date.now() + 60_000, + created_at: Date.now(), + updated_at: Date.now(), + created_by: 'tester', + artifacts_retired: 0, + released_at: null, + released_by: null, + gitops_generation_id: genA, + gitops_artifact_set_id: 'art-a-app-rec-wire', + gitops_source_acceptance_ref: 'acc-a-app-rec-wire', + }); + + // The restore fails before touching files, which is the classification the + // model has to get right: the previous workload is provably intact. + await StackUpdateRecoveryService.getInstance().compensateWithCandidate( + 'rec-wire-1', + async () => { throw new Error('compose unavailable'); }, + ); + + const target = store.getTarget('app-rec-wire', 1)!; + expect(target.recovery_phase).toBe('failed'); + expect(target.failure_stage).toBe('recovery'); + expect(target.failure_class).toBe('pre_mutation'); + expect(target.active_operation_stage).toBeNull(); + // The restore never completed, so nothing moved back to generation A. + expect(target.desired_generation_id).toBe('gen-b-app-rec-wire'); + expect(projectApplication('app-rec-wire', true).targets[0]?.runtime.status).toBe('recovery_failed'); + }); + + it('records a failed restore without moving success pointers', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedTwoGenerations('app-rec-fail', 'rec-fail-web'); + const before = store.getTarget('app-rec-fail', 1)!; + + tx.recoveryStarted({ + applicationId: 'app-rec-fail', + nodeId: 1, + recoveryRef: 'rec-fail', + recoveryGenerationId: 'gen-a-app-rec-fail', + envelope: env('op-rec-fail'), + }); + tx.recoveryFailed({ + applicationId: 'app-rec-fail', + nodeId: 1, + recoveryRef: 'rec-fail', + failureClass: 'post_mutation', + envelope: env('op-rec-fail'), + }); + + const target = store.getTarget('app-rec-fail', 1)!; + expect(target.recovery_phase).toBe('failed'); + expect(target.failure_stage).toBe('recovery'); + expect(target.failure_class).toBe('post_mutation'); + expect(target.desired_generation_id).toBe(before.desired_generation_id); + expect(target.active_operation_stage).toBeNull(); + + const projection = projectApplication('app-rec-fail', true); + if (projection.targetMode === 'not_applicable') throw new Error('expected an application'); + expect(projection.targets[0]?.runtime.status).toBe('recovery_failed'); + expect(projection.facets.source.status).toBe('recovery_failed'); + }); +}); + +function recover( + applicationId: string, + generationId: string, + artifactSetId: string, + acceptanceRef: string, +): void { + const tx = GitOpsTransitions.getInstance(); + tx.recoveryStarted({ + applicationId, + nodeId: 1, + recoveryRef: `rec-${applicationId}`, + recoveryGenerationId: generationId, + envelope: env(`op-${applicationId}`), + }); + tx.recoverySucceeded({ + applicationId, + nodeId: 1, + recoveryRef: `rec-${applicationId}`, + recoveryGenerationId: generationId, + proven: true, + gitopsBinding: 'bound', + capturedArtifactSetId: artifactSetId, + capturedSourceAcceptanceRef: acceptanceRef, + envelope: env(`op-${applicationId}`), + }); +} + +/** Apply generation A, then B, so the target has something to fall back to. */ +function seedTwoGenerations(applicationId: string, stackName: string): void { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app(applicationId, stackName), nodeId: 1, envelope: env(`op-act-${applicationId}`) }); + for (const label of ['a', 'b'] as const) { + const generationId = `gen-${label}-${applicationId}`; + store.insertGeneration(gen(generationId, applicationId)); + tx.fetchStarted(applicationId, env(`op-f-${label}-${applicationId}`)); + tx.fetched(applicationId, `sha-${label}`, env(`op-f-${label}-${applicationId}`)); + tx.candidateReady(applicationId, generationId, false, env(`op-c-${label}-${applicationId}`)); + tx.applied({ + applicationId, + generationId, + artifactSetId: `art-${label}-${applicationId}`, + sourceAcceptanceId: `acc-${label}-${applicationId}`, + authority: 'operator', + envelope: env(`op-a-${label}-${applicationId}`), + }); + } +} + +function env(operationId: string): EventEnvelope { + return { operationId, actor: 'tester', trigger: 'manual', at: Date.now() }; +} + +function app(id: string, stackName: string): GitOpsApplicationRow { + return { + id, + lifecycle_key: `direct:${stackName}`, + lifecycle_status: 'active', + target_mode: 'direct', + stack_name: stackName, + blueprint_id: null, + configured_repo_url: 'https://github.com/org/repo.git', + repo_identity_json: '{"host":"github.com","pathname":"/org/repo.git"}', + configured_ref: 'main', + compose_paths_json: '["compose.yml"]', + context_dir: null, + sync_env: 0, + env_path: null, + materialization_fingerprint: 'a'.repeat(64), + desired_commit_sha: null, + fetched_commit_sha: null, + candidate_generation_id: null, + accepted_generation_id: null, + candidate_plan_blocked: 0, + review_required: 0, + artifact_set_id: null, + latest_artifact_set_id: null, + intent_revision_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + rollout_authorization_ref: null, + legacy_combined_approval_ref: null, + preflight_fingerprint: null, + latest_operation_id: null, + active_operation_id: null, + active_operation_stage: null, + active_operation_at: null, + active_generation_id: null, + pause_at: null, + pause_reason: null, + partial_json: null, + failure_stage: null, + failure_class: null, + failure_at: null, + retry_at: null, + retry_count: 0, + suspended_at: null, + recovery_ref: null, + recovery_phase: null, + interruption_stage: null, + interruption_at: null, + interruption_operation_id: null, + interruption_generation_id: null, + evidence_fresh_at: null, + evidence_limitations_json: null, + created_at: 1, + updated_at: 1, + }; +} + +function gen(id: string, applicationId: string): GitOpsGenerationRow { + return { + id, + application_id: applicationId, + commit_sha: 'abc123', + repo_url: 'https://github.com/org/repo.git', + configured_ref: 'main', + repo_identity_json: '{"host":"github.com","pathname":"/org/repo.git"}', + manifest_version: 0, + candidate_dir: `generations/candidate-${id}`, + applied_dir: `generations/applied-${id}-0`, + expected_invocation_json: '{"composeFileOrder":[],"projectName":null,"projectDirectory":null,"envFileOrder":[]}', + materialization_fingerprint: 'a'.repeat(64), + validation_ok: 1, + plan_blocked: 0, + change_plan_fingerprint: null, + operation_id: `op-${id}`, + trigger: 'manual', + actor: 'tester', + previous_generation_id: null, + redacted_limitations_json: '[]', + created_at: 1, + }; +} diff --git a/backend/src/__tests__/gitops-repo-identity.test.ts b/backend/src/__tests__/gitops-repo-identity.test.ts new file mode 100644 index 00000000..440e56e1 --- /dev/null +++ b/backend/src/__tests__/gitops-repo-identity.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from 'vitest'; +import { + parseHttpsRepoUrl, + parseLegacyRepoUrl, + secretFreeRepoUrl, + serializeRepoIdentity, +} from '../services/gitops/repoIdentity'; +import { canonicalMaterialConfigJson, materializationFingerprint } from '../services/gitops/fingerprint'; + +describe('secret-free repository identity', () => { + it('rejects userinfo, query, fragment, and non-https urls', () => { + expect(parseHttpsRepoUrl('http://github.com/org/repo.git').ok).toBe(false); + const userinfo = parseHttpsRepoUrl('https://user:pass@github.com/org/repo.git'); + const query = parseHttpsRepoUrl('https://github.com/org/repo.git?token=1'); + const fragment = parseHttpsRepoUrl('https://github.com/org/repo.git#frag'); + expect(userinfo.ok ? null : userinfo.reason).toBe('userinfo'); + expect(query.ok ? null : query.reason).toBe('query'); + expect(fragment.ok ? null : fragment.reason).toBe('fragment'); + expect(parseHttpsRepoUrl('https://github.com/org/repo.git').ok).toBe(true); + }); + + it('serializes host and pathname only', () => { + const parsed = parseHttpsRepoUrl('https://github.com/org/repo.git'); + if (!parsed.ok) throw new Error('expected parse success'); + const identity = serializeRepoIdentity(parsed.url); + expect(identity).toEqual({ host: 'github.com', pathname: '/org/repo.git' }); + expect(secretFreeRepoUrl(identity)).toBe('https://github.com/org/repo.git'); + }); + + describe('legacy operational urls (migration only)', () => { + it('strips userinfo, query, and fragment instead of refusing the stack', () => { + for (const raw of [ + 'https://user:pass@github.com/org/repo.git', + 'https://github.com/org/repo.git?token=secret', + 'https://github.com/org/repo.git#frag', + 'https://user:pass@github.com/org/repo.git?token=secret#frag', + ]) { + const parsed = parseLegacyRepoUrl(raw); + if (!parsed.ok) throw new Error(`expected legacy parse success for ${raw}`); + expect({ host: parsed.url.host, pathname: parsed.url.pathname }).toEqual({ + host: 'github.com', + pathname: '/org/repo.git', + }); + expect(parsed.url.username).toBe(''); + expect(parsed.url.password).toBe(''); + expect(parsed.url.search).toBe(''); + expect(parsed.url.hash).toBe(''); + expect(secretFreeRepoUrl(serializeRepoIdentity(parsed.url))).toBe('https://github.com/org/repo.git'); + } + }); + + it('still refuses what has no storable identity', () => { + expect(parseLegacyRepoUrl('http://github.com/org/repo.git').ok).toBe(false); + expect(parseLegacyRepoUrl('not a url at all').ok).toBe(false); + expect(parseLegacyRepoUrl('').ok).toBe(false); + expect(parseLegacyRepoUrl(`https://github.com/${'x'.repeat(2100)}`).ok).toBe(false); + }); + }); + + it('fingerprints material config in the fixed key order', () => { + const json = canonicalMaterialConfigJson({ + repoIdentity: { host: 'github.com', pathname: '/org/repo.git' }, + configuredRef: 'main', + composePaths: ['compose.yml'], + contextDir: ' ', + syncEnv: false, + envPath: '.env', + }); + expect(json).toBe(JSON.stringify({ + composePaths: ['compose.yml'], + contextDir: null, + syncEnv: false, + envPath: null, + repoIdentity: { host: 'github.com', pathname: '/org/repo.git' }, + configuredRef: 'main', + })); + expect(materializationFingerprint({ + repoIdentity: { host: 'github.com', pathname: '/org/repo.git' }, + configuredRef: 'main', + composePaths: ['compose.yml'], + contextDir: null, + syncEnv: false, + envPath: null, + })).toMatch(/^[0-9a-f]{64}$/); + const synced = canonicalMaterialConfigJson({ + repoIdentity: { host: 'github.com', pathname: '/org/repo.git' }, + configuredRef: 'main', + composePaths: ['compose.yml'], + contextDir: null, + syncEnv: true, + envPath: '.env', + }); + expect(JSON.parse(synced).envPath).toBe('.env'); + expect(JSON.parse(synced).syncEnv).toBe(true); + }); +}); diff --git a/backend/src/__tests__/gitops-schema.test.ts b/backend/src/__tests__/gitops-schema.test.ts new file mode 100644 index 00000000..cde54bed --- /dev/null +++ b/backend/src/__tests__/gitops-schema.test.ts @@ -0,0 +1,334 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { isHubOnlyPath } from '../helpers/proxyExemptPaths'; +import { GitOpsStore, emptyTargetRow } from '../services/gitops/store'; +import { encodeArtifactEvidenceJson } from '../services/gitops/json'; +import type { GitOpsApplicationRow, GitOpsGenerationRow } from '../services/gitops/types'; + +describe('gitops schema', () => { + let tmpDir: string; + + beforeAll(async () => { + tmpDir = await setupTestDb(); + GitOpsStore.resetForTests(); + }); + + afterAll(() => { + cleanupTestDb(tmpDir); + }); + + it('creates gitops tables, recovery columns, and the schema version', async () => { + const { DatabaseService } = await import('../services/DatabaseService'); + const db = DatabaseService.getInstance().getDb(); + const tables = db.prepare( + "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'gitops_%' ORDER BY name", + ).all() as Array<{ name: string }>; + expect(tables.map((row) => row.name)).toEqual([ + 'gitops_applications', + 'gitops_approvals', + 'gitops_artifact_sets', + 'gitops_create_checkpoints', + 'gitops_generations', + 'gitops_history', + 'gitops_intent_revisions', + 'gitops_migration_checkpoints', + 'gitops_rollout_candidates', + 'gitops_target_current', + ]); + const version = db.prepare( + "SELECT value FROM global_settings WHERE key = 'gitops_schema_version'", + ).get() as { value: string }; + expect(version.value).toBe('1'); + const recoveryCols = new Set( + (db.pragma('table_info(stack_update_recovery_generations)') as Array<{ name: string }>).map((c) => c.name), + ); + expect(recoveryCols.has('gitops_generation_id')).toBe(true); + expect(recoveryCols.has('gitops_artifact_set_id')).toBe(true); + expect(recoveryCols.has('gitops_source_acceptance_ref')).toBe(true); + expect(recoveryCols.has('desired_target_generation_id')).toBe(false); + const appCols = new Set( + (db.pragma('table_info(gitops_applications)') as Array<{ name: string }>).map((c) => c.name), + ); + expect(appCols.has('desired_target_generation_id')).toBe(false); + expect(appCols.has('desired_commit_sha')).toBe(true); + const targetCols = new Set( + (db.pragma('table_info(gitops_target_current)') as Array<{ name: string }>).map((c) => c.name), + ); + expect(targetCols.has('desired_generation_id')).toBe(true); + expect(targetCols.has('candidate_generation_id')).toBe(true); + expect(targetCols.has('lkg_artifact_set_id')).toBe(true); + expect(targetCols.has('lkg_unavailable_at')).toBe(true); + expect(targetCols.has('lkg_unavailable_reason')).toBe(true); + const candidateCols = new Set( + (db.pragma('table_info(gitops_rollout_candidates)') as Array<{ name: string }>).map((c) => c.name), + ); + expect(candidateCols.has('source_acceptance_ref')).toBe(false); + expect(candidateCols.has('placement_approval_ref')).toBe(false); + expect(candidateCols.has('preflight_fingerprint')).toBe(false); + }); + + it('accepts recovery health triggers and keeps deployed_generation_id', async () => { + const { DatabaseService } = await import('../services/DatabaseService'); + const db = DatabaseService.getInstance(); + db.insertHealthGateRun({ + id: 'rec-1', + node_id: 1, + stack_name: 'web', + trigger_action: 'recovery', + status: 'observing', + reason: null, + window_seconds: 90, + containers_json: '[]', + started_at: 1, + ended_at: null, + created_by: 'tester', + target_scope: 'stack', + service_name: null, + failure_source: null, + deployed_generation_id: 'gen-a', + }); + const row = db.getHealthGateRun(1, 'web', 'rec-1'); + expect(row?.trigger_action).toBe('recovery'); + expect(row?.deployed_generation_id).toBe('gen-a'); + }); + + it('rejects invalid target recovery phases and LKG mismatches', async () => { + const { DatabaseService } = await import('../services/DatabaseService'); + const db = DatabaseService.getInstance().getDb(); + const store = GitOpsStore.getInstance(); + store.insertApplication(directApp('app-lkg', 'lkg-web')); + store.upsertTarget(emptyTargetRow('app-lkg', 1, 1)); + expect(() => { + db.prepare("UPDATE gitops_target_current SET recovery_phase = 'armed' WHERE application_id = 'app-lkg'").run(); + }).toThrow(); + expect(() => { + store.upsertTarget({ + ...emptyTargetRow('app-lkg', 1, 2), + lkg_generation_id: 'gen-missing', + lkg_artifact_set_id: 'art-missing', + }); + }).toThrow(/lkg_artifact_set_id/); + }); + + it('enforces one live Blueprint application across both Blueprint modes', async () => { + const store = GitOpsStore.getInstance(); + store.insertApplication(inlineApp('bp-inline', 7)); + expect(store.assertNoLiveBlueprintApplication(7).ok).toBe(false); + expect(() => store.insertApplication(blueprintApp('bp-git', 7))).toThrow(); + const { DatabaseService } = await import('../services/DatabaseService'); + DatabaseService.getInstance().getDb().prepare( + "UPDATE gitops_applications SET lifecycle_status = 'detached' WHERE id = 'bp-inline'", + ).run(); + store.insertApplication(blueprintApp('bp-git', 7)); + expect(store.getApplication('bp-git')?.target_mode).toBe('blueprint'); + }); + + it('round-trips recovery GitOps columns as null on legacy-shaped inserts', async () => { + const { DatabaseService } = await import('../services/DatabaseService'); + const db = DatabaseService.getInstance(); + db.insertStackUpdateRecoveryGeneration({ + id: 'recov-1', + node_id: 1, + stack_name: 'web', + status: 'candidate', + phase: 'captured', + is_current: 1, + backup_slot_id: null, + content_path: null, + operation_kind: null, + override_path: null, + services_json: '[]', + health_gate_id: null, + gate_retain_until: null, + artifact_expires_at: null, + operation_lease_expires_at: null, + created_at: 1, + updated_at: 1, + created_by: 'tester', + artifacts_retired: 0, + released_at: null, + released_by: null, + }); + const row = db.getStackUpdateRecoveryGeneration('recov-1'); + expect(row?.gitops_generation_id ?? null).toBeNull(); + expect(row?.gitops_artifact_set_id ?? null).toBeNull(); + expect(row?.gitops_source_acceptance_ref ?? null).toBeNull(); + db.insertStackUpdateRecoveryGeneration({ + id: 'recov-2', + node_id: 1, + stack_name: 'web', + status: 'candidate', + phase: 'captured', + is_current: 0, + backup_slot_id: null, + content_path: null, + operation_kind: null, + override_path: null, + services_json: '[]', + health_gate_id: null, + gate_retain_until: null, + artifact_expires_at: null, + operation_lease_expires_at: null, + created_at: 2, + updated_at: 2, + created_by: 'tester', + artifacts_retired: 0, + released_at: null, + released_by: null, + gitops_generation_id: 'gen-a', + gitops_artifact_set_id: 'art-a', + gitops_source_acceptance_ref: 'acc-a', + }); + const bound = db.getStackUpdateRecoveryGeneration('recov-2'); + expect(bound?.gitops_generation_id).toBe('gen-a'); + expect(bound?.gitops_artifact_set_id).toBe('art-a'); + expect(bound?.gitops_source_acceptance_ref).toBe('acc-a'); + }); + + it('enforces one live Direct application per stack and frees the name on tombstone', async () => { + const store = GitOpsStore.getInstance(); + store.insertApplication(directApp('dup-first', 'dup-web')); + expect(() => store.insertApplication(directApp('dup-second', 'dup-web'))).toThrow(); + const { DatabaseService } = await import('../services/DatabaseService'); + DatabaseService.getInstance().getDb().prepare( + "UPDATE gitops_applications SET lifecycle_status = 'deleted' WHERE id = 'dup-first'", + ).run(); + store.insertApplication(directApp('dup-second', 'dup-web')); + expect(store.getApplication('dup-second')?.stack_name).toBe('dup-web'); + }); + + it('keeps blueprints and node-labels hub-only and git-sources proxyable', () => { + expect(isHubOnlyPath('/api/blueprints')).toBe(true); + expect(isHubOnlyPath('/api/blueprints/1')).toBe(true); + expect(isHubOnlyPath('/api/node-labels')).toBe(true); + expect(isHubOnlyPath('/api/node-labels/1')).toBe(true); + expect(isHubOnlyPath('/api/git-sources')).toBe(false); + expect(isHubOnlyPath('/api/gitops/history')).toBe(false); + }); + + it('inserts unresolved artifact evidence without advancing authority', () => { + const store = GitOpsStore.getInstance(); + store.insertApplication(directApp('app-art', 'art-web')); + store.insertGeneration(generation('gen-art', 'app-art')); + store.insertArtifactSet({ + id: 'art-1', + generation_id: 'gen-art', + evidence_version: 1, + authoritative: 0, + qualification: 'unresolved', + evidence_json: encodeArtifactEvidenceJson({ kind: 'unresolved' }), + created_at: 1, + }); + expect(store.getArtifactSet('art-1')?.qualification).toBe('unresolved'); + }); +}); + +function directApp(id: string, stackName: string): GitOpsApplicationRow { + return { + id, + lifecycle_key: `direct:${stackName}`, + lifecycle_status: 'active', + target_mode: 'direct', + stack_name: stackName, + blueprint_id: null, + configured_repo_url: 'https://github.com/org/repo.git', + repo_identity_json: '{"host":"github.com","pathname":"/org/repo.git"}', + configured_ref: 'main', + compose_paths_json: '["compose.yml"]', + context_dir: null, + sync_env: 0, + env_path: null, + materialization_fingerprint: 'a'.repeat(64), + desired_commit_sha: null, + fetched_commit_sha: null, + candidate_generation_id: null, + accepted_generation_id: null, + candidate_plan_blocked: 0, + review_required: 0, + artifact_set_id: null, + latest_artifact_set_id: null, + intent_revision_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + rollout_authorization_ref: null, + legacy_combined_approval_ref: null, + preflight_fingerprint: null, + latest_operation_id: null, + active_operation_id: null, + active_operation_stage: null, + active_operation_at: null, + active_generation_id: null, + pause_at: null, + pause_reason: null, + partial_json: null, + failure_stage: null, + failure_class: null, + failure_at: null, + retry_at: null, + retry_count: 0, + suspended_at: null, + recovery_ref: null, + recovery_phase: null, + interruption_stage: null, + interruption_at: null, + interruption_operation_id: null, + interruption_generation_id: null, + evidence_fresh_at: null, + evidence_limitations_json: null, + created_at: 1, + updated_at: 1, + }; +} + +function inlineApp(id: string, blueprintId: number): GitOpsApplicationRow { + return { + ...directApp(id, 'unused'), + lifecycle_key: `blueprint:${blueprintId}`, + target_mode: 'inline_blueprint', + stack_name: null, + blueprint_id: blueprintId, + configured_repo_url: null, + repo_identity_json: null, + configured_ref: null, + compose_paths_json: null, + materialization_fingerprint: null, + }; +} + +function blueprintApp(id: string, blueprintId: number): GitOpsApplicationRow { + return { + ...directApp(id, 'unused'), + lifecycle_key: `blueprint:${blueprintId}`, + target_mode: 'blueprint', + stack_name: null, + blueprint_id: blueprintId, + configured_repo_url: 'https://github.com/org/repo.git', + }; +} + +function generation(id: string, applicationId: string): GitOpsGenerationRow { + return { + id, + application_id: applicationId, + commit_sha: 'abc123', + repo_url: 'https://github.com/org/repo.git', + configured_ref: 'main', + repo_identity_json: '{"host":"github.com","pathname":"/org/repo.git"}', + manifest_version: 0, + candidate_dir: 'generations/candidate-abc123', + applied_dir: 'generations/applied-abc123-0', + expected_invocation_json: '{"composeFileOrder":["compose.yml"],"projectName":null,"projectDirectory":null,"envFileOrder":[]}', + materialization_fingerprint: 'a'.repeat(64), + validation_ok: 1, + plan_blocked: 0, + change_plan_fingerprint: null, + operation_id: 'op-1', + trigger: 'manual', + actor: 'tester', + previous_generation_id: null, + redacted_limitations_json: '[]', + created_at: 1, + }; +} diff --git a/backend/src/__tests__/gitops-transitions.test.ts b/backend/src/__tests__/gitops-transitions.test.ts new file mode 100644 index 00000000..ebb9c2dc --- /dev/null +++ b/backend/src/__tests__/gitops-transitions.test.ts @@ -0,0 +1,641 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { DatabaseService } from '../services/DatabaseService'; +import { encodeArtifactEvidenceJson } from '../services/gitops/json'; +import { GitOpsStore } from '../services/gitops/store'; +import { GitOpsTransitions, type EventEnvelope } from '../services/gitops/transitions'; +import { projectApplication } from '../services/gitops/derive'; +import type { GitOpsApplicationRow, GitOpsGenerationRow } from '../services/gitops/types'; + +describe('gitops transitions', () => { + let tmpDir: string; + + beforeAll(async () => { + tmpDir = await setupTestDb(); + GitOpsStore.resetForTests(); + GitOpsTransitions.resetForTests(); + }); + + afterAll(() => { + cleanupTestDb(tmpDir); + }); + + it('binds desired+applied and source acceptance on Direct apply', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + const env = envelope('op-apply'); + tx.activateDirect({ application: app('app-apply', 'apply-web'), nodeId: 1, envelope: env }); + store.insertGeneration(gen('gen-apply', 'app-apply')); + tx.fetchStarted('app-apply', envelope('op-fetch')); + tx.fetched('app-apply', 'deadbeef', envelope('op-fetch')); + tx.candidateReady('app-apply', 'gen-apply', false, envelope('op-cand')); + tx.applyStarted('app-apply', 'gen-apply', envelope('op-apply')); + tx.applied({ + applicationId: 'app-apply', + generationId: 'gen-apply', + artifactSetId: 'art-apply', + sourceAcceptanceId: 'acc-apply', + authority: 'operator', + envelope: env, + }); + const application = store.getApplication('app-apply')!; + const target = store.getTarget('app-apply', 1)!; + expect(application.accepted_generation_id).toBe('gen-apply'); + expect(application.source_acceptance_ref).toBe('acc-apply'); + expect(target.desired_generation_id).toBe('gen-apply'); + expect(target.applied_generation_id).toBe('gen-apply'); + expect(target.candidate_generation_id).toBeNull(); + expect(target.source_acceptance_ref).toBe('acc-apply'); + expect(target.expected_artifact_set_id).toBe('art-apply'); + expect(store.resolveApprovalRef('acc-apply', { + kind: 'source_acceptance', + applicationId: 'app-apply', + generationId: 'gen-apply', + })?.authoritative).toBe(1); + expect(() => tx.applied({ + applicationId: 'app-apply', + generationId: 'gen-other', + artifactSetId: 'art-x', + sourceAcceptanceId: 'acc-x', + authority: 'operator', + envelope: envelope('op-apply-2'), + })).toThrow(/not the current candidate/); + }); + + it('advances expected only on first exact after unaccepted rows', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedApplied('app-art', 'art-web', 'gen-art', 'art-v1', 'acc-art'); + tx.recordArtifactEvidence({ + applicationId: 'app-art', + generationId: 'gen-art', + artifactSetId: 'art-v2', + evidenceVersion: 2, + qualification: 'unavailable', + evidenceJson: encodeArtifactEvidenceJson({ kind: 'unavailable' }), + authoritative: 0, + envelope: envelope('op-art-2'), + }); + tx.recordArtifactEvidence({ + applicationId: 'app-art', + generationId: 'gen-art', + artifactSetId: 'art-v3', + evidenceVersion: 3, + qualification: 'exact', + evidenceJson: encodeArtifactEvidenceJson({ kind: 'exact', identity: 'sha256:aaa' }), + authoritative: 0, + envelope: envelope('op-art-3'), + }); + const application = store.getApplication('app-art')!; + expect(application.artifact_set_id).toBe('art-v3'); + expect(application.latest_artifact_set_id).toBe('art-v3'); + tx.recordArtifactEvidence({ + applicationId: 'app-art', + generationId: 'gen-art', + artifactSetId: 'art-v4', + evidenceVersion: 4, + qualification: 'stale', + evidenceJson: encodeArtifactEvidenceJson({ kind: 'stale', identity: 'sha256:aaa' }), + authoritative: 0, + envelope: envelope('op-art-4'), + }); + expect(store.getApplication('app-art')?.artifact_set_id).toBe('art-v3'); + expect(store.getApplication('app-art')?.latest_artifact_set_id).toBe('art-v4'); + tx.recordArtifactEvidence({ + applicationId: 'app-art', + generationId: 'gen-art', + artifactSetId: 'art-v5', + evidenceVersion: 5, + qualification: 'exact', + evidenceJson: encodeArtifactEvidenceJson({ kind: 'exact', identity: 'sha256:bbb' }), + authoritative: 0, + envelope: envelope('op-art-5'), + }); + expect(store.getApplication('app-art')?.artifact_set_id).toBe('art-v3'); + expect(store.getApplication('app-art')?.latest_artifact_set_id).toBe('art-v5'); + expect(store.getTarget('app-art', 1)?.expected_artifact_set_id).toBe('art-v3'); + expect(store.getTarget('app-art', 1)?.latest_artifact_set_id).toBe('art-v5'); + }); + + it('clears fetch failure on successful fetch and keeps accepted pointers', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedApplied('app-fail', 'fail-web', 'gen-fail', 'art-fail', 'acc-fail'); + tx.fetchStarted('app-fail', envelope('op-fail')); + tx.fetchFailed('app-fail', envelope('op-fail')); + expect(store.getApplication('app-fail')?.failure_stage).toBe('fetch'); + expect(store.getApplication('app-fail')?.accepted_generation_id).toBe('gen-fail'); + tx.fetchStarted('app-fail', envelope('op-fail-2')); + tx.fetched('app-fail', 'cafebabe', envelope('op-fail-2')); + const application = store.getApplication('app-fail')!; + expect(application.failure_stage).toBeNull(); + expect(application.desired_commit_sha).toBe('cafebabe'); + expect(application.accepted_generation_id).toBe('gen-fail'); + expect(store.getTarget('app-fail', 1)?.applied_generation_id).toBe('gen-fail'); + }); + + it('rejects a candidate whose fingerprint no longer matches configuration', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-fp', 'fp-web'), nodeId: 1, envelope: envelope('op-act-fp') }); + store.insertGeneration(gen('gen-fp', 'app-fp')); + DatabaseService.getInstance().getDb().prepare( + "UPDATE gitops_applications SET materialization_fingerprint = ? WHERE id = 'app-fp'", + ).run('b'.repeat(64)); + expect(() => tx.candidateReady('app-fp', 'gen-fp', false, envelope('op-c-fp'))).toThrow(/fingerprint/); + DatabaseService.getInstance().getDb().prepare( + "UPDATE gitops_applications SET materialization_fingerprint = ? WHERE id = 'app-fp'", + ).run('a'.repeat(64)); + tx.fetchStarted('app-fp', envelope('op-f-fp')); + tx.fetched('app-fp', 'abc123', envelope('op-f-fp')); + tx.candidateReady('app-fp', 'gen-fp', false, envelope('op-c-fp2')); + DatabaseService.getInstance().getDb().prepare( + "UPDATE gitops_applications SET materialization_fingerprint = ? WHERE id = 'app-fp'", + ).run('c'.repeat(64)); + expect(() => tx.applyStarted('app-fp', 'gen-fp', envelope('op-a-fp'))).toThrow(/fingerprint/); + }); + + it('refuses to replace the candidate while an apply is in flight', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-race', 'race-web'), nodeId: 1, envelope: envelope('op-act-race') }); + store.insertGeneration(gen('gen-race-a', 'app-race')); + store.insertGeneration(gen('gen-race-b', 'app-race')); + tx.fetchStarted('app-race', envelope('op-f-race')); + tx.fetched('app-race', 'abc123', envelope('op-f-race')); + tx.candidateReady('app-race', 'gen-race-a', false, envelope('op-c-race-a')); + tx.applyStarted('app-race', 'gen-race-a', envelope('op-a-race')); + expect(() => tx.candidateReady('app-race', 'gen-race-b', false, envelope('op-c-race-b'))) + .toThrow(/apply is in flight/); + expect(store.getApplication('app-race')?.candidate_generation_id).toBe('gen-race-a'); + tx.applied({ + applicationId: 'app-race', + generationId: 'gen-race-a', + artifactSetId: 'art-race', + sourceAcceptanceId: 'acc-race', + authority: 'operator', + envelope: envelope('op-a-race'), + }); + expect(store.getApplication('app-race')?.accepted_generation_id).toBe('gen-race-a'); + }); + + it('rejects re-accepting a generation that is already accepted', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedApplied('app-reapply', 'reapply-web', 'gen-reapply', 'art-reapply', 'acc-reapply'); + tx.candidateReady('app-reapply', 'gen-reapply', false, envelope('op-c-reapply-2')); + expect(() => tx.applied({ + applicationId: 'app-reapply', + generationId: 'gen-reapply', + artifactSetId: 'art-reapply-2', + sourceAcceptanceId: 'acc-reapply-2', + authority: 'operator', + envelope: envelope('op-a-reapply-2'), + })).toThrow(/already accepted/); + expect(store.getArtifactSet('art-reapply-2')).toBeUndefined(); + }); + + it('surfaces an invalid stored observation as a limitation instead of a clean unknown', () => { + const store = GitOpsStore.getInstance(); + seedApplied('app-obs', 'obs-web', 'gen-obs', 'art-obs', 'acc-obs'); + DatabaseService.getInstance().getDb().prepare( + "UPDATE gitops_target_current SET observed_artifact_identity_json = ? WHERE application_id = 'app-obs'", + ).run('{"kind":"nonsense"}'); + const projection = mustProject('app-obs'); + expect(projection.limitations.map((l) => l.code)).toContain('artifact_observation_invalid'); + expect(store.getTarget('app-obs', 1)?.observed_artifact_identity_json).toBe('{"kind":"nonsense"}'); + }); + + it('advances the fetched SHA on an invalid commit without minting a candidate', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-inv', 'inv-web'), nodeId: 1, envelope: envelope('op-act-inv') }); + tx.fetchStarted('app-inv', envelope('op-f-inv')); + tx.fetchedInvalid('app-inv', 'bad1234', envelope('op-f-inv')); + const application = store.getApplication('app-inv')!; + expect(application.desired_commit_sha).toBe('bad1234'); + expect(application.fetched_commit_sha).toBe('bad1234'); + expect(application.candidate_generation_id).toBeNull(); + expect(application.failure_stage).toBe('validation'); + expect(mustProject('app-inv').facets.source.status).toBe('source_failed'); + }); + + it('exposes a blocked candidate without allowing it to apply', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-blk', 'blk-web'), nodeId: 1, envelope: envelope('op-act-blk') }); + store.insertGeneration({ ...gen('gen-blk', 'app-blk'), plan_blocked: 1 }); + tx.fetchStarted('app-blk', envelope('op-f-blk')); + tx.fetched('app-blk', 'abc123', envelope('op-f-blk')); + tx.sourceConflictBlocker('app-blk', 'gen-blk', envelope('op-b-blk')); + expect(store.getApplication('app-blk')?.candidate_plan_blocked).toBe(1); + expect(store.getTarget('app-blk', 1)?.candidate_generation_id).toBe('gen-blk'); + const projection = mustProject('app-blk'); + expect(projection.facets.source.status).toBe('source_conflict_blocker'); + expect(projection.availableActions).not.toContain('apply'); + expect(() => tx.applyStarted('app-blk', 'gen-blk', envelope('op-a-blk'))).toThrow(/blocked/); + }); + + it('dismisses a candidate without touching what is already applied', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedApplied('app-dis', 'dis-web', 'gen-dis', 'art-dis', 'acc-dis'); + store.insertGeneration(gen('gen-dis-2', 'app-dis')); + tx.candidateReady('app-dis', 'gen-dis-2', false, envelope('op-c-dis')); + tx.dismissed('app-dis', envelope('op-d-dis')); + const application = store.getApplication('app-dis')!; + expect(application.candidate_generation_id).toBeNull(); + expect(application.accepted_generation_id).toBe('gen-dis'); + expect(store.getTarget('app-dis', 1)?.applied_generation_id).toBe('gen-dis'); + expect(store.getTarget('app-dis', 1)?.candidate_generation_id).toBeNull(); + }); + + it('refuses to dismiss while an operation is in flight', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-dis2', 'dis2-web'), nodeId: 1, envelope: envelope('op-act-dis2') }); + store.insertGeneration(gen('gen-dis2', 'app-dis2')); + tx.fetchStarted('app-dis2', envelope('op-f-dis2')); + tx.fetched('app-dis2', 'abc123', envelope('op-f-dis2')); + tx.candidateReady('app-dis2', 'gen-dis2', false, envelope('op-c-dis2')); + tx.applyStarted('app-dis2', 'gen-dis2', envelope('op-a-dis2')); + expect(() => tx.dismissed('app-dis2', envelope('op-d-dis2'))).toThrow(/in flight/); + expect(store.getApplication('app-dis2')?.candidate_generation_id).toBe('gen-dis2'); + }); + + it('invalidates a staged candidate when the material configuration changes', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedApplied('app-cfg', 'cfg-web', 'gen-cfg', 'art-cfg', 'acc-cfg'); + store.insertGeneration(gen('gen-cfg-2', 'app-cfg')); + tx.candidateReady('app-cfg', 'gen-cfg-2', false, envelope('op-c-cfg')); + tx.configChangedPendingCleared({ + applicationId: 'app-cfg', + identity: { + repoUrl: 'https://github.com/org/other.git', + repoIdentityJson: '{"host":"github.com","pathname":"/org/other.git"}', + configuredRef: 'release', + }, + material: { + composePathsJson: '["compose.yml","compose.prod.yml"]', + contextDir: null, + syncEnv: 0, + envPath: null, + fingerprint: 'd'.repeat(64), + }, + envelope: envelope('op-cfg'), + }); + const application = store.getApplication('app-cfg')!; + expect(application.configured_ref).toBe('release'); + expect(application.materialization_fingerprint).toBe('d'.repeat(64)); + expect(application.desired_commit_sha).toBeNull(); + expect(application.candidate_generation_id).toBeNull(); + // The workload that is running did not change because the config did. + expect(application.accepted_generation_id).toBe('gen-cfg'); + expect(store.getTarget('app-cfg', 1)?.applied_generation_id).toBe('gen-cfg'); + const projection = mustProject('app-cfg'); + expect(projection.facets.source.status).toBe('source_reconcile_required'); + expect(projection.availableActions).toContain('fetch'); + }); + + it('records deploy failures without moving the deployed generation', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedApplied('app-dep', 'dep-web', 'gen-dep', 'art-dep', 'acc-dep'); + tx.deployStarted('app-dep', 1, 'gen-dep', envelope('op-dep-1')); + tx.deployUnbound('app-dep', 1, 'gen-dep', envelope('op-dep-1')); + let target = store.getTarget('app-dep', 1)!; + expect(target.deployed_generation_id).toBeNull(); + expect(target.failure_class).toBe('unbound'); + expect(mustProject('app-dep').targets[0]?.runtime.status).toBe('failed_previous_workload_intact'); + + tx.deployStarted('app-dep', 1, 'gen-dep', envelope('op-dep-2')); + tx.deployFailed('app-dep', 1, 'post_mutation', envelope('op-dep-2')); + target = store.getTarget('app-dep', 1)!; + expect(target.deployed_generation_id).toBeNull(); + expect(target.failure_class).toBe('post_mutation'); + expect(mustProject('app-dep').targets[0]?.runtime.status).toBe('failed_after_mutation'); + + // A later success clears the failure in the same move as the pointer. + tx.deployStarted('app-dep', 1, 'gen-dep', envelope('op-dep-3')); + tx.deployBound('app-dep', 1, 'gen-dep', envelope('op-dep-3')); + target = store.getTarget('app-dep', 1)!; + expect(target.deployed_generation_id).toBe('gen-dep'); + expect(target.failure_stage).toBeNull(); + }); + + it('promotes healthy and last-known-good only for the generation the run watched', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedApplied('app-hl', 'hl-web', 'gen-hl', 'art-hl', 'acc-hl'); + tx.deployStarted('app-hl', 1, 'gen-hl', envelope('op-hl-dep')); + tx.deployBound('app-hl', 1, 'gen-hl', envelope('op-hl-dep')); + + // A verdict for a generation that is not the deployed one proves nothing. + tx.healthFinalized({ + applicationId: 'app-hl', + nodeId: 1, + healthRunId: 'run-stale', + healthStatus: 'passed', + deployedGenerationId: 'gen-other', + targetScope: 'stack', + envelope: envelope('op-hl-stale'), + }); + expect(store.getTarget('app-hl', 1)?.healthy_generation_id).toBeNull(); + + // Nor does a service-scoped run, which never observed the whole stack. + tx.healthFinalized({ + applicationId: 'app-hl', + nodeId: 1, + healthRunId: 'run-service', + healthStatus: 'passed', + deployedGenerationId: 'gen-hl', + targetScope: 'service', + envelope: envelope('op-hl-service'), + }); + expect(store.getTarget('app-hl', 1)?.healthy_generation_id).toBeNull(); + + // Nor does a failure. + tx.healthFinalized({ + applicationId: 'app-hl', + nodeId: 1, + healthRunId: 'run-failed', + healthStatus: 'failed', + deployedGenerationId: 'gen-hl', + targetScope: 'stack', + envelope: envelope('op-hl-failed'), + }); + expect(store.getTarget('app-hl', 1)?.healthy_generation_id).toBeNull(); + + tx.healthFinalized({ + applicationId: 'app-hl', + nodeId: 1, + healthRunId: 'run-pass', + healthStatus: 'passed', + deployedGenerationId: 'gen-hl', + targetScope: 'stack', + envelope: envelope('op-hl-pass'), + }); + const target = store.getTarget('app-hl', 1)!; + expect(target.healthy_generation_id).toBe('gen-hl'); + expect(target.lkg_generation_id).toBe('gen-hl'); + // The expected artifact belongs to this generation, so it is kept as the + // qualification evidence for the last-known-good. + expect(target.lkg_artifact_set_id).toBe('art-hl'); + expect(target.lkg_unavailable_at).toBeNull(); + const projection = mustProject('app-hl'); + expect(projection.targets[0]?.runtime.status).toBe('synced_and_healthy'); + expect(projection.targets[0]?.lkg.status).not.toBe('none'); + }); + + it('keeps the last-known-good generation when its artifact belongs elsewhere', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedApplied('app-lkg', 'lkg-web', 'gen-lkg', 'art-lkg', 'acc-lkg'); + tx.deployStarted('app-lkg', 1, 'gen-lkg', envelope('op-lkg-dep')); + tx.deployBound('app-lkg', 1, 'gen-lkg', envelope('op-lkg-dep')); + // Clear the expectation so the promotion has no artifact to qualify with. + DatabaseService.getInstance().getDb().prepare( + "UPDATE gitops_target_current SET expected_artifact_set_id = NULL WHERE application_id = 'app-lkg'", + ).run(); + + tx.healthFinalized({ + applicationId: 'app-lkg', + nodeId: 1, + healthRunId: 'run-lkg', + healthStatus: 'passed', + deployedGenerationId: 'gen-lkg', + targetScope: 'stack', + envelope: envelope('op-lkg-pass'), + }); + + const target = store.getTarget('app-lkg', 1)!; + // The generation is still good; only its executable identity is unproven. + expect(target.lkg_generation_id).toBe('gen-lkg'); + expect(target.lkg_artifact_set_id).toBeNull(); + expect(mustProject('app-lkg').targets[0]?.lkg.status).toBe('available'); + }); + + it('tombstones an application and its target, and never reactivates it', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedApplied('app-tomb', 'tomb-web', 'gen-tomb', 'art-tomb', 'acc-tomb'); + tx.targetTombstoned('app-tomb', 1, envelope('op-tomb')); + tx.applicationTombstoned('app-tomb', 'detached', envelope('op-tomb')); + const application = store.getApplication('app-tomb')!; + expect(application.lifecycle_status).toBe('detached'); + // Configured identity survives as a frozen fact. + expect(application.configured_repo_url).toBe('https://github.com/org/repo.git'); + expect(application.desired_commit_sha).toBe('abc123'); + expect(store.getTarget('app-tomb', 1)?.target_status).toBe('tombstoned'); + expect(store.getLiveDirectApplication('tomb-web')).toBeUndefined(); + expect(() => tx.applicationTombstoned('app-tomb', 'deleted', envelope('op-tomb-2'))) + .toThrow(/already tombstoned/); + expect(mustProject('app-tomb').facets.source.status).toBe('not_live'); + }); + + it('retires every live target on a node without touching its applications', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedApplied('app-node-a', 'node-a-web', 'gen-node-a', 'art-node-a', 'acc-node-a'); + seedApplied('app-node-b', 'node-b-web', 'gen-node-b', 'art-node-b', 'acc-node-b'); + + tx.tombstoneNodeTargets(1, envelope('op-node-del')); + + expect(store.getTarget('app-node-a', 1)?.target_status).toBe('tombstoned'); + expect(store.getTarget('app-node-b', 1)?.target_status).toBe('tombstoned'); + // The applications still describe real stacks, so they stay live. + expect(store.getApplication('app-node-a')?.lifecycle_status).toBe('active'); + expect(store.getApplication('app-node-b')?.lifecycle_status).toBe('active'); + // Replaying finds nothing left to retire. + expect(tx.tombstoneNodeTargets(1, envelope('op-node-del-2')).historyIds).toHaveLength(0); + }); + + it('rejects terminal events with no matching operation', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-guard', 'guard-web'), nodeId: 1, envelope: envelope('op-act-guard') }); + store.insertGeneration(gen('gen-guard', 'app-guard')); + expect(() => tx.applyFailed('app-guard', 'apply', envelope('op-g1'))) + .toThrow(/no matching apply operation/); + expect(() => tx.deployStarted('app-guard', 1, 'gen-guard', envelope('op-g2'))) + .toThrow(/not applied/); + expect(() => tx.deployBound('app-guard', 1, 'gen-guard', envelope('op-g3'))) + .toThrow(/no matching deploy operation/); + expect(store.getTarget('app-guard', 1)?.deployed_generation_id).toBeNull(); + }); + + it('writes one history row per transition with the bound identity', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-hist', 'hist-web'), nodeId: 1, envelope: envelope('op-act-hist') }); + store.insertGeneration(gen('gen-hist', 'app-hist')); + tx.fetchStarted('app-hist', envelope('op-f-hist')); + tx.fetched('app-hist', 'abc123', envelope('op-f-hist')); + tx.candidateReady('app-hist', 'gen-hist', false, envelope('op-c-hist')); + const applied = tx.applied({ + applicationId: 'app-hist', + generationId: 'gen-hist', + artifactSetId: 'art-hist', + sourceAcceptanceId: 'acc-hist', + authority: 'operator', + envelope: envelope('op-a-hist'), + }); + expect(applied.replayed).toBe(false); + expect(applied.historyIds).toHaveLength(1); + const row = DatabaseService.getInstance().getDb().prepare( + 'SELECT stage, outcome, dedupe_target, generation_id, artifact_set_id, source_acceptance_ref, node_id FROM gitops_history WHERE id = ?', + ).get(applied.historyIds[0]) as Record; + expect(row.stage).toBe('applied'); + expect(row.outcome).toBe('committed'); + expect(row.dedupe_target).toBe('app'); + expect(row.generation_id).toBe('gen-hist'); + expect(row.artifact_set_id).toBe('art-hist'); + expect(row.source_acceptance_ref).toBe('acc-hist'); + expect(row.node_id).toBeNull(); + }); + + it('interrupts live apply and binds deploy after applied', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-int', 'int-web'), nodeId: 1, envelope: envelope('op-act-int') }); + store.insertGeneration(gen('gen-int', 'app-int')); + tx.fetchStarted('app-int', envelope('op-f-int')); + tx.fetched('app-int', 'abc123', envelope('op-f-int')); + tx.candidateReady('app-int', 'gen-int', false, envelope('op-c-int')); + tx.applyStarted('app-int', 'gen-int', envelope('op-apply-live')); + expect(mustProject('app-int').facets.source.status).toBe('applying'); + tx.interruptActiveOperations('app-int', envelope('op-boot')); + expect(mustProject('app-int').facets.source.status).toBe('source_unknown'); + tx.applied({ + applicationId: 'app-int', + generationId: 'gen-int', + artifactSetId: 'art-int', + sourceAcceptanceId: 'acc-int', + authority: 'operator', + envelope: envelope('op-a-int'), + }); + tx.deployStarted('app-int', 1, 'gen-int', envelope('op-dep')); + tx.deployBound('app-int', 1, 'gen-int', envelope('op-dep')); + expect(store.getTarget('app-int', 1)?.deployed_generation_id).toBe('gen-int'); + expect(store.getTarget('app-int', 1)?.failure_stage).toBeNull(); + }); +}); + +function mustProject(applicationId: string) { + const projection = projectApplication(applicationId, false); + if (projection.targetMode === 'not_applicable') throw new Error('expected application'); + return projection; +} + +function seedApplied( + applicationId: string, + stackName: string, + generationId: string, + artifactSetId: string, + sourceAcceptanceId: string, +): void { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app(applicationId, stackName), nodeId: 1, envelope: envelope(`op-act-${applicationId}`) }); + store.insertGeneration(gen(generationId, applicationId)); + tx.fetchStarted(applicationId, envelope(`op-f-${applicationId}`)); + tx.fetched(applicationId, 'abc123', envelope(`op-f-${applicationId}`)); + tx.candidateReady(applicationId, generationId, false, envelope(`op-c-${applicationId}`)); + tx.applied({ + applicationId, + generationId, + artifactSetId, + sourceAcceptanceId, + authority: 'operator', + envelope: envelope(`op-a-${applicationId}`), + }); +} + +function envelope(operationId: string): EventEnvelope { + return { operationId, actor: 'tester', trigger: 'manual', at: Date.now() }; +} + +function app(id: string, stackName: string): GitOpsApplicationRow { + return { + id, + lifecycle_key: `direct:${stackName}`, + lifecycle_status: 'active', + target_mode: 'direct', + stack_name: stackName, + blueprint_id: null, + configured_repo_url: 'https://github.com/org/repo.git', + repo_identity_json: '{"host":"github.com","pathname":"/org/repo.git"}', + configured_ref: 'main', + compose_paths_json: '["compose.yml"]', + context_dir: null, + sync_env: 0, + env_path: null, + materialization_fingerprint: 'a'.repeat(64), + desired_commit_sha: null, + fetched_commit_sha: null, + candidate_generation_id: null, + accepted_generation_id: null, + candidate_plan_blocked: 0, + review_required: 0, + artifact_set_id: null, + latest_artifact_set_id: null, + intent_revision_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + rollout_authorization_ref: null, + legacy_combined_approval_ref: null, + preflight_fingerprint: null, + latest_operation_id: null, + active_operation_id: null, + active_operation_stage: null, + active_operation_at: null, + active_generation_id: null, + pause_at: null, + pause_reason: null, + partial_json: null, + failure_stage: null, + failure_class: null, + failure_at: null, + retry_at: null, + retry_count: 0, + suspended_at: null, + recovery_ref: null, + recovery_phase: null, + interruption_stage: null, + interruption_at: null, + interruption_operation_id: null, + interruption_generation_id: null, + evidence_fresh_at: null, + evidence_limitations_json: null, + created_at: 1, + updated_at: 1, + }; +} + +function gen(id: string, applicationId: string): GitOpsGenerationRow { + return { + id, + application_id: applicationId, + commit_sha: 'abc123', + repo_url: 'https://github.com/org/repo.git', + configured_ref: 'main', + repo_identity_json: '{"host":"github.com","pathname":"/org/repo.git"}', + manifest_version: 0, + candidate_dir: `generations/candidate-${id}`, + applied_dir: `generations/applied-${id}-0`, + expected_invocation_json: '{"composeFileOrder":[],"projectName":null,"projectDirectory":null,"envFileOrder":[]}', + materialization_fingerprint: 'a'.repeat(64), + validation_ok: 1, + plan_blocked: 0, + change_plan_fingerprint: null, + operation_id: `op-${id}`, + trigger: 'manual', + actor: 'tester', + previous_generation_id: null, + redacted_limitations_json: '[]', + created_at: 1, + }; +} diff --git a/backend/src/__tests__/health-gate-prepare.test.ts b/backend/src/__tests__/health-gate-prepare.test.ts index 3a4a3cd4..de1e055c 100644 --- a/backend/src/__tests__/health-gate-prepare.test.ts +++ b/backend/src/__tests__/health-gate-prepare.test.ts @@ -52,7 +52,7 @@ vi.mock('../services/DatabaseService', () => ({ .sort((a, b) => b.started_at - a.started_at); return matches[0] ? { ...matches[0] } : undefined; }, - markInterruptedHealthGateRuns: () => 0, + listObservingHealthGateRuns: () => [], addNotificationHistory: (_nodeId: number, item: { category?: string; message: string; level: string }) => ({ ...item, id: 1, is_read: false }), }), }, @@ -209,7 +209,7 @@ describe('prepare / beginPrepared nullability', () => { }); it('persists an immediate unknown past the concurrency cap', async () => { - for (let i = 0; i < 25; i++) svc().beginStack(0, `stack-${i}`, 'update', 'tester'); + for (let i = 0; i < 25; i++) svc().beginStack(0, `stack-${i}`, 'update', 'tester', { deployedGenerationId: null }); const token = await prepareService([{ id: 'p1', name: 'web-app-1', service: 'app' }]); svc().attachExpectedImage(token, 'sha256:app'); const result = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); diff --git a/backend/src/__tests__/health-gate-service.test.ts b/backend/src/__tests__/health-gate-service.test.ts index 0ba4458c..b56590b8 100644 --- a/backend/src/__tests__/health-gate-service.test.ts +++ b/backend/src/__tests__/health-gate-service.test.ts @@ -9,7 +9,7 @@ interface StoredRun { id: string; node_id: number; stack_name: string; - trigger_action: 'update' | 'deploy' | 'service_update' | 'service_restore'; + trigger_action: 'update' | 'deploy' | 'service_update' | 'service_restore' | 'recovery'; status: 'observing' | 'passed' | 'failed' | 'unknown'; reason: string | null; window_seconds: number; @@ -20,13 +20,17 @@ interface StoredRun { target_scope: 'stack' | 'service'; service_name: string | null; failure_source: 'primary' | 'collateral' | null; + deployed_generation_id?: string | null; } const { state } = vi.hoisted(() => ({ state: { runs: new Map(), + recoveries: new Map(), activity: [] as Array<{ category?: string; message: string; level: string }>, settings: {} as Record, + /** Run id whose finalize write should fail, for the per-row sweep guard. */ + failFinalizeFor: null as string | null, listContainers: vi.fn(), inspect: vi.fn(), renderConfig: vi.fn(), @@ -39,6 +43,7 @@ vi.mock('../services/DatabaseService', () => ({ getGlobalSettings: () => state.settings, insertHealthGateRun: (run: StoredRun) => { state.runs.set(run.id, { ...run }); }, finalizeHealthGateRun: (id: string, status: StoredRun['status'], reason: string | null, endedAt: number, containersJson: string, failureSource: StoredRun['failure_source'] = null) => { + if (state.failFinalizeFor === id) throw new Error('row is unreadable'); const run = state.runs.get(id); if (run) Object.assign(run, { status, reason, ended_at: endedAt, containers_json: containersJson, failure_source: failureSource }); }, @@ -52,16 +57,17 @@ vi.mock('../services/DatabaseService', () => ({ .sort((a, b) => b.started_at - a.started_at); return matches[0] ? { ...matches[0] } : undefined; }, - markInterruptedHealthGateRuns: (reason: string, endedAt: number) => { - let n = 0; - for (const run of state.runs.values()) { - if (run.status === 'observing') { - Object.assign(run, { status: 'unknown', reason, ended_at: endedAt }); - n++; - } - } - return n; + getStackUpdateRecoveryGeneration: (id: string) => { + const row = state.recoveries.get(id); + return row ? { ...row } : undefined; }, + updateStackUpdateRecoveryGeneration: (id: string, patch: { health_gate_id?: string | null }) => { + const row = state.recoveries.get(id); + if (row) Object.assign(row, patch); + }, + listObservingHealthGateRuns: () => [...state.runs.values()] + .filter(run => run.status === 'observing') + .map(run => ({ ...run })), addNotificationHistory: (_nodeId: number, item: { category?: string; message: string; level: string }) => { state.activity.push(item); return { ...item, id: state.activity.length, is_read: false }; @@ -163,6 +169,8 @@ async function ticks(n: number): Promise { beforeEach(() => { vi.useFakeTimers(); state.runs.clear(); + state.recoveries.clear(); + state.failFinalizeFor = null; state.activity.length = 0; state.settings = { health_gate_enabled: '1', health_gate_window_seconds: '30' }; state.listContainers.mockReset(); @@ -181,7 +189,7 @@ afterEach(() => { describe('HealthGateService verdicts', () => { it('passes at the window end when containers stay running', async () => { - const id = svc().beginStack(0, 'web', 'update', 'tester'); + const id = svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); expect(id).toBeTruthy(); await ticks(3); // 15s: still observing expect(latest().status).toBe('observing'); @@ -191,7 +199,7 @@ describe('HealthGateService verdicts', () => { }); it('fails fast when a container exits', async () => { - svc().beginStack(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); await ticks(1); // baseline setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 1, restartPolicy: 'unless-stopped' }]); await ticks(1); @@ -207,7 +215,7 @@ describe('HealthGateService verdicts', () => { { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' }, ]); - svc().beginStack(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); await ticks(1); setContainers([ { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, @@ -229,7 +237,7 @@ describe('HealthGateService verdicts', () => { state: 'running', restartPolicy: 'no', }, ]); - svc().beginStack(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); await ticks(1); setContainers([ { @@ -248,7 +256,7 @@ describe('HealthGateService verdicts', () => { { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' }, ]); - svc().beginStack(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); await ticks(1); setContainers([ { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, @@ -269,7 +277,7 @@ describe('HealthGateService verdicts', () => { { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' }, ]); - svc().beginStack(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); await ticks(1); setContainers([ { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, @@ -290,7 +298,7 @@ describe('HealthGateService verdicts', () => { { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' }, ]); - svc().beginStack(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); await ticks(1); setContainers([ { @@ -310,7 +318,7 @@ describe('HealthGateService verdicts', () => { { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' }, ]); - svc().beginStack(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); await ticks(1); setContainers([ { @@ -327,7 +335,7 @@ describe('HealthGateService verdicts', () => { }); it('fails when exit 0 has unless-stopped restart policy', async () => { - svc().beginStack(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); await ticks(1); setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 0, restartPolicy: 'unless-stopped' }]); await ticks(1); @@ -336,7 +344,7 @@ describe('HealthGateService verdicts', () => { }); it('fails when exit 0 has always restart policy', async () => { - svc().beginStack(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); await ticks(1); setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 0, restartPolicy: 'always' }]); await ticks(1); @@ -345,7 +353,7 @@ describe('HealthGateService verdicts', () => { }); it('fails closed when exit code is null on an exited container', async () => { - svc().beginStack(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); await ticks(1); setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: null, restartPolicy: 'no' }]); await ticks(1); @@ -354,7 +362,7 @@ describe('HealthGateService verdicts', () => { }); it('fails when a one-shot exits non-zero', async () => { - svc().beginStack(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); await ticks(1); setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 1, restartPolicy: 'no' }]); await ticks(1); @@ -363,7 +371,7 @@ describe('HealthGateService verdicts', () => { }); it('fails fast when a healthcheck reports unhealthy', async () => { - svc().beginStack(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); await ticks(1); setContainers([{ id: 'aaa', name: 'web-app-1', health: 'unhealthy' }]); await ticks(1); @@ -372,7 +380,7 @@ describe('HealthGateService verdicts', () => { }); it('detects a restart loop via container replacement (new id)', async () => { - svc().beginStack(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); await ticks(1); setContainers([{ id: 'bbb', name: 'web-app-1' }]); await ticks(1); // restart 1 observed; carried as new baseline @@ -388,7 +396,7 @@ describe('HealthGateService verdicts', () => { }); it('detects a restart loop via RestartCount and StartedAt movement', async () => { - svc().beginStack(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); await ticks(1); setContainers([{ id: 'aaa', name: 'web-app-1', restartCount: 1 }]); await ticks(1); @@ -403,7 +411,7 @@ describe('HealthGateService verdicts', () => { }); it('tolerates a one-poll disappearance but fails on two consecutive misses', async () => { - svc().beginStack(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); await ticks(1); // baseline setContainers([]); // one missed poll: tolerated await ticks(1); @@ -411,7 +419,7 @@ describe('HealthGateService verdicts', () => { await ticks(5); // through the 30s window expect(latest().status).toBe('passed'); - const second = svc().beginStack(0, 'web', 'update', 'tester')!; + const second = svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null })!; await ticks(1); setContainers([]); await ticks(2); // two consecutive misses: disappeared @@ -421,7 +429,7 @@ describe('HealthGateService verdicts', () => { }); it('fails when a container is stuck restarting across consecutive polls', async () => { - svc().beginStack(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); await ticks(1); setContainers([{ id: 'aaa', name: 'web-app-1', state: 'restarting' }]); await ticks(2); @@ -430,7 +438,7 @@ describe('HealthGateService verdicts', () => { }); it('goes unknown after three consecutive docker errors', async () => { - svc().beginStack(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); await ticks(1); state.listContainers.mockRejectedValue(new Error('socket gone')); await ticks(3); @@ -439,7 +447,7 @@ describe('HealthGateService verdicts', () => { }); it('resolves unknown when every docker observe hangs', async () => { - svc().beginStack(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); // A wedged socket never settles. The per-observe timeout turns each poll // into an error, and three in a row finalize the gate unknown instead of // observing forever on a pending promise. @@ -452,7 +460,7 @@ describe('HealthGateService verdicts', () => { }); it('recovers from a transient observe timeout instead of finalizing', async () => { - svc().beginStack(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); // One observe wedges and times out (a single strike), then the socket // recovers; the gate must keep observing, not give up at one error. state.listContainers.mockImplementationOnce(() => new Promise(() => {})); @@ -465,7 +473,7 @@ describe('HealthGateService verdicts', () => { }); it('runs polls single-flight: no second observe until the first settles', async () => { - svc().beginStack(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); let release: (value: Array<{ Id: string; Names: string[]; State: string }>) => void = () => {}; state.listContainers.mockImplementationOnce(() => new Promise(resolve => { release = resolve; })); // Advance past a second poll interval while the first observe is still @@ -480,7 +488,7 @@ describe('HealthGateService verdicts', () => { }); it('ends unknown when a healthcheck is still starting at the window end', async () => { - svc().beginStack(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); setContainers([{ id: 'aaa', name: 'web-app-1', health: 'starting' }]); await ticks(7); expect(latest().status).toBe('unknown'); @@ -489,7 +497,7 @@ describe('HealthGateService verdicts', () => { it('goes unknown when no containers ever appear', async () => { setContainers([]); - svc().beginStack(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); await ticks(4); expect(latest().status).toBe('unknown'); expect(latest().reason).toContain('no containers'); @@ -501,7 +509,7 @@ describe('HealthGateService lifecycle', () => { // A poll is mid-await on Docker when a newer update supersedes the gate; // when the await resolves with healthy containers, the superseded run // must keep its terminal unknown verdict. - const first = svc().beginStack(0, 'web', 'update', 'tester')!; + const first = svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null })!; await ticks(2); // baseline established, healthy let releasePoll: (value: Array<{ Id: string; Names: string[]; State: string }>) => void = () => {}; @@ -510,7 +518,7 @@ describe('HealthGateService lifecycle', () => { ); const straddlingPoll = vi.advanceTimersByTimeAsync(5_000); // poll now awaiting Docker - const second = svc().beginStack(0, 'web', 'update', 'tester')!; + const second = svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null })!; expect(svc().getReport(0, 'web', first).status).toBe('unknown'); releasePoll([{ Id: 'aaa', Names: ['/web-app-1'], State: 'running' }]); @@ -525,10 +533,10 @@ describe('HealthGateService lifecycle', () => { }); it('supersede finalizes the old run as unknown, clears its timer, and getRun still resolves it', async () => { - const first = svc().beginStack(0, 'web', 'update', 'tester')!; + const first = svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null })!; await ticks(1); const timersBefore = vi.getTimerCount(); - const second = svc().beginStack(0, 'web', 'update', 'tester')!; + const second = svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null })!; expect(vi.getTimerCount()).toBe(timersBefore); // old interval cleared, new one added const superseded = svc().getReport(0, 'web', first); @@ -552,9 +560,145 @@ describe('HealthGateService lifecycle', () => { expect(state.runs.get('stale')!.reason).toContain('restarted'); }); + it('start() finalizes every interrupted run even when one of them is unreadable', () => { + state.runs.set('bad', { + id: 'bad', node_id: 0, stack_name: 'web', trigger_action: 'update', status: 'observing', + reason: null, window_seconds: 30, containers_json: '[]', started_at: 1, ended_at: null, created_by: null, + target_scope: 'stack', service_name: null, failure_source: null, + }); + state.runs.set('good', { + id: 'good', node_id: 0, stack_name: 'other', trigger_action: 'recovery', status: 'observing', + reason: null, window_seconds: 30, containers_json: '[]', started_at: 1, ended_at: null, created_by: null, + target_scope: 'stack', service_name: null, failure_source: null, + }); + // One row that cannot be written must not cost every later row its verdict. + // A bulk sweep is what this replaced, and it told the model about none of + // them; finalizing per row is only better if one bad row stays contained. + state.failFinalizeFor = 'bad'; + + svc().start(); + + expect(state.runs.get('bad')!.status).toBe('observing'); + expect(state.runs.get('good')!.status).toBe('unknown'); + }); +}); + +describe('HealthGateService recovery reservations', () => { + /** A reserved, committed recovery run as `compensateWithCandidate` leaves it. */ + function reserve(recoveryRef = 'rec-1', stackName = 'web') { + state.recoveries.set(recoveryRef, { id: recoveryRef, health_gate_id: null }); + return svc().reserveRecoveryRun({ + recoveryRef, + nodeId: 0, + stackName, + deployedGenerationId: 'gen-1', + actor: 'system:recovery', + }); + } + + it('writes the run and links it to the recovery generation', () => { + const result = reserve(); + expect(result.outcome).toBe('reserved'); + expect(result.runId).toBeTruthy(); + + const run = state.runs.get(result.runId!)!; + expect(run.trigger_action).toBe('recovery'); + expect(run.status).toBe('observing'); + // The generation is on the row, so the verdict is attributed to what this + // run was recorded as observing rather than to whatever is current later. + expect(run.deployed_generation_id).toBe('gen-1'); + expect(state.recoveries.get('rec-1')!.health_gate_id).toBe(result.runId); + // Reserving is a write, not an observation: no timer yet. + expect(vi.getTimerCount()).toBe(0); + }); + + it('reuses the run a replayed recovery already owns', () => { + const first = reserve(); + const second = svc().reserveRecoveryRun({ + recoveryRef: 'rec-1', + nodeId: 0, + stackName: 'web', + deployedGenerationId: 'gen-1', + actor: 'system:recovery', + }); + expect(second.outcome).toBe('replayed'); + expect(second.runId).toBe(first.runId); + expect(state.runs.size).toBe(1); + }); + + it('reserves nothing when the gate is disabled', () => { + state.settings.health_gate_enabled = '0'; + const result = reserve(); + expect(result).toEqual({ outcome: 'disabled', runId: null }); + expect(state.runs.size).toBe(0); + }); + + it('arms a reserved run without inserting a second one, and is idempotent', async () => { + const { runId } = reserve(); + svc().armReservedRun(runId!, 0, 'web'); + expect(state.runs.size).toBe(1); + + // Arming the run that is already the active gate must not supersede it. + svc().armReservedRun(runId!, 0, 'web'); + expect(state.runs.size).toBe(1); + expect(state.runs.get(runId!)!.status).toBe('observing'); + + await ticks(7); + expect(state.runs.get(runId!)!.status).toBe('passed'); + }); + + it('supersedes a conflicting stack gate rather than observing twice', async () => { + const older = svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); + const { runId } = reserve(); + svc().armReservedRun(runId!, 0, 'web'); + + expect(state.runs.get(older!)!.status).toBe('unknown'); + expect(state.runs.get(older!)!.reason).toContain('superseded'); + await ticks(7); + expect(state.runs.get(runId!)!.status).toBe('passed'); + }); + + it('refuses to arm a run that is not a reserved stack recovery', () => { + const { runId } = reserve(); + state.runs.get(runId!)!.trigger_action = 'update'; + expect(() => svc().armReservedRun(runId!, 0, 'web')).toThrow(/reserved stack recovery/); + + expect(() => svc().armReservedRun('no-such-run', 0, 'web')).toThrow(/not found/); + }); + + it('refuses to arm anything once the service is stopped', () => { + const { runId } = reserve(); + svc().stop(); + expect(() => svc().armReservedRun(runId!, 0, 'web')).toThrow(/not started/); + svc().start(); + }); + + it('writes off a reservation nothing could arm', () => { + const { runId } = reserve(); + svc().abandonReservedRun(runId!, 0, 'web', 'could not arm: too many concurrent observations'); + + const run = state.runs.get(runId!)!; + expect(run.status).toBe('unknown'); + expect(run.reason).toContain('could not arm'); + // Writing it off twice must not reopen or rewrite it. + svc().abandonReservedRun(runId!, 0, 'web', 'second attempt'); + expect(state.runs.get(runId!)!.reason).toContain('could not arm'); + }); + + it('never arms a reservation that outlived its process', () => { + const { runId } = reserve(); + // A restart: the row is still observing, and nothing in memory owns it. + svc().stop(); + svc().start(); + + expect(state.runs.get(runId!)!.status).toBe('unknown'); + expect(state.runs.get(runId!)!.reason).toContain('restarted'); + expect(vi.getTimerCount()).toBe(0); + }); + it('no-ops when disabled but still records the update_started event', () => { state.settings.health_gate_enabled = '0'; - const id = svc().beginStack(0, 'web', 'update', 'tester'); + const id = svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); expect(id).toBeNull(); expect(state.runs.size).toBe(0); expect(state.activity.some(a => a.category === 'update_started')).toBe(true); @@ -562,24 +706,24 @@ describe('HealthGateService lifecycle', () => { }); it('records update_started for update triggers but not deploy triggers', () => { - svc().beginStack(0, 'web', 'deploy', 'tester'); + svc().beginStack(0, 'web', 'deploy', 'tester', { deployedGenerationId: null }); expect(state.activity.some(a => a.category === 'update_started')).toBe(false); - svc().beginStack(0, 'web', 'update', 'tester'); + svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null }); expect(state.activity.some(a => a.category === 'update_started')).toBe(true); }); it('refuses to begin before start() so shutdown cannot leak timers', () => { svc().stop(); - expect(svc().beginStack(0, 'web', 'update', 'tester')).toBeNull(); + expect(svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null })).toBeNull(); expect(vi.getTimerCount()).toBe(0); svc().start(); }); it('persists an immediate unknown past the concurrency cap', () => { for (let i = 0; i < 25; i++) { - svc().beginStack(0, `stack-${i}`, 'update', 'tester'); + svc().beginStack(0, `stack-${i}`, 'update', 'tester', { deployedGenerationId: null }); } - const overCap = svc().beginStack(0, 'one-too-many', 'update', 'tester')!; + const overCap = svc().beginStack(0, 'one-too-many', 'update', 'tester', { deployedGenerationId: null })!; const report = svc().getReport(0, 'one-too-many', overCap); expect(report.status).toBe('unknown'); expect(report.reason).toContain('concurrent'); @@ -587,10 +731,10 @@ describe('HealthGateService lifecycle', () => { it('clamps the configured window into its valid range and falls back on garbage', () => { state.settings.health_gate_window_seconds = '99999'; - const a = svc().beginStack(0, 'web', 'update', 'tester')!; + const a = svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null })!; expect(svc().getReport(0, 'web', a).windowSeconds).toBe(600); state.settings.health_gate_window_seconds = 'banana'; - const b = svc().beginStack(0, 'web', 'update', 'tester')!; + const b = svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null })!; expect(svc().getReport(0, 'web', b).windowSeconds).toBe(90); }); @@ -601,7 +745,7 @@ describe('HealthGateService lifecycle', () => { }); it('stop() finalizes in-flight gates as unknown with zero timers left', async () => { - const id = svc().beginStack(0, 'web', 'update', 'tester')!; + const id = svc().beginStack(0, 'web', 'update', 'tester', { deployedGenerationId: null })!; await ticks(1); svc().stop(); expect(vi.getTimerCount()).toBe(0); diff --git a/backend/src/__tests__/helpers/gitopsFixtures.ts b/backend/src/__tests__/helpers/gitopsFixtures.ts new file mode 100644 index 00000000..36b611e0 --- /dev/null +++ b/backend/src/__tests__/helpers/gitopsFixtures.ts @@ -0,0 +1,76 @@ +/** + * Shared GitOps row fixtures for route and projection tests. + * + * Type-only imports, so this module pulls no service in at load time and stays + * safe to import statically from a test file whose singletons are only wired up + * once setupTestDb has run. + */ +import type { GitOpsApplicationRow } from '../../services/gitops/types'; + +/** + * A minimal live Direct application row. + * + * Every column is spelled out because the row type mirrors the table, so a + * partial object would not type-check and a cast would let a schema change land + * without a compile error here. Only the identifiers vary between tests; the + * rest is the quiet, freshly activated state a Direct attachment starts in. + */ +export function directApplicationFixture(id: string, stackName: string): GitOpsApplicationRow { + const now = Date.now(); + return { + id, + lifecycle_key: `direct:${stackName}`, + lifecycle_status: 'active', + target_mode: 'direct', + stack_name: stackName, + blueprint_id: null, + configured_repo_url: 'https://github.com/example/repo.git', + repo_identity_json: '{"host":"github.com","pathname":"/example/repo.git"}', + configured_ref: 'main', + compose_paths_json: '["compose.yaml"]', + context_dir: null, + sync_env: 0, + env_path: null, + materialization_fingerprint: 'a'.repeat(64), + desired_commit_sha: null, + fetched_commit_sha: null, + candidate_generation_id: null, + accepted_generation_id: null, + candidate_plan_blocked: 0, + review_required: 0, + artifact_set_id: null, + latest_artifact_set_id: null, + intent_revision_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + rollout_authorization_ref: null, + legacy_combined_approval_ref: null, + preflight_fingerprint: null, + latest_operation_id: null, + active_operation_id: null, + active_operation_stage: null, + active_operation_at: null, + active_generation_id: null, + pause_at: null, + pause_reason: null, + partial_json: null, + failure_stage: null, + failure_class: null, + failure_at: null, + retry_at: null, + retry_count: 0, + suspended_at: null, + recovery_ref: null, + recovery_phase: null, + interruption_stage: null, + interruption_at: null, + interruption_operation_id: null, + interruption_generation_id: null, + evidence_fresh_at: null, + evidence_limitations_json: null, + created_at: now, + updated_at: now, + }; +} diff --git a/backend/src/__tests__/hub-only-guard.test.ts b/backend/src/__tests__/hub-only-guard.test.ts index 939987c0..3a67cd0c 100644 --- a/backend/src/__tests__/hub-only-guard.test.ts +++ b/backend/src/__tests__/hub-only-guard.test.ts @@ -257,4 +257,33 @@ describe('hubOnlyGuard', () => { expect(res.body?.code).not.toBe('HUB_ONLY_ENDPOINT'); }); + + it('rejects /api/blueprints with 403 when nodeId targets a remote node', async () => { + const res = await request(app) + .get('/api/blueprints') + .set('Authorization', authHeader) + .set('x-node-id', String(remoteNodeId)); + + expect(res.status).toBe(403); + expect(res.body?.code).toBe('HUB_ONLY_ENDPOINT'); + }); + + it('rejects /api/node-labels with 403 when nodeId targets a remote node', async () => { + const res = await request(app) + .get('/api/node-labels') + .set('Authorization', authHeader) + .set('x-node-id', String(remoteNodeId)); + + expect(res.status).toBe(403); + expect(res.body?.code).toBe('HUB_ONLY_ENDPOINT'); + }); + + it('does not treat /api/git-sources as hub-only', async () => { + const res = await request(app) + .get('/api/git-sources') + .set('Authorization', authHeader) + .set('x-node-id', String(remoteNodeId)); + + expect(res.body?.code).not.toBe('HUB_ONLY_ENDPOINT'); + }); }); diff --git a/backend/src/__tests__/image-updates-routes.test.ts b/backend/src/__tests__/image-updates-routes.test.ts index 1505e5eb..7bbcd53c 100644 --- a/backend/src/__tests__/image-updates-routes.test.ts +++ b/backend/src/__tests__/image-updates-routes.test.ts @@ -761,7 +761,7 @@ describe('POST /api/auto-update/execute', () => { .mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }] as never); const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage') .mockResolvedValue({ hasUpdate: true, digestUpdate: true } as never); - const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null }); + const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const gateSpy = vi.spyOn(PolicyEnforcement, 'enforcePolicyPreDeploy').mockResolvedValue({ ok: false, bypassed: false, @@ -808,7 +808,7 @@ describe('POST /api/auto-update/execute', () => { .mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }] as never); const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage') .mockResolvedValue({ hasUpdate: true, digestUpdate: true } as never); - const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null }); + const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const recheckSpy = vi.spyOn(ImageUpdateService.getInstance(), 'recheckStack') .mockImplementation(async () => { callOrder.push('recheckStack'); @@ -826,7 +826,7 @@ describe('POST /api/auto-update/execute', () => { expect(res.status).toBe(200); expect(updateSpy).toHaveBeenCalledWith('auto-upd-gate', undefined, true); expect(recheckSpy).toHaveBeenCalledWith(nodeId, 'auto-upd-gate'); - expect(beginSpy).toHaveBeenCalledWith(nodeId, 'auto-upd-gate', 'update', `auto-update:${TEST_USERNAME}`); + expect(beginSpy).toHaveBeenCalledWith(nodeId, 'auto-upd-gate', 'update', `auto-update:${TEST_USERNAME}`, { deployedGenerationId: null }); expect(callOrder.indexOf('beginStack')).toBeLessThan(callOrder.indexOf('recheckStack')); } finally { containersSpy.mockRestore(); @@ -848,7 +848,7 @@ describe('POST /api/auto-update/execute', () => { .mockResolvedValue([{ Id: 'c1', Image: 'nginx:1.2.3' }] as never); const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage') .mockResolvedValue({ hasUpdate: true, digestUpdate: false, tagUpdate: true } as never); - const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null }); + const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const recheckSpy = vi.spyOn(ImageUpdateService.getInstance(), 'recheckStack'); const clearSpy = vi.spyOn(DatabaseService.getInstance(), 'clearStackUpdateStatus'); try { @@ -883,7 +883,7 @@ describe('POST /api/auto-update/execute', () => { const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage') .mockResolvedValueOnce({ hasUpdate: true, digestUpdate: true, tagUpdate: false } as never) .mockResolvedValueOnce({ hasUpdate: false, error: 'registry timeout', checkStatus: 'failed' } as never); - const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null }); + const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const recheckSpy = vi.spyOn(ImageUpdateService.getInstance(), 'recheckStack'); try { const res = await request(app) @@ -913,7 +913,7 @@ describe('POST /api/auto-update/execute', () => { .mockResolvedValue([{ Id: 'c1', Image: 'nginx:latest' }] as never); const checkSpy = vi.spyOn(ImageUpdateService.getInstance(), 'checkImage') .mockResolvedValue({ hasUpdate: true, digestUpdate: true, tagUpdate: false } as never); - const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null }); + const updateSpy = vi.spyOn(ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const recheckSpy = vi.spyOn(ImageUpdateService.getInstance(), 'recheckStack') .mockResolvedValue({ outcome: 'still_present', warning: null } as never); const clearSpy = vi.spyOn(DatabaseService.getInstance(), 'clearStackUpdateStatus'); diff --git a/backend/src/__tests__/proxy-pilot-agent-role-header.test.ts b/backend/src/__tests__/proxy-pilot-agent-role-header.test.ts index 97a639ec..2e246548 100644 --- a/backend/src/__tests__/proxy-pilot-agent-role-header.test.ts +++ b/backend/src/__tests__/proxy-pilot-agent-role-header.test.ts @@ -80,6 +80,26 @@ describe('pilot-agent-mode proxy role header parity', () => { expect(captured?.[PROXY_ROLE_HEADER]).toBe('deployer'); }); + it('strips conditional request headers on the gitops identity hop', async () => { + captured = null; + const res = await request(app) + .get('/api/git-sources') + .set('Authorization', personas.deployer.bearer) + .set('x-node-id', String(pilotNodeId)) + // A remote answering this with 304 would let the client keep a cached + // page the hub never re-filtered, so the revalidation question must + // never reach the remote. + .set('If-None-Match', 'W/"cached-upstream"') + .set('Accept', 'application/json'); + expect(res.status).toBe(200); + expect(captured).not.toBeNull(); + expect(captured?.['if-none-match']).toBeUndefined(); + // Unrelated headers still travel. + expect(captured?.['accept']).toBe('application/json'); + // The answer itself must not be cacheable under any validator. + expect(res.headers['cache-control']).toBe('no-store'); + }); + it('overwrites a smuggled admin header with the real deployer role on pilot path', async () => { captured = null; const res = await request(app) diff --git a/backend/src/__tests__/scheduler-service.test.ts b/backend/src/__tests__/scheduler-service.test.ts index c1c294e3..38fb9567 100644 --- a/backend/src/__tests__/scheduler-service.test.ts +++ b/backend/src/__tests__/scheduler-service.test.ts @@ -55,7 +55,7 @@ const { mockStartContainer: vi.fn().mockResolvedValue(undefined), mockStopContainer: vi.fn().mockResolvedValue(undefined), mockPruneSystem: vi.fn().mockResolvedValue({ success: true, reclaimedBytes: 0 }), - mockUpdateStack: vi.fn().mockResolvedValue({ recoveryId: null }), + mockUpdateStack: vi.fn().mockResolvedValue({ recoveryId: null, deployedGenerationId: null }), mockGetStacks: vi.fn().mockResolvedValue([]), mockGetStackContent: vi.fn().mockResolvedValue(''), mockGetEnvContent: vi.fn().mockResolvedValue(''), @@ -912,7 +912,7 @@ describe('SchedulerService - executeUpdate', () => { await SchedulerService.getInstance().triggerTask(83); - expect(beginSpy).toHaveBeenCalledWith(1, 'web-app', 'update', 'system:scheduler'); + expect(beginSpy).toHaveBeenCalledWith(1, 'web-app', 'update', 'system:scheduler', { deployedGenerationId: null }); expect(mockRecheckStack).toHaveBeenCalledWith(1, 'web-app'); expect(callOrder.indexOf('beginStack')).toBeLessThan(callOrder.indexOf('recheckStack')); } finally { diff --git a/backend/src/__tests__/service-scoped-update-routes.test.ts b/backend/src/__tests__/service-scoped-update-routes.test.ts index 5811e4e9..18e2ab31 100644 --- a/backend/src/__tests__/service-scoped-update-routes.test.ts +++ b/backend/src/__tests__/service-scoped-update-routes.test.ts @@ -150,7 +150,7 @@ describe('OrchestratorResult to HTTP mapping', () => { const res = await request(app) .post('/api/stacks/web/services/app/restore') .set('Cookie', adminCookie) - .send({ recoveryId: 'rec-2' }); + .send({ deployedGenerationId: null, recoveryId: 'rec-2' }); expect(res.status).toBe(200); expect(res.body).toMatchObject({ serviceName: 'app', healthGateId: 'hg-2', recoveryId: 'rec-2' }); expect(mockExecute).toHaveBeenCalledTimes(1); diff --git a/backend/src/__tests__/stack-bulk-routes.test.ts b/backend/src/__tests__/stack-bulk-routes.test.ts index f62454c8..0ac64f5b 100644 --- a/backend/src/__tests__/stack-bulk-routes.test.ts +++ b/backend/src/__tests__/stack-bulk-routes.test.ts @@ -260,7 +260,7 @@ describe('POST /api/stacks/bulk execution', () => { }); it('handles update action (paid tier) by calling ComposeService.updateStack', async () => { - mockUpdateStack.mockResolvedValue({ recoveryId: null }); + mockUpdateStack.mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const { LicenseService } = await import('../services/LicenseService'); const tierSpy = vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); try { @@ -287,7 +287,7 @@ describe('POST /api/stacks/bulk execution', () => { policy: { id: 1, name: 'block-criticals', node_id: null, node_identity: '', stack_pattern: null, max_severity: 'HIGH', block_on_deploy: 1, block_on_severity: 1, block_on_kev: 0, block_on_fixable: 0, enabled: 1, replicated_from_control: 0, created_at: Date.now(), updated_at: Date.now() }, violations: [{ imageRef: 'nginx:latest', severity: 'CRITICAL', criticalCount: 3, highCount: 0, kevCount: 0, fixableCount: 0, reasons: ['severity'], scanId: 1 }], }); - mockUpdateStack.mockResolvedValue({ recoveryId: null }); + mockUpdateStack.mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); try { const res = await request(app) .post('/api/stacks/bulk') diff --git a/backend/src/__tests__/stack-op-lock-routes.test.ts b/backend/src/__tests__/stack-op-lock-routes.test.ts index 5a6aa98a..a07dbd95 100644 --- a/backend/src/__tests__/stack-op-lock-routes.test.ts +++ b/backend/src/__tests__/stack-op-lock-routes.test.ts @@ -98,7 +98,7 @@ afterAll(() => { }); beforeEach(async () => { - mockDeployStack.mockReset().mockResolvedValue({ recoveryId: null }); + mockDeployStack.mockReset().mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); mockRunCommand.mockReset(); mockRunDown.mockReset(); mockUpdateStack.mockReset(); @@ -128,7 +128,7 @@ function deferred(): Deferred { describe('Stack lifecycle mutex', () => { it('returns 409 with stack_op_in_progress when a deploy is already running', async () => { - const gate = deferred<{ recoveryId: string | null }>(); + const gate = deferred<{ recoveryId: string | null; deployedGenerationId: string | null }>(); mockDeployStack.mockImplementationOnce(() => gate.promise); const first = request(app) @@ -152,20 +152,20 @@ describe('Stack lifecycle mutex', () => { expect(second.body.error).toMatch(/already deploying/i); expect(typeof second.body.inProgress.startedAt).toBe('number'); - gate.resolve({ recoveryId: null }); + gate.resolve({ recoveryId: null, deployedGenerationId: null }); const firstRes = await first; expect(firstRes.status).toBe(200); }); it('releases the lock after a successful deploy so the next request acquires', async () => { - mockDeployStack.mockResolvedValueOnce({ recoveryId: null }); + mockDeployStack.mockResolvedValueOnce({ recoveryId: null, deployedGenerationId: null }); const first = await request(app) .post('/api/stacks/web/deploy') .set('Cookie', authCookie) .send({ skip_scan: true }); expect(first.status).toBe(200); - mockDeployStack.mockResolvedValueOnce({ recoveryId: null }); + mockDeployStack.mockResolvedValueOnce({ recoveryId: null, deployedGenerationId: null }); const second = await request(app) .post('/api/stacks/web/deploy') .set('Cookie', authCookie) @@ -181,7 +181,7 @@ describe('Stack lifecycle mutex', () => { .send({ skip_scan: true }); expect(first.status).toBe(500); - mockDeployStack.mockResolvedValueOnce({ recoveryId: null }); + mockDeployStack.mockResolvedValueOnce({ recoveryId: null, deployedGenerationId: null }); const second = await request(app) .post('/api/stacks/web/deploy') .set('Cookie', authCookie) @@ -190,7 +190,7 @@ describe('Stack lifecycle mutex', () => { }); it('blocks restart while a deploy is in flight on the same stack', async () => { - const gate = deferred<{ recoveryId: string | null }>(); + const gate = deferred<{ recoveryId: string | null; deployedGenerationId: string | null }>(); mockDeployStack.mockImplementationOnce(() => gate.promise); const deploy = request(app) @@ -207,12 +207,12 @@ describe('Stack lifecycle mutex', () => { expect(restart.body.code).toBe('stack_op_in_progress'); expect(restart.body.inProgress.action).toBe('deploy'); - gate.resolve({ recoveryId: null }); + gate.resolve({ recoveryId: null, deployedGenerationId: null }); await deploy; }); it('allows concurrent ops on different stacks', async () => { - const gate = deferred<{ recoveryId: string | null }>(); + const gate = deferred<{ recoveryId: string | null; deployedGenerationId: string | null }>(); mockDeployStack.mockImplementation(() => gate.promise); const webDeploy = request(app) @@ -229,7 +229,7 @@ describe('Stack lifecycle mutex', () => { .then(r => r); await vi.waitFor(() => expect(mockDeployStack).toHaveBeenCalledTimes(2)); - gate.resolve({ recoveryId: null }); + gate.resolve({ recoveryId: null, deployedGenerationId: null }); const [webRes, apiRes] = await Promise.all([webDeploy, apiDeploy]); expect(webRes.status).toBe(200); expect(apiRes.status).toBe(200); diff --git a/backend/src/__tests__/stack-self-protected-routes.test.ts b/backend/src/__tests__/stack-self-protected-routes.test.ts index db7330d5..cfc2c96c 100644 --- a/backend/src/__tests__/stack-self-protected-routes.test.ts +++ b/backend/src/__tests__/stack-self-protected-routes.test.ts @@ -164,7 +164,7 @@ describe('self stack lifecycle refusal', () => { }); it('allows update on a non-self stack', async () => { - mockUpdateStack.mockResolvedValue({ recoveryId: null }); + mockUpdateStack.mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const res = await request(app) .post('/api/stacks/web/update') .set('Cookie', authCookie); @@ -177,7 +177,7 @@ describe('self stack lifecycle refusal', () => { describe('POST /api/stacks/bulk self stack skip', () => { beforeEach(() => { stubSelfProject('sencho'); - mockUpdateStack.mockResolvedValue({ recoveryId: null }); + mockUpdateStack.mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); mockGetContainersByStack.mockResolvedValue([{ Id: 'c1' }]); }); diff --git a/backend/src/__tests__/stack-update-orchestrator.test.ts b/backend/src/__tests__/stack-update-orchestrator.test.ts index 7e58775f..303c2952 100644 --- a/backend/src/__tests__/stack-update-orchestrator.test.ts +++ b/backend/src/__tests__/stack-update-orchestrator.test.ts @@ -35,7 +35,7 @@ function spec(name: string): EffectiveServiceSpec { beforeEach(() => { state.updateStack.mockReset(); - state.updateStack.mockResolvedValue({ recoveryId: null }); + state.updateStack.mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); state.model = null; }); @@ -85,7 +85,7 @@ describe('StackUpdateOrchestrator stack branch', () => { { nodeId: 0, stackName: 'web', target: { scope: 'stack' }, trigger: 'manual', actor: 'tester' }, { atomic: true, terminalWs: null }, ); - expect(result).toEqual({ kind: 'stack_compose_done', recoveryId: null }); + expect(result).toEqual({ kind: 'stack_compose_done', recoveryId: null, deployedGenerationId: null }); expect(state.updateStack).toHaveBeenCalledWith('web', undefined, true); // recoveryId is forwarded from ComposeService.updateStack }); diff --git a/backend/src/__tests__/stack-update-post-recheck.test.ts b/backend/src/__tests__/stack-update-post-recheck.test.ts index 8a21c3e2..0c24e5fc 100644 --- a/backend/src/__tests__/stack-update-post-recheck.test.ts +++ b/backend/src/__tests__/stack-update-post-recheck.test.ts @@ -118,7 +118,7 @@ beforeEach(() => { mockExecute.mockImplementation(async () => { callOrder.push('execute'); - return { kind: 'stack_compose_done', recoveryId: null }; + return { kind: 'stack_compose_done', recoveryId: null, deployedGenerationId: null }; }); mockBeginStack.mockImplementation(() => { callOrder.push('beginStack'); diff --git a/backend/src/__tests__/stack-update-recovery-service.test.ts b/backend/src/__tests__/stack-update-recovery-service.test.ts index 18856c72..bfeab10e 100644 --- a/backend/src/__tests__/stack-update-recovery-service.test.ts +++ b/backend/src/__tests__/stack-update-recovery-service.test.ts @@ -140,7 +140,7 @@ vi.mock('fs/promises', () => ({ rm: (p: string, opts?: unknown) => mockRm(p, opts), })); -import { DatabaseService } from '../services/DatabaseService'; +import { DatabaseService, type StackUpdateRecoveryGenerationRow } from '../services/DatabaseService'; import { StackUpdateRecoveryService } from '../services/StackUpdateRecoveryService'; describe('StackUpdateRecoveryService', () => { @@ -172,8 +172,12 @@ describe('StackUpdateRecoveryService', () => { mockTag.mockImplementation(async () => { order.push('tag'); }); mockWriteFile.mockImplementation(async () => { order.push('write'); }); + let inserted: StackUpdateRecoveryGenerationRow | undefined; const spyInsert = vi.spyOn(DatabaseService.prototype, 'insertStackUpdateRecoveryGeneration') - .mockImplementation(() => { order.push('insert'); }); + .mockImplementation((row) => { + inserted = row; + order.push('insert'); + }); vi.spyOn(DatabaseService.prototype, 'getGlobalSettings').mockReturnValue({}); await StackUpdateRecoveryService.getInstance().captureCandidate({ @@ -185,6 +189,11 @@ describe('StackUpdateRecoveryService', () => { expect(order.indexOf('validate')).toBeLessThan(order.indexOf('tag')); expect(order.indexOf('tag')).toBeLessThan(order.indexOf('write')); expect(order.indexOf('write')).toBeLessThan(order.indexOf('insert')); + expect(inserted).toMatchObject({ + gitops_generation_id: null, + gitops_artifact_set_id: null, + gitops_source_acceptance_ref: null, + }); spyInsert.mockRestore(); }); diff --git a/backend/src/__tests__/stackRouteAuth.test.ts b/backend/src/__tests__/stackRouteAuth.test.ts index 2fc5418e..c6efe976 100644 --- a/backend/src/__tests__/stackRouteAuth.test.ts +++ b/backend/src/__tests__/stackRouteAuth.test.ts @@ -21,6 +21,15 @@ describe('classifyStackApiPath', () => { expect(classifyStackApiPath('GET', '/stacks/web/git-source')).toEqual({ kind: 'named-stack', stackName: 'web', action: 'stack:read', }); + // Load-bearing: an unclassified named-stack path is refused before the + // admin bypass, so a missing rule here 403s this route on every remote + // node for every caller. + expect(classifyStackApiPath('GET', '/stacks/web/git-source/history')).toEqual({ + kind: 'named-stack', stackName: 'web', action: 'stack:read', + }); + expect(classifyStackApiPath('GET', '/stacks/web/git-source/manifest')).toEqual({ + kind: 'named-stack', stackName: 'web', action: 'stack:read', + }); expect(classifyStackApiPath('POST', '/stacks/web/drift/recheck')).toEqual({ kind: 'named-stack', stackName: 'web', action: 'stack:read', }); diff --git a/backend/src/__tests__/stacks-failure-notifications.test.ts b/backend/src/__tests__/stacks-failure-notifications.test.ts index 52fb5755..6bef2f99 100644 --- a/backend/src/__tests__/stacks-failure-notifications.test.ts +++ b/backend/src/__tests__/stacks-failure-notifications.test.ts @@ -142,10 +142,10 @@ afterAll(() => { }); beforeEach(() => { - mockDeployStack.mockReset().mockResolvedValue({ recoveryId: null }); + mockDeployStack.mockReset().mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); mockRunCommand.mockReset(); mockRunDown.mockReset(); - mockUpdateStack.mockReset().mockResolvedValue({ recoveryId: null }); + mockUpdateStack.mockReset().mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); mockGetContainersByStack.mockReset(); mockRestartContainer.mockReset(); mockStopContainer.mockReset(); @@ -234,7 +234,7 @@ describe('deploy_failure notification on /deploy error', () => { }); it('uses trusted proxy tier headers for remote atomic deploys', async () => { - mockDeployStack.mockResolvedValue({ recoveryId: null }); + mockDeployStack.mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const token = jwt.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' }); const res = await request(app) @@ -260,18 +260,18 @@ describe('health gate begin call sites', () => { }); it('begins a gate after a manual deploy and returns its id', async () => { - mockDeployStack.mockResolvedValue({ recoveryId: null }); + mockDeployStack.mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const res = await request(app) .post('/api/stacks/myapp/deploy') .set('Cookie', authCookie) .send({ skip_scan: true }); expect(res.status).toBe(200); - expect(beginSpy).toHaveBeenCalledWith(expect.any(Number), 'myapp', 'deploy', 'testadmin'); + expect(beginSpy).toHaveBeenCalledWith(expect.any(Number), 'myapp', 'deploy', 'testadmin', { deployedGenerationId: null }); expect(res.body.healthGateId).toBe('gate-123'); }); it('links the deploy recovery generation to the observing gate', async () => { - mockDeployStack.mockResolvedValue({ recoveryId: 'rec-deploy' }); + mockDeployStack.mockResolvedValue({ deployedGenerationId: null, recoveryId: 'rec-deploy' }); const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService'); const linkSpy = vi.spyOn(StackUpdateRecoveryService.getInstance(), 'linkGateOrRetain'); const res = await request(app) @@ -284,25 +284,25 @@ describe('health gate begin call sites', () => { }); it('begins a gate after a manual update and returns its id', async () => { - mockUpdateStack.mockResolvedValue({ recoveryId: null }); + mockUpdateStack.mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const res = await request(app) .post('/api/stacks/myapp/update') .set('Cookie', authCookie) .send({ skip_scan: true }); expect(res.status).toBe(200); - expect(beginSpy).toHaveBeenCalledWith(expect.any(Number), 'myapp', 'update', 'testadmin'); + expect(beginSpy).toHaveBeenCalledWith(expect.any(Number), 'myapp', 'update', 'testadmin', { deployedGenerationId: null }); expect(res.body.healthGateId).toBe('gate-123'); }); it('begins a gate per stack in a bulk update and carries ids in the results', async () => { - mockUpdateStack.mockResolvedValue({ recoveryId: null }); + mockUpdateStack.mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const res = await request(app) .post('/api/stacks/bulk') .set('Cookie', authCookie) .send({ action: 'update', stackNames: ['myapp', 'webapp'] }); expect(res.status).toBe(200); - expect(beginSpy).toHaveBeenCalledWith(expect.any(Number), 'myapp', 'update', 'testadmin'); - expect(beginSpy).toHaveBeenCalledWith(expect.any(Number), 'webapp', 'update', 'testadmin'); + expect(beginSpy).toHaveBeenCalledWith(expect.any(Number), 'myapp', 'update', 'testadmin', { deployedGenerationId: null }); + expect(beginSpy).toHaveBeenCalledWith(expect.any(Number), 'webapp', 'update', 'testadmin', { deployedGenerationId: null }); const items = res.body.results as Array<{ stackName: string; ok: boolean; healthGateId?: string | null }>; expect(items).toHaveLength(2); for (const item of items) { @@ -318,7 +318,7 @@ describe('health gate begin call sites', () => { }); it('never begins a gate for the rollback recovery path', async () => { - mockDeployStack.mockResolvedValue({ recoveryId: null }); + mockDeployStack.mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const res = await request(app) .post('/api/stacks/myapp/rollback') .set('Cookie', authCookie); @@ -416,7 +416,7 @@ describe('failure classification on deploy/update error responses', () => { describe('post-deploy scan opt-out', () => { it('does not trigger a post-deploy scan when skip_scan is true', async () => { - mockDeployStack.mockResolvedValue({ recoveryId: null }); + mockDeployStack.mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const res = await request(app) .post('/api/stacks/myapp/deploy') @@ -530,7 +530,7 @@ describe('deploy_failure notification on /update error', () => { }); it('uses trusted proxy tier headers for remote atomic updates', async () => { - mockUpdateStack.mockResolvedValue({ recoveryId: null }); + mockUpdateStack.mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const token = jwt.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' }); const res = await request(app) diff --git a/backend/src/__tests__/stacks-from-git-skip-scan.test.ts b/backend/src/__tests__/stacks-from-git-skip-scan.test.ts index fa4f622e..c4090345 100644 --- a/backend/src/__tests__/stacks-from-git-skip-scan.test.ts +++ b/backend/src/__tests__/stacks-from-git-skip-scan.test.ts @@ -123,7 +123,7 @@ beforeEach(() => { envWritten: false, warnings: [], }); - mockDeployStack.mockResolvedValue({ recoveryId: null }); + mockDeployStack.mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); mockIsTrivyAvailable.mockReturnValue(true); mockListContainers.mockResolvedValue([{ Image: 'nginx:latest' }]); mockGetImageDigest.mockResolvedValue(null); diff --git a/backend/src/__tests__/webhooks-trigger.test.ts b/backend/src/__tests__/webhooks-trigger.test.ts index 267decb0..977e1b8c 100644 --- a/backend/src/__tests__/webhooks-trigger.test.ts +++ b/backend/src/__tests__/webhooks-trigger.test.ts @@ -461,14 +461,14 @@ describe('WebhookService.execute: health gate begin call sites', () => { const { HealthGateService } = await import('../services/HealthGateService'); vi.spyOn(policyGate, 'assertPolicyGateAllows').mockResolvedValue(undefined); vi.spyOn(fs.FileSystemService.prototype, 'getStacks').mockResolvedValue([stack]); - vi.spyOn(compose.ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: 'rec-hook' }); + vi.spyOn(compose.ComposeService.prototype, 'deployStack').mockResolvedValue({ deployedGenerationId: null, recoveryId: 'rec-hook' }); const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-hook'); const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService'); const linkSpy = vi.spyOn(StackUpdateRecoveryService.getInstance(), 'linkGateOrRetain'); const result = await WebhookService.getInstance().execute(webhook, 'deploy', 'test', true); expect(result.success).toBe(true); - expect(beginSpy).toHaveBeenCalledWith(nodeId, stack, 'deploy', 'system:webhook'); + expect(beginSpy).toHaveBeenCalledWith(nodeId, stack, 'deploy', 'system:webhook', { deployedGenerationId: null }); expect(linkSpy).toHaveBeenCalledWith('rec-hook', 'gate-hook'); }); @@ -484,11 +484,11 @@ describe('WebhookService.execute: health gate begin call sites', () => { const { HealthGateService } = await import('../services/HealthGateService'); vi.spyOn(policyGate, 'assertPolicyGateAllows').mockResolvedValue(undefined); vi.spyOn(fs.FileSystemService.prototype, 'getStacks').mockResolvedValue([stack]); - vi.spyOn(compose.ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null }); + vi.spyOn(compose.ComposeService.prototype, 'updateStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-hook'); const result = await WebhookService.getInstance().execute(webhook, 'pull', 'test', true); expect(result.success).toBe(true); - expect(beginSpy).toHaveBeenCalledWith(nodeId, stack, 'update', 'system:webhook'); + expect(beginSpy).toHaveBeenCalledWith(nodeId, stack, 'update', 'system:webhook', { deployedGenerationId: null }); }); }); diff --git a/backend/src/bootstrap/startup.ts b/backend/src/bootstrap/startup.ts index 2361eab2..5fa15cc9 100644 --- a/backend/src/bootstrap/startup.ts +++ b/backend/src/bootstrap/startup.ts @@ -29,6 +29,11 @@ import { PilotTunnelManager } from '../services/PilotTunnelManager'; import { PilotMetrics } from '../services/PilotMetrics'; import { invalidateRemoteMetaCache } from '../helpers/cacheInvalidation'; import { sweepStaleTempDirs as sweepStaleGitTempDirs, sweepGitManifestOrphans } from '../services/GitSourceService'; +import { assertCreatesSettled, reclassifyInterruptedOperations, resolveInterruptedCreates } from '../services/gitops/createRecovery'; +import { loadMigrationManifests, migrateDirectGitStacks, migrateInlineBlueprints } from '../services/gitops/migrate'; +import { setGitOpsEventSink } from '../services/gitops/publish'; +import { NotificationService } from '../services/NotificationService'; +import { sanitizeForLog } from '../utils/safeLog'; import { PORT } from '../helpers/constants'; import { LOW_MEMORY_FLOOR_BYTES } from '../utils/spawnErrors'; @@ -137,6 +142,18 @@ export async function startServer(server: Server): Promise { // Initialize the license service before any tier-gated code can run. LicenseService.getInstance().initialize(); + // Announce committed GitOps transitions from here on. This has to precede + // every reconcile and migration pass below, because all of them write + // history: the deletion reconcile tombstones applications and targets, and + // it awaits inside its loop, so a drain would otherwise land while the sink + // was still absent. Those rows would then be counted and never signalled, + // and the unannounced warning would fire on every boot that has a prepared + // deletion intent, which is the fastest way to teach an operator to ignore + // it when it means something. Nothing is connected this early, so announcing + // costs nothing; the closure resolves the notification service lazily, so it + // can be installed as soon as the database is up. + setGitOpsEventSink((event) => NotificationService.getInstance().broadcastEvent(event)); + // Deletion-intent reconciliation must finish before mutation-capable // background services or HTTP accept traffic that could recreate a stack // name still covered by a prepared/ready tombstone. @@ -145,6 +162,7 @@ export async function startServer(server: Server): Promise { } catch (err) { console.error('[Startup] Deployed stack deletion reconcile failed:', (err as Error).message); } + // Interrupted rollback restores must finish before mutation-capable services // or HTTP accept traffic. Fail closed: rethrow so unresolved intents never // leave mutators or HTTP accepting writes. @@ -156,6 +174,69 @@ export async function startServer(server: Server): Promise { } StackUpdateRecoveryService.getInstance().start(); + // Interrupted creates are settled here, ahead of the background mutators and + // the HTTP bind below, because a scheduler or webhook that fired first could + // act on a stack whose ownership is still undecided. Fail closed for the same + // reason the restore reconcile above does: a create left unresolved can leave + // a half-built stack directory that the deploy path cannot tell apart from a + // finished one, and starting anyway would let a mutator act on it. + try { + const settled = await resolveInterruptedCreates(); + for (const entry of settled) { + console.log(`[GitOps] Interrupted create for ${sanitizeForLog(entry.stackName)}: ${entry.outcome}`); + } + assertCreatesSettled(settled); + } catch (err) { + console.error('[GitOps] Interrupted-create recovery failed:', err instanceof Error ? err.stack ?? err.message : String(err)); + throw err; + } + // Operations the last process never finished are reclassified as unknown. + // Without this an interrupted fetch or apply reports as still running for + // ever and the stack is offered no actions at all. + try { + const reclassified = reclassifyInterruptedOperations(); + if (reclassified > 0) { + console.log(`[GitOps] Reclassified ${reclassified} interrupted operation(s) as unknown`); + } + } catch (err) { + console.error('[GitOps] Interrupted-operation reclassification failed:', err instanceof Error ? err.stack ?? err.message : String(err)); + } + + // Git stacks that predate the revision state model are brought into it here, + // after interrupted work is settled so migration never races a half-finished + // create, and before any mutation service can act on a stack the model does + // not yet describe. + try { + await loadMigrationManifests(); + const migrated = migrateDirectGitStacks().filter((entry) => entry.outcome !== 'skipped_current'); + for (const entry of migrated) { + console.log(`[GitOps] Migrated Git stack ${sanitizeForLog(entry.stackName)}: ${entry.outcome}`); + } + } catch (err) { + console.error('[GitOps] Migration of pre-existing Git stacks failed:', err instanceof Error ? err.stack ?? err.message : String(err)); + } + + // Blueprints migrate separately, and a failure in one must not stop the + // other: they share nothing, and coupling them would let a single unreadable + // Git stack keep every Blueprint outside the model. + try { + const migratedBlueprints = migrateInlineBlueprints().filter((entry) => entry.outcome !== 'skipped_current'); + for (const entry of migratedBlueprints) { + console.log(`[GitOps] Migrated blueprint ${sanitizeForLog(entry.stackName)}: ${entry.outcome}`); + } + } catch (err) { + console.error('[GitOps] Migration of pre-existing blueprints failed:', err instanceof Error ? err.stack ?? err.message : String(err)); + } + + // The managed-area sweep follows. It preserves anything whose ownership it + // cannot prove, so a failure here can only leave files behind, never remove + // the wrong ones, and retrying next boot is safe. + try { + await sweepGitManifestOrphans(); + } catch (err) { + console.warn('[GitManifest] Managed-area sweep failed:', err instanceof Error ? err.message : String(err)); + } + // Synchronous starts: schedule background timers and continue. None of // these fire their first tick for at least a few seconds, so they // safely run alongside the async initializers below. @@ -203,9 +284,6 @@ export async function startServer(server: Server): Promise { sweepStaleGitTempDirs().catch((err) => { console.warn('[GitSource] Temp dir sweep failed:', (err as Error).message); }); - sweepGitManifestOrphans().catch((err) => { - console.warn('[GitManifest] Managed-area sweep failed:', (err as Error).message); - }); sweepStaleTrivyTempDirs().catch((err) => { console.warn('[Trivy] Temp dir sweep failed:', (err as Error).message); }); diff --git a/backend/src/helpers/gitopsHistoryPage.ts b/backend/src/helpers/gitopsHistoryPage.ts new file mode 100644 index 00000000..16b05b32 --- /dev/null +++ b/backend/src/helpers/gitopsHistoryPage.ts @@ -0,0 +1,290 @@ +import type { Request, Response } from 'express'; +import { DatabaseService } from '../services/DatabaseService'; +import { NodeRegistry } from '../services/NodeRegistry'; +import { GitOpsStore } from '../services/gitops/store'; +import { + HISTORY_DEFAULT_LIMIT, + HISTORY_MAX_LIMIT, + HISTORY_SCAN_CAP, + decodeHistoryCursor, + encodeHistoryCursor, + queryHistoryRows, + toHistoryItem, + type GitOpsHistoryCursor, + type GitOpsHistoryFilters, + type GitOpsHistoryItem, + type HistoryOutcome, +} from '../services/gitops/history'; +import { classifyHistoryRow, satisfiesGitOpsRead } from '../services/gitops/readAuth'; +import { stackResourceSet } from './gitopsResponse'; +import type { + GitOpsApplicationRow, + GitOpsHistoryRow, + GitOpsHistoryEvidenceFields, +} from '../services/gitops/types'; + +/** + * Accepted `outcome` values. + * + * Source of truth is the `gitops_history.outcome` CHECK constraint in + * `services/gitops/schema.ts`. Typed as `HistoryOutcome[]` so adding a value + * there and forgetting it here fails the build rather than making the new + * outcome quietly unfilterable. + */ +const OUTCOMES: readonly HistoryOutcome[] = [ + 'committed', 'failed', 'skipped', 'superseded', 'recovered', 'unknown', +]; + +function isOutcome(value: string): value is HistoryOutcome { + return (OUTCOMES as readonly string[]).includes(value); +} + +/** + * How a request's rows are authorized. + * + * A union rather than a flag because skipping the row classifier is only sound + * when the query is pinned to the exact resource the caller already proved. + * Carrying the stack name in the scope means the page builder derives that + * filter itself, so the unsafe combination (skip the classifier, do not pin the + * query) cannot be written. + * + * `authorized_stack` is for a route that proved `stack:read` on one stack up + * front. That grant exempts the rows of the application holding the name *now*, + * which is what keeps a stack's own entries visible to the operator who just + * proved they may read it, including while the stack is still being created and + * the row classifier would refuse it. Every other row on that name still goes + * through the classifier: a stack name outlives the applications that held it, + * and a grant on the current one is not evidence about an earlier one. + * + * What that closes, precisely: every predecessor needs `system:audit`, whether + * it was `deleted`, `detached`, still `creating`, or has lost its stack + * resource. So the classifier partitions a stack name between the application + * holding it now, whose rows the grant covers, and everyone who held it before, + * whose rows belong to the audit audience. `classifyHistoryRow` says why + * `detached` is in that second group rather than riding its files. + */ +export type HistoryScope = + | { kind: 'per_row' } + | { kind: 'authorized_stack'; stackName: string }; + +/** + * One history entry as the API returns it. + * + * The two extra fields are the owning instance's answers to questions only it + * can settle, and they exist so a hub can authorize this entry without holding + * the instance's database: whether the stack is really on disk, and what its + * application's lifecycle currently is. Both are validated fail-closed by the + * reader rather than trusted outright. + */ +export type GitOpsHistoryPageItem = GitOpsHistoryItem & GitOpsHistoryEvidenceFields; + +export type GitOpsHistoryPage = { + items: GitOpsHistoryPageItem[]; + nextCursor: string | null; +}; + +type FilterParse = + | { ok: true; filters: GitOpsHistoryFilters } + | { ok: false; message: string }; + +function stringParam(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +/** + * Read the caller's filters off the query string. + * + * A recognized filter carrying an unusable value is rejected rather than + * dropped. Silently ignoring it would answer "show me the failures" with the + * entire trail under a 200, and on an audit surface a superset reads as an + * answer rather than as a non-answer. + * + * `stackName` is intentionally absent: it is route-fixed by the per-stack + * scope and never caller-supplied. + */ +export function parseHistoryFilters(query: Request['query']): FilterParse { + const filters: GitOpsHistoryFilters = { + applicationId: stringParam(query.applicationId), + repoIdentity: stringParam(query.repoIdentity), + configuredRef: stringParam(query.configuredRef), + commitSha: stringParam(query.commitSha), + generationId: stringParam(query.generationId), + artifactSetId: stringParam(query.artifactSetId), + rolloutCandidateId: stringParam(query.rolloutCandidateId), + rolloutGenerationId: stringParam(query.rolloutGenerationId), + trigger: stringParam(query.trigger), + actor: stringParam(query.actor), + }; + + for (const key of ['blueprintId', 'nodeId'] as const) { + const raw = stringParam(query[key]); + if (raw === undefined) continue; + const parsed = Number(raw); + if (!Number.isSafeInteger(parsed)) { + return { ok: false, message: `${key} must be an integer` }; + } + filters[key] = parsed; + } + + const outcome = stringParam(query.outcome); + if (outcome !== undefined) { + if (!isOutcome(outcome)) { + return { ok: false, message: `outcome must be one of: ${OUTCOMES.join(', ')}` }; + } + filters.outcome = outcome; + } + + return { ok: true, filters }; +} + +/** + * Resolve the node filter a hub asked this instance to apply. + * + * A hub cannot name this instance's node ids, so when it wants history for the + * node it is talking to it sends `gitopsLocalTarget=1` and this instance + * resolves that to its own default node. + * + * Refused rather than ignored when it arrives without a proxied hop. Dropping + * it would answer a request for one node's rows with every node's rows under a + * 200, the same superset-reads-as-an-answer problem the filter parser refuses + * by name. It is worse here: the hub stamps one node id onto every row it + * rewrites, so rows belonging to another node would come back positively + * claiming to belong to this one. No legitimate caller sets it, since the hub + * strips any caller-supplied value before forwarding. + */ +export function resolveLocalTargetNodeId( + req: Request, +): { ok: true; nodeId: number | undefined } | { ok: false; message: string } { + if (stringParam(req.query.gitopsLocalTarget) !== '1') return { ok: true, nodeId: undefined }; + const proxied = req.machineAuthScope === 'node_proxy' || req.machineAuthScope === 'pilot_tunnel'; + if (!proxied) { + console.warn( + `[GitOps] Refused gitopsLocalTarget on a direct request (scope=${req.machineAuthScope ?? 'none'}).`, + ); + return { ok: false, message: 'gitopsLocalTarget is not accepted on a direct request' }; + } + return { ok: true, nodeId: NodeRegistry.getInstance().getDefaultNodeId() }; +} + +export function parseLimit(value: unknown): number { + const raw = stringParam(value); + if (raw === undefined) return HISTORY_DEFAULT_LIMIT; + const parsed = Number(raw); + if (!Number.isSafeInteger(parsed) || parsed < 1) return HISTORY_DEFAULT_LIMIT; + return Math.min(parsed, HISTORY_MAX_LIMIT); +} + +/** + * Build one authorized page of history. + * + * Rows are authorized individually after the query, so the cursor advances past + * every row *examined* rather than every row kept. A caller whose grants filter + * out most of a scan window still makes forward progress instead of re-reading + * the same rejected rows on the next request. The cursor therefore names a row + * the caller may not be able to read; it carries that row's timestamp and id + * and nothing else about it. + */ +function buildHistoryPage( + req: Request, + filters: GitOpsHistoryFilters, + limit: number, + cursor: GitOpsHistoryCursor | null, + present: Set, + scope: HistoryScope, +): GitOpsHistoryPage { + // Either scope can discard rows now, so both have to scan ahead of the page + // they are filling rather than stopping at it. + const rows = queryHistoryRows(DatabaseService.getInstance().getDb(), filters, cursor, HISTORY_SCAN_CAP); + + const store = GitOpsStore.getInstance(); + // The application a stack-read grant on this name covers, read from the store + // here beside the rows it authorizes rather than accepted from the route. + // + // The live lookup spans `active` and `creating`, which is the whole point: a + // create still in flight has no other way to show the operator its own + // history. Any predecessor is absent from it, so a predecessor's rows go to + // the classifier, which refuses every one of them on a stack grant. + // + // Direct mode only, which is all `getLiveDirectApplication` returns. A + // Blueprint-delivered stack therefore resolves to null here and has every row + // classified. That is fail-closed and correct while it is `active`, since the + // classifier grants those rows on the same `stack:read`. + const scopedApplicationId = scope.kind === 'authorized_stack' + ? store.getLiveDirectApplication(scope.stackName)?.id ?? null + : null; + // One lookup per application, not per row: a busy stack contributes many + // rows that all resolve to the same application. + const applications = new Map(); + const applicationFor = (id: string): GitOpsApplicationRow | undefined => { + if (!applications.has(id)) applications.set(id, store.getApplication(id)); + return applications.get(id); + }; + + const items: GitOpsHistoryPageItem[] = []; + let lastExamined: GitOpsHistoryRow | null = null; + let exhausted = true; + + for (const row of rows) { + if (items.length === limit) { + exhausted = false; + break; + } + lastExamined = row; + const stackResourcePresent = row.stack_name !== null && present.has(row.stack_name); + const applicationLifecycleStatus = applicationFor(row.application_id)?.lifecycle_status ?? null; + // Only the application the caller's grant actually names is exempt. A row + // from an earlier application on the same stack name is a different + // resource, and is classified like any other. + const coveredByScope = scopedApplicationId !== null && row.application_id === scopedApplicationId; + if (!coveredByScope) { + const requirement = classifyHistoryRow({ + stackName: row.stack_name, + applicationLifecycleStatus, + stackResourcePresent, + }); + if (!satisfiesGitOpsRead(req, requirement)) continue; + } + items.push({ ...toHistoryItem(row), stackResourcePresent, applicationLifecycleStatus }); + } + + // A full scan window means the table may hold more beyond it, so the caller + // is handed a cursor even when this page came back short. + const moreMayFollow = !exhausted || rows.length === HISTORY_SCAN_CAP; + return { + items, + nextCursor: moreMayFollow && lastExamined + ? encodeHistoryCursor({ createdAt: lastExamined.created_at, id: lastExamined.id }) + : null, + }; +} + +/** Answer a history request under the given scope. */ +export async function respondWithHistory( + req: Request, + res: Response, + scope: HistoryScope, +): Promise { + const parsed = parseHistoryFilters(req.query); + if (!parsed.ok) { + res.status(400).json({ error: parsed.message }); + return; + } + const cursorRaw = stringParam(req.query.cursor); + const cursor = cursorRaw === undefined ? null : decodeHistoryCursor(cursorRaw); + if (cursorRaw !== undefined && cursor === null) { + res.status(400).json({ error: 'Invalid page cursor. Restart from the first page.' }); + return; + } + const localTarget = resolveLocalTargetNodeId(req); + if (!localTarget.ok) { + res.status(400).json({ error: localTarget.message }); + return; + } + const filters: GitOpsHistoryFilters = { + ...parsed.filters, + ...(localTarget.nodeId === undefined ? {} : { nodeId: localTarget.nodeId }), + ...(scope.kind === 'authorized_stack' ? { stackName: scope.stackName } : {}), + }; + const present = await stackResourceSet(req.nodeId); + res.json(buildHistoryPage(req, filters, parseLimit(req.query.limit), cursor, present, scope)); +} diff --git a/backend/src/helpers/gitopsResponse.ts b/backend/src/helpers/gitopsResponse.ts new file mode 100644 index 00000000..87f48558 --- /dev/null +++ b/backend/src/helpers/gitopsResponse.ts @@ -0,0 +1,250 @@ +import { DatabaseService } from '../services/DatabaseService'; +import { FileSystemService } from '../services/FileSystemService'; +import { GitOpsStore } from '../services/gitops/store'; +import { missingBlueprintApplicationRevision, NOT_APPLICABLE_REVISION, projectApplication } from '../services/gitops/derive'; +import { sanitizeForLog } from '../utils/safeLog'; +import type { GitOpsApplicationRow, GitOpsRevisionProjection } from '../services/gitops/types'; + +export { NOT_APPLICABLE_REVISION }; + +/** + * Whether the health gate is switched off for this instance. + * + * Only the explicit `'0'` disables it, matching HealthGateService. The setting + * is seeded to `'1'` at schema init, so an absent row means a database whose + * seed did not run; reading that as enabled matches the seeded default. + */ +function healthGateDisabled(): boolean { + return DatabaseService.getInstance().getGlobalSettings()['health_gate_enabled'] === '0'; +} + +/** + * The Blueprint application that materialized a stack directory on this node. + * + * A Blueprint application is stored with `stack_name` NULL, because it + * describes a Blueprint rather than one placement of it, so no lookup by stack + * name can reach it. Meanwhile the reconciler materializes every Blueprint as a + * real stack directory named after the Blueprint, on each node it targets. So + * a stack-state surface asked about that directory has to bridge the two, or it + * reports "no GitOps here" about a stack GitOps is actively managing. + * + * The deployment row is what makes the bridge safe, and it has to be the right + * predicate rather than merely a present row. Blueprint names and stack names + * share one namespace, and `name_conflict` is written *precisely* when a stack + * of that name already exists on the node and Sencho does not own it. Treating + * that row as ownership would hand the unrelated stack's operator this + * Blueprint's repository, ref, and SHA pointers: the exact collision the bridge + * exists to rule out. `last_deployed_at` being set is what proves this + * Blueprint really did write that directory, and it also excludes `pending`, + * `pending_state_review`, and a first deploy that failed. Same predicate the + * delete and withdraw paths use. + * + * Only a live Blueprint application qualifies. A retired one has no stronger + * claim on the directory than anything else, and the Blueprint surface still + * reports it through projectBlueprintRevision. + * + * Three outcomes, not two, because "no Blueprint owns this" and "a Blueprint + * owns this and its application row is gone" must not look alike to the caller. + * The guards below establish ownership from the deployment row; reaching the + * lookup and missing therefore means a referential fault, and answering + * `unowned` there would let the caller fall through and report some older + * Direct application's repository and SHA as this directory's GitOps state. + * + * Known limit: this resolves on the instance holding the Blueprint rows, which + * is the hub. A Blueprint deployed to a remote node is materialized there by a + * file push, and the drift route for it executes on that remote, which has no + * blueprint, deployment, or application row of its own. So a Blueprint-owned + * stack on a remote node still projects not_applicable. + */ +type BlueprintOwnership = + | { kind: 'unowned' } + | { kind: 'owned'; application: GitOpsApplicationRow } + | { kind: 'owned_application_missing'; blueprintId: number }; + +function blueprintApplicationOwningStack(stackName: string, nodeId: number | undefined): BlueprintOwnership { + if (nodeId === undefined) return { kind: 'unowned' }; + const db = DatabaseService.getInstance(); + const blueprint = db.getBlueprintByName(stackName); + if (!blueprint) return { kind: 'unowned' }; + const deployment = db.getDeployment(blueprint.id, nodeId); + if (!deployment) return { kind: 'unowned' }; + if (deployment.last_deployed_at == null) return { kind: 'unowned' }; + if (deployment.status === 'name_conflict' || deployment.status === 'withdrawn') return { kind: 'unowned' }; + const application = GitOpsStore.getInstance().getLiveBlueprintApplication(blueprint.id); + if (!application) return { kind: 'owned_application_missing', blueprintId: blueprint.id }; + return { kind: 'owned', application }; +} + +/** + * The revision projection for a stack's own Direct Git attachment. + * + * Live applications only. Detach deletes the Git-source row and writes the + * tombstone in one transaction, so a source row beside a detached application + * is not a producible state, and the Git-source routes are the only callers. + * The detached case is reachable through the stack directory instead, which + * survives a detach, and projectManagedStackRevision below is what answers it. + * + * Used by the Git-source routes, which answer specifically about Direct + * attachment. They must not be answered with some other application's identity, + * so the Blueprint bridge above is deliberately not applied here. That also + * keeps them off the read classifier's lifecycle input. + */ +export function projectStackRevision(stackName: string): GitOpsRevisionProjection { + const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName); + if (!app) return NOT_APPLICABLE_REVISION; + return projectApplication(app.id, healthGateDisabled()); +} + +/** + * The revision projection for a stack's state, from whichever application + * manages the directory on this node. + * + * Resolution order is precedence, not preference. A live Direct application is + * the stack's own Git attachment and always wins. Failing that, a Blueprint may + * have materialized the directory. Failing both, a detached Direct application + * still describes what the stack was before it was detached, which is a + * different fact from never having been modelled and is what the source + * deriver's `not_live` status exists to report. + * + * A `deleted` application is deliberately never resolved. Deletion means the + * stack was removed, so any directory of that name now is a different stack, + * and reporting the old application's repository and SHA against it would + * disclose one stack's Git identity through another's name. `readAuth` excludes + * `deleted` from stack-grant reads for that same reason. + * + * Separate from projectStackRevision because the two answer different + * questions. "What Git source is attached to this stack" must never be answered + * with a Blueprint's identity; "what manages this stack" must be. + */ +export function projectManagedStackRevision(stackName: string, nodeId: number | undefined): GitOpsRevisionProjection { + const store = GitOpsStore.getInstance(); + const direct = store.getLiveDirectApplication(stackName); + if (direct) return projectApplication(direct.id, healthGateDisabled()); + + const owned = blueprintApplicationOwningStack(stackName, nodeId); + if (owned.kind === 'owned_application_missing') { + // Proven ownership with nothing to project. Falling through would answer + // with an unrelated application; the sentinel alone would call a fault a + // normal absence. Say both. + console.error( + '[GitOps] Blueprint %s deployed stack %s on node %s but has no live application row.', + sanitizeForLog(owned.blueprintId), sanitizeForLog(stackName), sanitizeForLog(nodeId ?? 'unknown'), + ); + return missingBlueprintApplicationRevision(owned.blueprintId, stackName); + } + if (owned.kind === 'owned') return scopeToNode(projectApplication(owned.application.id, healthGateDisabled()), nodeId); + + const detached = store.getDetachedDirectApplication(stackName); + if (!detached) return NOT_APPLICABLE_REVISION; + return projectApplication(detached.id, healthGateDisabled()); +} + +/** + * Narrow a Blueprint projection to the node being asked about. + * + * A Blueprint application's targets span every node it is placed on, and this + * route is authorized by a grant on one stack name, not by the fleet-wide read + * the Blueprint catalog requires. Reporting the whole roster here would answer + * a stack-scoped question with fleet-scoped placement. Scoping is also simply + * the right answer: the question is what manages this directory on this node. + */ +function scopeToNode(projection: GitOpsRevisionProjection, nodeId: number | undefined): GitOpsRevisionProjection { + if (projection.targetMode === 'not_applicable' || nodeId === undefined) return projection; + return { ...projection, targets: projection.targets.filter(target => target.nodeId === nodeId) }; +} + +/** + * The revision projection for a Blueprint's application. + * + * Live applications only. Blueprint retirement writes `deleted`, never + * `detached`, so there is no detached Blueprint state to report; a Blueprint + * that predates the model, or one migration has not brought in, projects + * `not_applicable` rather than throwing, so the catalog gets a uniform shape + * across rows. + */ +export function projectBlueprintRevision(blueprintId: number): GitOpsRevisionProjection { + const app = GitOpsStore.getInstance().getLiveBlueprintApplication(blueprintId); + if (!app) return NOT_APPLICABLE_REVISION; + return projectApplication(app.id, healthGateDisabled()); +} + +/** + * Revisions for the Blueprints a mutation actually moved, `blueprintId` ascending. + * + * Sorted here rather than at each call site because the callers hand over ids + * in the order their producer happened to visit them, which is a Map iteration + * order, not a contract. Duplicates are collapsed: a caller that reports the + * same Blueprint twice would otherwise put two copies of one projection on the + * wire and let a consumer count the same move twice. + */ +function projectBlueprintRevisions(blueprintIds: readonly number[]): GitOpsRevisionProjection[] { + return [...new Set(blueprintIds)].sort((a, b) => a - b).map(projectBlueprintRevision); +} + +/** + * Revisions to decorate a mutation that has already committed. + * + * Best effort on purpose, and the one place in this file that swallows + * anything. The write is done by the time this runs, so letting a projection + * fault escape would land in the route's own catch and answer a successful + * cordon, label, or node deletion with a 500. The operator would then retry a + * deletion that already happened and be told the node does not exist, or retry + * a create and be told the name is taken. A field the response can live without + * must not be able to invert what the response means. + * + * The failure is logged with the operation that produced it rather than + * dropped, and the field degrades to an empty list, which every consumer + * already handles: it is what a mutation that moved nothing returns. + * + * Read routes deliberately do not use this. There the revision is part of the + * answer, not a decoration on one, so a fault there should surface. + */ +export function projectCommittedRevisions( + blueprintIds: readonly number[], + operation: string, +): GitOpsRevisionProjection[] { + try { + return projectBlueprintRevisions(blueprintIds); + } catch (error) { + console.error('[GitOps] Revision projection failed after %s committed:', operation, error); + return []; + } +} + +/** + * One revision to decorate a mutation that has already committed. + * + * Same contract as projectCommittedRevisions, degrading to the not-applicable + * shape so the response keeps one field shape across every mutation. + */ +export function projectCommittedRevision( + blueprintId: number, + operation: string, +): GitOpsRevisionProjection { + try { + return projectBlueprintRevision(blueprintId); + } catch (error) { + console.error('[GitOps] Revision projection failed after %s committed:', operation, error); + return NOT_APPLICABLE_REVISION; + } +} + +/** + * Stack directories that exist on this instance right now. + * + * Read once per request. The list and history routes share one probe across + * every row; the per-stack route pays a full listing to answer a single + * membership test, which is the same cost its existence check already paid. + * + * Deliberately the strict listing. This set is the evidence behind + * `stackResourcePresent`, which decides whether a row can be authorized by a + * stack grant and which travels to other instances as a positive claim about + * the filesystem. The lenient variant answers a failed directory read with an + * empty list, which here would read as "every stack is gone": every row would + * silently fall to Admin and a scoped operator would receive an empty list and + * an empty audit trail, indistinguishable from having none. A read failure is + * raised so the caller can report it instead. + */ +export async function stackResourceSet(nodeId: number | undefined): Promise> { + return new Set(await FileSystemService.getInstance(nodeId).getStacksStrict()); +} diff --git a/backend/src/helpers/proxyExemptPaths.ts b/backend/src/helpers/proxyExemptPaths.ts index 49d4c409..c1c23e1d 100644 --- a/backend/src/helpers/proxyExemptPaths.ts +++ b/backend/src/helpers/proxyExemptPaths.ts @@ -28,6 +28,10 @@ export function isProxyExemptPath(path: string): boolean { // the local hub (centralized audit, fleet schedules, notification routing // rules, the admin-only aggregated logs feed and its stream counters) and // private registry credentials, which are stored and managed per instance. +// Blueprints and node labels are hub-owned too: the hub is the only instance +// that holds the desired-state definitions and the label set its placement +// selectors resolve against, so a proxied request would read or write a remote +// node's unrelated copy instead of the fleet's actual intent. // Routed to the local hub when nodeId resolves to local, but rejected when // nodeId resolves to a remote node so a script/curl call cannot trick the proxy // into forwarding the request across a node boundary. This matters for the logs @@ -57,6 +61,8 @@ export const HUB_ONLY_PREFIXES: readonly string[] = [ '/api/system/log-stream-metrics/', '/api/registries/', '/api/secrets/', + '/api/blueprints/', + '/api/node-labels/', ]; /** Returns true when the path is hub-only and must not be proxied to a remote node. */ diff --git a/backend/src/helpers/stackRouteAuth.ts b/backend/src/helpers/stackRouteAuth.ts index 75440791..2a06bac4 100644 --- a/backend/src/helpers/stackRouteAuth.ts +++ b/backend/src/helpers/stackRouteAuth.ts @@ -59,6 +59,8 @@ const EXACT_SUFFIX_RULES: readonly SuffixRule[] = [ { method: 'GET', suffix: '/files/permissions', action: 'stack:read' }, { method: 'GET', suffix: '/activity', action: 'stack:read' }, { method: 'GET', suffix: '/git-source', action: 'stack:read' }, + { method: 'GET', suffix: '/git-source/history', action: 'stack:read' }, + { method: 'GET', suffix: '/git-source/manifest', action: 'stack:read' }, // Edit { method: 'PUT', suffix: '', action: 'stack:edit' }, diff --git a/backend/src/index.ts b/backend/src/index.ts index 871e0318..3ed6faaa 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -52,6 +52,7 @@ import { nodesRouter } from './routes/nodes'; import { stacksRouter } from './routes/stacks'; import { stackActivityRouter } from './routes/stackActivity'; import { stackMetricsRouter } from './routes/stackMetrics'; +import { gitopsMetricsRouter } from './routes/gitopsMetrics'; import { fileExplorerMetricsRouter } from './routes/fileExplorerMetrics'; import { stackActivityMetricsRouter } from './routes/stackActivityMetrics'; import { secretsRouter } from './routes/secrets'; @@ -156,6 +157,7 @@ app.use('/api/nodes', nodesRouter); app.use('/api/stacks', stackActivityRouter); app.use('/api/stacks', stacksRouter); app.use('/api/stack-metrics', stackMetricsRouter); +app.use('/api/gitops-metrics', gitopsMetricsRouter); app.use('/api/file-explorer-metrics', fileExplorerMetricsRouter); app.use('/api/stack-activity-metrics', stackActivityMetricsRouter); diff --git a/backend/src/proxy/gitopsIdentityProxy.ts b/backend/src/proxy/gitopsIdentityProxy.ts new file mode 100644 index 00000000..41d16d36 --- /dev/null +++ b/backend/src/proxy/gitopsIdentityProxy.ts @@ -0,0 +1,639 @@ +import type { IncomingMessage } from 'http'; +import type { Readable } from 'stream'; +import zlib from 'zlib'; +import type { Request } from 'express'; +import { isRecord } from '../services/gitops/json'; +import { classifyHistoryRow, classifySourceRow } from '../services/gitops/readAuth'; +import type { GitOpsReadRequirement } from '../services/gitops/readAuth'; +import { sanitizeForLog } from '../utils/safeLog'; + +/** + * Ceiling on one decompressed identity response, in bytes. + * + * These routes return configuration pages, audit pages, and one drift report, + * not bulk payloads. A remote answering with more than this is either + * misbehaving or not the endpoint we think it is, and buffering it whole to + * rewrite node ids would hand a remote instance a way to exhaust hub memory. + * + * The drift pair is the one entry whose size scales with the stack rather than + * being bounded by configuration: it carries a finding per drifted service plus + * a capped 20-row ledger. That is still far short of this ceiling, but it is + * the reason the ceiling is a real bound here and a sanity check elsewhere, so + * raising it needs a drift report that genuinely outgrew it, not a hunch. + */ +export const IDENTITY_PROXY_MAX_BYTES = 1048576; + +/** + * How long this hop waits on a remote before giving up, in milliseconds. + * + * A remote that sends response headers and then stalls emits no end, no error, + * and no abort, so without a bound the hop would never settle and would pin the + * buffered body and both sockets for as long as the connection stayed open. + */ +export const IDENTITY_PROXY_TIMEOUT_MS = 30000; + +const HISTORY_ROUTES = [ + /^\/git-sources\/history\/?$/, + /^\/stacks\/[^/]+\/git-source\/history\/?$/, +]; + +/** + * Paths whose JSON carries node identities this hub has to correct, each with + * the methods that reach them. + * + * Per route rather than one blanket verb check because the drift pair is a GET + * and a POST over the same payload. A re-check that answered with the remote's + * own numbering while the GET beside it answered with the hub's would make the + * same object mean two different things depending on how it was asked for. + * + * The history pair is spread in rather than repeated, so the two lists cannot + * drift into disagreeing about what counts as history. + */ +const IDENTITY_ROUTES: readonly { pattern: RegExp; method: string }[] = [ + { pattern: /^\/git-sources\/?$/, method: 'GET' }, + { pattern: /^\/stacks\/[^/]+\/git-source\/?$/, method: 'GET' }, + ...HISTORY_ROUTES.map(pattern => ({ pattern, method: 'GET' })), + { pattern: /^\/stacks\/[^/]+\/drift\/?$/, method: 'GET' }, + { pattern: /^\/stacks\/[^/]+\/drift\/recheck\/?$/, method: 'POST' }, +]; + +/** + * Whether this request is one the hub buffers and rewrites. + * + * Deliberately narrow. Logs, downloads, and event streams must keep flowing + * through the streaming hop: buffering them to rewrite identities they do not + * carry would break streaming and cap responses that are legitimately large. + */ +export function isGitOpsIdentityJsonRoute(pathname: string, method: string): boolean { + return IDENTITY_ROUTES.some(route => route.method === method && route.pattern.test(pathname)); +} + +export function isGitOpsHistoryRoute(pathname: string): boolean { + return HISTORY_ROUTES.some(pattern => pattern.test(pathname)); +} + +/** + * Replace a remote's node id with the id this hub knows it by. + * + * A remote instance numbers its own nodes from one and has never heard of the + * hub's numbering, so every id it reports is a statement in its own namespace. + * Left alone, a hub joining two nodes would show two different machines as the + * same node. + * + * Only JSON numbers are replaced. A null is preserved, because "no node" is a + * fact the remote is entitled to state and inventing a node there would claim a + * placement that does not exist. Non-enumerated keys and every string field + * (application ids, stack names) are left exactly as received. + */ +function rewriteNodeId(container: unknown, nodeId: number): void { + if (!isRecord(container)) return; + if (typeof container.nodeId === 'number') container.nodeId = nodeId; +} + +function rewriteTargets(container: unknown, nodeId: number): void { + if (!isRecord(container)) return; + const targets = container.targets; + if (Array.isArray(targets)) { + for (const target of targets) rewriteNodeId(target, nodeId); + } + const drift = container.drift; + if (Array.isArray(drift)) { + for (const item of drift) { + if (!isRecord(item)) continue; + const affected = item.affectedTargets; + if (Array.isArray(affected)) { + for (const entry of affected) rewriteNodeId(entry, nodeId); + } + } + } +} + +/** + * Rewrite every node id inside one revision projection or recorded delta. + * + * The top-level id is rewritten too. A projection does not carry one, but the + * `before`/`after` deltas this also walks are an open record shape, so a delta + * naming a node would otherwise reach the client in the remote's numbering. + */ +function rewriteRevision(revision: unknown, nodeId: number): void { + if (!isRecord(revision)) return; + rewriteNodeId(revision, nodeId); + rewriteTargets(revision, nodeId); +} + +/** + * Rewrite the enumerated node-id positions in one parsed identity response. + * + * `gitopsRevisions` should not appear on these Direct Git routes; it is walked + * anyway so a response that nests one is corrected rather than passed through + * carrying a foreign node id. + */ +export function rewriteIdentityPayload(payload: unknown, nodeId: number): void { + if (Array.isArray(payload)) { + for (const row of payload) rewriteIdentityObject(row, nodeId); + return; + } + if (!isRecord(payload)) return; + const items = payload.items; + if (Array.isArray(items)) { + for (const item of items) { + rewriteIdentityObject(item, nodeId); + if (!isRecord(item)) continue; + rewriteRevision(item.before, nodeId); + rewriteRevision(item.after, nodeId); + } + return; + } + rewriteIdentityObject(payload, nodeId); +} + +function rewriteIdentityObject(row: unknown, nodeId: number): void { + if (!isRecord(row)) return; + rewriteNodeId(row, nodeId); + rewriteTargets(row, nodeId); + rewriteRevision(row.gitopsRevision, nodeId); + const revisions = row.gitopsRevisions; + if (Array.isArray(revisions)) { + for (const revision of revisions) rewriteRevision(revision, nodeId); + } +} + +/** + * Rework an identity request's query before it leaves the hub. + * + * Returns the query string to forward, or a refusal the hub answers itself. + * + * `gitopsLocalTarget` is synthesized here and nowhere else, so a caller-supplied + * one is always stripped first: it instructs the remote to filter to its own + * node, and a client that could set it would be steering another instance's + * query. + * + * A history request naming the node being proxied to is asking for that node's + * rows, which the remote can only express about itself, so the hub translates + * it. A request naming a different node is refused rather than forwarded: the + * remote would answer about itself and the page would look like an answer to a + * question nobody asked. + */ +export function prepareIdentityQuery( + search: URLSearchParams, + pathname: string, + hubNodeId: number | undefined, +): { kind: 'forward'; search: URLSearchParams } | { kind: 'refuse'; error: string } { + const forwarded = new URLSearchParams(search); + forwarded.delete('gitopsLocalTarget'); + const requestedNodeId = forwarded.get('nodeId'); + forwarded.delete('nodeId'); + + if (!isGitOpsHistoryRoute(pathname)) return { kind: 'forward', search: forwarded }; + + if (requestedNodeId !== null && (hubNodeId === undefined || requestedNodeId !== String(hubNodeId))) { + return { + kind: 'refuse', + error: 'History for another node cannot be read through this node. Select that node instead.', + }; + } + // Set even when the caller named no node. A request routed to this node is a + // question about this node, and the hub stamps one node id across every row + // it rewrites: without the filter the remote would answer with rows from all + // of its own nodes and they would come back claiming to belong to this one. + forwarded.set('gitopsLocalTarget', '1'); + return { kind: 'forward', search: forwarded }; +} + +/** + * Drop rows the caller may not read from an already-rewritten remote payload. + * + * Runs after the rewrite so the classifier sees this hub's node ids. Relative + * order is preserved, and a page filtered down to nothing still returns its + * envelope: `nextCursor` is the remote's own last examined row, so a caller + * whose grants reject a whole window keeps paging rather than concluding the + * history is empty. + */ +export function filterIdentityCollection( + payload: unknown, + keepRow: (row: unknown) => boolean, + keepItem: (item: unknown) => boolean, +): unknown { + if (Array.isArray(payload)) return payload.filter(keepRow); + if (isRecord(payload) && Array.isArray(payload.items)) { + return { ...payload, items: payload.items.filter(keepItem) }; + } + return payload; +} + +/** The hub's own node id for this hop, when one is set. */ +export function hubNodeIdFor(req: Request): number | undefined { + return typeof req.nodeId === 'number' ? req.nodeId : undefined; +} + +/** + * Apply this hub's read rules to a remote collection. + * + * The remote authorized its own rows for the machine account the hub proxies + * with, which says nothing about the person behind the request. So the hub + * re-decides every row against the signed-in user, using the same classifiers + * the local routes use. + * + * It classifies from what the owning instance stated (`stackResourcePresent`, + * `applicationLifecycleStatus`) because the hub has no application row for + * another instance's stacks. Both are validated fail-closed, and any + * `historyAuth`-style verdict a remote might volunteer is ignored. + * + * The honest limit: this catches a peer that is outdated, misconfigured, or + * simply not filtering, because anything it omits or malforms degrades to the + * Admin or audit bucket. It is not a defense against a hostile peer, which + * could state evidence that downgrades a row to a stack read on a name it + * chooses. A peer that far gone can fabricate the row contents anyway. + * + * Per-stack routes are not filtered here. Those were authorized by name before + * the hop, and re-filtering their rows would hide a stack's own entries from + * the operator who just proved they may read it. + */ +export function filterRemoteIdentityPayload( + pathname: string, + payload: unknown, + canRead: (requirement: GitOpsReadRequirement) => boolean, + nodeId: number, +): unknown { + // Only the two cross-stack collections are filtered here. + if (!/^\/git-sources(\/history)?\/?$/.test(pathname)) return payload; + + const filtered = filterRows(canRead, payload); + const received = countRows(payload); + const kept = countRows(filtered); + // Keeping nothing from a page that had rows is the signature of a remote + // whose response predates the evidence fields this classification needs. + // The client is told nothing (a withheld count discloses what it may not + // read), but an operator staring at an empty page needs the reason. + if (received > 0 && kept === 0) { + console.warn( + `[Proxy] GitOps identity filter kept 0 of ${received} rows from node ${nodeId}. ` + + 'Either the caller may read none of them, or that node is too old to report ' + + 'stackResourcePresent and applicationLifecycleStatus.', + ); + } + return filtered; +} + +function countRows(payload: unknown): number { + if (Array.isArray(payload)) return payload.length; + if (isRecord(payload) && Array.isArray(payload.items)) return payload.items.length; + return 0; +} + +function filterRows( + canRead: (requirement: GitOpsReadRequirement) => boolean, + payload: unknown, +): unknown { + return filterIdentityCollection( + payload, + (row) => isRecord(row) && canRead(classifySourceRow({ + stackName: row.stack_name, + gitopsRevision: row.gitopsRevision, + stackResourcePresent: row.stackResourcePresent, + })), + (item) => isRecord(item) && canRead(classifyHistoryRow({ + stackName: item.stackName, + applicationLifecycleStatus: item.applicationLifecycleStatus, + stackResourcePresent: item.stackResourcePresent, + })), + ); +} + +/** + * Headers that describe one connection's framing and must not be replayed. + * + * The hub decodes and rewrites the body, so the upstream's length and encoding + * describe bytes that no longer exist. Forwarding them would frame the response + * as something it is not. + */ +const HOP_BY_HOP_HEADERS = [ + 'content-length', 'content-encoding', 'transfer-encoding', 'connection', + 'keep-alive', 'proxy-connection', 'te', 'trailer', 'upgrade', +]; + +/** + * End-to-end headers that stay meaningful after the body is rewritten. + * + * Deliberately excludes every cache validator and cacheability header. The + * upstream validators describe the remote's unfiltered representation, while + * the body the hub sends is rewritten and filtered for one caller; letting a + * client pair the two would let a cached page outlive the authorization it was + * filtered under. This hop answers `no-store` instead, so nothing downstream + * retains a filtered page to revalidate with. + */ +const FORWARDED_HEADERS = [ + 'location', 'retry-after', 'x-sencho-proxy', +]; + +/** + * Request headers that ask an upstream to answer from its cache. + * + * Stripped before forwarding on every identity route. A remote answering 304 + * would hand back a status the hub relays without a body, and the client would + * keep serving the page it cached under the remote's validator, which was + * never filtered by this hub. Without the strip, a permission revoked between + * two reads would not take effect until the remote's content actually changed. + */ +export const CONDITIONAL_REQUEST_HEADERS = [ + 'if-none-match', 'if-modified-since', 'if-match', 'if-unmodified-since', +] as const; + +/** Remove every conditional request header from one outgoing request. */ +export function stripConditionalRequestHeaders(target: { removeHeader(name: string): unknown }): void { + for (const header of CONDITIONAL_REQUEST_HEADERS) target.removeHeader(header); +} + +export type IdentityTerminalKind = + | 'rewrite' + | 'passthrough' + | 'too_large' + | 'decompress_error' + | 'parse_error' + | 'rewrite_failed' + | 'upstream_failed' + | 'downstream_close'; + +/** + * What each terminal does: answer with the remote's status, answer with one the + * hub generates, or write nothing at all. + * + * Total rather than partial, and the single source for all three decisions this + * hop makes per terminal (log, timing outcome, response). A partial table meant + * a missing entry silently read as "use the upstream status", so a ninth kind + * added later would inherit the remote's 200 for a body the hub could not read. + * Here the compiler demands the answer. + * + * `rewrite_failed` is a 500 rather than a 502 on purpose: everything it covers + * runs on this instance, so blaming the remote would send an operator to check + * a node that did nothing wrong. + */ +type IdentityDisposition = + | { respond: 'silent' } + | { respond: 'upstream' } + | { respond: 'generated'; status: number; body: { error: string; code: string } }; + +const TERMINALS: Record = { + rewrite: { respond: 'upstream' }, + passthrough: { respond: 'upstream' }, + downstream_close: { respond: 'silent' }, + too_large: { + respond: 'generated', + status: 502, + body: { error: 'Remote GitOps response too large', code: 'gitops_proxy_too_large' }, + }, + decompress_error: { + respond: 'generated', + status: 502, + body: { error: 'Remote GitOps response could not be decoded', code: 'gitops_proxy_decompress_failed' }, + }, + parse_error: { + respond: 'generated', + status: 502, + body: { error: 'Remote GitOps response was not valid JSON', code: 'gitops_proxy_unparseable' }, + }, + rewrite_failed: { + respond: 'generated', + status: 500, + body: { error: 'This instance could not process the GitOps response', code: 'gitops_proxy_rewrite_failed' }, + }, + upstream_failed: { + respond: 'generated', + status: 502, + body: { error: 'Remote GitOps response failed', code: 'gitops_proxy_upstream_failed' }, + }, +}; + +/** Whether a terminal represents a failure worth reporting to the operator. */ +export function isIdentityFailure(kind: IdentityTerminalKind): boolean { + return TERMINALS[kind].respond === 'generated'; +} + +/** Decode one upstream body according to its declared encoding. */ +function decodeStream(proxyRes: IncomingMessage): Readable { + const encoding = String(proxyRes.headers['content-encoding'] ?? '').toLowerCase().trim(); + if (encoding === 'gzip' || encoding === 'x-gzip') return proxyRes.pipe(zlib.createGunzip()); + if (encoding === 'deflate') return proxyRes.pipe(zlib.createInflate()); + if (encoding === 'br') return proxyRes.pipe(zlib.createBrotliDecompress()); + return proxyRes; +} + +export type IdentityResponseHooks = { + /** Rewrite and optionally filter a parsed 200/201 body. Returns what to send. */ + transform: (payload: unknown) => unknown; + /** Runs exactly once, whatever the outcome. */ + finalizeTiming: (kind: IdentityTerminalKind) => void; +}; + +/** + * The downstream response, as this handler actually uses it. + * + * Structural rather than the Express type so the terminal rules can be tested + * against a plain object. An Express response satisfies it as-is. + */ +export type IdentityResponseSink = { + headersSent: boolean; + writableEnded: boolean; + statusCode: number; + removeHeader(name: string): void; + setHeader(name: string, value: number | string | readonly string[]): unknown; + end(body?: Buffer): unknown; + on(event: 'close', listener: () => void): unknown; +}; + +/** + * Write the one answer a terminal calls for. + * + * Split from the settling logic so the response rules can be read, and tested, + * without a stream in the picture. Everything it needs is passed in. + */ +export function writeTerminal( + res: IdentityResponseSink, + proxyRes: IncomingMessage, + kind: IdentityTerminalKind, + body?: Buffer, +): void { + const disposition = TERMINALS[kind]; + if (disposition.respond === 'silent') return; + if (res.headersSent || res.writableEnded) { + // Unreachable while this hop owns the response, so if it ever fires the + // client is left with a half-written body and no other trace. + console.warn(`[Proxy] GitOps identity hop could not answer (kind=${kind}): the response was already sent.`); + return; + } + + for (const header of HOP_BY_HOP_HEADERS) res.removeHeader(header); + for (const header of FORWARDED_HEADERS) { + const value = proxyRes.headers[header]; + if (value !== undefined) res.setHeader(header, value); + } + // Every answer this hop writes is rewritten or filtered for one caller, so + // nothing downstream may retain it: a stored page keyed to no validator is + // exactly how a stale authorized view survives its own permission change. + res.setHeader('cache-control', 'no-store'); + + if (disposition.respond === 'generated') { + // A hub-generated failure never borrows the upstream status: reporting our + // own inability to read the response as the remote's 200 would call a + // truncated or undecodable body a successful answer. + const generatedBody = Buffer.from(JSON.stringify(disposition.body)); + res.statusCode = disposition.status; + res.setHeader('content-type', 'application/json; charset=utf-8'); + res.setHeader('content-length', String(generatedBody.length)); + res.end(generatedBody); + return; + } + + const status = proxyRes.statusCode ?? 502; + res.statusCode = status; + // 204 and 304 carry no body by definition. A 304 still gets the no-store + // answer above rather than the remote's validators: relaying them would let + // a client keep serving a cached page this hub never re-filtered. + if (status === 204 || status === 304 || body === undefined || body.length === 0) { + res.end(); + return; + } + if (kind === 'rewrite') { + res.setHeader('content-type', 'application/json; charset=utf-8'); + } else { + const upstreamType = proxyRes.headers['content-type']; + if (upstreamType !== undefined) res.setHeader('content-type', upstreamType); + } + res.setHeader('content-length', String(body.length)); + res.end(body); +} + +/** + * Buffer, rewrite, and answer one identity response. + * + * The hub has to hold the whole body to correct node ids inside it, which is + * why this hop exists separately from the streaming one. Everything here is + * arranged so exactly one terminal answer is written: a response that is too + * large, fails to decode, dies upstream, or loses its client all converge on + * the same single-shot responder, and duplicate events are dropped rather than + * writing a second time onto a finished response. + */ +export function handleIdentityResponse( + proxyRes: IncomingMessage, + res: IdentityResponseSink, + hooks: IdentityResponseHooks, +): void { + const chunks: Buffer[] = []; + let total = 0; + let settled = false; + let decoded: Readable | undefined; + + const finish = (kind: IdentityTerminalKind, body?: Buffer, cause?: unknown): void => { + if (settled) return; + settled = true; + if (proxyRes.readable) proxyRes.destroy(); + // The decompressor holds a native zlib context that piping alone does not + // release, and the buffered chunks are dead once a terminal is chosen. + // Both matter most on `too_large`, the one path a remote can trigger at + // will. + if (decoded !== undefined && decoded !== proxyRes) decoded.destroy(); + if (kind !== 'rewrite') chunks.length = 0; + + // Reported unconditionally. The timing hook below is a developer-mode + // diagnostic that carries no error detail and does not arm for these + // routes, so without this an operator sees a 502 in the browser and finds + // nothing whatsoever in the hub's log to explain it. + if (isIdentityFailure(kind)) { + console.error( + `[Proxy] GitOps identity hop failed: kind=${kind} upstreamStatus=${proxyRes.statusCode ?? 'none'} ` + + `bytes=${total} encoding=${sanitizeForLog(String(proxyRes.headers['content-encoding'] ?? 'identity'))}`, + cause === undefined ? '' : cause, + ); + } + // Guarded because it runs after the response is marked settled but before + // anything is written: a throw here would leave the client waiting on a + // response no later terminal can produce. + try { + hooks.finalizeTiming(kind); + } catch (timingError) { + console.error('[Proxy] GitOps identity timing hook threw:', timingError); + } + + writeTerminal(res, proxyRes, kind, body); + }; + + // A client that hangs up mid-flight ends the hop without an answer: there is + // nobody left to write to, and the upstream is dropped rather than left + // filling a buffer nobody will read. + res.on('close', () => { + if (!res.writableEnded) finish('downstream_close'); + }); + + // The cause is threaded through rather than discarded: a certificate error, + // a reset connection, and a timeout each need a different fix and would + // otherwise collapse into one indistinguishable message. + proxyRes.on('error', (error) => finish('upstream_failed', undefined, error)); + // A premature close is an upstream failure, not an empty success: answering + // 200 with a truncated body would report a partial page as a whole one. + proxyRes.on('aborted', () => finish('upstream_failed', undefined, 'upstream closed before the body ended')); + + try { + decoded = decodeStream(proxyRes); + } catch (error) { + finish('decompress_error', undefined, error); + return; + } + + decoded.on('error', (error) => finish('decompress_error', undefined, error)); + decoded.on('data', (chunk: Buffer) => { + if (settled) return; + total += chunk.length; + if (total > IDENTITY_PROXY_MAX_BYTES) { + finish('too_large'); + return; + } + chunks.push(chunk); + }); + decoded.on('end', () => { + if (settled) return; + // Checked rather than waiting for `aborted`, which races the end of the + // decoded stream. Without this a body that stopped early parses as broken + // JSON and gets reported as a malformed response, sending an operator to + // inspect the remote's output when the connection is what failed. + if (!proxyRes.complete) { + finish('upstream_failed', undefined, 'upstream ended before the body was complete'); + return; + } + const raw = Buffer.concat(chunks); + const status = proxyRes.statusCode ?? 502; + // Only a successful JSON body is rewritten. Redirects keep their Location, + // and anything else is returned as the bytes the remote sent. + if (status !== 200 && status !== 201) { + finish('passthrough', raw); + return; + } + let parsed: unknown; + try { + parsed = JSON.parse(raw.toString('utf8')); + } catch (error) { + // Not passed through. Every intercepted route answers with JSON on success, so a + // 200 that will not parse is a body the hub could not read, exactly like + // one it could not decompress. Relaying it under the remote's success + // status would hand the client an unrewritten, unauthorized payload and + // call it an answer. A captive portal or an error page served at 200 is + // the usual cause. + finish('parse_error', undefined, error); + return; + } + // Only the rewrite itself is guarded. `finish` must stay outside, because + // it marks the response settled on its first line: a throw from writing the + // response would otherwise be "recovered" by a second finish that returns + // immediately, leaving the client hanging with nothing logged. + let rewritten: Buffer; + try { + rewritten = Buffer.from(JSON.stringify(hooks.transform(parsed))); + } catch (error) { + // Everything in transform runs on this instance, so this is a hub fault + // and is reported as one. The error object is logged whole; for a bug on + // our own side the stack is the diagnostic. + finish('rewrite_failed', undefined, error); + return; + } + finish('rewrite', rewritten); + }); +} diff --git a/backend/src/proxy/remoteNodeProxy.ts b/backend/src/proxy/remoteNodeProxy.ts index 10e44715..0d2589db 100644 --- a/backend/src/proxy/remoteNodeProxy.ts +++ b/backend/src/proxy/remoteNodeProxy.ts @@ -1,5 +1,18 @@ import type { Request, Response, NextFunction, RequestHandler } from 'express'; import { createProxyMiddleware } from 'http-proxy-middleware'; +import type { OnProxyEvent } from 'http-proxy-middleware'; +import { + IDENTITY_PROXY_TIMEOUT_MS, + isIdentityFailure, + filterRemoteIdentityPayload, + handleIdentityResponse, + hubNodeIdFor, + isGitOpsIdentityJsonRoute, + prepareIdentityQuery, + rewriteIdentityPayload, + stripConditionalRequestHeaders, +} from './gitopsIdentityProxy'; +import { satisfiesGitOpsRead } from '../services/gitops/readAuth'; import { NodeRegistry } from '../services/NodeRegistry'; import { PROXY_TIER_HEADER, @@ -108,174 +121,251 @@ function finalizeProxyTiming(req: Request, outcome: ProxyTimingOutcome): void { * Per-request target resolution is handled via the `router` option. */ export function createRemoteProxyMiddleware(): RequestHandler { - const proxy = createProxyMiddleware({ + // Shared by both hops. The identity hop replaces only `proxyRes`, so + // credential stripping, role and tier assertion, and error handling stay + // identical however a request is forwarded. + const sharedOn: OnProxyEvent = { + proxyReq: (proxyReq, req) => { + // Strip headers that must not reach the remote instance: + // - x-node-id: remote Sencho treats all requests as local + // - cookie: the browser's sencho_token is signed with THIS instance's JWT secret; + // the remote would try to verify it with its own secret and return 401. + // Authentication is handled exclusively via the Bearer token below. + proxyReq.removeHeader('x-node-id'); + proxyReq.removeHeader('cookie'); + // Pilot-agent targets carry an empty token; see NodeRegistry.getProxyTarget. + if (req.proxyTarget?.apiToken) { + proxyReq.setHeader('Authorization', `Bearer ${req.proxyTarget.apiToken}`); + } + // Distributed License Enforcement: assert the main instance's license + // tier to the remote node so tier-gated routes honor the main's + // license instead of the node's local (likely Community) tier. The + // remote's authMiddleware only trusts these headers when the request + // carries a valid node_proxy JWT. The cached snapshot here invalidates + // on activate / deactivate / validate so the headers track license + // state changes within one proxy call. + const headers = LicenseService.getInstance().getProxyHeaders(); + proxyReq.setHeader(PROXY_TIER_HEADER, headers.tier); + // Forward the signed-in user's role so the remote enforces their RBAC + // rather than treating every proxied request as admin. Strip first so a + // browser/API client cannot smuggle the header through the gateway, then + // re-set from the authenticated session (authGate runs before this proxy, + // so req.user is always resolved here). + proxyReq.removeHeader(PROXY_ROLE_HEADER); + if (req.proxyElevatedRole) { + proxyReq.setHeader(PROXY_ROLE_HEADER, req.proxyElevatedRole); + } else if (req.user?.role) { + proxyReq.setHeader(PROXY_ROLE_HEADER, req.user.role); + } + // Deploy provenance: always strip client-supplied values, then set + // interactive manual + authenticated username for proxied browser/API + // deploys. Background machine callers do not go through this gateway + // with browser credentials; they set headers on direct machine HTTP. + proxyReq.removeHeader(PROXY_DEPLOY_SOURCE_HEADER); + proxyReq.removeHeader(PROXY_DEPLOY_ACTOR_HEADER); + proxyReq.setHeader(PROXY_DEPLOY_SOURCE_HEADER, 'manual'); + if (req.user?.username) { + proxyReq.setHeader(PROXY_DEPLOY_ACTOR_HEADER, req.user.username); + } + // Scoped stack evidence: always strip client-supplied values, then + // attach hub-built evidence when the gate stashed elevation for this hop. + proxyReq.removeHeader(PROXY_SCOPED_STACK_NAME_HEADER); + proxyReq.removeHeader(PROXY_SCOPED_STACK_ACTIONS_HEADER); + if (req.proxyScopedStackEvidence) { + proxyReq.setHeader(PROXY_SCOPED_STACK_NAME_HEADER, req.proxyScopedStackEvidence.stackName); + proxyReq.setHeader( + PROXY_SCOPED_STACK_ACTIONS_HEADER, + formatScopedStackActionsHeader(req.proxyScopedStackEvidence.actions), + ); + } + // Strip the ?nodeId= query param so the remote's nodeContextMiddleware + // doesn't reject the request with 404 ("Node X not found") - the remote + // has no record of the gateway's node IDs and should treat the request + // as local. This affects endpoints like EventSource /api/containers/:id/logs + // that pass nodeId as a query param rather than the x-node-id header. + if (req.gitopsIdentity !== undefined) { + // The identity hop already decided this query, including whether the + // remote is being asked to filter to its own node. Apply it whole so + // the generic strip below cannot undo that decision. + const [pathname] = proxyReq.path.split('?'); + proxyReq.path = pathname + (req.gitopsIdentity.query ? `?${req.gitopsIdentity.query}` : ''); + // A remote answering a conditional request with 304 would bypass the + // hub's rewrite and per-row filtering entirely, so the client is never + // allowed to ask the remote to revalidate. Identity-hop only: the + // streaming forwards keep client conditionals, which optimistic- + // concurrency file writes depend on. + stripConditionalRequestHeaders(proxyReq); + } else if (proxyReq.path.includes('nodeId=')) { + const [pathname, qs] = proxyReq.path.split('?'); + const params = new URLSearchParams(qs || ''); + params.delete('nodeId'); + // Hub-synthesized only; a caller must never smuggle one to a remote. + params.delete('gitopsLocalTarget'); + const newQs = params.toString(); + proxyReq.path = pathname + (newQs ? `?${newQs}` : ''); + } + // Body forwarding: conditionalJsonParser skips parsing for remote + // requests (see middleware/jsonParser.ts), so req's raw stream is + // usually intact and http-proxy's req.pipe(proxyReq) forwards it. + // When a gate must inspect JSON (POST /alerts), we buffer into + // req.rawBody first; rewrite that buffer here because the stream is + // already consumed. + if (req.rawBody) { + proxyReq.removeHeader('Transfer-Encoding'); + proxyReq.removeHeader('Content-Length'); + if (!proxyReq.getHeader('Content-Type')) { + proxyReq.setHeader('Content-Type', 'application/json'); + } + proxyReq.setHeader('Content-Length', req.rawBody.length); + proxyReq.write(req.rawBody); + } + }, + proxyRes: (proxyRes, req) => { + // Mark every response forwarded from a remote node with a sentinel + // header. The frontend (apiFetch / fetchForNode) checks this before + // firing the global 'sencho-unauthorized' event: a 401 from a remote + // means the stored api_token for that node is invalid, not that the + // user's own session expired. Without this distinction, any node with + // a bad token causes an immediate logout loop. + proxyRes.headers['x-sencho-proxy'] = '1'; + // Record upstream status and time-to-first-byte only; the log is + // finalized on the downstream finish/close so an abort mid-body is not + // mislabeled as success. + const timing = proxyTimings.get(req); + if (timing) { + timing.upstreamStatus = proxyRes.statusCode; + timing.ttfbMs = Date.now() - timing.startedAt; + } + // Hub fleet aggregation is local-only. A successful remote full-stack + // Apply or update-preview reconcile must drop the hub cache so the next + // fleet poll does not revive a verified-cleared card from a stale entry. + const status = proxyRes.statusCode ?? 0; + if ( + req.method === 'POST' + && status >= 200 + && status < 300 + && (isFullStackUpdatePath(req.path) || isUpdatePreviewPath(req.path)) + ) { + invalidateFleetUpdateCache(); + } + // Successful remote stack DELETE: clear hub grants for this (node, stack) + // only. Failed / non-2xx responses must preserve assignments. Use the + // gate-stashed classification: pathRewrite mutates req.url before this + // callback, so re-running classifyStackApiPath(req.path) would miss. + if (req.method === 'DELETE' && status >= 200 && status < 300) { + const route = req.proxyNamedStackRoute; + if (route?.action === 'stack:delete') { + try { + DatabaseService.getInstance().deleteRoleAssignmentsByStack(req.nodeId, route.stackName); + } catch (cleanupErr) { + console.warn( + '[Proxy] Failed to clear role assignments after remote stack delete:', + getErrorMessage(cleanupErr, 'unknown'), + ); + } + } + } + }, + error: (err, req, proxyRes) => { + // Finalize the hop timing with an error outcome before the existing + // 502 handling; the logged guard keeps the later finish/close a no-op. + finalizeProxyTiming(req, 'error'); + console.error('[Proxy] Remote node error:', getErrorMessage(err, 'unknown')); + const path = req.originalUrl || req.url; + if (req.method === 'POST' && /^\/api\/stacks\/[^/]+\/(?:deploy|update|services\/[^/]+\/(?:update|restore))(?:\?|$)/.test(path)) { + try { + DatabaseService.getInstance().insertAuditLog({ + timestamp: Date.now(), + username: req.user?.username ?? 'unknown', + method: req.method, + path, + status_code: 502, + node_id: req.nodeId, + ip_address: req.ip ?? '', + summary: `remote deploy proxy error: ${redactSensitiveText(getErrorMessage(err, 'unknown'))}`, + }); + } catch (auditErr) { + console.warn('[Proxy] Failed to record remote deploy proxy error:', getErrorMessage(auditErr, 'unknown')); + } + } + // proxyRes can be either a ServerResponse (HTTP) or a raw Socket + // (WS/TCP errors). Only attempt to send an HTTP 502 if it is a + // proper ServerResponse with a headersSent flag; otherwise silently + // drop (the socket will be destroyed). + const res = proxyRes as { headersSent?: boolean; status?: (n: number) => { json: (b: unknown) => void } }; + if (typeof res?.headersSent === 'boolean' && !res.headersSent && typeof res.status === 'function') { + res.status(502).json({ + error: 'Remote node is unreachable. Check the API URL and ensure Sencho is running on that host.', + }); + } + }, +}; + + const baseOptions = { target: 'http://localhost:0', // placeholder - overridden per-request by router changeOrigin: true, - router: (req) => req.proxyTarget?.apiUrl.replace(/\/$/, ''), + router: (req: Request) => req.proxyTarget?.apiUrl.replace(/\/$/, ''), // When mounted at app.use('/api/', ...), Express strips the '/api/' prefix from // req.url before the middleware sees it. Re-add it so the remote Sencho instance // receives the full path (e.g. '/stats' becomes '/api/stats'). - pathRewrite: (path) => '/api' + path, + pathRewrite: (path: string) => '/api' + path, + }; + + const proxy = createProxyMiddleware({ ...baseOptions, on: sharedOn }); + + /** + * The identity hop: buffers the response so node ids inside it can be + * corrected to this hub's numbering before the client sees them. + * + * Separate from the streaming hop rather than a mode of it, because + * buffering is exactly what the streaming hop must never do. Logs, event + * streams, and downloads keep flowing through that one untouched. + */ + const identityProxy = createProxyMiddleware({ + ...baseOptions, + selfHandleResponse: true, + // Bounded so a remote that sends headers and then stalls cannot pin the + // buffered body and both sockets indefinitely. No pathFilter: the + // dispatcher already gated the route, and a second predicate derived from + // a normalized pathname could disagree and hand a remote request to a + // local handler. + proxyTimeout: IDENTITY_PROXY_TIMEOUT_MS, on: { - proxyReq: (proxyReq, req) => { - // Strip headers that must not reach the remote instance: - // - x-node-id: remote Sencho treats all requests as local - // - cookie: the browser's sencho_token is signed with THIS instance's JWT secret; - // the remote would try to verify it with its own secret and return 401. - // Authentication is handled exclusively via the Bearer token below. - proxyReq.removeHeader('x-node-id'); - proxyReq.removeHeader('cookie'); - // Pilot-agent targets carry an empty token; see NodeRegistry.getProxyTarget. - if (req.proxyTarget?.apiToken) { - proxyReq.setHeader('Authorization', `Bearer ${req.proxyTarget.apiToken}`); - } - // Distributed License Enforcement: assert the main instance's license - // tier to the remote node so tier-gated routes honor the main's - // license instead of the node's local (likely Community) tier. The - // remote's authMiddleware only trusts these headers when the request - // carries a valid node_proxy JWT. The cached snapshot here invalidates - // on activate / deactivate / validate so the headers track license - // state changes within one proxy call. - const headers = LicenseService.getInstance().getProxyHeaders(); - proxyReq.setHeader(PROXY_TIER_HEADER, headers.tier); - // Forward the signed-in user's role so the remote enforces their RBAC - // rather than treating every proxied request as admin. Strip first so a - // browser/API client cannot smuggle the header through the gateway, then - // re-set from the authenticated session (authGate runs before this proxy, - // so req.user is always resolved here). - proxyReq.removeHeader(PROXY_ROLE_HEADER); - if (req.proxyElevatedRole) { - proxyReq.setHeader(PROXY_ROLE_HEADER, req.proxyElevatedRole); - } else if (req.user?.role) { - proxyReq.setHeader(PROXY_ROLE_HEADER, req.user.role); - } - // Deploy provenance: always strip client-supplied values, then set - // interactive manual + authenticated username for proxied browser/API - // deploys. Background machine callers do not go through this gateway - // with browser credentials; they set headers on direct machine HTTP. - proxyReq.removeHeader(PROXY_DEPLOY_SOURCE_HEADER); - proxyReq.removeHeader(PROXY_DEPLOY_ACTOR_HEADER); - proxyReq.setHeader(PROXY_DEPLOY_SOURCE_HEADER, 'manual'); - if (req.user?.username) { - proxyReq.setHeader(PROXY_DEPLOY_ACTOR_HEADER, req.user.username); - } - // Scoped stack evidence: always strip client-supplied values, then - // attach hub-built evidence when the gate stashed elevation for this hop. - proxyReq.removeHeader(PROXY_SCOPED_STACK_NAME_HEADER); - proxyReq.removeHeader(PROXY_SCOPED_STACK_ACTIONS_HEADER); - if (req.proxyScopedStackEvidence) { - proxyReq.setHeader(PROXY_SCOPED_STACK_NAME_HEADER, req.proxyScopedStackEvidence.stackName); - proxyReq.setHeader( - PROXY_SCOPED_STACK_ACTIONS_HEADER, - formatScopedStackActionsHeader(req.proxyScopedStackEvidence.actions), - ); - } - // Strip the ?nodeId= query param so the remote's nodeContextMiddleware - // doesn't reject the request with 404 ("Node X not found") - the remote - // has no record of the gateway's node IDs and should treat the request - // as local. This affects endpoints like EventSource /api/containers/:id/logs - // that pass nodeId as a query param rather than the x-node-id header. - if (proxyReq.path.includes('nodeId=')) { - const [pathname, qs] = proxyReq.path.split('?'); - const params = new URLSearchParams(qs || ''); - params.delete('nodeId'); - const newQs = params.toString(); - proxyReq.path = pathname + (newQs ? `?${newQs}` : ''); - } - // Body forwarding: conditionalJsonParser skips parsing for remote - // requests (see middleware/jsonParser.ts), so req's raw stream is - // usually intact and http-proxy's req.pipe(proxyReq) forwards it. - // When a gate must inspect JSON (POST /alerts), we buffer into - // req.rawBody first; rewrite that buffer here because the stream is - // already consumed. - if (req.rawBody) { - proxyReq.removeHeader('Transfer-Encoding'); - proxyReq.removeHeader('Content-Length'); - if (!proxyReq.getHeader('Content-Type')) { - proxyReq.setHeader('Content-Type', 'application/json'); - } - proxyReq.setHeader('Content-Length', req.rawBody.length); - proxyReq.write(req.rawBody); - } - }, - proxyRes: (proxyRes, req) => { - // Mark every response forwarded from a remote node with a sentinel - // header. The frontend (apiFetch / fetchForNode) checks this before - // firing the global 'sencho-unauthorized' event: a 401 from a remote - // means the stored api_token for that node is invalid, not that the - // user's own session expired. Without this distinction, any node with - // a bad token causes an immediate logout loop. + ...sharedOn, + proxyRes: (proxyRes, req, res) => { proxyRes.headers['x-sencho-proxy'] = '1'; - // Record upstream status and time-to-first-byte only; the log is - // finalized on the downstream finish/close so an abort mid-body is not - // mislabeled as success. const timing = proxyTimings.get(req); if (timing) { timing.upstreamStatus = proxyRes.statusCode; timing.ttfbMs = Date.now() - timing.startedAt; } - // Hub fleet aggregation is local-only. A successful remote full-stack - // Apply or update-preview reconcile must drop the hub cache so the next - // fleet poll does not revive a verified-cleared card from a stale entry. - const status = proxyRes.statusCode ?? 0; - if ( - req.method === 'POST' - && status >= 200 - && status < 300 - && (isFullStackUpdatePath(req.path) || isUpdatePreviewPath(req.path)) - ) { - invalidateFleetUpdateCache(); - } - // Successful remote stack DELETE: clear hub grants for this (node, stack) - // only. Failed / non-2xx responses must preserve assignments. Use the - // gate-stashed classification: pathRewrite mutates req.url before this - // callback, so re-running classifyStackApiPath(req.path) would miss. - if (req.method === 'DELETE' && status >= 200 && status < 300) { - const route = req.proxyNamedStackRoute; - if (route?.action === 'stack:delete') { - try { - DatabaseService.getInstance().deleteRoleAssignmentsByStack(req.nodeId, route.stackName); - } catch (cleanupErr) { - console.warn( - '[Proxy] Failed to clear role assignments after remote stack delete:', - getErrorMessage(cleanupErr, 'unknown'), - ); - } - } - } - }, - error: (err, req, proxyRes) => { - // Finalize the hop timing with an error outcome before the existing - // 502 handling; the logged guard keeps the later finish/close a no-op. - finalizeProxyTiming(req, 'error'); - console.error('[Proxy] Remote node error:', getErrorMessage(err, 'unknown')); - const path = req.originalUrl || req.url; - if (req.method === 'POST' && /^\/api\/stacks\/[^/]+\/(?:deploy|update|services\/[^/]+\/(?:update|restore))(?:\?|$)/.test(path)) { - try { - DatabaseService.getInstance().insertAuditLog({ - timestamp: Date.now(), - username: req.user?.username ?? 'unknown', - method: req.method, - path, - status_code: 502, - node_id: req.nodeId, - ip_address: req.ip ?? '', - summary: `remote deploy proxy error: ${redactSensitiveText(getErrorMessage(err, 'unknown'))}`, - }); - } catch (auditErr) { - console.warn('[Proxy] Failed to record remote deploy proxy error:', getErrorMessage(auditErr, 'unknown')); - } - } - // proxyRes can be either a ServerResponse (HTTP) or a raw Socket - // (WS/TCP errors). Only attempt to send an HTTP 502 if it is a - // proper ServerResponse with a headersSent flag; otherwise silently - // drop (the socket will be destroyed). - const res = proxyRes as { headersSent?: boolean; status?: (n: number) => { json: (b: unknown) => void } }; - if (typeof res?.headersSent === 'boolean' && !res.headersSent && typeof res.status === 'function') { - res.status(502).json({ - error: 'Remote node is unreachable. Check the API URL and ensure Sencho is running on that host.', - }); - } + const nodeId = hubNodeIdFor(req); + handleIdentityResponse(proxyRes, res, { + transform: (payload) => { + // Without an id there is nothing to rewrite the remote's numbering + // to, and passing its ids through unchanged is the exact defect + // this hop exists to prevent. Refuse rather than answer with + // identities from another instance's namespace. + if (nodeId === undefined) throw new Error('proxied request has no node id to rewrite identities to'); + const identity = req.gitopsIdentity; + if (identity === undefined) throw new Error('identity hop ran without a stashed pre-rewrite path'); + // Correct identities first so the collection filter classifies + // against this hub's node ids rather than the remote's. + rewriteIdentityPayload(payload, nodeId); + return filterRemoteIdentityPayload( + // No fallback to `req.path`: pathRewrite has already prefixed + // `/api` by now, so falling back would match no collection and + // ship every remote row unfiltered under a 200. + identity.preRewritePath, + payload, + (requirement) => satisfiesGitOpsRead(req, requirement), + nodeId, + ); + }, + finalizeTiming: (kind) => { + finalizeProxyTiming(req, isIdentityFailure(kind) ? 'error' : 'ok'); + }, + }); }, }, }); @@ -598,6 +688,27 @@ export function createRemoteProxyMiddleware(): RequestHandler { } req.proxyTarget = target; + + // Identity routes are reworked before the hop, not during it: a request + // naming a node this instance cannot answer for is refused here rather + // than forwarded, so the remote never answers about itself a question + // that was asked about somebody else. + if (isGitOpsIdentityJsonRoute(req.path, req.method)) { + const prepared = prepareIdentityQuery( + new URLSearchParams(req.url.split('?')[1] ?? ''), + req.path, + hubNodeIdFor(req), + ); + if (prepared.kind === 'refuse') { + res.status(400).json({ error: prepared.error, code: 'gitops_history_node_mismatch' }); + return; + } + req.gitopsIdentity = { query: prepared.search.toString(), preRewritePath: req.path }; + beginProxyTiming(req, res); + identityProxy(req, res, next); + return; + } + beginProxyTiming(req, res); proxy(req, res, next); }; diff --git a/backend/src/routes/OPERATIONAL_PERMISSIONS.md b/backend/src/routes/OPERATIONAL_PERMISSIONS.md index 8ba06238..79e3a1f0 100644 --- a/backend/src/routes/OPERATIONAL_PERMISSIONS.md +++ b/backend/src/routes/OPERATIONAL_PERMISSIONS.md @@ -34,6 +34,9 @@ check. Bulk routes must authorize every valid target before starting any work. | Security policies, suppressions, and acknowledgements | `stack:read` | n/a | `stack:edit` | n/a | n/a | global collection | | Docker resource inventory and orphan reads | `stack:read` | n/a | n/a | n/a | n/a | global read | | Network topology and inspection | `node:read` | n/a | n/a | n/a | n/a | global read | +| `/api/git-sources` | per row: exact `stack:read`, else Admin | n/a | n/a | n/a | n/a | a row reduces to its own stack when that stack is live and present on disk; otherwise Admin, because the row is live Git configuration rather than a record of events | +| `/api/git-sources/history` and `/api/stacks/:stackName/git-source/history` | per row: exact `stack:read`, else `system:audit` | n/a | n/a | n/a | n/a | history entries are an audit trail, so an entry that cannot be tied to a readable stack falls to the audit permission rather than Admin. The per-stack route is authorized whole at `stack:read` by suffix rule instead of per row | +| `/api/gitops-metrics` | Admin | n/a | n/a | n/a | n/a | in-process counters keyed only on transition stage and outcome; see the boundary below | ## Preserved system boundaries @@ -44,6 +47,13 @@ image, volume, network, resource, and fleet pruning. Reset-anchor and mesh-wide membership cascades remain Admin-only because their effects are broader than one ordinary node or stack permission check can safely authorize. +In-process diagnostic counters are Admin-only for a different reason: they +describe the instance rather than any one stack or node, so there is no resource +identity to scope an operational grant against. Their payloads are aggregate by +construction, naming no stack, node, repository, or actor. Anyone wanting to +know what happened to a particular stack reads the history routes above, which +authorize per row. + ## Frontend parity Navigation and controls use `can()` with the same action and resource identity. diff --git a/backend/src/routes/blueprints.ts b/backend/src/routes/blueprints.ts index 76a53e59..92aae891 100644 --- a/backend/src/routes/blueprints.ts +++ b/backend/src/routes/blueprints.ts @@ -4,6 +4,7 @@ import { requireBody } from '../middleware/tierGates'; import { requirePermission } from '../middleware/permissions'; import { DatabaseService, + type Blueprint, type BlueprintSelector, type DriftMode, } from '../services/DatabaseService'; @@ -26,6 +27,13 @@ import { parseConfirmableActionsBody, serializeApprovedBlast, } from '../services/blueprintApproval'; +import { + commitBlueprintCreate, + commitBlueprintDelete, + commitBlueprintPin, + commitBlueprintUpdate, +} from '../services/gitops/blueprintProducers'; +import { projectBlueprintRevision, projectCommittedRevision } from '../helpers/gitopsResponse'; import { isValidStackName } from '../utils/validation'; import { parseIntParam } from '../utils/parseIntParam'; import { isDebugEnabled } from '../utils/debug'; @@ -51,6 +59,18 @@ interface BlueprintBody { enabled?: unknown; } +/** + * Nodes a Blueprint currently asks for, as the reconciler computes them. + * + * Passed into the revision-state producers rather than imported by them: the + * reconciler reaches that layer, so importing it back would close a cycle. + */ +function desiredNodeIdsFor(blueprint: Blueprint): number[] { + return BlueprintReconciler.getInstance() + .listDesiredNodes(blueprint, DatabaseService.getInstance().getNodes()) + .map(node => node.id); +} + function parseSelector(raw: unknown): { ok: true; selector: BlueprintSelector } | { ok: false; error: string } { if (!raw || typeof raw !== 'object') return { ok: false, error: 'selector is required' }; const obj = raw as Record; @@ -132,6 +152,7 @@ function summarizeBlueprint(blueprintId: number) { statusCounts: counts, effectiveApproval: auth?.effectiveApproval ?? 'pending', unauthorizedActions: auth?.unauthorizedActions ?? [], + gitopsRevision: projectBlueprintRevision(blueprintId), }; } @@ -150,6 +171,7 @@ blueprintsRouter.get('/', (req: Request, res: Response): void => { deploymentTotal: deployments.length, effectiveApproval: auth?.effectiveApproval ?? 'pending', unauthorizedActions: auth?.unauthorizedActions ?? [], + gitopsRevision: projectBlueprintRevision(b.id), }; }); res.json(summaries); @@ -176,7 +198,7 @@ blueprintsRouter.post('/', (req: Request, res: Response): void => { try { const composeContent = body.compose_content as string; const analysis = BlueprintAnalyzer.analyze(composeContent); - const blueprint = DatabaseService.getInstance().createBlueprint({ + const blueprint = commitBlueprintCreate({ name: (body.name as string).trim(), description: typeof body.description === 'string' ? body.description : null, compose_content: composeContent, @@ -186,8 +208,8 @@ blueprintsRouter.post('/', (req: Request, res: Response): void => { classification_reasons: analysis.reasons, enabled: body.enabled === undefined ? true : Boolean(body.enabled), created_by: req.user?.username ?? null, - }); - res.status(201).json(blueprint); + }, desiredNodeIdsFor); + res.status(201).json({ ...blueprint, gitopsRevision: projectCommittedRevision(blueprint.id, 'blueprint create') }); } catch (error) { if (isSqliteUniqueViolation(error)) { res.status(409).json({ error: 'A blueprint with that name already exists' }); @@ -281,9 +303,9 @@ blueprintsRouter.put('/:id', (req: Request, res: Response): void => { updates.enabled = next; } try { - const updated = DatabaseService.getInstance().updateBlueprint(id, updates); + const { blueprint: updated } = commitBlueprintUpdate(id, updates, req.user?.username ?? null, desiredNodeIdsFor); if (!updated) { res.status(404).json({ error: 'Blueprint not found' }); return; } - res.json(updated); + res.json({ ...updated, gitopsRevision: projectCommittedRevision(id, 'blueprint update') }); } catch (error) { if (isSqliteUniqueViolation(error)) { res.status(409).json({ error: 'A blueprint with that name already exists' }); @@ -351,7 +373,7 @@ blueprintsRouter.delete('/:id', async (req: Request, res: Response): Promise { console.warn('[Blueprints] post-pin reconcileOne failed:', err); }); } - res.json(updated); + res.json({ ...updated, gitopsRevision }); } catch (error) { console.error('[Blueprints] Pin error:', error); res.status(500).json({ error: 'Failed to update blueprint pin' }); diff --git a/backend/src/routes/gitSources.ts b/backend/src/routes/gitSources.ts index 07092a31..f06e37c8 100644 --- a/backend/src/routes/gitSources.ts +++ b/backend/src/routes/gitSources.ts @@ -1,20 +1,24 @@ import { Router, type Request, type Response } from 'express'; -import { GitSourceService } from '../services/GitSourceService'; +import { GitSourceService, type PublicGitSource } from '../services/GitSourceService'; +import type { GitOpsRevisionProjection } from '../services/gitops/types'; import { GitProjectManifestService } from '../services/GitProjectManifestService'; import { FileSystemService } from '../services/FileSystemService'; import { DatabaseService } from '../services/DatabaseService'; import { CryptoService } from '../services/CryptoService'; -import { checkPermission, requirePermission } from '../middleware/permissions'; +import { requirePermission } from '../middleware/permissions'; +import { classifySourceRow, satisfiesGitOpsRead } from '../services/gitops/readAuth'; +import { NOT_APPLICABLE_REVISION, projectStackRevision, stackResourceSet } from '../helpers/gitopsResponse'; +import { respondWithHistory } from '../helpers/gitopsHistoryPage'; import { invalidateNodeCaches } from '../helpers/cacheInvalidation'; import { triggerPostDeployScan } from '../helpers/policyGate'; import { parseComposeSelection, defaultEnvPath } from '../helpers/gitSourceSelection'; import { isValidGitSourcePath, isValidStackName } from '../utils/validation'; import { sendGitSourceError, webhookPullStatus } from '../utils/gitSourceHttp'; import { sanitizeForLog } from '../utils/safeLog'; +import { repoUrlRejectionMessage } from '../services/gitops/repoIdentity'; // Reasonable upper bounds so a caller cannot flood the service with huge // payloads. Generous compared to anything a real Git provider emits. -const MAX_REPO_URL_LENGTH = 2048; const MAX_BRANCH_LENGTH = 256; const MAX_ENV_PATH_LENGTH = 1024; const MAX_TOKEN_LENGTH = 8192; @@ -35,12 +39,9 @@ async function handleBrowse(req: Request, res: Response, storedToken: string | n res.status(400).json({ error: 'branch is required' }); return; } - if (!/^https:\/\//i.test(repo_url)) { - res.status(400).json({ error: 'Only HTTPS repository URLs are supported' }); - return; - } - if (repo_url.length > MAX_REPO_URL_LENGTH) { - res.status(400).json({ error: 'repo_url is too long' }); + const repoUrlError = repoUrlRejectionMessage(repo_url); + if (repoUrlError) { + res.status(400).json({ error: repoUrlError }); return; } if (branch.length > MAX_BRANCH_LENGTH) { @@ -75,15 +76,49 @@ export const gitSourcesRouter = Router(); gitSourcesRouter.get('/', async (req: Request, res: Response): Promise => { try { const all = GitSourceService.getInstance().list(); + const present = await stackResourceSet(req.nodeId); // Filter to the subset of stacks the caller can read. Keeps scoped - // Admiral roles from discovering git config for stacks outside their grant. - const visible = all.filter(src => checkPermission(req, 'stack:read', 'stack', src.stack_name)); + // roles from discovering git config for stacks outside their grant. + // A row we cannot tie to a live, on-disk stack falls back to Admin, so a + // source whose application is missing or half-created is never authorized + // by a stack grant that may since have been reassigned. + const visible: Array = []; + for (const src of all) { + const gitopsRevision = projectStackRevision(src.stack_name); + const stackResourcePresent = present.has(src.stack_name); + const requirement = classifySourceRow({ + stackName: src.stack_name, + gitopsRevision, + stackResourcePresent, + }); + if (!satisfiesGitOpsRead(req, requirement)) continue; + visible.push({ ...src, gitopsRevision, stackResourcePresent }); + } res.json(visible); } catch (error) { sendGitSourceError(res, error); } }); +/** + * Cross-stack GitOps history for this instance. + * + * Every row is authorized on its own, so this returns the operator's own + * stacks for a scoped role and every row on this instance for an Admin. + * History is instance-local: a remote node's rows are read by proxying this + * same route to that node. + */ +gitSourcesRouter.get('/history', async (req: Request, res: Response): Promise => { + try { + await respondWithHistory(req, res, { kind: 'per_row' }); + } catch (error) { + sendGitSourceError(res, error); + } +}); + // Create-mode repo browse (no stack yet): gated by the same permission as // creating a stack from Git. gitSourcesRouter.post('/browse', async (req: Request, res: Response): Promise => { @@ -108,6 +143,10 @@ stackGitSourceRouter.get('/:stackName/git-source', async (req: Request, res: Res try { const gitSources = GitSourceService.getInstance(); const source = gitSources.get(stackName); + // Only this instance can say whether the stack's directory is really here, + // so the answer travels with the response rather than being inferred by a + // hub that has never seen the filesystem. + const stackResourcePresent = (await stackResourceSet(req.nodeId)).has(stackName); if (source) { // The managed-project manifest summary rides the source branch; the // unlinked {linked:false} shape below is unchanged. Heal-on-read may @@ -118,6 +157,8 @@ stackGitSourceRouter.get('/:stackName/git-source', async (req: Request, res: Res ...refreshed, manifest_state: manifest?.state ?? refreshed.manifest_state, manifest, + gitopsRevision: projectStackRevision(stackName), + stackResourcePresent, }); return; } @@ -126,12 +167,36 @@ stackGitSourceRouter.get('/:stackName/git-source', async (req: Request, res: Res // dashboard probes this endpoint for every stack, so returning 404 here // would paint a console error for every unlinked stack; answer 200 with // a discriminator instead and reserve 404 for the stack-not-found case. - const stacks = await FileSystemService.getInstance(req.nodeId).getStacks(); - if (!stacks.includes(stackName)) { + if (!stackResourcePresent) { res.status(404).json({ error: 'Stack not found' }); return; } - res.json({ linked: false }); + res.json({ + linked: false, + gitopsRevision: NOT_APPLICABLE_REVISION, + stackResourcePresent, + }); + } catch (error) { + sendGitSourceError(res, error); + } +}); + +/** + * GitOps history for one stack. + * + * The stack read below covers the application holding this name now. Rows from + * an application that held it earlier are a different resource and are + * authorized per row, so a reused stack name cannot expose its predecessor. + */ +stackGitSourceRouter.get('/:stackName/git-source/history', async (req: Request, res: Response): Promise => { + const stackName = req.params.stackName as string; + if (!isValidStackName(stackName)) { + res.status(400).json({ error: 'Invalid stack name' }); + return; + } + if (!requirePermission(req, res, 'stack:read', 'stack', stackName)) return; + try { + await respondWithHistory(req, res, { kind: 'authorized_stack', stackName }); } catch (error) { sendGitSourceError(res, error); } @@ -181,12 +246,9 @@ stackGitSourceRouter.put('/:stackName/git-source', async (req: Request, res: Res res.status(400).json({ error: 'auto_deploy_on_apply must be a boolean' }); return; } - if (!/^https:\/\//i.test(repo_url)) { - res.status(400).json({ error: 'Only HTTPS repository URLs are supported' }); - return; - } - if (repo_url.length > MAX_REPO_URL_LENGTH) { - res.status(400).json({ error: 'repo_url is too long' }); + const repoUrlError = repoUrlRejectionMessage(repo_url); + if (repoUrlError) { + res.status(400).json({ error: repoUrlError }); return; } if (branch.length > MAX_BRANCH_LENGTH) { @@ -404,7 +466,7 @@ stackGitSourceRouter.post('/:stackName/git-source/dismiss-pending', async (req: } if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return; try { - GitSourceService.getInstance().dismissPending(stackName); + GitSourceService.getInstance().dismissPending(stackName, req.user?.username ?? 'unknown'); res.json({ success: true }); } catch (error) { sendGitSourceError(res, error); diff --git a/backend/src/routes/gitopsMetrics.ts b/backend/src/routes/gitopsMetrics.ts new file mode 100644 index 00000000..676f184f --- /dev/null +++ b/backend/src/routes/gitopsMetrics.ts @@ -0,0 +1,24 @@ +import { Router, type Request, type Response } from 'express'; +import { requireAdmin } from '../middleware/tierGates'; +import { GitOpsMetricsService } from '../services/GitOpsMetricsService'; + +export const gitopsMetricsRouter = Router(); + +/** + * Admin-only snapshot of in-process GitOps transition counters. + * + * Instance-local rather than hub-only, so selecting a node answers with that + * node's counters: the transitions being counted happen wherever the stack + * lives, and a hub-only reading would report the hub's own activity under + * every node's name. + * + * Mounted at /api/gitops-metrics after the global auth gate, alongside + * /api/stack-metrics, which this mirrors. Admin rather than an operational + * permission for the same reason that one is: these are process diagnostics + * about the instance, not a record of any one stack's work, so there is no + * stack or node to scope a grant against. + */ +gitopsMetricsRouter.get('/', (req: Request, res: Response): void => { + if (!requireAdmin(req, res)) return; + res.json({ entries: GitOpsMetricsService.getInstance().snapshot() }); +}); diff --git a/backend/src/routes/imageUpdates.ts b/backend/src/routes/imageUpdates.ts index c6ea4af5..00cdb216 100644 --- a/backend/src/routes/imageUpdates.ts +++ b/backend/src/routes/imageUpdates.ts @@ -560,8 +560,8 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp // Health observation starts immediately after Compose; registry recheck is // isolated so a verification failure cannot turn Compose success into a failure. - const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`); const orchResult = lock.result; + const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', `auto-update:${req.user?.username ?? 'scheduler'}`, { deployedGenerationId: orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.deployedGenerationId : null }); const recoveryId = orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null; if (recoveryId) { const { StackUpdateRecoveryService } = await import('../services/StackUpdateRecoveryService'); diff --git a/backend/src/routes/nodeLabels.ts b/backend/src/routes/nodeLabels.ts index 8f4b8058..5e87c537 100644 --- a/backend/src/routes/nodeLabels.ts +++ b/backend/src/routes/nodeLabels.ts @@ -5,9 +5,29 @@ import { requirePermission } from '../middleware/permissions'; import { DatabaseService } from '../services/DatabaseService'; import { NodeLabelService } from '../services/NodeLabelService'; import { parseIntParam } from '../utils/parseIntParam'; +import { BlueprintReconciler } from '../services/BlueprintReconciler'; +import { recordPlacementShift, snapshotPlacementWith } from '../services/gitops/nodePlacementProducers'; +import { projectCommittedRevisions } from '../helpers/gitopsResponse'; export const nodeLabelsRouter = Router(); +/** + * Placement as it currently resolves, for every Blueprint. + * + * Taken either side of a label write so the recording covers only the + * Blueprints the label actually moved. A label no selector mentions moves + * nothing, and minting for it would invalidate acknowledgements fleet-wide over + * an edit no node can observe. + */ +function snapshotPlacement() { + const db = DatabaseService.getInstance(); + const nodes = db.getNodes(); + return snapshotPlacementWith( + (blueprint) => BlueprintReconciler.getInstance().listDesiredNodes(blueprint, nodes).map(n => n.id), + db.listBlueprints(), + ); +} + nodeLabelsRouter.use(authMiddleware); nodeLabelsRouter.get('/', (req: Request, res: Response): void => { @@ -62,12 +82,24 @@ nodeLabelsRouter.post('/:nodeId', (req: Request, res: Response): void => { res.status(404).json({ error: 'Node not found' }); return; } - const result = NodeLabelService.getInstance().addLabel(nodeId, label); - if (!result.ok) { - res.status(400).json(result.error); + // The label and the placement it moves commit together, so a recording + // failure cannot leave a fleet selecting on a label nothing recorded. + const { added, moved } = DatabaseService.getInstance().getDb().transaction(() => { + const before = snapshotPlacement(); + const added = NodeLabelService.getInstance().addLabel(nodeId, label); + const moved = added.ok + ? recordPlacementShift(before, snapshotPlacement(), req.user?.username ?? null, 'node_label_add') + : []; + return { added, moved }; + })(); + if (!added.ok) { + res.status(400).json(added.error); return; } - res.status(201).json({ nodeId, label: result.label }); + // Projected after the commit, so the revisions describe what the label + // write actually left behind. A label no selector mentions moves nothing + // and reports an empty list rather than every Blueprint in the fleet. + res.status(201).json({ nodeId, label: added.label, gitopsRevisions: projectCommittedRevisions(moved, 'node label add') }); } catch (error) { console.error('[NodeLabels] Add error:', error); res.status(500).json({ error: 'Failed to add label' }); @@ -85,7 +117,14 @@ nodeLabelsRouter.delete('/:nodeId/:label', (req: Request, res: Response): void = return; } try { - const removed = NodeLabelService.getInstance().removeLabel(nodeId, label); + const removed = DatabaseService.getInstance().getDb().transaction(() => { + const before = snapshotPlacement(); + const gone = NodeLabelService.getInstance().removeLabel(nodeId, label); + if (gone) { + recordPlacementShift(before, snapshotPlacement(), req.user?.username ?? null, 'node_label_remove'); + } + return gone; + })(); if (!removed) { res.status(404).json({ error: 'Label assignment not found' }); return; diff --git a/backend/src/routes/nodes.ts b/backend/src/routes/nodes.ts index 74798101..2697a9e6 100644 --- a/backend/src/routes/nodes.ts +++ b/backend/src/routes/nodes.ts @@ -24,6 +24,9 @@ import { toPublicNode } from '../helpers/publicNode'; import { isDebugEnabled } from '../utils/debug'; import { sanitizeForLog } from '../utils/safeLog'; import { logDebugTiming } from '../utils/requestTiming'; +import { BlueprintReconciler } from '../services/BlueprintReconciler'; +import { recordPlacementShift, snapshotPlacementWith } from '../services/gitops/nodePlacementProducers'; +import { projectCommittedRevisions } from '../helpers/gitopsResponse'; const NODE_SCOPE_MESSAGE = 'API tokens cannot manage nodes.'; const REMOTE_META_CACHE_TTL = 3 * 60 * 1000; @@ -117,6 +120,30 @@ function mintPilotEnrollment(nodeId: number, req: Request): { token: string; exp export const nodesRouter = Router(); +/** + * Placement as it currently resolves, for every Blueprint. + * + * Taken either side of a cordon, and today the two snapshots are always equal, + * so a cordon revises nothing and reports no revisions. That is the correct + * answer, not a gap: a cordon suppresses *new* placements, while this snapshot + * reports what each Blueprint asks for, and those are different questions. + * `listDesiredNodes` accordingly does not read `cordoned` at all, and the + * reconciler applies the cordon filter later, when it decides what to place. + * + * The comparison is kept rather than short-circuited because it is the same + * shared helper the label routes use, where placement genuinely does move, and + * because it is what would start reporting correctly if the desired set ever + * became cordon-aware. Cheap either way: two in-memory selector evaluations. + */ +function snapshotBlueprintPlacement() { + const db = DatabaseService.getInstance(); + const nodes = db.getNodes(); + return snapshotPlacementWith( + (blueprint) => BlueprintReconciler.getInstance().listDesiredNodes(blueprint, nodes).map(n => n.id), + db.listBlueprints(), + ); +} + nodesRouter.get('/', async (req: Request, res: Response) => { if (!requirePermission(req, res, 'node:read')) return; const startedAt = Date.now(); @@ -426,17 +453,15 @@ nodesRouter.delete('/:id', async (req: Request, res: Response) => { // Local-socket nodes: ready tombstone + recovery-row retirement in the same // transaction as the node delete, then sweep tags/paths. Remote hub records // create no Docker cleanup tombstone. - if (existing.type === 'local') { - await DeployedStackDeletionService.getInstance().deleteLocalNode(id); - } else { - DatabaseService.getInstance().deleteNode(id); - } + const movedBlueprints = existing.type === 'local' + ? await DeployedStackDeletionService.getInstance().deleteLocalNode(id) + : DeployedStackDeletionService.getInstance().deleteNodeWithGitOps(id); NodeRegistry.getInstance().evictConnection(id); NodeRegistry.getInstance().notifyNodeRemoved(id); CacheService.getInstance().invalidate(`${REMOTE_META_NAMESPACE}:${id}`); FleetUpdateTrackerService.getInstance().delete(id); console.log(`[Nodes] Deleted node ${id} ("${sanitizeForLog(existing.name)}")`); - res.json({ success: true }); + res.json({ success: true, gitopsRevisions: projectCommittedRevisions(movedBlueprints, 'node delete') }); } catch (error: unknown) { const message = error instanceof Error ? error.message : ''; if (message.includes('Cannot delete the only local node')) { @@ -476,9 +501,17 @@ nodesRouter.post('/:id/cordon', (req: Request, res: Response) => { res.status(404).json({ error: 'Node not found' }); return; } - const updated = DatabaseService.getInstance().setNodeCordoned(id, true, reason); + // The cordon and the placement it moves commit together. + const { node: updated, moved } = DatabaseService.getInstance().getDb().transaction(() => { + const before = snapshotBlueprintPlacement(); + const node = DatabaseService.getInstance().setNodeCordoned(id, true, reason); + const moved = existing.cordoned + ? [] + : recordPlacementShift(before, snapshotBlueprintPlacement(), req.user?.username ?? null, 'node_cordon'); + return { node, moved }; + })(); if (isDebugEnabled()) console.log('[Federation:diag] cordoned node=%s reasonLen=%s', sanitizeForLog(id), sanitizeForLog(reason?.length ?? 0)); - res.set('cache-control', 'no-store').json(updated); + res.set('cache-control', 'no-store').json({ ...updated, gitopsRevisions: projectCommittedRevisions(moved, 'node cordon') }); } catch (error: unknown) { console.error('Failed to cordon node:', error); res.status(500).json({ error: error instanceof Error ? error.message : 'Failed to cordon node' }); @@ -500,8 +533,15 @@ nodesRouter.post('/:id/uncordon', (req: Request, res: Response) => { res.status(404).json({ error: 'Node not found' }); return; } - const updated = DatabaseService.getInstance().setNodeCordoned(id, false, null); - res.set('cache-control', 'no-store').json(updated); + const { node: updated, moved } = DatabaseService.getInstance().getDb().transaction(() => { + const before = snapshotBlueprintPlacement(); + const node = DatabaseService.getInstance().setNodeCordoned(id, false, null); + const moved = existing.cordoned + ? recordPlacementShift(before, snapshotBlueprintPlacement(), req.user?.username ?? null, 'node_uncordon') + : []; + return { node, moved }; + })(); + res.set('cache-control', 'no-store').json({ ...updated, gitopsRevisions: projectCommittedRevisions(moved, 'node uncordon') }); } catch (error: unknown) { console.error('Failed to uncordon node:', error); res.status(500).json({ error: error instanceof Error ? error.message : 'Failed to uncordon node' }); diff --git a/backend/src/routes/stacks.ts b/backend/src/routes/stacks.ts index 197558cd..8bc84caf 100644 --- a/backend/src/routes/stacks.ts +++ b/backend/src/routes/stacks.ts @@ -23,6 +23,7 @@ import { buildDetectionDisabledPreview, } from '../services/UpdatePreviewService'; import { GitSourceService, GitSourceError, repoHost as gitRepoHost } from '../services/GitSourceService'; +import { repoUrlRejectionMessage } from '../services/gitops/repoIdentity'; import { enforcePolicyPreDeploy } from '../services/PolicyEnforcement'; import { buildStackDriftReport, type DriftFindingKind, type StackDriftReport } from '../services/DriftDetectionService'; import { DriftLedgerService, type DriftTemporal } from '../services/DriftLedgerService'; @@ -86,6 +87,8 @@ import { import { getActiveCapabilities, STACK_DOWN_REMOVE_VOLUMES_CAPABILITY, SERVICE_SCOPED_UPDATE_CAPABILITY } from '../services/CapabilityRegistry'; import { ServiceUpdateRecoveryService } from '../services/ServiceUpdateRecoveryService'; import { classifyStackApiPath } from '../helpers/stackRouteAuth'; +import { projectManagedStackRevision } from '../helpers/gitopsResponse'; +import type { GitOpsRevisionProjection } from '../services/gitops/types'; // Authenticated users with edit permission can write arbitrarily large compose // files. Refuse to YAML.parse anything beyond this bound so a malformed (or @@ -601,7 +604,7 @@ async function runStackBulkOp( triggerPostDeployScan(stackName, req.nodeId).catch(err => console.error('[Security] Post-deploy scan failed for %s:', sanitizeForLog(stackName), err), ); - const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null); + const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null, { deployedGenerationId: orchResult.kind === 'stack_compose_done' ? orchResult.deployedGenerationId : null }); const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null; linkStackUpdateRecoveryGate(recoveryId, healthGateId); return { stackName, ok: true, healthGateId }; @@ -1110,11 +1113,9 @@ stacksRouter.post('/from-git', async (req: Request, res: Response) => { return res.status(400).json({ error: 'auto_deploy_on_apply must be a boolean' }); } const resolvedAuthType = auth_type === 'token' ? 'token' : 'none'; - if (!/^https:\/\//i.test(repo_url)) { - return res.status(400).json({ error: 'Only HTTPS repository URLs are supported' }); - } - if (repo_url.length > 2048) { - return res.status(400).json({ error: 'repo_url is too long' }); + const repoUrlError = repoUrlRejectionMessage(repo_url); + if (repoUrlError) { + return res.status(400).json({ error: repoUrlError }); } if (branch.length > 256) { return res.status(400).json({ error: 'branch is too long' }); @@ -1339,7 +1340,12 @@ async function buildDriftPayload( nodeId: number, stackName: string, reconcile: boolean, -): Promise { +): Promise { const report = await buildStackDriftReport(nodeId, stackName); // Only the on-disk read is best-effort: an unreadable compose is already surfaced // by the report as a parse error, so temporal degrades to neutral. computeTemporal @@ -1371,7 +1377,17 @@ async function buildDriftPayload( // not this passive read, so surface when that was: the Drift tab labels the history // "checked {time ago}" and a stale finding reads as history, not current truth. const lastCheckedAt = DatabaseService.getInstance().getStackDossier(nodeId, stackName)?.last_drift_check_at ?? null; - return { ...report, temporal, ledger, lastCheckedAt }; + // Additive and separate on purpose. The ledger above is the compose-versus-runtime + // record this tab has always shown; the revision carries the GitOps drift classes, + // which are derived state and are never written into stack_drift_findings. + // + // Deliberately not guarded, matching computeTemporal above: on a read, the + // revision is part of the answer rather than decoration on one, so a fault + // reading it surfaces as a 500 instead of a projection that quietly reports + // less state than exists. Mutation routes take the opposite side, because + // there the write has already committed and a decoration must not be able to + // report it as failed. + return { ...report, temporal, ledger, lastCheckedAt, gitopsRevision: projectManagedStackRevision(stackName, nodeId) }; } stacksRouter.get('/:stackName/drift', async (req: Request, res: Response) => { @@ -1837,7 +1853,7 @@ stacksRouter.post('/:stackName/deploy', async (req: Request, res: Response) => { dlog(`[Stacks] Deploy completed: ${sanitizeForLog(stackName)}`); if (debug) console.debug(`[Stacks:debug] Deploy finished in ${Date.now() - t0}ms`); ok = true; - const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'deploy', req.user?.username ?? null); + const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'deploy', req.user?.username ?? null, { deployedGenerationId: deployResult.deployedGenerationId }); linkStackUpdateRecoveryGate(deployResult.recoveryId, healthGateId); res.json({ message: 'Deployed successfully', healthGateId }); notifyActionSuccess('deploy_success', `${stackName} deployed`, stackName, req.user?.username ?? 'system'); @@ -2410,7 +2426,7 @@ stacksRouter.post('/:stackName/update', async (req: Request, res: Response) => { // Health observation starts immediately after Compose; registry recheck is // isolated so a verification failure cannot turn Compose success into 500. ok = true; - const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null); + const healthGateId = HealthGateService.getInstance().beginStack(req.nodeId, stackName, 'update', req.user?.username ?? null, { deployedGenerationId: orchResult.kind === 'stack_compose_done' ? orchResult.deployedGenerationId : null }); const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null; linkStackUpdateRecoveryGate(recoveryId, healthGateId); @@ -2499,14 +2515,14 @@ stacksRouter.post('/:stackName/rollback', async (req: Request, res: Response) => try { const rolledBack = await recoverySvc.compensateWithCandidate( currentGen.id, - async (overridePath, invocation) => { - await ComposeService.getInstance(req.nodeId).composeUpWithRecoveryOverride( - stackName, - overridePath, - getTerminalWs(req.get(DEPLOY_SESSION_HEADER)), - invocation, - ); - }, + // Returns the Compose result rather than swallowing it, so a proven + // restore can bind its deployed pointer and open a health run. + (overridePath, invocation) => ComposeService.getInstance(req.nodeId).composeUpWithRecoveryOverride( + stackName, + overridePath, + getTerminalWs(req.get(DEPLOY_SESSION_HEADER)), + invocation, + ), buildPolicyGateOptions(req, { actor: req.user?.username ?? 'system' }), ); if (!rolledBack) { diff --git a/backend/src/services/BlueprintReconciler.ts b/backend/src/services/BlueprintReconciler.ts index b49ac6f8..2edc8a89 100644 --- a/backend/src/services/BlueprintReconciler.ts +++ b/backend/src/services/BlueprintReconciler.ts @@ -21,6 +21,8 @@ import { applyClearStaleGuard, buildBlueprintPreview, } from './blueprintPreviewProjection'; +import { commitBlueprintDeploymentCause } from './gitops/blueprintDeploymentProducers'; +import { GitOpsStore } from './gitops/store'; const RECONCILER_INTERVAL_MS = 60_000; const RECONCILER_INITIAL_DELAY_MS = 5_000; @@ -134,6 +136,8 @@ export interface ReconcileDecision { check: Node[]; stateReview: Node[]; evictBlocked: Node[]; + /** Nodes whose canonical target is severed; no automatic action may run. */ + severedNodeIds: number[]; } /** @@ -324,25 +328,21 @@ export class BlueprintReconciler { switch (action) { case 'await_state_review': { const existing = DatabaseService.getInstance().getDeployment(blueprint.id, node.id); - DatabaseService.getInstance().upsertDeployment({ - blueprint_id: blueprint.id, - node_id: node.id, + commitBlueprintDeploymentCause('await_state_review', blueprint.id, node.id, { status: 'pending_state_review', last_checked_at: Date.now(), drift_summary: existing ? 'Stateful blueprint revision change awaits operator confirmation' : 'Stateful blueprint awaiting operator confirmation before first deploy', - }); + }, null); return { ...base, status: 'ok' }; } case 'await_evict_confirm': { - DatabaseService.getInstance().upsertDeployment({ - blueprint_id: blueprint.id, - node_id: node.id, + commitBlueprintDeploymentCause('await_evict_confirm', blueprint.id, node.id, { status: 'evict_blocked', last_checked_at: Date.now(), drift_summary: 'Stateful blueprint eviction requires operator confirmation', - }); + }, null); return { ...base, status: 'ok' }; } case 'clear_reversed_evict': @@ -365,14 +365,12 @@ export class BlueprintReconciler { const driftResult = await svc.checkForDrift(blueprint, node); if (!driftResult.drifted) return { ...base, status: 'ok' }; const reason = driftResult.reason ?? 'unknown drift'; - DatabaseService.getInstance().upsertDeployment({ - blueprint_id: blueprint.id, - node_id: node.id, + commitBlueprintDeploymentCause('drift_observed', blueprint.id, node.id, { status: 'drifted', last_checked_at: Date.now(), last_drift_at: Date.now(), drift_summary: reason, - }); + }, null); // observe/suggest/enforce: notify path via handleDrift still respects drift_mode await this.handleDrift(blueprint, node, reason); return { ...base, status: 'ok' }; @@ -511,18 +509,34 @@ export class BlueprintReconciler { const deploymentByNode = new Map(); for (const dep of existingDeployments) deploymentByNode.set(dep.node_id, dep); + // A tombstoned target is a placement the model has severed (withdraw, + // node delete). Redeploying onto one would run the workload while the + // projection insists the target is gone, so automatic placement skips + // it. Only an explicit deploy re-opens the placement, and that revival + // is recorded by the transition itself. + const gitopsApp = GitOpsStore.getInstance().getLiveBlueprintApplication(blueprint.id); + const severedNodes = new Set( + gitopsApp + ? GitOpsStore.getInstance().listTargets(gitopsApp.id) + .filter((t) => t.target_status === 'tombstoned') + .map((t) => t.node_id) + : [], + ); + const decision: ReconcileDecision = { deploy: [], withdraw: [], check: [], stateReview: [], evictBlocked: [], + severedNodeIds: [...severedNodes], }; // Desired but not active or stale for (const node of desiredNodes) { const dep = deploymentByNode.get(node.id); if (!dep) { + if (severedNodes.has(node.id)) continue; // Cordon filter: skip new placements onto cordoned nodes. // Pin always wins, so the pinned node is exempt. Existing // deployments below are untouched: cordon does not evict. @@ -553,6 +567,7 @@ export class BlueprintReconciler { continue; } if (dep.applied_revision !== blueprint.revision) { + if (severedNodes.has(node.id)) continue; if (blueprint.classification === 'stateful' || blueprint.classification === 'unknown') { decision.stateReview.push(node); } else { @@ -561,6 +576,7 @@ export class BlueprintReconciler { continue; } if (dep.status === 'failed' || dep.status === 'pending') { + if (severedNodes.has(node.id)) continue; decision.deploy.push(node); continue; } @@ -623,12 +639,10 @@ export class BlueprintReconciler { return; } } - DatabaseService.getInstance().upsertDeployment({ - blueprint_id: blueprint.id, - node_id: node.id, + commitBlueprintDeploymentCause('drift_enforce_start', blueprint.id, node.id, { status: 'correcting', last_checked_at: Date.now(), - }); + }, null); const result = await BlueprintService.getInstance().deployToNode(blueprint, node); if (result.status !== 'active') { notifications.dispatchAlert( diff --git a/backend/src/services/BlueprintService.ts b/backend/src/services/BlueprintService.ts index 73f4b161..5c9fd39f 100644 --- a/backend/src/services/BlueprintService.ts +++ b/backend/src/services/BlueprintService.ts @@ -1,718 +1,732 @@ -import path from 'path'; -import { promises as fsPromises } from 'fs'; -import axios, { AxiosError } from 'axios'; -import { - DatabaseService, - type Blueprint, - type BlueprintDeployment, - type BlueprintDeploymentStatus, - type Node, -} from './DatabaseService'; -import { ComposeService } from './ComposeService'; -import { StackOpLockService, stackOpSkipMessage, type StackOpAction } from './StackOpLockService'; -import { DeployedStackDeletionService } from './DeployedStackDeletionService'; -import { FileSystemService } from './FileSystemService'; -import { NodeRegistry } from './NodeRegistry'; -import { PROXY_TIER_HEADER, deployProvenanceHeaders } from './license-headers'; -import { LicenseService } from './LicenseService'; -import { assertPolicyGateAllows, buildSystemPolicyGateOptions, describePolicyBlock, triggerPostDeployScan } from '../helpers/policyGate'; -import { enforcePolicyForImageRefs } from './PolicyEnforcement'; -import { BlueprintAnalyzer } from './BlueprintAnalyzer'; -import { sanitizeForLog } from '../utils/safeLog'; -import { isPathWithinBase } from '../utils/validation'; -import { - BLUEPRINT_MARKER_FILENAME, - parseBlueprintMarker, - type BlueprintMarker, -} from '../helpers/blueprintMarker'; -/** On-disk compose name for Blueprint applies. Must match createStack scaffold and Sencho discovery priority. */ -const COMPOSE_FILENAME = 'compose.yaml'; -const REMOTE_HTTP_TIMEOUT_MS = 30_000; - -function isDeveloperModeEnabled(): boolean { - try { - return DatabaseService.getInstance().getGlobalSettings().developer_mode === '1'; - } catch { - return false; - } -} - -function diagnosticLog(message: string, fields: Record): void { - if (!isDeveloperModeEnabled()) return; - const safeFields = Object.fromEntries( - Object.entries(fields).map(([key, value]) => [key, typeof value === 'string' ? sanitizeForLog(value) : value]), - ); - console.info(`[BlueprintService:diag] ${message}`, safeFields); -} - -export type { BlueprintMarker }; - -export class BlueprintNameConflictError extends Error { - readonly code = 'name_conflict' as const; - constructor(message: string) { - super(message); - this.name = 'BlueprintNameConflictError'; - } -} - -/** Thrown when a remote node lacks the atomic apply/withdraw endpoints. */ -export class BlueprintRemoteUpgradeRequiredError extends Error { - readonly code = 'remote_upgrade_required' as const; - constructor(message: string) { - super(message); - this.name = 'BlueprintRemoteUpgradeRequiredError'; - } -} - -/** Thrown when ownership cannot be verified (non-ENOENT I/O or remote probe failure). */ -export class BlueprintOwnershipProbeError extends Error { - readonly code = 'ownership_probe_failed' as const; - constructor(message: string) { - super(message); - this.name = 'BlueprintOwnershipProbeError'; - } -} - -export interface DeployOutcome { - status: BlueprintDeploymentStatus; - error?: string; -} - -type LocalMarkerRead = - | { kind: 'missing' } - | { kind: 'present'; marker: BlueprintMarker } - | { kind: 'failed'; error: string }; - -/** - * BlueprintService is the orchestration layer between the reconciler and the - * concrete deploy/withdraw primitives. It owns: - * - per-target marker-file management (writes, reads, validates ownership) - * - name-conflict guard (refuses apply/withdraw when the directory lacks a matching - * `.blueprint.json` for this blueprint ID) - * - local deploy via ComposeService + FileSystemService - * - remote deploy via direct HTTP calls to the remote Sencho instance - * - per-(blueprint,node) concurrency lock so overlapping ticks don't collide - * - * The reconciler decides *what* needs to happen; this service performs it. - */ -export class BlueprintService { - private static instance: BlueprintService | null = null; - private readonly inflight = new Set(); - - static getInstance(): BlueprintService { - if (!BlueprintService.instance) { - BlueprintService.instance = new BlueprintService(); - } - return BlueprintService.instance; - } - - private constructor() { /* singleton */ } - - private lockKey(blueprintId: number, nodeId: number): string { - return `${blueprintId}:${nodeId}`; - } - - private acquireLock(blueprintId: number, nodeId: number): boolean { - const key = this.lockKey(blueprintId, nodeId); - if (this.inflight.has(key)) return false; - this.inflight.add(key); - return true; - } - - private releaseLock(blueprintId: number, nodeId: number): void { - this.inflight.delete(this.lockKey(blueprintId, nodeId)); - } - - private buildMarker(blueprint: Blueprint): BlueprintMarker { - return { - blueprintId: blueprint.id, - revision: blueprint.revision, - lastApplied: Date.now(), - }; - } - - private setStatus( - blueprintId: number, - nodeId: number, - status: BlueprintDeploymentStatus, - extras: Partial<{ - applied_revision: number | null; - last_deployed_at: number | null; - last_drift_at: number | null; - drift_summary: string | null; - last_error: string | null; - }> = {}, - ): BlueprintDeployment { - return DatabaseService.getInstance().upsertDeployment({ - blueprint_id: blueprintId, - node_id: nodeId, - status, - last_checked_at: Date.now(), - ...extras, - }); - } - - /** - * Read the marker file from a target node. Returns null when missing, - * malformed, or unreadable. The reconciler treats null as "we do not - * own this directory" and refuses to touch it. - */ - async readMarker(blueprintName: string, node: Node): Promise { - try { - if (node.type === 'local') { - const markerRead = await this.readLocalMarkerFromDisk(node.id, blueprintName); - return markerRead.kind === 'present' ? markerRead.marker : null; - } - const target = NodeRegistry.getInstance().getProxyTarget(node.id); - if (!target) return null; - const url = `${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(blueprintName)}/files/content?path=${encodeURIComponent(BLUEPRINT_MARKER_FILENAME)}`; - const res = await axios.get(url, { - headers: this.remoteHeaders(target.apiToken), - timeout: REMOTE_HTTP_TIMEOUT_MS, - validateStatus: () => true, - }); - if (res.status !== 200) return null; - const body = res.data; - const content = typeof body === 'string' ? body : (typeof body?.content === 'string' ? body.content : null); - if (content == null) return null; - return parseBlueprintMarker(content); - } catch { - return null; - } - } - - /** - * Returns true when a stack directory by this name exists on the target - * node and the on-disk marker is missing, malformed, or references a - * different blueprint ID. Throws BlueprintOwnershipProbeError when the - * directory or marker cannot be probed (non-ENOENT I/O or remote list failure). - */ - async hasNameConflict(blueprintName: string, node: Node, blueprintId: number): Promise { - if (node.type === 'local') { - const baseDir = NodeRegistry.getInstance().getComposeDir(node.id); - const stackDir = path.resolve(baseDir, blueprintName); - if (!isPathWithinBase(stackDir, baseDir)) return true; - try { - const stat = await fsPromises.stat(stackDir); - if (!stat.isDirectory()) return false; - } catch (err) { - const code = (err as NodeJS.ErrnoException).code; - if (code === 'ENOENT') return false; - throw BlueprintService.ownershipProbeError(blueprintName, BlueprintService.formatError(err)); - } - const markerRead = await this.readLocalMarkerFromDisk(node.id, blueprintName); - if (markerRead.kind === 'failed') { - throw BlueprintService.ownershipProbeError(blueprintName, markerRead.error); - } - return markerRead.kind === 'missing' || markerRead.marker.blueprintId !== blueprintId; - } - const target = NodeRegistry.getInstance().getProxyTarget(node.id); - if (!target) { - throw new BlueprintOwnershipProbeError( - `Cannot verify stack ownership on remote node "${node.name}": no proxy target configured`, - ); - } - const baseUrl = target.apiUrl.replace(/\/$/, ''); - let listRes; - try { - listRes = await axios.get(`${baseUrl}/api/stacks`, { - headers: this.remoteHeaders(target.apiToken), - timeout: REMOTE_HTTP_TIMEOUT_MS, - validateStatus: () => true, - }); - } catch (err) { - throw new BlueprintOwnershipProbeError( - `Cannot verify stack ownership on remote node "${node.name}": ${BlueprintService.formatError(err)}`, - ); - } - if (listRes.status !== 200) { - throw new BlueprintOwnershipProbeError( - `Cannot verify stack ownership on remote node "${node.name}" (HTTP ${listRes.status})`, - ); - } - const stacks = Array.isArray(listRes.data) ? listRes.data as Array<{ name?: string }> : []; - const exists = stacks.some(s => s?.name === blueprintName); - if (!exists) return false; - const marker = await this.readMarker(blueprintName, node); - return marker == null || marker.blueprintId !== blueprintId; - } - - /** Read and parse a local on-disk marker without going through the remote HTTP path. */ - private async readLocalMarkerFromDisk(nodeId: number, stackName: string): Promise { - try { - // Canonical js/path-injection barrier inline with the read sink. - const baseResolved = path.resolve(NodeRegistry.getInstance().getComposeDir(nodeId)); - const safePath = path.resolve(baseResolved, stackName, BLUEPRINT_MARKER_FILENAME); - if (!safePath.startsWith(baseResolved + path.sep)) { - return { kind: 'failed', error: 'Invalid stack path for blueprint marker' }; - } - const content = await fsPromises.readFile(safePath, 'utf-8'); - const marker = parseBlueprintMarker(content); - if (!marker) return { kind: 'missing' }; - return { kind: 'present', marker }; - } catch (err) { - const code = (err as NodeJS.ErrnoException).code; - if (code === 'ENOENT') return { kind: 'missing' }; - return { kind: 'failed', error: BlueprintService.formatError(err) }; - } - } - - /** - * Deploy this blueprint to the given target node. Caller must have already - * resolved that the target should receive this blueprint (selector match - * passed, no state-review pending, etc.). This method handles the - * name-conflict guard and the local/remote dispatch. - */ - async deployToNode(blueprint: Blueprint, node: Node): Promise { - if (!this.acquireLock(blueprint.id, node.id)) { - return { status: 'pending' }; - } - const started = Date.now(); - console.info('[BlueprintService] deploy start blueprint=%s node=%s type=%s revision=%s', - sanitizeForLog(blueprint.name), node.id, node.type, blueprint.revision); - diagnosticLog('deploy inputs', { - blueprintId: blueprint.id, - blueprintName: blueprint.name, - nodeId: node.id, - nodeType: node.type, - revision: blueprint.revision, - classification: blueprint.classification, - driftMode: blueprint.drift_mode, - }); - try { - this.setStatus(blueprint.id, node.id, 'deploying'); - if (await this.hasNameConflict(blueprint.name, node, blueprint.id)) { - this.setStatus(blueprint.id, node.id, 'name_conflict', { - last_error: `A stack named "${blueprint.name}" already exists on this node and is not managed by Sencho.`, - }); - console.warn('[BlueprintService] deploy name conflict blueprint=%s node=%s durationMs=%s', - sanitizeForLog(blueprint.name), node.id, Date.now() - started); - return { status: 'name_conflict', error: 'name_conflict' }; - } - const marker = this.buildMarker(blueprint); - if (node.type === 'local') { - diagnosticLog('deploy branch', { blueprintId: blueprint.id, nodeId: node.id, target: 'local' }); - await this.deployLocal(blueprint, node, marker); - } else { - diagnosticLog('deploy branch', { blueprintId: blueprint.id, nodeId: node.id, target: 'remote' }); - await this.deployRemote(blueprint, node, marker); - } - this.setStatus(blueprint.id, node.id, 'active', { - applied_revision: blueprint.revision, - last_deployed_at: Date.now(), - last_drift_at: null, - drift_summary: null, - last_error: null, - }); - console.info('[BlueprintService] deploy complete blueprint=%s node=%s durationMs=%s', - sanitizeForLog(blueprint.name), node.id, Date.now() - started); - return { status: 'active' }; - } catch (err) { - if (err instanceof BlueprintNameConflictError) { - this.setStatus(blueprint.id, node.id, 'name_conflict', { last_error: err.message }); - console.warn('[BlueprintService] deploy name conflict blueprint=%s node=%s durationMs=%s', - sanitizeForLog(blueprint.name), node.id, Date.now() - started); - return { status: 'name_conflict', error: 'name_conflict' }; - } - const message = BlueprintService.formatError(err); - this.setStatus(blueprint.id, node.id, 'failed', { last_error: message }); - console.error('[BlueprintService] deploy failed blueprint=%s node=%s durationMs=%s error=%s', - sanitizeForLog(blueprint.name), node.id, Date.now() - started, sanitizeForLog(message)); - return { status: 'failed', error: message }; - } finally { - this.releaseLock(blueprint.id, node.id); - } - } - - /** - * Withdraw a blueprint from the target node: docker compose down, delete - * the directory. Caller must have already cleared the eviction guard - * (stateful blueprints require explicit operator confirmation). - */ - async withdrawFromNode(blueprint: Blueprint, node: Node): Promise { - if (!this.acquireLock(blueprint.id, node.id)) { - return { status: 'pending' }; - } - const started = Date.now(); - console.info('[BlueprintService] withdraw start blueprint=%s node=%s type=%s', - sanitizeForLog(blueprint.name), node.id, node.type); - diagnosticLog('withdraw inputs', { - blueprintId: blueprint.id, - blueprintName: blueprint.name, - nodeId: node.id, - nodeType: node.type, - classification: blueprint.classification, - }); - try { - this.setStatus(blueprint.id, node.id, 'withdrawing'); - // Ownership is validated on the node that owns the stack, inside the delete lock. - if (node.type === 'local') { - diagnosticLog('withdraw branch', { blueprintId: blueprint.id, nodeId: node.id, target: 'local' }); - const localOutcome = await this.withdrawLocal(blueprint, node); - if (localOutcome.status !== 'withdrawn') return localOutcome; - } else { - diagnosticLog('withdraw branch', { blueprintId: blueprint.id, nodeId: node.id, target: 'remote' }); - const remoteOutcome = await this.withdrawRemote(blueprint, node); - if (remoteOutcome.status !== 'withdrawn') return remoteOutcome; - } - DatabaseService.getInstance().deleteDeployment(blueprint.id, node.id); - console.info('[BlueprintService] withdraw complete blueprint=%s node=%s durationMs=%s', - sanitizeForLog(blueprint.name), node.id, Date.now() - started); - return { status: 'withdrawn' }; - } catch (err) { - const message = BlueprintService.formatError(err); - this.setStatus(blueprint.id, node.id, 'failed', { last_error: `withdraw failed: ${message}` }); - console.error('[BlueprintService] withdraw failed blueprint=%s node=%s durationMs=%s error=%s', - sanitizeForLog(blueprint.name), node.id, Date.now() - started, sanitizeForLog(message)); - return { status: 'failed', error: message }; - } finally { - this.releaseLock(blueprint.id, node.id); - } - } - - /** - * Inspect the actual state of a deployment on its node and report - * whether it has drifted from the desired state. The reconciler decides - * what to do with the result based on drift_mode. - */ - async checkForDrift(blueprint: Blueprint, node: Node): Promise<{ drifted: boolean; reason?: string }> { - try { - const marker = await this.readMarker(blueprint.name, node); - if (!marker) { - return { drifted: true, reason: 'marker file missing on node' }; - } - if (marker.blueprintId !== blueprint.id) { - return { drifted: true, reason: 'marker references a different blueprint' }; - } - if (marker.revision !== blueprint.revision) { - return { drifted: true, reason: `revision drift (node has ${marker.revision}, blueprint is ${blueprint.revision})` }; - } - // Check container state - const containerState = await this.containerHealth(blueprint.name, node); - if (!containerState.allRunning) { - return { drifted: true, reason: containerState.detail }; - } - return { drifted: false }; - } catch (err) { - return { drifted: true, reason: BlueprintService.formatError(err) }; - } - } - - private async containerHealth(blueprintName: string, node: Node): Promise<{ allRunning: boolean; detail: string }> { - try { - // Docker Compose normalizes the project name to lowercase. Match the same canonical form. - const projectName = blueprintName.toLowerCase(); - if (node.type === 'local') { - const docker = NodeRegistry.getInstance().getDocker(node.id); - const containers = await docker.listContainers({ - all: true, - filters: { label: [`com.docker.compose.project=${projectName}`] }, - }); - if (containers.length === 0) return { allRunning: false, detail: 'no containers running for this blueprint' }; - const notRunning = containers.filter(c => c.State !== 'running'); - if (notRunning.length > 0) { - const first = notRunning[0]; - return { allRunning: false, detail: `container "${first.Names[0] ?? first.Id.slice(0, 12)}" is ${first.State}` }; - } - return { allRunning: true, detail: '' }; - } - const target = NodeRegistry.getInstance().getProxyTarget(node.id); - if (!target) return { allRunning: false, detail: 'remote node not reachable (no proxy target)' }; - const url = `${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(blueprintName)}/containers`; - const res = await axios.get(url, { - headers: this.remoteHeaders(target.apiToken), - timeout: REMOTE_HTTP_TIMEOUT_MS, - validateStatus: () => true, - }); - if (res.status !== 200) { - return { allRunning: false, detail: `remote stack lookup returned HTTP ${res.status}` }; - } - const list = Array.isArray(res.data) ? res.data as Array<{ State?: string; Names?: string[]; Id?: string }> : []; - if (list.length === 0) return { allRunning: false, detail: 'remote stack has no containers' }; - const notRunning = list.filter(c => (c.State ?? '') !== 'running'); - if (notRunning.length > 0) { - const first = notRunning[0]; - return { allRunning: false, detail: `remote container "${first.Names?.[0] ?? first.Id?.slice(0, 12)}" is ${first.State}` }; - } - return { allRunning: true, detail: '' }; - } catch (err) { - return { allRunning: false, detail: BlueprintService.formatError(err) }; - } - } - - // ---- local primitives ---- - - /** Returns whether the stack directory exists. Throws on non-ENOENT I/O. */ - private async stackDirExists(nodeId: number, blueprintName: string): Promise { - const baseDir = NodeRegistry.getInstance().getComposeDir(nodeId); - const stackDir = path.resolve(baseDir, blueprintName); - if (!isPathWithinBase(stackDir, baseDir)) { - throw new BlueprintOwnershipProbeError(`Invalid stack path for "${blueprintName}"`); - } - try { - const stat = await fsPromises.stat(stackDir); - return stat.isDirectory(); - } catch (err) { - const code = (err as NodeJS.ErrnoException).code; - if (code === 'ENOENT') return false; - throw new BlueprintOwnershipProbeError( - `Cannot access stack directory "${blueprintName}": ${BlueprintService.formatError(err)}`, - ); - } - } - - private async deployLocal(blueprint: Blueprint, node: Node, marker: BlueprintMarker): Promise { - const imageRefs = BlueprintAnalyzer.extractImageRefs(blueprint.compose_content); - const gate = await enforcePolicyForImageRefs(blueprint.name, node.id, imageRefs, { - bypass: false, - actor: 'blueprint-reconciler', - auditMethod: 'POST', - auditPath: `/api/blueprints/${blueprint.id}/apply`, - }, undefined, true); - if (!gate.ok) { - throw new Error(describePolicyBlock(gate.policy, gate.violations)); - } - - const outcome = await this.applyLocalUnderLock( - node.id, - blueprint.name, - blueprint.compose_content, - JSON.stringify(marker, null, 2), - `/api/blueprints/${blueprint.id}/deployments/${node.id}`, - ); - if (!outcome.ran) { - throw new Error(stackOpSkipMessage(blueprint.name, outcome.existingAction)); - } - triggerPostDeployScan(blueprint.name, node.id).catch(err => { - console.error('[BlueprintService] post-deploy scan failed for "%s" on node %s: %s', - sanitizeForLog(blueprint.name), node.id, sanitizeForLog(BlueprintService.formatError(err))); - }); - } - - /** - * Create the stack if needed, write the compose file, run the deploy policy - * gate and deploy, then write the marker, all under the per-stack operation - * lock. The marker is written only after a successful deploy so a failed - * apply cannot claim an applied revision that never ran. Runs on the node - * that owns the stack: deployLocal calls it for the hub's own node, and the - * /api/blueprints/apply-local route calls it on a remote node receiving a - * blueprint apply from its hub. On lock conflict nothing is written and - * { ran: false } is returned. - */ - async applyLocalUnderLock( - nodeId: number, - stackName: string, - composeContent: string, - markerContent: string, - auditPath: string, - ): Promise<{ ran: true } | { ran: false; existingAction: StackOpAction }> { - const expected = parseBlueprintMarker(markerContent); - if (!expected) { - throw new Error('Invalid blueprint marker'); - } - const fs = FileSystemService.getInstance(nodeId); - const lock = await StackOpLockService.getInstance().runExclusive( - nodeId, stackName, 'deploy', 'system', - async () => { - let createdStack = false; - if (await this.stackDirExists(nodeId, stackName)) { - const existing = await this.readLocalMarkerFromDisk(nodeId, stackName); - if (existing.kind === 'failed') { - throw new BlueprintOwnershipProbeError( - `Cannot verify ownership of stack "${stackName}": ${existing.error}`, - ); - } - if (existing.kind === 'missing' || existing.marker.blueprintId !== expected.blueprintId) { - throw new BlueprintNameConflictError( - `A stack named "${stackName}" already exists on this node and is not managed by this blueprint.`, - ); - } - } else { - await fs.createStack(stackName); - createdStack = true; - } - await fs.writeStackFile(stackName, COMPOSE_FILENAME, composeContent); - // Clear lower-priority compose siblings so discovery cannot shadow compose.yaml. - await fs.removeAlternateRootComposeFiles(stackName); - try { - await assertPolicyGateAllows( - stackName, - nodeId, - buildSystemPolicyGateOptions('blueprint', { auditPath }), - ); - await ComposeService.getInstance(nodeId).deployStack( - stackName, - undefined, - false, - { source: 'blueprint', actor: 'system:blueprint' }, - ); - await fs.writeStackFile(stackName, BLUEPRINT_MARKER_FILENAME, markerContent); - } catch (err) { - if (createdStack) { - try { - await fs.deleteStack(stackName); - } catch (cleanupErr) { - console.warn( - '[BlueprintService] Failed to roll back newly created stack "%s" after apply error: %s', - sanitizeForLog(stackName), - sanitizeForLog(BlueprintService.formatError(cleanupErr)), - ); - } - } - throw err; - } - }, - ); - return lock.ran ? { ran: true } : { ran: false, existingAction: lock.existing.action }; - } - - private async withdrawLocal(blueprint: Blueprint, node: Node): Promise { - const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({ - nodeId: node.id, - stackName: blueprint.name, - pruneVolumes: false, - actor: 'system:blueprint', - requireBlueprintId: blueprint.id, - }); - if (result.ok) { - return { status: 'withdrawn' }; - } - if (result.code === 'name_conflict') { - this.setStatus(blueprint.id, node.id, 'name_conflict', { last_error: result.error }); - return { status: 'name_conflict' }; - } - this.setStatus(blueprint.id, node.id, 'failed', { last_error: result.error }); - return { status: 'failed', error: result.error }; - } - - // ---- remote primitives ---- - - private remoteHeaders(apiToken: string): Record { - const proxy = LicenseService.getInstance().getProxyHeaders(); - return { - Authorization: `Bearer ${apiToken}`, - [PROXY_TIER_HEADER]: proxy.tier, - 'Content-Type': 'application/json', - ...deployProvenanceHeaders('blueprint', 'system:blueprint'), - }; - } - - private async deployRemote(blueprint: Blueprint, node: Node, marker: BlueprintMarker): Promise { - const target = NodeRegistry.getInstance().getProxyTarget(node.id); - if (!target) throw new Error(`Remote node "${node.name}" has no proxy target configured`); - const baseUrl = target.apiUrl.replace(/\/$/, ''); - const headers = this.remoteHeaders(target.apiToken); - - // Atomic apply: the remote validates ownership and writes under its stack lock. - const res = await axios.post( - `${baseUrl}/api/blueprints/apply-local`, - { - stackName: blueprint.name, - composeContent: blueprint.compose_content, - markerContent: JSON.stringify(marker, null, 2), - }, - { headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true }, - ); - if (res.status === 404) { - throw new BlueprintRemoteUpgradeRequiredError( - `Remote node "${node.name}" does not support atomic blueprint apply (/api/blueprints/apply-local). Upgrade that Sencho instance, then retry.`, - ); - } - if (res.status === 409) { - if (BlueprintService.extractApiCode(res.data) === 'name_conflict') { - throw new BlueprintNameConflictError( - BlueprintService.extractApiError(res.data) - || `A stack named "${blueprint.name}" already exists on this node and is not managed by this blueprint.`, - ); - } - throw new Error(`blueprint apply skipped: ${BlueprintService.extractApiError(res.data) || 'another operation is already in progress'}`); - } - if (res.status >= 400) { - throw new Error(`blueprint apply: HTTP ${res.status} ${BlueprintService.extractApiError(res.data)}`); - } - } - - private async withdrawRemote(blueprint: Blueprint, node: Node): Promise { - const target = NodeRegistry.getInstance().getProxyTarget(node.id); - if (!target) throw new Error(`Remote node "${node.name}" has no proxy target configured`); - const baseUrl = target.apiUrl.replace(/\/$/, ''); - const headers = this.remoteHeaders(target.apiToken); - - let res; - try { - res = await axios.post( - `${baseUrl}/api/blueprints/withdraw-local`, - { stackName: blueprint.name, blueprintId: blueprint.id }, - { headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true }, - ); - } catch (err) { - const message = BlueprintService.formatError(err); - this.setStatus(blueprint.id, node.id, 'failed', { last_error: message }); - return { status: 'failed', error: message }; - } - - if (res.status === 404) { - throw new BlueprintRemoteUpgradeRequiredError( - `Remote node "${node.name}" does not support atomic blueprint withdraw (/api/blueprints/withdraw-local). Upgrade that Sencho instance, then retry.`, - ); - } - if (res.status === 200) { - DatabaseService.getInstance().deleteRoleAssignmentsByStack(node.id, blueprint.name); - return { status: 'withdrawn' }; - } - if (res.status === 409) { - const error = BlueprintService.extractApiError(res.data) || 'withdraw refused'; - if (BlueprintService.extractApiCode(res.data) === 'name_conflict') { - this.setStatus(blueprint.id, node.id, 'name_conflict', { last_error: error }); - return { status: 'name_conflict' }; - } - // stack_op_in_progress and any other 409: match local withdraw lock-conflict → failed - this.setStatus(blueprint.id, node.id, 'failed', { last_error: error }); - return { status: 'failed', error }; - } - const message = `blueprint withdraw: HTTP ${res.status} ${BlueprintService.extractApiError(res.data)}`; - this.setStatus(blueprint.id, node.id, 'failed', { last_error: message }); - return { status: 'failed', error: message }; - } - - static parseMarker(content: string): BlueprintMarker | null { - return parseBlueprintMarker(content); - } - - private static ownershipProbeError(blueprintName: string, detail: string): BlueprintOwnershipProbeError { - return new BlueprintOwnershipProbeError( - `Cannot verify stack ownership for "${blueprintName}": ${detail}`, - ); - } - - static extractApiCode(body: unknown): string { - if (!body || typeof body !== 'object') return ''; - const code = (body as Record).code; - return typeof code === 'string' ? code : ''; - } - - static formatError(err: unknown): string { - if (axios.isAxiosError(err)) { - const ax = err as AxiosError<{ error?: string; message?: string }>; - if (ax.response?.data) { - const body = ax.response.data; - if (body && typeof body === 'object') { - if (typeof body.error === 'string') return body.error; - if (typeof body.message === 'string') return body.message; - } - } - if (ax.code) return `${ax.code}: ${ax.message}`; - return ax.message; - } - if (err instanceof Error) return err.message; - return String(err); - } - - static extractApiError(body: unknown): string { - if (!body || typeof body !== 'object') return ''; - const obj = body as Record; - if (typeof obj.error === 'string') return obj.error; - if (typeof obj.message === 'string') return obj.message; - return ''; - } -} +import path from 'path'; +import { promises as fsPromises } from 'fs'; +import axios, { AxiosError } from 'axios'; +import { + DatabaseService, + type Blueprint, + type BlueprintDeployment, + type BlueprintDeploymentStatus, + type Node, +} from './DatabaseService'; +import { ComposeService } from './ComposeService'; +import { StackOpLockService, stackOpSkipMessage, type StackOpAction } from './StackOpLockService'; +import { DeployedStackDeletionService } from './DeployedStackDeletionService'; +import { FileSystemService } from './FileSystemService'; +import { NodeRegistry } from './NodeRegistry'; +import { PROXY_TIER_HEADER, deployProvenanceHeaders } from './license-headers'; +import { LicenseService } from './LicenseService'; +import { assertPolicyGateAllows, buildSystemPolicyGateOptions, describePolicyBlock, triggerPostDeployScan } from '../helpers/policyGate'; +import { enforcePolicyForImageRefs } from './PolicyEnforcement'; +import { BlueprintAnalyzer } from './BlueprintAnalyzer'; +import { sanitizeForLog } from '../utils/safeLog'; +import { isPathWithinBase } from '../utils/validation'; +import { + BLUEPRINT_MARKER_FILENAME, + parseBlueprintMarker, + type BlueprintMarker, +} from '../helpers/blueprintMarker'; +import { + commitBlueprintDeploymentCause, + commitBlueprintDeploymentRemoved, + type BlueprintDeploymentCause, +} from './gitops/blueprintDeploymentProducers'; +/** On-disk compose name for Blueprint applies. Must match createStack scaffold and Sencho discovery priority. */ +const COMPOSE_FILENAME = 'compose.yaml'; +const REMOTE_HTTP_TIMEOUT_MS = 30_000; + +function isDeveloperModeEnabled(): boolean { + try { + return DatabaseService.getInstance().getGlobalSettings().developer_mode === '1'; + } catch { + return false; + } +} + +function diagnosticLog(message: string, fields: Record): void { + if (!isDeveloperModeEnabled()) return; + const safeFields = Object.fromEntries( + Object.entries(fields).map(([key, value]) => [key, typeof value === 'string' ? sanitizeForLog(value) : value]), + ); + console.info(`[BlueprintService:diag] ${message}`, safeFields); +} + +export type { BlueprintMarker }; + +export class BlueprintNameConflictError extends Error { + readonly code = 'name_conflict' as const; + constructor(message: string) { + super(message); + this.name = 'BlueprintNameConflictError'; + } +} + +/** Thrown when a remote node lacks the atomic apply/withdraw endpoints. */ +export class BlueprintRemoteUpgradeRequiredError extends Error { + readonly code = 'remote_upgrade_required' as const; + constructor(message: string) { + super(message); + this.name = 'BlueprintRemoteUpgradeRequiredError'; + } +} + +/** Thrown when ownership cannot be verified (non-ENOENT I/O or remote probe failure). */ +export class BlueprintOwnershipProbeError extends Error { + readonly code = 'ownership_probe_failed' as const; + constructor(message: string) { + super(message); + this.name = 'BlueprintOwnershipProbeError'; + } +} + +export interface DeployOutcome { + status: BlueprintDeploymentStatus; + error?: string; +} + +type LocalMarkerRead = + | { kind: 'missing' } + | { kind: 'present'; marker: BlueprintMarker } + | { kind: 'failed'; error: string }; + +/** + * BlueprintService is the orchestration layer between the reconciler and the + * concrete deploy/withdraw primitives. It owns: + * - per-target marker-file management (writes, reads, validates ownership) + * - name-conflict guard (refuses apply/withdraw when the directory lacks a matching + * `.blueprint.json` for this blueprint ID) + * - local deploy via ComposeService + FileSystemService + * - remote deploy via direct HTTP calls to the remote Sencho instance + * - per-(blueprint,node) concurrency lock so overlapping ticks don't collide + * + * The reconciler decides *what* needs to happen; this service performs it. + */ +export class BlueprintService { + private static instance: BlueprintService | null = null; + private readonly inflight = new Set(); + + static getInstance(): BlueprintService { + if (!BlueprintService.instance) { + BlueprintService.instance = new BlueprintService(); + } + return BlueprintService.instance; + } + + private constructor() { /* singleton */ } + + private lockKey(blueprintId: number, nodeId: number): string { + return `${blueprintId}:${nodeId}`; + } + + private acquireLock(blueprintId: number, nodeId: number): boolean { + const key = this.lockKey(blueprintId, nodeId); + if (this.inflight.has(key)) return false; + this.inflight.add(key); + return true; + } + + private releaseLock(blueprintId: number, nodeId: number): void { + this.inflight.delete(this.lockKey(blueprintId, nodeId)); + } + + private buildMarker(blueprint: Blueprint): BlueprintMarker { + return { + blueprintId: blueprint.id, + revision: blueprint.revision, + lastApplied: Date.now(), + }; + } + + /** + * Write the deployment row, recording what caused the move. + * + * The cause is explicit because it cannot be recovered from the status: a + * deploy that failed and a withdraw that failed both land on `failed`, and + * they mean opposite things about whether the deployment is still there. + */ + private setStatus( + blueprintId: number, + nodeId: number, + status: BlueprintDeploymentStatus, + cause: BlueprintDeploymentCause, + extras: Partial<{ + applied_revision: number | null; + last_deployed_at: number | null; + last_drift_at: number | null; + drift_summary: string | null; + last_error: string | null; + }> = {}, + ): BlueprintDeployment { + return commitBlueprintDeploymentCause(cause, blueprintId, nodeId, { + status, + last_checked_at: Date.now(), + ...extras, + }, null); + } + + /** + * Read the marker file from a target node. Returns null when missing, + * malformed, or unreadable. The reconciler treats null as "we do not + * own this directory" and refuses to touch it. + */ + async readMarker(blueprintName: string, node: Node): Promise { + try { + if (node.type === 'local') { + const markerRead = await this.readLocalMarkerFromDisk(node.id, blueprintName); + return markerRead.kind === 'present' ? markerRead.marker : null; + } + const target = NodeRegistry.getInstance().getProxyTarget(node.id); + if (!target) return null; + const url = `${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(blueprintName)}/files/content?path=${encodeURIComponent(BLUEPRINT_MARKER_FILENAME)}`; + const res = await axios.get(url, { + headers: this.remoteHeaders(target.apiToken), + timeout: REMOTE_HTTP_TIMEOUT_MS, + validateStatus: () => true, + }); + if (res.status !== 200) return null; + const body = res.data; + const content = typeof body === 'string' ? body : (typeof body?.content === 'string' ? body.content : null); + if (content == null) return null; + return parseBlueprintMarker(content); + } catch { + return null; + } + } + + /** + * Returns true when a stack directory by this name exists on the target + * node and the on-disk marker is missing, malformed, or references a + * different blueprint ID. Throws BlueprintOwnershipProbeError when the + * directory or marker cannot be probed (non-ENOENT I/O or remote list failure). + */ + async hasNameConflict(blueprintName: string, node: Node, blueprintId: number): Promise { + if (node.type === 'local') { + const baseDir = NodeRegistry.getInstance().getComposeDir(node.id); + const stackDir = path.resolve(baseDir, blueprintName); + if (!isPathWithinBase(stackDir, baseDir)) return true; + try { + const stat = await fsPromises.stat(stackDir); + if (!stat.isDirectory()) return false; + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT') return false; + throw BlueprintService.ownershipProbeError(blueprintName, BlueprintService.formatError(err)); + } + const markerRead = await this.readLocalMarkerFromDisk(node.id, blueprintName); + if (markerRead.kind === 'failed') { + throw BlueprintService.ownershipProbeError(blueprintName, markerRead.error); + } + return markerRead.kind === 'missing' || markerRead.marker.blueprintId !== blueprintId; + } + const target = NodeRegistry.getInstance().getProxyTarget(node.id); + if (!target) { + throw new BlueprintOwnershipProbeError( + `Cannot verify stack ownership on remote node "${node.name}": no proxy target configured`, + ); + } + const baseUrl = target.apiUrl.replace(/\/$/, ''); + let listRes; + try { + listRes = await axios.get(`${baseUrl}/api/stacks`, { + headers: this.remoteHeaders(target.apiToken), + timeout: REMOTE_HTTP_TIMEOUT_MS, + validateStatus: () => true, + }); + } catch (err) { + throw new BlueprintOwnershipProbeError( + `Cannot verify stack ownership on remote node "${node.name}": ${BlueprintService.formatError(err)}`, + ); + } + if (listRes.status !== 200) { + throw new BlueprintOwnershipProbeError( + `Cannot verify stack ownership on remote node "${node.name}" (HTTP ${listRes.status})`, + ); + } + const stacks = Array.isArray(listRes.data) ? listRes.data as Array<{ name?: string }> : []; + const exists = stacks.some(s => s?.name === blueprintName); + if (!exists) return false; + const marker = await this.readMarker(blueprintName, node); + return marker == null || marker.blueprintId !== blueprintId; + } + + /** Read and parse a local on-disk marker without going through the remote HTTP path. */ + private async readLocalMarkerFromDisk(nodeId: number, stackName: string): Promise { + try { + // Canonical js/path-injection barrier inline with the read sink. + const baseResolved = path.resolve(NodeRegistry.getInstance().getComposeDir(nodeId)); + const safePath = path.resolve(baseResolved, stackName, BLUEPRINT_MARKER_FILENAME); + if (!safePath.startsWith(baseResolved + path.sep)) { + return { kind: 'failed', error: 'Invalid stack path for blueprint marker' }; + } + const content = await fsPromises.readFile(safePath, 'utf-8'); + const marker = parseBlueprintMarker(content); + if (!marker) return { kind: 'missing' }; + return { kind: 'present', marker }; + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT') return { kind: 'missing' }; + return { kind: 'failed', error: BlueprintService.formatError(err) }; + } + } + + /** + * Deploy this blueprint to the given target node. Caller must have already + * resolved that the target should receive this blueprint (selector match + * passed, no state-review pending, etc.). This method handles the + * name-conflict guard and the local/remote dispatch. + */ + async deployToNode(blueprint: Blueprint, node: Node): Promise { + if (!this.acquireLock(blueprint.id, node.id)) { + return { status: 'pending' }; + } + const started = Date.now(); + console.info('[BlueprintService] deploy start blueprint=%s node=%s type=%s revision=%s', + sanitizeForLog(blueprint.name), node.id, node.type, blueprint.revision); + diagnosticLog('deploy inputs', { + blueprintId: blueprint.id, + blueprintName: blueprint.name, + nodeId: node.id, + nodeType: node.type, + revision: blueprint.revision, + classification: blueprint.classification, + driftMode: blueprint.drift_mode, + }); + try { + this.setStatus(blueprint.id, node.id, 'deploying', 'deploy_start'); + if (await this.hasNameConflict(blueprint.name, node, blueprint.id)) { + this.setStatus(blueprint.id, node.id, 'name_conflict', 'name_conflict', { + last_error: `A stack named "${blueprint.name}" already exists on this node and is not managed by Sencho.`, + }); + console.warn('[BlueprintService] deploy name conflict blueprint=%s node=%s durationMs=%s', + sanitizeForLog(blueprint.name), node.id, Date.now() - started); + return { status: 'name_conflict', error: 'name_conflict' }; + } + const marker = this.buildMarker(blueprint); + if (node.type === 'local') { + diagnosticLog('deploy branch', { blueprintId: blueprint.id, nodeId: node.id, target: 'local' }); + await this.deployLocal(blueprint, node, marker); + } else { + diagnosticLog('deploy branch', { blueprintId: blueprint.id, nodeId: node.id, target: 'remote' }); + await this.deployRemote(blueprint, node, marker); + } + this.setStatus(blueprint.id, node.id, 'active', 'deploy_ack', { + applied_revision: blueprint.revision, + last_deployed_at: Date.now(), + last_drift_at: null, + drift_summary: null, + last_error: null, + }); + console.info('[BlueprintService] deploy complete blueprint=%s node=%s durationMs=%s', + sanitizeForLog(blueprint.name), node.id, Date.now() - started); + return { status: 'active' }; + } catch (err) { + if (err instanceof BlueprintNameConflictError) { + this.setStatus(blueprint.id, node.id, 'name_conflict', 'name_conflict', { last_error: err.message }); + console.warn('[BlueprintService] deploy name conflict blueprint=%s node=%s durationMs=%s', + sanitizeForLog(blueprint.name), node.id, Date.now() - started); + return { status: 'name_conflict', error: 'name_conflict' }; + } + const message = BlueprintService.formatError(err); + this.setStatus(blueprint.id, node.id, 'failed', 'deploy_fail', { last_error: message }); + console.error('[BlueprintService] deploy failed blueprint=%s node=%s durationMs=%s error=%s', + sanitizeForLog(blueprint.name), node.id, Date.now() - started, sanitizeForLog(message)); + return { status: 'failed', error: message }; + } finally { + this.releaseLock(blueprint.id, node.id); + } + } + + /** + * Withdraw a blueprint from the target node: docker compose down, delete + * the directory. Caller must have already cleared the eviction guard + * (stateful blueprints require explicit operator confirmation). + */ + async withdrawFromNode(blueprint: Blueprint, node: Node): Promise { + if (!this.acquireLock(blueprint.id, node.id)) { + return { status: 'pending' }; + } + const started = Date.now(); + console.info('[BlueprintService] withdraw start blueprint=%s node=%s type=%s', + sanitizeForLog(blueprint.name), node.id, node.type); + diagnosticLog('withdraw inputs', { + blueprintId: blueprint.id, + blueprintName: blueprint.name, + nodeId: node.id, + nodeType: node.type, + classification: blueprint.classification, + }); + try { + this.setStatus(blueprint.id, node.id, 'withdrawing', 'withdraw_start'); + // Ownership is validated on the node that owns the stack, inside the delete lock. + if (node.type === 'local') { + diagnosticLog('withdraw branch', { blueprintId: blueprint.id, nodeId: node.id, target: 'local' }); + const localOutcome = await this.withdrawLocal(blueprint, node); + if (localOutcome.status !== 'withdrawn') return localOutcome; + } else { + diagnosticLog('withdraw branch', { blueprintId: blueprint.id, nodeId: node.id, target: 'remote' }); + const remoteOutcome = await this.withdrawRemote(blueprint, node); + if (remoteOutcome.status !== 'withdrawn') return remoteOutcome; + } + commitBlueprintDeploymentRemoved(blueprint.id, node.id, null); + console.info('[BlueprintService] withdraw complete blueprint=%s node=%s durationMs=%s', + sanitizeForLog(blueprint.name), node.id, Date.now() - started); + return { status: 'withdrawn' }; + } catch (err) { + const message = BlueprintService.formatError(err); + this.setStatus(blueprint.id, node.id, 'failed', 'withdraw_fail', { last_error: `withdraw failed: ${message}` }); + console.error('[BlueprintService] withdraw failed blueprint=%s node=%s durationMs=%s error=%s', + sanitizeForLog(blueprint.name), node.id, Date.now() - started, sanitizeForLog(message)); + return { status: 'failed', error: message }; + } finally { + this.releaseLock(blueprint.id, node.id); + } + } + + /** + * Inspect the actual state of a deployment on its node and report + * whether it has drifted from the desired state. The reconciler decides + * what to do with the result based on drift_mode. + */ + async checkForDrift(blueprint: Blueprint, node: Node): Promise<{ drifted: boolean; reason?: string }> { + try { + const marker = await this.readMarker(blueprint.name, node); + if (!marker) { + return { drifted: true, reason: 'marker file missing on node' }; + } + if (marker.blueprintId !== blueprint.id) { + return { drifted: true, reason: 'marker references a different blueprint' }; + } + if (marker.revision !== blueprint.revision) { + return { drifted: true, reason: `revision drift (node has ${marker.revision}, blueprint is ${blueprint.revision})` }; + } + // Check container state + const containerState = await this.containerHealth(blueprint.name, node); + if (!containerState.allRunning) { + return { drifted: true, reason: containerState.detail }; + } + return { drifted: false }; + } catch (err) { + return { drifted: true, reason: BlueprintService.formatError(err) }; + } + } + + private async containerHealth(blueprintName: string, node: Node): Promise<{ allRunning: boolean; detail: string }> { + try { + // Docker Compose normalizes the project name to lowercase. Match the same canonical form. + const projectName = blueprintName.toLowerCase(); + if (node.type === 'local') { + const docker = NodeRegistry.getInstance().getDocker(node.id); + const containers = await docker.listContainers({ + all: true, + filters: { label: [`com.docker.compose.project=${projectName}`] }, + }); + if (containers.length === 0) return { allRunning: false, detail: 'no containers running for this blueprint' }; + const notRunning = containers.filter(c => c.State !== 'running'); + if (notRunning.length > 0) { + const first = notRunning[0]; + return { allRunning: false, detail: `container "${first.Names[0] ?? first.Id.slice(0, 12)}" is ${first.State}` }; + } + return { allRunning: true, detail: '' }; + } + const target = NodeRegistry.getInstance().getProxyTarget(node.id); + if (!target) return { allRunning: false, detail: 'remote node not reachable (no proxy target)' }; + const url = `${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(blueprintName)}/containers`; + const res = await axios.get(url, { + headers: this.remoteHeaders(target.apiToken), + timeout: REMOTE_HTTP_TIMEOUT_MS, + validateStatus: () => true, + }); + if (res.status !== 200) { + return { allRunning: false, detail: `remote stack lookup returned HTTP ${res.status}` }; + } + const list = Array.isArray(res.data) ? res.data as Array<{ State?: string; Names?: string[]; Id?: string }> : []; + if (list.length === 0) return { allRunning: false, detail: 'remote stack has no containers' }; + const notRunning = list.filter(c => (c.State ?? '') !== 'running'); + if (notRunning.length > 0) { + const first = notRunning[0]; + return { allRunning: false, detail: `remote container "${first.Names?.[0] ?? first.Id?.slice(0, 12)}" is ${first.State}` }; + } + return { allRunning: true, detail: '' }; + } catch (err) { + return { allRunning: false, detail: BlueprintService.formatError(err) }; + } + } + + // ---- local primitives ---- + + /** Returns whether the stack directory exists. Throws on non-ENOENT I/O. */ + private async stackDirExists(nodeId: number, blueprintName: string): Promise { + // Inline containment barrier at the stat sink. The scanner does not + // credit the wrapped isPathWithinBase helper, so the check has to sit + // with the call it protects. + const baseResolved = path.resolve(NodeRegistry.getInstance().getComposeDir(nodeId)); + const stackDir = path.resolve(baseResolved, blueprintName); + if (!stackDir.startsWith(baseResolved + path.sep)) { + throw new BlueprintOwnershipProbeError(`Invalid stack path for "${blueprintName}"`); + } + try { + const stat = await fsPromises.stat(stackDir); + return stat.isDirectory(); + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT') return false; + throw new BlueprintOwnershipProbeError( + `Cannot access stack directory "${blueprintName}": ${BlueprintService.formatError(err)}`, + ); + } + } + + private async deployLocal(blueprint: Blueprint, node: Node, marker: BlueprintMarker): Promise { + const imageRefs = BlueprintAnalyzer.extractImageRefs(blueprint.compose_content); + const gate = await enforcePolicyForImageRefs(blueprint.name, node.id, imageRefs, { + bypass: false, + actor: 'blueprint-reconciler', + auditMethod: 'POST', + auditPath: `/api/blueprints/${blueprint.id}/apply`, + }, undefined, true); + if (!gate.ok) { + throw new Error(describePolicyBlock(gate.policy, gate.violations)); + } + + const outcome = await this.applyLocalUnderLock( + node.id, + blueprint.name, + blueprint.compose_content, + JSON.stringify(marker, null, 2), + `/api/blueprints/${blueprint.id}/deployments/${node.id}`, + ); + if (!outcome.ran) { + throw new Error(stackOpSkipMessage(blueprint.name, outcome.existingAction)); + } + triggerPostDeployScan(blueprint.name, node.id).catch(err => { + console.error('[BlueprintService] post-deploy scan failed for "%s" on node %s: %s', + sanitizeForLog(blueprint.name), node.id, sanitizeForLog(BlueprintService.formatError(err))); + }); + } + + /** + * Create the stack if needed, write the compose file, run the deploy policy + * gate and deploy, then write the marker, all under the per-stack operation + * lock. The marker is written only after a successful deploy so a failed + * apply cannot claim an applied revision that never ran. Runs on the node + * that owns the stack: deployLocal calls it for the hub's own node, and the + * /api/blueprints/apply-local route calls it on a remote node receiving a + * blueprint apply from its hub. On lock conflict nothing is written and + * { ran: false } is returned. + */ + async applyLocalUnderLock( + nodeId: number, + stackName: string, + composeContent: string, + markerContent: string, + auditPath: string, + ): Promise<{ ran: true } | { ran: false; existingAction: StackOpAction }> { + const expected = parseBlueprintMarker(markerContent); + if (!expected) { + throw new Error('Invalid blueprint marker'); + } + const fs = FileSystemService.getInstance(nodeId); + const lock = await StackOpLockService.getInstance().runExclusive( + nodeId, stackName, 'deploy', 'system', + async () => { + let createdStack = false; + if (await this.stackDirExists(nodeId, stackName)) { + const existing = await this.readLocalMarkerFromDisk(nodeId, stackName); + if (existing.kind === 'failed') { + throw new BlueprintOwnershipProbeError( + `Cannot verify ownership of stack "${stackName}": ${existing.error}`, + ); + } + if (existing.kind === 'missing' || existing.marker.blueprintId !== expected.blueprintId) { + throw new BlueprintNameConflictError( + `A stack named "${stackName}" already exists on this node and is not managed by this blueprint.`, + ); + } + } else { + await fs.createStack(stackName); + createdStack = true; + } + await fs.writeStackFile(stackName, COMPOSE_FILENAME, composeContent); + // Clear lower-priority compose siblings so discovery cannot shadow compose.yaml. + await fs.removeAlternateRootComposeFiles(stackName); + try { + await assertPolicyGateAllows( + stackName, + nodeId, + buildSystemPolicyGateOptions('blueprint', { auditPath }), + ); + await ComposeService.getInstance(nodeId).deployStack( + stackName, + undefined, + false, + { source: 'blueprint', actor: 'system:blueprint' }, + ); + await fs.writeStackFile(stackName, BLUEPRINT_MARKER_FILENAME, markerContent); + } catch (err) { + if (createdStack) { + try { + await fs.deleteStack(stackName); + } catch (cleanupErr) { + console.warn( + '[BlueprintService] Failed to roll back newly created stack "%s" after apply error: %s', + sanitizeForLog(stackName), + sanitizeForLog(BlueprintService.formatError(cleanupErr)), + ); + } + } + throw err; + } + }, + ); + return lock.ran ? { ran: true } : { ran: false, existingAction: lock.existing.action }; + } + + private async withdrawLocal(blueprint: Blueprint, node: Node): Promise { + const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({ + nodeId: node.id, + stackName: blueprint.name, + pruneVolumes: false, + actor: 'system:blueprint', + requireBlueprintId: blueprint.id, + }); + if (result.ok) { + return { status: 'withdrawn' }; + } + if (result.code === 'name_conflict') { + this.setStatus(blueprint.id, node.id, 'name_conflict', 'withdraw_name_conflict', { last_error: result.error }); + return { status: 'name_conflict' }; + } + this.setStatus(blueprint.id, node.id, 'failed', 'withdraw_fail', { last_error: result.error }); + return { status: 'failed', error: result.error }; + } + + // ---- remote primitives ---- + + private remoteHeaders(apiToken: string): Record { + const proxy = LicenseService.getInstance().getProxyHeaders(); + return { + Authorization: `Bearer ${apiToken}`, + [PROXY_TIER_HEADER]: proxy.tier, + 'Content-Type': 'application/json', + ...deployProvenanceHeaders('blueprint', 'system:blueprint'), + }; + } + + private async deployRemote(blueprint: Blueprint, node: Node, marker: BlueprintMarker): Promise { + const target = NodeRegistry.getInstance().getProxyTarget(node.id); + if (!target) throw new Error(`Remote node "${node.name}" has no proxy target configured`); + const baseUrl = target.apiUrl.replace(/\/$/, ''); + const headers = this.remoteHeaders(target.apiToken); + + // Atomic apply: the remote validates ownership and writes under its stack lock. + const res = await axios.post( + `${baseUrl}/api/blueprints/apply-local`, + { + stackName: blueprint.name, + composeContent: blueprint.compose_content, + markerContent: JSON.stringify(marker, null, 2), + }, + { headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true }, + ); + if (res.status === 404) { + throw new BlueprintRemoteUpgradeRequiredError( + `Remote node "${node.name}" does not support atomic blueprint apply (/api/blueprints/apply-local). Upgrade that Sencho instance, then retry.`, + ); + } + if (res.status === 409) { + if (BlueprintService.extractApiCode(res.data) === 'name_conflict') { + throw new BlueprintNameConflictError( + BlueprintService.extractApiError(res.data) + || `A stack named "${blueprint.name}" already exists on this node and is not managed by this blueprint.`, + ); + } + throw new Error(`blueprint apply skipped: ${BlueprintService.extractApiError(res.data) || 'another operation is already in progress'}`); + } + if (res.status >= 400) { + throw new Error(`blueprint apply: HTTP ${res.status} ${BlueprintService.extractApiError(res.data)}`); + } + } + + private async withdrawRemote(blueprint: Blueprint, node: Node): Promise { + const target = NodeRegistry.getInstance().getProxyTarget(node.id); + if (!target) throw new Error(`Remote node "${node.name}" has no proxy target configured`); + const baseUrl = target.apiUrl.replace(/\/$/, ''); + const headers = this.remoteHeaders(target.apiToken); + + let res; + try { + res = await axios.post( + `${baseUrl}/api/blueprints/withdraw-local`, + { stackName: blueprint.name, blueprintId: blueprint.id }, + { headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true }, + ); + } catch (err) { + const message = BlueprintService.formatError(err); + this.setStatus(blueprint.id, node.id, 'failed', 'withdraw_fail', { last_error: message }); + return { status: 'failed', error: message }; + } + + if (res.status === 404) { + throw new BlueprintRemoteUpgradeRequiredError( + `Remote node "${node.name}" does not support atomic blueprint withdraw (/api/blueprints/withdraw-local). Upgrade that Sencho instance, then retry.`, + ); + } + if (res.status === 200) { + DatabaseService.getInstance().deleteRoleAssignmentsByStack(node.id, blueprint.name); + return { status: 'withdrawn' }; + } + if (res.status === 409) { + const error = BlueprintService.extractApiError(res.data) || 'withdraw refused'; + if (BlueprintService.extractApiCode(res.data) === 'name_conflict') { + this.setStatus(blueprint.id, node.id, 'name_conflict', 'withdraw_name_conflict', { last_error: error }); + return { status: 'name_conflict' }; + } + // stack_op_in_progress and any other 409: match local withdraw lock-conflict → failed + this.setStatus(blueprint.id, node.id, 'failed', 'withdraw_fail', { last_error: error }); + return { status: 'failed', error }; + } + const message = `blueprint withdraw: HTTP ${res.status} ${BlueprintService.extractApiError(res.data)}`; + this.setStatus(blueprint.id, node.id, 'failed', 'withdraw_fail', { last_error: message }); + return { status: 'failed', error: message }; + } + + static parseMarker(content: string): BlueprintMarker | null { + return parseBlueprintMarker(content); + } + + private static ownershipProbeError(blueprintName: string, detail: string): BlueprintOwnershipProbeError { + return new BlueprintOwnershipProbeError( + `Cannot verify stack ownership for "${blueprintName}": ${detail}`, + ); + } + + static extractApiCode(body: unknown): string { + if (!body || typeof body !== 'object') return ''; + const code = (body as Record).code; + return typeof code === 'string' ? code : ''; + } + + static formatError(err: unknown): string { + if (axios.isAxiosError(err)) { + const ax = err as AxiosError<{ error?: string; message?: string }>; + if (ax.response?.data) { + const body = ax.response.data; + if (body && typeof body === 'object') { + if (typeof body.error === 'string') return body.error; + if (typeof body.message === 'string') return body.message; + } + } + if (ax.code) return `${ax.code}: ${ax.message}`; + return ax.message; + } + if (err instanceof Error) return err.message; + return String(err); + } + + static extractApiError(body: unknown): string { + if (!body || typeof body !== 'object') return ''; + const obj = body as Record; + if (typeof obj.error === 'string') return obj.error; + if (typeof obj.message === 'string') return obj.message; + return ''; + } +} diff --git a/backend/src/services/ComposeService.ts b/backend/src/services/ComposeService.ts index b425e94f..3899c442 100644 --- a/backend/src/services/ComposeService.ts +++ b/backend/src/services/ComposeService.ts @@ -24,6 +24,9 @@ import { authoredComposeFileArgs, authoredComposeEnvFileArgs } from '../utils/au import type { RollbackInvocationRecord } from '../types/rollbackGeneration'; import { parseMissingRequiredVars } from '../helpers/envVarParse'; import { redactSensitiveText, sanitizeForLog } from '../utils/safeLog'; +import { randomUUID } from 'crypto'; +import { GitOpsStore } from './gitops/store'; +import { GitOpsTransitions } from './gitops/transitions'; import { pathsMatch, resolveHostBindPath } from '../utils/composePathMapping'; import { loadStackBuildServices } from './ImageUpdateService'; import { resolveMissingExternalNetworks } from './network/resolveMissingExternalNetworks'; @@ -139,6 +142,16 @@ function getComposeStallTimeoutMs(): number { * In the Distributed API model, remote node compose operations are handled * by the remote Sencho instance. This service only executes commands locally. */ +/** + * Evidence that a Compose mutation actually ran. + * + * Returned rather than inferred from a resolved promise because the recovery + * path takes its Compose step as a callback: a caller that restores some other + * way resolves identically, and binding the deployed pointer on that would + * claim a workload nobody launched. + */ +export type ComposeMutationResult = { mutatedByCompose: true }; + export class ComposeService { private baseDir: string; private nodeId: number; @@ -659,12 +672,70 @@ export class ComposeService { recordCreatedNetworks('info'); } + /** + * Open a GitOps deploy operation for this stack, or nothing when there is no + * generation to bind. + * + * Returns closures rather than ids so the caller cannot terminate an + * operation it never started. A stack with no live application, or one whose + * target has nothing applied, has no deploy identity to record, so the whole + * thing is a no-op. Recording never fails the deploy: the store describes + * what happened, it does not make it happen. + */ + private beginGitOpsDeploy(stackName: string): { + generationId: string; + bound: () => void; + failed: (failureClass: 'pre_mutation' | 'post_mutation') => void; + } | null { + try { + const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName); + if (!app || app.lifecycle_status !== 'active') return null; + const target = GitOpsStore.getInstance().getTarget(app.id, this.nodeId); + const generationId = target?.applied_generation_id; + if (!target || target.target_status !== 'active' || !generationId) return null; + + const tx = GitOpsTransitions.getInstance(); + const envelope = { operationId: randomUUID(), actor: 'system:compose', trigger: 'deploy', at: Date.now() }; + const record = (what: string, write: () => void): boolean => { + try { + write(); + return true; + } catch (error) { + console.error( + '[GitOps] Could not record deploy %s for %s (application %s, generation %s):', + what, + sanitizeForLog(stackName), + app.id, + generationId, + error instanceof Error ? error.stack ?? error.message : String(error), + ); + return false; + } + }; + if (!record('start', () => tx.deployStarted(app.id, this.nodeId, generationId, envelope))) { + return null; + } + return { + generationId, + bound: () => record('binding', () => tx.deployBound(app.id, this.nodeId, generationId, envelope)), + failed: (failureClass) => record('failure', () => tx.deployFailed(app.id, this.nodeId, failureClass, envelope)), + }; + } catch (error) { + console.error( + '[GitOps] Could not open a deploy operation for %s:', + sanitizeForLog(stackName), + error instanceof Error ? error.message : String(error), + ); + return null; + } + } + async deployStack( stackName: string, ws?: WebSocket, atomic?: boolean, ctx?: DeployInvocationContext, - ): Promise<{ recoveryId: string | null }> { + ): Promise<{ recoveryId: string | null; deployedGenerationId: string | null }> { await this.assertRequiredEnvPresent(stackName); await this.assertSafePilotBindMapping(stackName); await this.ensureExternalNetworksForDeploy(stackName, ctx); @@ -705,6 +776,12 @@ export class ComposeService { } } + // ComposeService is the only producer of deploy events: every deploy path + // (manual, bulk, Git auto-deploy, App Store, scheduler, webhook) funnels + // through here, so recording it anywhere else would double-count. + const gitopsDeploy = this.beginGitOpsDeploy(stackName); + let composeHandedOff = false; + try { try { const dockerController = DockerController.getInstance(this.nodeId); @@ -718,7 +795,9 @@ export class ComposeService { } await this.withRegistryAuth(async (env) => { - await this.execute('docker', await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), stackDir, ws, true, env, getComposeStallTimeoutMs()); + const args = await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']); + composeHandedOff = true; + await this.execute('docker', args, stackDir, ws, true, env, getComposeStallTimeoutMs()); }, sendOutput); // Post-Deploy Health Probe @@ -751,7 +830,11 @@ export class ComposeService { } } if (debug) console.debug(`[ComposeService:debug] deployStack completed in ${Date.now() - t0}ms`, { stackName }); + gitopsDeploy?.bound(); } catch (deployError) { + // Classified by whether Compose was handed the mutation. Only a failure + // before that leaves the previous workload provably intact. + gitopsDeploy?.failed(composeHandedOff ? 'post_mutation' : 'pre_mutation'); if (atomic && recoverySvc && handedOff && recoveryId) { sendOutput('\n=== Deployment failed - restoring previous runtime from recovery generation ===\n'); const generationId = recoveryId; @@ -793,7 +876,7 @@ export class ComposeService { console.warn('[ComposeService] Exposure refresh failed after deploy for %s:', sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(err, 'unknown'))); } - return { recoveryId }; + return { recoveryId, deployedGenerationId: gitopsDeploy?.generationId ?? null }; } streamLogs(stackName: string, ws: WebSocket) { @@ -942,7 +1025,7 @@ export class ComposeService { overridePath: string, ws?: WebSocket, invocation?: RollbackInvocationRecord | null, - ): Promise { + ): Promise { const stackDir = path.join(this.baseDir, stackName); const sendOutput = (data: string) => { if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); @@ -963,6 +1046,7 @@ export class ComposeService { getComposeStallTimeoutMs(), ); }, sendOutput); + return { mutatedByCompose: true }; } /** @@ -1121,7 +1205,7 @@ export class ComposeService { stackName: string, ws?: WebSocket, atomic?: boolean, - ): Promise<{ recoveryId: string | null }> { + ): Promise<{ recoveryId: string | null; deployedGenerationId: string | null }> { await this.assertRequiredEnvPresent(stackName); await this.assertSafePilotBindMapping(stackName); const stackDir = path.join(this.baseDir, stackName); @@ -1132,6 +1216,12 @@ export class ComposeService { if (ws && ws.readyState === WebSocket.OPEN) ws.send(data); }; + // Opened once the update is committed to recreating containers, not at the + // top: an update that fails during capture or classification never reached + // Compose, so there is no deploy to record. + let gitopsDeploy: ReturnType = null; + let composeHandedOff = false; + // Dynamic import avoids a static cycle (recovery imports getComposeCommandTimeoutMs). const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); const recoverySvc = StackUpdateRecoveryService.getInstance(); @@ -1218,13 +1308,15 @@ export class ComposeService { } } + gitopsDeploy = this.beginGitOpsDeploy(stackName); await this.withRegistryAuth(async (env) => { sendOutput('=== Recreating containers ===\n'); - await this.execute( - 'docker', - await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']), - stackDir, ws, true, env, getComposeStallTimeoutMs(), - ); + const args = await this.authoredComposeArgs(stackName, ['up', '-d', '--remove-orphans']); + // Set only once Compose is genuinely about to receive the mutation: + // reading compose args or resolving registry auth can still fail with + // the previous workload provably intact. + composeHandedOff = true; + await this.execute('docker', args, stackDir, ws, true, env, getComposeStallTimeoutMs()); }, sendOutput); // Immediate verification probe @@ -1279,7 +1371,9 @@ export class ComposeService { if (debug) { console.debug(`[ComposeService:debug] updateStack completed in ${Date.now() - t0}ms`, { stackName }); } + gitopsDeploy?.bound(); } catch (updateError) { + gitopsDeploy?.failed(composeHandedOff ? 'post_mutation' : 'pre_mutation'); if (!handedOff && recoveryId) { await recoverySvc.abandon(recoveryId); recoveryId = null; @@ -1315,7 +1409,7 @@ export class ComposeService { sanitizeForLog(getErrorMessage(err, 'unknown')), ); } - return { recoveryId }; + return { recoveryId, deployedGenerationId: gitopsDeploy?.generationId ?? null }; } /** diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index e02705dc..34efc3e0 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -20,6 +20,7 @@ import { sanitizeForLog } from '../utils/safeLog'; import type { GitSourceManifestState } from '../types/gitProjectManifest'; import type { RollbackOperationKind } from '../types/rollbackGeneration'; import { collectImageIds, parseServicesJsonStrict } from './recoveryServicesJson'; +import { GITOPS_SCHEMA_SQL } from './gitops/schema'; export type { SnapshotFileReadResult } from '../helpers/snapshotFileDecrypt'; export type { RollbackOperationKind } from '../types/rollbackGeneration'; @@ -220,13 +221,13 @@ export interface StackExposureRow { computed_at: number; } -/** One post-update health gate observation run. */ +/** One health-gate observation run, keyed by `trigger_action`. */ export interface HealthGateRunRow { id: string; node_id: number; stack_name: string; /** Named trigger_action because TRIGGER is reserved in SQLite. */ - trigger_action: 'update' | 'deploy' | 'service_update' | 'service_restore'; + trigger_action: 'update' | 'deploy' | 'service_update' | 'service_restore' | 'recovery'; status: 'observing' | 'passed' | 'failed' | 'unknown'; reason: string | null; window_seconds: number; @@ -239,6 +240,11 @@ export interface HealthGateRunRow { target_scope: 'stack' | 'service'; service_name: string | null; failure_source: 'primary' | 'collateral' | null; + /** + * Reserved for the GitOps deploy path: the generation live when this run + * started. No writer populates it yet, so it is currently always null. + */ + deployed_generation_id?: string | null; } /** Pre-update image snapshot enabling a manual per-service restore after a service-scoped update. */ @@ -269,6 +275,9 @@ export interface StackUpdateRecoveryGenerationRow { /** Set when an operator manually released rollback protection early (see releaseStackUpdateRecoveryGeneration). */ released_at: number | null; released_by: string | null; + gitops_generation_id?: string | null; + gitops_artifact_set_id?: string | null; + gitops_source_acceptance_ref?: string | null; } /** Durable cleanup tombstone for stack/node deletion artifact sweep. */ @@ -1161,6 +1170,7 @@ export class DatabaseService { this.migrateGitSourceMultiFile(); this.migrateGitSourceManifest(); this.migrateGitSourceChangePlan(); + this.migrateGitOpsRecoveryColumns(); this.migrateNodeUpdateSkips(); this.migrateStackAlertServiceScope(); @@ -1777,7 +1787,7 @@ export class DatabaseService { id TEXT PRIMARY KEY, node_id INTEGER NOT NULL, stack_name TEXT NOT NULL, - trigger_action TEXT NOT NULL CHECK (trigger_action IN ('update','deploy','service_update','service_restore')), + trigger_action TEXT NOT NULL CHECK (trigger_action IN ('update','deploy','service_update','service_restore','recovery')), status TEXT NOT NULL CHECK (status IN ('observing','passed','failed','unknown')), reason TEXT, window_seconds INTEGER NOT NULL, @@ -1906,6 +1916,8 @@ export class DatabaseService { CREATE INDEX IF NOT EXISTS idx_secret_pushes_node ON secret_pushes(node_id, stack_name); `); + this.db.exec(GITOPS_SCHEMA_SQL); + // Apply migrations safely (ignore if columns already exist) const maybeAddCol = (table: string, col: string, def: string) => { try { this.db.prepare(`ALTER TABLE ${table} ADD COLUMN ${col} ${def}`).run(); } catch (e) { /* ignore */ } @@ -2098,6 +2110,7 @@ export class DatabaseService { // behave); admins who want a strict absolute session ceiling can turn // it off in Settings > Users. stmt.run('session_sliding_refresh', '1'); + stmt.run('gitops_schema_version', '1'); // Seed the default local node if none exists const nodeCount = (this.db.prepare('SELECT COUNT(*) as count FROM nodes').get() as any)?.count || 0; @@ -2177,10 +2190,11 @@ export class DatabaseService { } /** - * Rebuild health_gate_runs when the installed CHECK still only allows - * update|deploy or target/failure columns are missing. Idempotent and - * restart-safe: drops a stale temporary table, then rebuilds in one - * better-sqlite3 transaction so an interrupted startup cannot leave + * Rebuild health_gate_runs when the CHECK lacks service_update, + * service_restore, or recovery, or when target/failure columns are missing. + * After the CHECK is current, ensure deployed_generation_id exists. + * Idempotent and restart-safe: drops a stale temporary table, then rebuilds + * in one better-sqlite3 transaction so an interrupted startup cannot leave * CREATE TABLE health_gate_runs_new blocking the next boot. */ private migrateHealthGateTargetSchema(): void { @@ -2192,7 +2206,11 @@ export class DatabaseService { const hasTarget = colNames.has('target_scope'); const hasFailure = colNames.has('failure_source'); const hasWideTrigger = tableSql.includes('service_update') && tableSql.includes('service_restore'); - if (hasTarget && hasFailure && hasWideTrigger) return; + const hasRecoveryTrigger = tableSql.includes("'recovery'"); + if (hasTarget && hasFailure && hasWideTrigger && hasRecoveryTrigger) { + this.ensureHealthGateDeployedGenerationColumn(); + return; + } // A previous crash between CREATE and RENAME leaves this temp table behind. this.db.exec('DROP TABLE IF EXISTS health_gate_runs_new'); @@ -2200,6 +2218,7 @@ export class DatabaseService { const targetExpr = hasTarget ? 'target_scope' : "'stack'"; const serviceExpr = colNames.has('service_name') ? 'service_name' : 'NULL'; const failureExpr = hasFailure ? 'failure_source' : 'NULL'; + const deployedExpr = colNames.has('deployed_generation_id') ? 'deployed_generation_id' : 'NULL'; this.db.transaction(() => { this.db.exec(` @@ -2207,7 +2226,7 @@ export class DatabaseService { id TEXT PRIMARY KEY, node_id INTEGER NOT NULL, stack_name TEXT NOT NULL, - trigger_action TEXT NOT NULL CHECK (trigger_action IN ('update','deploy','service_update','service_restore')), + trigger_action TEXT NOT NULL CHECK (trigger_action IN ('update','deploy','service_update','service_restore','recovery')), status TEXT NOT NULL CHECK (status IN ('observing','passed','failed','unknown')), reason TEXT, window_seconds INTEGER NOT NULL, @@ -2217,23 +2236,36 @@ export class DatabaseService { created_by TEXT, target_scope TEXT NOT NULL DEFAULT 'stack' CHECK (target_scope IN ('stack','service')), service_name TEXT, - failure_source TEXT CHECK (failure_source IS NULL OR failure_source IN ('primary','collateral')) + failure_source TEXT CHECK (failure_source IS NULL OR failure_source IN ('primary','collateral')), + deployed_generation_id TEXT NULL ); INSERT INTO health_gate_runs_new ( id, node_id, stack_name, trigger_action, status, reason, window_seconds, - containers_json, started_at, ended_at, created_by, target_scope, service_name, failure_source + containers_json, started_at, ended_at, created_by, target_scope, service_name, + failure_source, deployed_generation_id ) SELECT id, node_id, stack_name, trigger_action, status, reason, window_seconds, containers_json, started_at, ended_at, created_by, - ${targetExpr}, ${serviceExpr}, ${failureExpr} + ${targetExpr}, ${serviceExpr}, ${failureExpr}, ${deployedExpr} FROM health_gate_runs; DROP TABLE health_gate_runs; ALTER TABLE health_gate_runs_new RENAME TO health_gate_runs; CREATE INDEX IF NOT EXISTS idx_health_gate_runs_node_stack ON health_gate_runs(node_id, stack_name, started_at); + CREATE INDEX IF NOT EXISTS idx_health_gate_runs_deployed_gen + ON health_gate_runs(node_id, stack_name, deployed_generation_id); `); })(); + this.ensureHealthGateDeployedGenerationColumn(); + } + + private ensureHealthGateDeployedGenerationColumn(): void { + this.tryAddColumn('health_gate_runs', 'deployed_generation_id', 'TEXT'); + this.db.exec(` + CREATE INDEX IF NOT EXISTS idx_health_gate_runs_deployed_gen + ON health_gate_runs(node_id, stack_name, deployed_generation_id); + `); } private migrateEncryptNodeTokens(): void { @@ -2550,6 +2582,12 @@ export class DatabaseService { this.tryAddColumn('stack_git_sources', 'last_plan_outcome', 'TEXT'); } + private migrateGitOpsRecoveryColumns(): void { + this.tryAddColumn('stack_update_recovery_generations', 'gitops_generation_id', 'TEXT'); + this.tryAddColumn('stack_update_recovery_generations', 'gitops_artifact_set_id', 'TEXT'); + this.tryAddColumn('stack_update_recovery_generations', 'gitops_source_acceptance_ref', 'TEXT'); + } + private migrateGitSourceMultiFile(): void { this.tryAddColumn('stack_git_sources', 'compose_paths', 'TEXT'); this.tryAddColumn('stack_git_sources', 'context_dir', 'TEXT'); @@ -4054,12 +4092,14 @@ export class DatabaseService { this.db.prepare( `INSERT INTO health_gate_runs (id, node_id, stack_name, trigger_action, status, reason, window_seconds, containers_json, - started_at, ended_at, created_by, target_scope, service_name, failure_source) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + started_at, ended_at, created_by, target_scope, service_name, failure_source, + deployed_generation_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ).run( run.id, run.node_id, run.stack_name, run.trigger_action, run.status, run.reason, run.window_seconds, run.containers_json, run.started_at, run.ended_at, run.created_by, run.target_scope, run.service_name, run.failure_source, + run.deployed_generation_id ?? null, ); // Bounded history: keep only the 10 most recent runs per stack. this.db.prepare( @@ -4099,11 +4139,17 @@ export class DatabaseService { } /** Finalize runs left observing by a previous process (startup sweep). */ - public markInterruptedHealthGateRuns(reason: string, endedAt: number): number { - const result = this.db.prepare( - "UPDATE health_gate_runs SET status = 'unknown', reason = ?, ended_at = ? WHERE status = 'observing'" - ).run(reason, endedAt); - return result.changes; + /** + * Runs a previous process left observing. + * + * Returned as rows rather than swept with one UPDATE because each has to be + * finalized individually: the verdict is what the revision state listens + * for, and a bulk update moves the rows while telling the model nothing. + */ + public listObservingHealthGateRuns(): HealthGateRunRow[] { + return this.db.prepare( + "SELECT * FROM health_gate_runs WHERE status = 'observing'" + ).all() as HealthGateRunRow[]; } // --- Service Update Recovery --- @@ -4234,14 +4280,17 @@ export class DatabaseService { id, node_id, stack_name, status, phase, is_current, backup_slot_id, content_path, operation_kind, override_path, services_json, health_gate_id, gate_retain_until, artifact_expires_at, operation_lease_expires_at, created_at, updated_at, - created_by, artifacts_retired - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + created_by, artifacts_retired, gitops_generation_id, gitops_artifact_set_id, + gitops_source_acceptance_ref + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ).run( row.id, row.node_id, row.stack_name, row.status, row.phase, row.is_current, row.backup_slot_id, row.content_path ?? null, row.operation_kind ?? null, row.override_path, row.services_json, row.health_gate_id, row.gate_retain_until, row.artifact_expires_at, row.operation_lease_expires_at, row.created_at, row.updated_at, row.created_by, row.artifacts_retired ?? 0, + row.gitops_generation_id ?? null, row.gitops_artifact_set_id ?? null, + row.gitops_source_acceptance_ref ?? null, ); } diff --git a/backend/src/services/DeployedStackDeletionService.ts b/backend/src/services/DeployedStackDeletionService.ts index af92d054..96f5f2d0 100644 --- a/backend/src/services/DeployedStackDeletionService.ts +++ b/backend/src/services/DeployedStackDeletionService.ts @@ -28,6 +28,8 @@ import { BLUEPRINT_MARKER_FILENAME, parseBlueprintMarker, } from '../helpers/blueprintMarker'; +import { GitOpsStore } from './gitops/store'; +import { GitOpsTransitions } from './gitops/transitions'; import { scrapeRollbackTagsLenient } from './recoveryServicesJson'; /** @@ -329,6 +331,54 @@ export class DeployedStackDeletionService { } /** Ready transaction, secondary DB/RBAC cleanup, mesh opt-out, sweep, invalidate. */ + /** + * Commit the deletion and retire the stack's GitOps application together. + * + * One transaction, because a deleted stack with a live application would keep + * claiming a stack name that no longer exists, and would block re-creating it + * through the unique live-application index. The tombstone is driven from + * here rather than from inside DatabaseService so the store keeps its + * transitions, and its history, in one place. + */ + private commitDeletionReady(intentId: string, nodeId: number, stackName: string): boolean { + const db = DatabaseService.getInstance(); + return db.getDb().transaction(() => { + if (!db.commitStackDeletionReadyTransaction(intentId, nodeId, stackName)) return false; + const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName); + if (!app) return true; + const tx = GitOpsTransitions.getInstance(); + const envelope = { + operationId: intentId, + actor: 'system:stack-deletion', + trigger: 'delete', + at: Date.now(), + }; + // The files are already gone by the time this runs, and the startup + // reconciler loops over every prepared intent. A rejected tombstone must + // fail this one deletion, not throw an opaque driver error out of a + // deletion that already succeeded on disk, and not abandon the intents + // that follow it. + try { + for (const target of GitOpsStore.getInstance().listTargets(app.id)) { + if (target.target_status !== 'active') continue; + tx.targetTombstoned(app.id, target.node_id, envelope); + } + tx.applicationTombstoned(app.id, 'deleted', envelope); + } catch (error) { + console.error( + '[GitOps] Could not retire the application for deleted stack %s (application %s):', + sanitizeForLog(stackName), app.id, + error instanceof Error ? error.stack ?? error.message : String(error), + ); + return false; + } + // A create that never settled leaves a checkpoint whose application is + // now gone; drop it so boot recovery does not retry it for ever. + GitOpsStore.getInstance().deleteCreateCheckpoint(app.id); + return true; + })(); + } + private async finalizeLogicalDeletion( input: DeleteDeployedStackInput, intentId: string, @@ -336,7 +386,7 @@ export class DeployedStackDeletionService { const { nodeId, stackName } = input; const db = DatabaseService.getInstance(); - if (!db.commitStackDeletionReadyTransaction(intentId, nodeId, stackName)) { + if (!this.commitDeletionReady(intentId, nodeId, stackName)) { return { ok: false, code: 'db_failed', @@ -524,17 +574,67 @@ export class DeployedStackDeletionService { ); } + /** + * Remove a node and retire the GitOps targets that lived on it, together. + * + * The tombstones have to be written while the target rows still exist, and in + * the same transaction as the delete, or a failure part-way through would + * leave targets pointing at a node that is gone. Applications are left live: + * a Direct application still describes a real stack, and a Blueprint one may + * have targets on other nodes. + * + * Returns the Blueprints that lost a target here, so the caller can report + * what the deletion moved. Read inside the transaction and before the + * tombstone, because afterwards no active target row remains to trace back to + * an application. Direct applications contribute nothing: they carry no + * `blueprint_id`. + */ + public deleteNodeWithGitOps( + nodeId: number, + localCleanup?: { tombstoneId: string; tags: string[]; overridePaths: string[] }, + ): number[] { + const db = DatabaseService.getInstance(); + return db.getDb().transaction(() => { + const store = GitOpsStore.getInstance(); + const blueprintIds: number[] = []; + for (const target of store.listActiveTargetsForNode(nodeId)) { + const application = store.getApplication(target.application_id); + if (!application) { + // No foreign key backs this column and the database runs without + // cascade, so an orphaned target is possible, and this is the only + // path that would ever look at one. Collapsing it into the Direct + // case below would make a referential fault read as the normal + // outcome. The tombstone still retires the row either way. + console.error( + '[DeployedStackDeletion] Orphaned GitOps target on node %s: application %s is missing.', + sanitizeForLog(nodeId), + sanitizeForLog(target.application_id), + ); + continue; + } + if (application.blueprint_id !== null) blueprintIds.push(application.blueprint_id); + } + GitOpsTransitions.getInstance().tombstoneNodeTargets(nodeId, { + operationId: localCleanup?.tombstoneId ?? randomUUID(), + actor: 'system:node-deletion', + trigger: 'node_delete', + at: Date.now(), + }); + db.deleteNode(nodeId, localCleanup); + return blueprintIds; + })(); + } + /** * Delete a local-socket node with an atomic ready tombstone, then sweep. * Remote node records call DatabaseService.deleteNode without cleanup. */ - public async deleteLocalNode(nodeId: number): Promise { + public async deleteLocalNode(nodeId: number): Promise { const db = DatabaseService.getInstance(); const node = db.getNode(nodeId); if (!node) throw new Error('Node not found'); if (node.type !== 'local') { - db.deleteNode(nodeId); - return; + return this.deleteNodeWithGitOps(nodeId); } // Preserve Docker + compose dir before the row disappears so sweep never // targets a remote default node. @@ -542,7 +642,7 @@ export class DeployedStackDeletionService { const composeDir = FileSystemService.getInstance(nodeId).getBaseDir(); const { tags, overridePaths } = this.collectNodeArtifacts(nodeId); const tombstoneId = randomUUID(); - db.deleteNode(nodeId, { tombstoneId, tags, overridePaths }); + const blueprintIds = this.deleteNodeWithGitOps(nodeId, { tombstoneId, tags, overridePaths }); NodeRegistry.getInstance().evictConnection(nodeId); try { await this.sweepReadyIntent(tombstoneId, { docker, composeDir }); @@ -554,6 +654,7 @@ export class DeployedStackDeletionService { sanitizeForLog(getErrorMessage(error, 'unknown')), ); } + return blueprintIds; } /** @@ -585,7 +686,7 @@ export class DeployedStackDeletionService { } if (!dirExists) { - if (!db.commitStackDeletionReadyTransaction(intent.id, nodeId, stackName)) { + if (!this.commitDeletionReady(intent.id, nodeId, stackName)) { console.warn( '[DeployedStackDeletion] Startup ready commit failed for %s/%s', nodeId, diff --git a/backend/src/services/GitOpsMetricsService.ts b/backend/src/services/GitOpsMetricsService.ts new file mode 100644 index 00000000..aa7d77c3 --- /dev/null +++ b/backend/src/services/GitOpsMetricsService.ts @@ -0,0 +1,70 @@ +/** + * Counters for GitOps revision transitions, one increment per history row that + * was actually inserted. + * + * Process-local and in-memory, the same bargain StackOpMetricsService makes: a + * restart clears the counters, and persisting them would put a write on every + * transition for very little operator value. The durable record is + * `gitops_history` itself, which these counters only summarise. + * + * The keyspace is finite by construction. `GitOpsHistoryStage` is a closed + * union of everything a producer can write and `HistoryOutcome` is a closed + * CHECK set of six, so the map cannot exceed their product however much traffic + * arrives. No identity, stack name, node, actor, or repository is recorded: + * a counter that carried those would be an audit trail with no retention rules + * and no authorization, which is what the history API is for. + */ +import type { GitOpsHistoryStage, HistoryOutcome } from './gitops/history'; + +export interface GitOpsMetricEntry { + stage: GitOpsHistoryStage; + outcome: HistoryOutcome; + count: number; +} + +export class GitOpsMetricsService { + private static instance: GitOpsMetricsService; + private readonly buckets = new Map(); + + public static getInstance(): GitOpsMetricsService { + if (!GitOpsMetricsService.instance) { + GitOpsMetricsService.instance = new GitOpsMetricsService(); + } + return GitOpsMetricsService.instance; + } + + public static resetForTests(): void { + this.instance = new GitOpsMetricsService(); + } + + /** + * Count one transition. + * + * Called once per newly inserted history row, never on a dedupe replay: a + * replay is the same transition arriving twice, and counting it would report + * retries as activity. + */ + public record(stage: GitOpsHistoryStage, outcome: HistoryOutcome): void { + const key = `${stage}:${outcome}`; + const bucket = this.buckets.get(key); + if (bucket) { + bucket.count += 1; + return; + } + this.buckets.set(key, { stage, outcome, count: 1 }); + } + + /** + * Every bucket that has been touched, ordered by stage then outcome. + * + * Untouched pairs are absent rather than zero. Ordering is stable so an + * operator pulling this twice can diff the two responses directly. Each + * bucket is copied, so a reader cannot edit the counters through the + * snapshot it was handed. + */ + public snapshot(): GitOpsMetricEntry[] { + return [...this.buckets.values()] + .map((bucket) => ({ ...bucket })) + .sort((a, b) => a.stage.localeCompare(b.stage) || a.outcome.localeCompare(b.outcome)); + } +} diff --git a/backend/src/services/GitProjectManifestService.ts b/backend/src/services/GitProjectManifestService.ts index 5bf78d5b..df4138eb 100644 --- a/backend/src/services/GitProjectManifestService.ts +++ b/backend/src/services/GitProjectManifestService.ts @@ -27,6 +27,7 @@ import { authoredComposeFileArgs, authoredComposeEnvFileArgs } from '../utils/au import { collectManifestFilePaths } from '../helpers/manifestFilePaths'; import { sanitizeForLog } from '../utils/safeLog'; import { isPathWithinBase, isValidStackName } from '../utils/validation'; +import { isRealPathAtManagedLocation, managedAreaBase } from './gitops/managedPaths'; import type { BuildContextPlan, ComposeInputEntry, @@ -1226,10 +1227,29 @@ export class GitProjectManifestService { // says nothing about recency, and the manifest's previousDir is what a // crash restore reads from. const keep = new Set([keepBase, previousDir ? path.basename(previousDir) : null].filter((v): v is string => v !== null)); + const areaBase = managedAreaBase(); for (const entry of entries) { if (!entry.isDirectory() || !entry.name.startsWith('applied-')) continue; if (keep.has(entry.name)) continue; - await fs.promises.rm(path.join(dir, entry.name), { recursive: true, force: true }); + const abs = path.resolve(dir, entry.name); + // Inline containment barrier at the removal sink (see + // `managedAreaBase`): the analyzer credits this literal comparison, + // not the positional check below it. + if (!abs.startsWith(areaBase + path.sep)) { + console.warn(`[GitManifest] refusing to prune ${sanitizeForLog(entry.name)} for ${sanitizeForLog(stackName)}: it resolves outside the managed area`); + continue; + } + // Positional barrier at the recursive-delete sink. This runs on + // every successful apply, exactly where a link planted over one + // generation name would do the most damage, and lexical + // containment cannot see it. Refusal skips the entry rather than + // failing an apply that already committed; retention keeps + // everything recoverable either way. + if (!await isRealPathAtManagedLocation(abs)) { + console.warn(`[GitManifest] refusing to prune ${sanitizeForLog(entry.name)} for ${sanitizeForLog(stackName)}: it is not at its own location in the managed area`); + continue; + } + await fs.promises.rm(abs, { recursive: true, force: true }); } } @@ -1344,9 +1364,24 @@ export class GitProjectManifestService { try { const entries = await fs.promises.readdir(dir, { withFileTypes: true }); const now = Date.now(); + const areaBase = managedAreaBase(); for (const entry of entries) { if (!entry.isDirectory() || !entry.name.startsWith('candidate-')) continue; - const abs = path.join(dir, entry.name); + const abs = path.resolve(dir, entry.name); + // Inline containment barrier at the removal sink (see + // `managedAreaBase`): the analyzer credits this literal + // comparison, not the positional check below it. + if (!abs.startsWith(areaBase + path.sep)) { + console.warn(`[GitManifest] refusing to reap orphan candidate ${sanitizeForLog(entry.name)} for ${sanitizeForLog(stackName)}: it resolves outside the managed area`); + continue; + } + // Same positional barrier as generation pruning: the boot sweep + // reaps candidate directories nobody claims, which is precisely + // the kind of unattended delete a planted link would steer. + if (!await isRealPathAtManagedLocation(abs)) { + console.warn(`[GitManifest] refusing to reap orphan candidate ${sanitizeForLog(entry.name)} for ${sanitizeForLog(stackName)}: it is not at its own location in the managed area`); + continue; + } const complete = await fs.promises .access(path.join(abs, CANDIDATE_COMPLETE_MARKER)) .then(() => true) @@ -1475,7 +1510,17 @@ export class GitProjectManifestService { } await fs.promises.rm(markerPath, { force: true }); if (!parsed.managedAreaExisted) { - await fs.promises.rm(root, { recursive: true, force: true }); + // Inline containment barrier at the removal sink (see + // `managedAreaBase`), then the positional check. Refusal leaves the + // area in place for the operator rather than deleting through a + // redirected root; the restore itself already succeeded, so + // recovery still reports success. + if (!path.resolve(root).startsWith(managedAreaBase() + path.sep) + || !await isRealPathAtManagedLocation(root)) { + console.warn(`[GitManifest] restored ${sanitizeForLog(stackName)} but left its managed area in place: it is not at its own location in the managed area`); + } else { + await fs.promises.rm(root, { recursive: true, force: true }); + } } return true; } @@ -1509,6 +1554,18 @@ export class GitProjectManifestService { async stageManagedAreaForDetach(stackName: string): Promise { const root = this.managedRoot(stackName); const staged = this.detachStagedRoot(stackName); + // Inline containment barrier at the removal sink (see + // `managedAreaBase`): the analyzer credits this literal comparison, + // not the positional check below it. + if (!path.resolve(staged).startsWith(managedAreaBase() + path.sep)) { + throw new Error(`refusing to restage the managed area for ${sanitizeForLog(stackName)}: a stale staged area resolves outside the managed area`); + } + // Positional barrier on the stale-staged cleanup. Thrown rather than + // skipped because the rename below needs this path back, and detaching + // over an unverifiable directory silently is the outcome to avoid. + if (!await isRealPathAtManagedLocation(staged)) { + throw new Error(`refusing to restage the managed area for ${sanitizeForLog(stackName)}: a stale staged area is not at its own location in the managed area`); + } await fs.promises.rm(staged, { recursive: true, force: true }); try { await fs.promises.rename(root, staged); @@ -1529,8 +1586,19 @@ export class GitProjectManifestService { /** Delete a staged managed area after the database row is gone. */ async finalizeStagedDetach(stackName: string): Promise { + const staged = this.detachStagedRoot(stackName); + // Same refusal semantics as `deleteManagedArea`: false leaves the area + // in place for an operator rather than deleting through a redirected + // root now that nothing else references it. The inline containment + // barrier (see `managedAreaBase`) is what the analyzer credits; the + // positional check carries the property. + if (!path.resolve(staged).startsWith(managedAreaBase() + path.sep) + || !await isRealPathAtManagedLocation(staged)) { + console.warn(`[GitManifest] refusing to delete staged area for ${sanitizeForLog(stackName)}: it is not at its own location in the managed area`); + return false; + } try { - await fs.promises.rm(this.detachStagedRoot(stackName), { recursive: true, force: true }); + await fs.promises.rm(staged, { recursive: true, force: true }); return true; } catch (e) { console.warn('[GitManifest] staged detach cleanup failed:', sanitizeForLog(stackName), (e as Error).message); @@ -1546,6 +1614,16 @@ export class GitProjectManifestService { */ async deleteManagedArea(stackName: string): Promise { const root = this.managedRoot(stackName); + // Same refusal semantics as the rm failure below: false means the area + // survived, which is what keeps a detach from dropping its row while + // generations it cannot account for are still on disk. The inline + // containment barrier (see `managedAreaBase`) is what the analyzer + // credits; the positional check carries the property. + if (!path.resolve(root).startsWith(managedAreaBase() + path.sep) + || !await isRealPathAtManagedLocation(root)) { + console.warn(`[GitManifest] refusing to delete managed area for ${sanitizeForLog(stackName)}: it is not at its own location in the managed area`); + return false; + } try { await fs.promises.rm(root, { recursive: true, force: true }); return true; diff --git a/backend/src/services/GitSourceService.ts b/backend/src/services/GitSourceService.ts index 6a64c5bd..1f64dcee 100644 --- a/backend/src/services/GitSourceService.ts +++ b/backend/src/services/GitSourceService.ts @@ -28,6 +28,20 @@ import type { ComposeInputEntry, GitProjectManifest, GitSourceManifestState, Inv import type { GitChangePlan, PublicGitChangePlan, GitChangePlanCounts, PublicGitChangePlanOperation } from '../types/gitChangePlan'; import { GIT_CHANGE_PLAN_SCHEMA_VERSION } from '../types/gitChangePlan'; import type { NotificationCategory } from './NotificationService'; +import { GitOpsStore } from './gitops/store'; +import { GitOpsTransitions, GitOpsTransitionError } from './gitops/transitions'; +import { + buildCreateCheckpointRow, + buildDirectApplicationRow, + buildGenerationRow, + directSourceIdentity, + newGitOpsId, + stackManagedRoot, +} from './gitops/directApplication'; +import type { GitOpsApplicationRow } from './gitops/types'; +import { appliedRelPathFor, candidateRelPathForSha, deleteStagingMarker, readStagingMarker, writeStagingMarker } from './gitops/createStagingMarker'; +import { cleanupUnclaimedManagedRoot, removeOperationOwnedPaths } from './gitops/createCleanup'; +import { managedAreaBase } from './gitops/managedPaths'; import type { GitHttpRequest, GitHttpResponse, HttpClient } from 'isomorphic-git/http/node'; // isomorphic-git is the heaviest dependency in the backend (~5 MB) and only @@ -180,7 +194,8 @@ export type GitSourceErrorCode = | 'PLAN_FINGERPRINT_REQUIRED' | 'PLAN_BLOCKED' | 'LEGACY_PENDING' - | 'PLAN_UNAVAILABLE'; + | 'PLAN_UNAVAILABLE' + | 'OPERATION_IN_FLIGHT'; export class GitSourceError extends Error { constructor( @@ -758,31 +773,88 @@ export class GitSourceService { (existing.context_dir ?? null) !== input.contextDir ); - db.upsertGitSource({ - stack_name: input.stackName, - repo_url: input.repoUrl, + const gitopsConfig = { + repoUrl: input.repoUrl, branch: input.branch, - compose_path: input.composePaths[0], - compose_paths: input.composePaths, - context_dir: input.contextDir, - sync_env: input.syncEnv, - env_path: resolvedEnvPath, - auth_type: input.authType, - encrypted_token: encryptedToken, - auto_apply_on_webhook: input.autoApplyOnWebhook, - auto_deploy_on_apply: input.autoDeployOnApply, - last_applied_commit_sha: existing?.last_applied_commit_sha ?? null, - last_applied_content_hash: existing?.last_applied_content_hash ?? null, - pending_commit_sha: existing?.pending_commit_sha ?? null, - pending_compose_content: existing?.pending_compose_content ?? null, - pending_env_content: existing?.pending_env_content ?? null, - pending_fetched_at: existing?.pending_fetched_at ?? null, - last_debounce_at: existing?.last_debounce_at ?? null, - }); + composePaths: input.composePaths, + contextDir: input.contextDir, + syncEnv: input.syncEnv, + envPath: resolvedEnvPath, + }; + const gitopsIdentity = directSourceIdentity(gitopsConfig); - if (configChanged) { - db.clearGitSourcePending(input.stackName); - } + // The source row, the pending clear, and the GitOps transition commit + // together. Clearing pending without invalidating the candidate would + // leave the model offering an apply for files the operator can no + // longer produce. + db.getDb().transaction(() => { + db.upsertGitSource({ + stack_name: input.stackName, + repo_url: input.repoUrl, + branch: input.branch, + compose_path: input.composePaths[0], + compose_paths: input.composePaths, + context_dir: input.contextDir, + sync_env: input.syncEnv, + env_path: resolvedEnvPath, + auth_type: input.authType, + encrypted_token: encryptedToken, + auto_apply_on_webhook: input.autoApplyOnWebhook, + auto_deploy_on_apply: input.autoDeployOnApply, + last_applied_commit_sha: existing?.last_applied_commit_sha ?? null, + last_applied_content_hash: existing?.last_applied_content_hash ?? null, + pending_commit_sha: existing?.pending_commit_sha ?? null, + pending_compose_content: existing?.pending_compose_content ?? null, + pending_env_content: existing?.pending_env_content ?? null, + pending_fetched_at: existing?.pending_fetched_at ?? null, + last_debounce_at: existing?.last_debounce_at ?? null, + }); + + if (configChanged) { + db.clearGitSourcePending(input.stackName); + } + + const app = this.gitopsApplicationFor(input.stackName); + const envelope = this.gitopsEnvelope(crypto.randomUUID(), 'system:git-source', 'configure'); + if (!app && !existing && !this.gitopsNameHeld(input.stackName)) { + // Linking a stack that already exists. Nothing is fetched or + // accepted yet, so the application starts live with no desired + // commit and the projection asks for a fetch. + GitOpsTransitions.getInstance().activateDirect({ + application: buildDirectApplicationRow({ + id: newGitOpsId(), + stackName: input.stackName, + config: gitopsConfig, + identity: gitopsIdentity, + lifecycleStatus: 'active', + at: envelope.at, + }), + nodeId: NodeRegistry.getInstance().getDefaultNodeId(), + envelope, + }); + return; + } + // Credential-only and policy-only edits change nothing material, so + // they leave the candidate and every accepted pointer alone. + if (app && configChanged) { + GitOpsTransitions.getInstance().configChangedPendingCleared({ + applicationId: app.id, + identity: { + repoUrl: gitopsIdentity.repoUrl, + repoIdentityJson: JSON.stringify(gitopsIdentity.identity), + configuredRef: input.branch, + }, + material: { + composePathsJson: JSON.stringify([...input.composePaths]), + contextDir: input.contextDir, + syncEnv: input.syncEnv ? 1 : 0, + envPath: resolvedEnvPath, + fingerprint: gitopsIdentity.fingerprint, + }, + envelope, + }); + } + })(); return this.get(input.stackName)!; } @@ -933,7 +1005,24 @@ export class GitSourceService { await rollbackAndThrow('Managed project data disappeared during detach'); } try { - DatabaseService.getInstance().deleteGitSource(stackName); + // The source row and the GitOps tombstones commit together, so + // a detached stack can never leave a live application pointing + // at a source that no longer exists. Configured identity and + // SHA pointers survive on the tombstone as frozen facts, and a + // later reattach mints a new application rather than reviving + // this one. + const gitopsApp = this.gitopsApplicationFor(stackName); + DatabaseService.getInstance().getDb().transaction(() => { + DatabaseService.getInstance().deleteGitSource(stackName); + if (!gitopsApp) return; + const tx = GitOpsTransitions.getInstance(); + const envelope = this.gitopsEnvelope(crypto.randomUUID(), 'system:git-source', 'detach'); + for (const target of GitOpsStore.getInstance().listTargets(gitopsApp.id)) { + if (target.target_status !== 'active') continue; + tx.targetTombstoned(gitopsApp.id, target.node_id, envelope); + } + tx.applicationTombstoned(gitopsApp.id, 'detached', envelope); + })(); } catch (e) { await rollbackAndThrow('Could not commit the Git source removal', e); } @@ -1777,12 +1866,131 @@ export class GitSourceService { * reads last_debounce_at while it is still unset on every request, slips * past the gate, and clones once per request. */ + /** + * The GitOps application tracking this stack, or null when there is none. + * + * Stacks that predate the revision-state model have no application until + * migration runs, so every producer is a no-op for them rather than + * inventing an application from configuration alone. + */ + private gitopsApplicationFor(stackName: string): GitOpsApplicationRow | null { + const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName); + return app && app.lifecycle_status === 'active' ? app : null; + } + + /** + * Whether any application still holds this stack name. + * + * Wider than `gitopsApplicationFor`, which only reports usable applications. + * A `creating` row left by a crash the boot sweep could not settle still + * occupies the unique live-application index, so activating over it would + * fail the whole save with an internal constraint message. + */ + private gitopsNameHeld(stackName: string): boolean { + return !!GitOpsStore.getInstance().getLiveDirectApplication(stackName); + } + + private gitopsEnvelope(operationId: string, actor: string, trigger: string) { + return { operationId, actor, trigger, at: Date.now() }; + } + + /** + * Record a GitOps transition without letting it break the operation it + * describes. + * + * The store is the record of what happened, not the mechanism that makes it + * happen, so a rejected transition must not fail a fetch or an apply that + * has already touched the filesystem. The rejection is logged loudly + * because it means the recorded state has drifted from reality. + */ + private recordGitOps(stackName: string, what: string, write: () => void): boolean { + try { + write(); + return true; + } catch (error) { + console.error( + `[GitOps] Could not record ${what} for ${sanitizeForLog(stackName)}:`, + error instanceof Error ? error.stack ?? error.message : String(error), + ); + return false; + } + } + + /** + * Close an operation whose terminal transition was rejected. + * + * A start that never terminates leaves the source reporting work in + * progress and offering no actions, permanently. When the real terminal + * event cannot be recorded, the next best truth is that we lost track: + * clear the operation and stamp a failure the projection can render, so the + * operator sees an error they can retry instead of a spinner. + */ + private abandonGitOpsOperation( + stackName: string, + applicationId: string, + envelope: ReturnType, + ): void { + this.recordGitOps(stackName, 'lost-track fallback', () => { + GitOpsTransitions.getInstance().applyFailed(applicationId, 'bookkeeping_rejected', envelope); + }); + } + private async pullLocked(stackName: string, actor: string): Promise { const db = DatabaseService.getInstance(); const src = db.getGitSource(stackName); if (!src) throw new GitSourceError('GIT_ERROR', 'No Git source configured for this stack.'); + const gitopsApp = this.gitopsApplicationFor(stackName); + const gitopsOperationId = crypto.randomUUID(); + const gitopsEnv = this.gitopsEnvelope(gitopsOperationId, actor, 'pull'); + // A fetch that starts and never terminates is worse than one that is + // never recorded: fetchStarted refuses to open a second operation, so + // every later pull silently stops being tracked until a restart. + let fetchOpen = false; + if (gitopsApp) { + fetchOpen = this.recordGitOps(stackName, 'fetch start', () => { + GitOpsTransitions.getInstance().fetchStarted(gitopsApp.id, gitopsEnv); + }); + } + const closeFetch = (): void => { + if (!gitopsApp || !fetchOpen) return; + fetchOpen = false; + this.recordGitOps(stackName, 'fetch failure', () => { + GitOpsTransitions.getInstance().fetchFailed(gitopsApp.id, gitopsEnv); + }); + }; + try { + return await this.pullLockedBody(stackName, actor, src, { + app: gitopsApp, + operationId: gitopsOperationId, + envelope: gitopsEnv, + markSettled: () => { fetchOpen = false; }, + abandon: closeFetch, + }); + } catch (e) { + closeFetch(); + throw e; + } + } + + private async pullLockedBody( + stackName: string, + actor: string, + src: StackGitSource, + gitops: { + app: GitOpsApplicationRow | null; + operationId: string; + envelope: ReturnType; + markSettled: () => void; + abandon: () => void; + }, + ): Promise { + const db = DatabaseService.getInstance(); const diag = isDebugEnabled(); + const gitopsApp = gitops.app; + const gitopsOperationId = gitops.operationId; + const gitopsEnv = gitops.envelope; + if (diag) { console.log(`[GitSource:diag] pull start stack=${stackName} branch=${src.branch} host=${repoHost(src.repo_url)}`); } @@ -1792,7 +2000,9 @@ export class GitSourceService { // Object holder: property access is not narrowed by control-flow // analysis, so the closure assignment below stays visible. const materialization: { value: MaterializationResult | null } = { value: null }; - const fetched = await this.fetchFromGit({ + // Every throw from here on, including this fetch, is closed by the + // caller's handler, so nothing is recorded locally. + const fetched: FetchResult = await this.fetchFromGit({ repoUrl: src.repo_url, branch: src.branch, composePaths: src.compose_paths, @@ -1835,6 +2045,80 @@ export class GitSourceService { }); } + // Record what this fetch resolved before the pending blob is written, + // so the durable pointers and the operational pending store agree. + if (gitopsApp) { + const outcomeRecorded = this.recordGitOps(stackName, 'fetch outcome', () => { + const tx = GitOpsTransitions.getInstance(); + // One transaction: a candidate that exists without the fetch + // that produced it would let a later apply accept the wrong + // generation while the projection reports the older commit. + DatabaseService.getInstance().getDb().transaction(() => { + if (!validation.ok) { + tx.fetchedInvalid(gitopsApp.id, fetched.commitSha, gitopsEnv); + return; + } + tx.fetched(gitopsApp.id, fetched.commitSha, gitopsEnv); + if (!materialization.value) return; + const identity = directSourceIdentity({ + repoUrl: src.repo_url, + branch: src.branch, + composePaths: src.compose_paths, + contextDir: src.context_dir, + syncEnv: src.sync_env, + envPath: src.env_path, + }); + // A pull that resolves to exactly what the live candidate + // already proposes (same commit, source fingerprint, plan + // verdict) must not mint a lookalike generation and rewrite the + // candidate pointers. The staged generation stands; only the + // fetch above is new. A candidate for a different commit, or no + // candidate at all, mints anew: staging after an apply is a new + // dispatch cycle and needs its own generation to accept. + const staged = gitopsApp.candidate_generation_id + ? GitOpsStore.getInstance().getGeneration(gitopsApp.candidate_generation_id) + : undefined; + if ( + staged && + staged.commit_sha === fetched.commitSha && + staged.materialization_fingerprint === identity.fingerprint && + staged.plan_blocked === (plan?.blocked === true ? 1 : 0) + ) { + return; + } + const generationId = newGitOpsId(); + const nextManifestVersion = (prior?.manifestVersion ?? 0) + 1; + GitOpsStore.getInstance().insertGeneration(buildGenerationRow({ + id: generationId, + applicationId: gitopsApp.id, + commitSha: fetched.commitSha, + identity, + configuredRef: src.branch, + candidateRelPath: materialization.value.candidateRelPath, + appliedRelPath: appliedRelPathFor(fetched.commitSha, nextManifestVersion), + manifestVersion: nextManifestVersion, + // The candidate's own invocation. Recording the prior + // generation's would attribute one generation's facts to + // another, which is the whole failure this model prevents. + expectedInvocation: plan?.candidateInvocation ?? prior?.project.invocation ?? null, + changePlanFingerprint: plan?.fingerprint ?? null, + operationId: gitopsOperationId, + trigger: gitopsEnv.trigger, + actor, + at: gitopsEnv.at, + planBlocked: plan?.blocked === true, + })); + if (plan?.blocked) tx.sourceConflictBlocker(gitopsApp.id, generationId, gitopsEnv); + else tx.candidateReady(gitopsApp.id, generationId, false, gitopsEnv); + })(); + gitops.markSettled(); + }); + // The pull itself succeeded; the files and the pending blob are + // real. Closing the operation is what stops the source reporting a + // fetch in flight for ever and locking out every later pull. + if (!outcomeRecorded) gitops.abandon(); + } + const publicPlan = plan ? GitChangePlanService.getInstance().toPublic(plan) : null; const summary = plan ? GitChangePlanService.getInstance().toPendingSummary(plan) : null; db.setGitSourcePending( @@ -1947,10 +2231,47 @@ export class GitSourceService { } /** Body of apply(); assumes the caller already holds Git mutex + shared stack lock. */ + /** + * Apply a pending pull, recording the attempt as a GitOps operation. + * + * The wrapper exists so a throw anywhere in the body still closes the + * operation. An apply that started and never terminated would leave the + * source projecting `applying` until the next restart reclassified it as an + * interruption, which reads as "still working" when nothing is. + */ private async applyLocked( stackName: string, commitSha: string, opts: GitApplyOpts, + ): Promise<{ applied: boolean; deployed: boolean; deployError?: string; recoveryId?: string }> { + const started: { app: GitOpsApplicationRow | null; env: ReturnType | null; settled: boolean } = { + app: null, + env: null, + settled: false, + }; + try { + return await this.applyLockedBody(stackName, commitSha, opts, started); + } catch (e) { + if (started.app && started.env && !started.settled) { + const app = started.app; + const env = started.env; + this.recordGitOps(stackName, 'apply failure', () => { + GitOpsTransitions.getInstance().applyFailed( + app.id, + e instanceof GitSourceError ? e.code : 'apply', + env, + ); + }); + } + throw e; + } + } + + private async applyLockedBody( + stackName: string, + commitSha: string, + opts: GitApplyOpts, + started: { app: GitOpsApplicationRow | null; env: ReturnType | null; settled: boolean }, ): Promise<{ applied: boolean; deployed: boolean; deployError?: string; recoveryId?: string }> { const diag = isDebugEnabled(); const db = DatabaseService.getInstance(); @@ -1994,6 +2315,34 @@ export class GitSourceService { const actor = opts.actor ?? 'system:git-source'; let recoveryId: string | undefined; + // The apply is bound to the candidate the pull recorded. Without one + // there is nothing to accept, so the acceptance below is skipped rather + // than inventing a generation from the pending blob. + const gitopsApp = this.gitopsApplicationFor(stackName); + // The candidate must be the generation built from the commit being + // applied. Without this check a candidate left behind by a swallowed + // fetch outcome would be accepted for files from a different commit, + // and the projection would confidently report the wrong one. + const candidateId = gitopsApp?.candidate_generation_id ?? null; + const candidateGeneration = candidateId + ? GitOpsStore.getInstance().getGeneration(candidateId) + : undefined; + const gitopsGenerationId = candidateGeneration?.commit_sha === commitSha ? candidateId : null; + if (candidateId && !gitopsGenerationId) { + console.warn( + '[GitOps] Skipping acceptance for %s: the recorded candidate is not built from %s', + sanitizeForLog(stackName), sanitizeForLog(commitSha.slice(0, 7)), + ); + } + const gitopsEnv = this.gitopsEnvelope(pending.operationId, actor, 'apply'); + if (gitopsApp && gitopsGenerationId) { + this.recordGitOps(stackName, 'apply start', () => { + GitOpsTransitions.getInstance().applyStarted(gitopsApp.id, gitopsGenerationId, gitopsEnv); + started.app = gitopsApp; + started.env = gitopsEnv; + }); + } + let appliedSpec: GitSourceAppliedSpec | null; if (pending.candidateRelPath !== null && pending.inventory !== null) { // ── Complete-project path (v4 pending) ─────────────────────────── @@ -2255,6 +2604,25 @@ export class GitSourceService { db.markGitSourceApplied(stackName, commitSha, hash); db.setGitSourceAppliedSpec(stackName, appliedSpec); + // The files are on disk and the source row now points at this commit, + // so this is where the generation becomes the accepted one. + if (gitopsApp && gitopsGenerationId) { + const recorded = this.recordGitOps(stackName, 'acceptance', () => { + GitOpsTransitions.getInstance().applied({ + applicationId: gitopsApp.id, + generationId: gitopsGenerationId, + artifactSetId: newGitOpsId(), + sourceAcceptanceId: newGitOpsId(), + authority: actor === 'system:webhook' ? 'configured_policy' : 'operator', + envelope: gitopsEnv, + }); + }); + started.settled = true; + // The files are on disk either way. What we can still control is + // not leaving the operation open when the acceptance was rejected. + if (!recorded) this.abandonGitOpsOperation(stackName, gitopsApp.id, gitopsEnv); + } + const shouldDeploy = opts.deploy ?? src.auto_deploy_on_apply; if (diag) console.log('[GitSource:diag] apply wrote stack=%s sha=%s deploy=%s', sanitizeForLog(stackName), sanitizeForLog(commitSha.slice(0, 7)), sanitizeForLog(shouldDeploy)); @@ -2289,7 +2657,7 @@ export class GitSourceService { await finalizeRecoveryCurrent(recoveryId, false); } // Shared stack lock already held as git_apply for capture→deploy. - await ComposeService.getInstance(nodeId).deployStack( + const autoDeploy = await ComposeService.getInstance(nodeId).deployStack( stackName, undefined, undefined, @@ -2305,6 +2673,7 @@ export class GitSourceService { stackName, 'deploy', 'system:git-source', + { deployedGenerationId: autoDeploy.deployedGenerationId }, ); if (recoveryId) { recoverySvc.linkGateOrRetain(recoveryId, healthGateId); @@ -2354,7 +2723,28 @@ export class GitSourceService { return { applied: true, deployed: false, recoveryId }; } - public dismissPending(stackName: string): void { + public dismissPending(stackName: string, actor?: string): void { + const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName); + if (app?.candidate_generation_id) { + try { + GitOpsTransitions.getInstance().dismissed( + app.id, + this.gitopsEnvelope(crypto.randomUUID(), actor ?? 'system:git-source', 'dismiss'), + ); + } catch (error) { + // The refusal is the outcome the operator must see. Swallowing it + // here would leave the projection offering a candidate they just + // declined, which is exactly the stale state dismissal exists to + // prevent. + if (error instanceof GitOpsTransitionError) { + throw new GitSourceError( + 'OPERATION_IN_FLIGHT', + `Cannot dismiss the pending update for ${stackName}: ${error.message}`, + ); + } + throw error; + } + } DatabaseService.getInstance().clearGitSourcePending(stackName); } @@ -2380,25 +2770,69 @@ export class GitSourceService { throw new GitSourceError('GIT_ERROR', 'Auto-deploy requires auto-apply-on-webhook to be enabled.'); } + const gitopsOperationId = crypto.randomUUID(); + // Inline containment barrier at the stat sink. CodeQL does not + // credit the wrapped isPathWithinBase helper, so resolve against the + // managed-area base and check containment right here. + const areaBase = managedAreaBase(); + const managedRoot = path.resolve(stackManagedRoot(input.stackName)); + if (!managedRoot.startsWith(areaBase + path.sep)) { + throw new GitSourceError('GIT_ERROR', 'Invalid stack path'); + } + // Whether the managed root is ours to delete is decided once, here, + // before anything can create it. Cleanup later reads this answer + // rather than re-probing a directory it may itself have made. + const rootPreexisted = existsSync(managedRoot); + const gitopsIdentity = directSourceIdentity({ + repoUrl: input.repoUrl, + branch: input.branch, + composePaths: input.composePaths, + contextDir: input.contextDir, + syncEnv: input.syncEnv, + envPath: input.envPath, + }); + const staged: { candidateRelPath: string | null } = { candidateRelPath: null }; + // 1. Fetch from git BEFORE touching disk or DB. If the fetch // fails there is nothing to clean up. The onClone hook stages // the complete-project candidate inside the clone lifecycle. const manifestSvc = GitProjectManifestService.getInstance(); const materialization: { value: MaterializationResult | null } = { value: null }; - const fetched = await this.fetchFromGit({ - repoUrl: input.repoUrl, - branch: input.branch, - composePaths: input.composePaths, - envPath: input.syncEnv ? input.envPath : null, - token: input.token, - onClone: async (cloneDir, commitSha, envContent) => { - materialization.value = await this.buildMaterialization(input.stackName, cloneDir, commitSha, { - compose_paths: input.composePaths, - context_dir: input.contextDir, - sync_env: input.syncEnv, - }, envContent); - }, - }); + let fetched: FetchResult; + try { + fetched = await this.fetchFromGit({ + repoUrl: input.repoUrl, + branch: input.branch, + composePaths: input.composePaths, + envPath: input.syncEnv ? input.envPath : null, + token: input.token, + onClone: async (cloneDir, commitSha, envContent) => { + // The candidate path is recorded before the build that + // creates it, so a crash mid-build still names exactly one + // directory this operation owns. + staged.candidateRelPath = candidateRelPathForSha(commitSha); + await writeStagingMarker(managedRoot, { + schemaVersion: 1, + operationId: gitopsOperationId, + rootPreexisted, + candidateRelPath: staged.candidateRelPath, + createdAt: Date.now(), + }); + materialization.value = await this.buildMaterialization(input.stackName, cloneDir, commitSha, { + compose_paths: input.composePaths, + context_dir: input.contextDir, + sync_env: input.syncEnv, + }, envContent); + }, + }); + } catch (e) { + // Materialization refuses routinely, not just on crashes. The + // marker has to come off with the staged files, or it would + // claim this managed area against every later attempt and make + // the stack name uncreatable until the next restart. + await this.cleanupStagedCreate(managedRoot, staged.candidateRelPath, rootPreexisted); + throw e; + } // 2. Validate against the same `docker compose config` check the // apply path uses. Reject before creating anything on disk. @@ -2406,6 +2840,7 @@ export class GitSourceService { ? materialization.value.validation : await this.validateCompose(fetched.composeFiles, fetched.envContent, input.contextDir); if (!validation.ok) { + await this.cleanupStagedCreate(managedRoot, staged.candidateRelPath, rootPreexisted); throw new GitSourceError('GIT_ERROR', `Compose validation failed: ${validation.error}`); } @@ -2420,6 +2855,12 @@ export class GitSourceService { // state instead of 'absent'. let completeProjectManifest: GitProjectManifest | null = null; let recordedCreatePlan: GitChangePlan | null = null; + // Set once the activation transaction commits. Before that there is + // nothing in the database to tear down; after it, cleanup has to go + // through create_failed so the checkpoint and tombstone stay + // consistent with what was removed from disk. + let gitopsApplicationId: string | null = null; + let gitopsCommitted = false; try { let appliedSpec: GitSourceAppliedSpec | null; if (materialization.value) { @@ -2511,10 +2952,88 @@ export class GitSourceService { completeProjectManifest = manifest; } + // 3b. Persist the GitOps identity of this create. Everything + // above is still reversible by deleting files; from here on + // recovery is driven by the checkpoint instead of guesswork. + if (completeProjectManifest && materialization.value && staged.candidateRelPath) { + const applicationId = newGitOpsId(); + const envelope = { + operationId: gitopsOperationId, + actor: 'system:git-source', + trigger: 'create', + at: Date.now(), + }; + const sourceConfig = { + repoUrl: input.repoUrl, + branch: input.branch, + composePaths: input.composePaths, + contextDir: input.contextDir, + syncEnv: input.syncEnv, + envPath: input.envPath, + }; + GitOpsTransitions.getInstance().activateCreateFromGit({ + application: buildDirectApplicationRow({ + id: applicationId, + stackName: input.stackName, + config: sourceConfig, + identity: gitopsIdentity, + lifecycleStatus: 'creating', + at: envelope.at, + }), + nodeId: NodeRegistry.getInstance().getDefaultNodeId(), + commitSha: fetched.commitSha, + generation: buildGenerationRow({ + id: newGitOpsId(), + applicationId, + commitSha: fetched.commitSha, + identity: gitopsIdentity, + configuredRef: input.branch, + candidateRelPath: staged.candidateRelPath, + appliedRelPath: appliedRelPathFor(fetched.commitSha, completeProjectManifest.manifestVersion), + manifestVersion: completeProjectManifest.manifestVersion, + expectedInvocation: completeProjectManifest.project.invocation, + changePlanFingerprint: recordedCreatePlan?.fingerprint ?? null, + operationId: gitopsOperationId, + trigger: envelope.trigger, + actor: envelope.actor, + at: envelope.at, + }), + checkpoint: buildCreateCheckpointRow({ + applicationId, + stackName: input.stackName, + operationId: gitopsOperationId, + config: sourceConfig, + identity: gitopsIdentity, + authType: input.authType, + encryptedToken: input.authType === 'token' && input.token + ? this.crypto.encrypt(input.token) + : null, + autoApplyOnWebhook: input.autoApplyOnWebhook, + autoDeployOnApply: input.autoDeployOnApply, + commitSha: fetched.commitSha, + createdManagedRoot: !rootPreexisted, + at: envelope.at, + }), + envelope, + }); + gitopsApplicationId = applicationId; + await deleteStagingMarker(managedRoot); + } + await fsSvc.createStack(input.stackName); stackCreated = true; + if (gitopsApplicationId) { + GitOpsStore.getInstance().updateCreateCheckpoint( + gitopsApplicationId, { phase: 'stack_created' }, Date.now(), + ); + } if (completeProjectManifest && materialization.value) { + if (gitopsApplicationId) { + GitOpsStore.getInstance().updateCreateCheckpoint( + gitopsApplicationId, { phase: 'promoting' }, Date.now(), + ); + } await manifestSvc.promoteGeneration(input.stackName, { sha: fetched.commitSha, candidateRelPath: materialization.value.candidateRelPath, @@ -2523,6 +3042,13 @@ export class GitSourceService { adoptExistingMaterializedPaths: 'all', }); appliedSpec = this.deriveAppliedSpec(input.composePaths, input.contextDir); + if (gitopsApplicationId) { + GitOpsStore.getInstance().updateCreateCheckpoint( + gitopsApplicationId, + { phase: 'manifest_committed', appliedSpecJson: JSON.stringify(appliedSpec) }, + Date.now(), + ); + } } else { appliedSpec = await this.materialize( input.stackName, fetched.composeFiles, input.contextDir, input.syncEnv, fetched.envContent, null, @@ -2537,6 +3063,11 @@ export class GitSourceService { ? this.crypto.encrypt(input.token) : null; const hash = this.hashContent(fetched.composeFiles, fetched.envContent); + // The source row, the applied pointers, and the checkpoint + // advance together. This commit is the success boundary: once + // it lands the stack is live, and any later error is reported + // without deleting anything. + const commitCreate = db.getDb().transaction(() => { db.upsertGitSource({ stack_name: input.stackName, repo_url: input.repoUrl, @@ -2570,6 +3101,38 @@ export class GitSourceService { completeProjectManifest.generation.appliedDir, ); } + if (gitopsApplicationId) { + const checkpoint = GitOpsStore.getInstance().getCreateCheckpoint(gitopsApplicationId); + if (!checkpoint?.generation_id) { + throw new GitSourceError('GIT_ERROR', 'Create checkpoint lost its generation before acceptance.'); + } + GitOpsTransitions.getInstance().applied({ + applicationId: gitopsApplicationId, + generationId: checkpoint.generation_id, + artifactSetId: newGitOpsId(), + sourceAcceptanceId: newGitOpsId(), + authority: 'operator', + envelope: { + operationId: gitopsOperationId, + actor: 'system:git-source', + trigger: 'create', + at: Date.now(), + }, + activateCreating: true, + }); + GitOpsStore.getInstance().updateCreateCheckpoint( + gitopsApplicationId, { phase: 'pointers_committed' }, Date.now(), + ); + } + }); + commitCreate(); + gitopsCommitted = true; + // The checkpoint has done its job. Dropping it here keeps the + // boot sweep reporting only genuine interruptions, and stops a + // copy of the encrypted token living past the create. + if (gitopsApplicationId) { + GitOpsStore.getInstance().deleteCreateCheckpoint(gitopsApplicationId); + } rowInserted = true; const operationId = crypto.randomUUID(); @@ -2601,6 +3164,31 @@ export class GitSourceService { } return { source, commitSha: fetched.commitSha, envWritten, warnings: fetched.warnings }; } catch (e) { + // Past the success boundary the stack is live and owned by the + // operator. A later error is reported, never compensated: the + // leftover marker or checkpoint is finished by the boot sweep. + if (gitopsCommitted) { + const detail = e instanceof Error ? e.message : String(e); + console.error( + `[GitSource] Create for ${sanitizeForLog(input.stackName)} succeeded but a later step failed:`, + detail, + ); + this.recordGitActivity( + input.stackName, + 'git_create', + `Git create for ${input.stackName} completed, but a follow-up step failed: ${detail}`, + 'system:git-source', + 'warning', + ); + // Say plainly that the stack exists. The raw downstream + // error reads as a failed create, and an operator acting on + // it retries and hits "stack already exists", which looks + // like corruption rather than success. + throw new GitSourceError( + 'GIT_ERROR', + `The stack was created from Git, but a follow-up step failed: ${detail}`, + ); + } // Roll back any partial on-disk state so the caller can retry // cleanly. The DB row is only inserted at step 4, so an error // earlier leaves nothing to clean in the DB. @@ -2616,7 +3204,10 @@ export class GitSourceService { // for a non-existence reason) must never lose its previous // applied generations to someone else's rollback: when the stack // dir was NOT created by us, remove only the candidate we staged. - if (stackCreated) { + if (stackCreated && !rootPreexisted) { + // Only legal because this operation created the managed + // root. A root that predated the create holds retained + // generations of its own and is cleaned path by path below. await GitProjectManifestService.getInstance().deleteManagedArea(input.stackName); } else if (materialization.value) { const dataDir = process.env.DATA_DIR || path.join(process.cwd(), 'data'); @@ -2632,11 +3223,61 @@ export class GitSourceService { if (rowInserted) { db.deleteGitSource(input.stackName); } + // Filesystem cleanup has to succeed before the tombstone, so a + // create whose files could not be removed keeps its checkpoint + // and is retried by the next boot rather than being recorded as + // cleanly failed. + if (gitopsApplicationId) { + try { + await removeOperationOwnedPaths({ + stackManagedRoot: managedRoot, + candidateRelPath: staged.candidateRelPath, + appliedRelPath: completeProjectManifest?.generation.appliedDir ?? null, + ownsManagedRoot: !rootPreexisted, + }); + GitOpsTransitions.getInstance().createFailed( + gitopsApplicationId, + e instanceof GitSourceError ? e.code : 'create', + { operationId: gitopsOperationId, actor: 'system:git-source', trigger: 'create', at: Date.now() }, + ); + } catch (cleanupErr) { + console.error( + `[GitSource] Could not finish tearing down the failed create for ${sanitizeForLog(input.stackName)}; leaving it for the next boot sweep:`, + cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr), + ); + } + } throw e; } }); } + /** + * Remove what a create staged before any GitOps row existed. + * + * Best-effort by design: the caller is already throwing the real error, and + * a leftover directory here is picked up by the boot sweep, which has the + * marker to tell it what this operation owned. + */ + private async cleanupStagedCreate( + managedRoot: string, + candidateRelPath: string | null, + rootPreexisted: boolean, + ): Promise { + try { + await removeOperationOwnedPaths({ + stackManagedRoot: managedRoot, + candidateRelPath, + ownsManagedRoot: !rootPreexisted, + }); + } catch (error) { + console.warn( + '[GitSource] Could not remove the staged create area:', + error instanceof Error ? error.message : String(error), + ); + } + } + /** * Boot sweep for every managed-project area, under the per-stack lock: * crash-recovery restore, orphan candidates, and areas whose stack no @@ -2730,13 +3371,63 @@ export class GitSourceService { } return; } + // A managed area with no git-source row is not automatically an orphan. + // An in-flight or crashed create owns its area through a checkpoint or + // a creating application before the source row exists, so both count as + // known and must survive the sweep. + for (const checkpoint of GitOpsStore.getInstance().listCreateCheckpoints()) { + known.add(checkpoint.stack_name); + } + for (const app of GitOpsStore.getInstance().listCreatingDirectApplications()) { + if (app.stack_name) known.add(app.stack_name); + } + for (const entry of entries) { if (!entry.isDirectory() || known.has(entry.name)) continue; - if (entry.name.startsWith('.detach-') && known.has(entry.name.slice('.detach-'.length))) continue; + // Detach staging areas carry their own ownership proof in the + // detach journal, not a create marker. They are reaped exactly as + // before once their stack is gone. + if (entry.name.startsWith('.detach-')) { + if (known.has(entry.name.slice('.detach-'.length))) continue; + try { + await fsPromises.rm(path.join(managedRoot, entry.name), { recursive: true, force: true }); + } catch (e) { + console.error(`[GitManifest] could not remove staged detach area ${sanitizeForLog(entry.name)}:`, (e as Error).message); + } + continue; + } + // Nothing in the database claims this directory: no git-source row, + // no create checkpoint, no creating application. What happens next + // turns on the staging marker, and the distinction between its two + // failure states is the whole rule. + // + // valid an in-flight create owns this area. Remove only what + // that operation staged. + // missing nothing ever claimed it. This is the ordinary orphan a + // crashed stack deletion leaves behind, and reaping it is + // the long-standing behavior that keeps managed data from + // outliving its stack. + // corrupt something claimed it and we cannot read the claim. + // Preserve: an unexplained directory is far cheaper than a + // wrongly deleted generation. + const area = path.join(managedRoot, entry.name); try { - await fsPromises.rm(path.join(managedRoot, entry.name), { recursive: true, force: true }); + const marker = await readStagingMarker(area); + if (marker.state === 'corrupt') { + console.warn( + `[GitManifest] preserving unclaimed managed area ${sanitizeForLog(entry.name)}: its staging marker is unreadable (${marker.reason}), so ownership cannot be established`, + ); + continue; + } + if (marker.state === 'missing') { + await fsPromises.rm(area, { recursive: true, force: true }); + console.log(`[GitManifest] removed orphaned managed area ${sanitizeForLog(entry.name)}: no stack, no create, no marker claims it`); + continue; + } + const outcome = await cleanupUnclaimedManagedRoot(area, marker.marker); + console.log(`[GitManifest] unclaimed managed area ${sanitizeForLog(entry.name)}: ${outcome}`); } catch (e) { - console.error(`[GitManifest] could not remove orphaned area ${sanitizeForLog(entry.name)}:`, (e as Error).message); + console.error(`[GitManifest] could not clean unclaimed area ${sanitizeForLog(entry.name)}:`, (e as Error).message); } } } diff --git a/backend/src/services/HealthGateService.ts b/backend/src/services/HealthGateService.ts index a5c413df..291f99a9 100644 --- a/backend/src/services/HealthGateService.ts +++ b/backend/src/services/HealthGateService.ts @@ -9,6 +9,8 @@ import { isCleanOneShotCompletion } from '../utils/oneShotCompletion'; import { declaredFromEffectiveModel } from '../helpers/effectiveToDeclaredCompose'; import { parseEffectiveModel } from './preflight/effectiveModel'; import type { HealthGateContainer, HealthGateReport } from './updateGuard/types'; +import { GitOpsStore } from './gitops/store'; +import { GitOpsTransitions } from './gitops/transitions'; import { ComposeService, getComposeCommandTimeoutMs } from './ComposeService'; const POLL_INTERVAL_MS = 5_000; @@ -79,7 +81,7 @@ interface ActiveGate { /** 'stack' for the legacy post-mutation gate, 'service' for prepared gates. */ targetScope: 'stack' | 'service'; /** Named trigger persisted on the row. */ - trigger: 'update' | 'deploy' | 'service_update' | 'service_restore'; + trigger: 'update' | 'deploy' | 'service_update' | 'service_restore' | 'recovery'; /** Service gates only. */ serviceName: string | null; /** Single image id every primary replica must converge on (service gates). */ @@ -224,18 +226,40 @@ export class HealthGateService { return HealthGateService.instance; } - /** Sweep runs left observing by a previous process, then accept begin() calls. */ + /** + * Sweep runs left observing by a previous process, then accept begin() calls. + * + * Each row is finalized on its own rather than swept with one UPDATE, so the + * revision state hears a verdict for every one of them. A reserved recovery + * run is finalized here like any other: a reservation is only ever armed by + * the process that made it, so one that survived a restart has no timer and + * nothing left to observe. + */ public start(): void { this.started = true; + let interrupted: HealthGateRunRow[]; try { - const swept = DatabaseService.getInstance().markInterruptedHealthGateRuns( - 'Sencho restarted during observation', Date.now(), - ); - if (swept > 0) { - console.log(`[HealthGate] Marked ${swept} interrupted observation(s) as unknown`); - } + interrupted = DatabaseService.getInstance().listObservingHealthGateRuns(); } catch (error) { console.error('[HealthGate] Startup sweep failed:', getErrorMessage(error, 'unknown')); + return; + } + let finalized = 0; + for (const run of interrupted) { + // Per row, so one unreadable row cannot leave every later one observing + // for ever. + try { + this.finalizePersistedRun(run, 'Sencho restarted during observation'); + finalized++; + } catch (error) { + console.error( + '[HealthGate] Could not finalize interrupted run %s for %s:', + run.id, sanitizeForLog(run.stack_name), getErrorMessage(error, 'unknown'), + ); + } + } + if (finalized > 0) { + console.log(`[HealthGate] Marked ${finalized} interrupted observation(s) as unknown`); } } @@ -303,11 +327,22 @@ export class HealthGateService { * every gated update path gets the timeline marker even when the gate * itself is disabled. */ + /** + * Start a stack-scoped health observation. + * + * `binding.deployedGenerationId` is the generation the mutation that preceded + * this call actually deployed, or null when there was none. It is a required + * argument rather than something this method reads from current state, + * because a verdict is only meaningful for the generation the run watched: + * reading it later could bind a run to whatever happens to be deployed by + * then, and a pass would then promote a generation this run never observed. + */ public beginStack( nodeId: number, stackName: string, trigger: 'update' | 'deploy', actor: string | null, + binding: { deployedGenerationId: string | null }, ): string | null { // Refuses work outside the start()/stop() lifecycle so a late call during // shutdown cannot leave a dangling poll timer. @@ -343,6 +378,7 @@ export class HealthGateService { target_scope: 'stack', service_name: null, failure_source: null, + deployed_generation_id: binding.deployedGenerationId, }; if (this.active.size >= MAX_CONCURRENT_GATES) { @@ -386,14 +422,136 @@ export class HealthGateService { } } + /** + * Claim a health run for a proven, bound recovery, inside the caller's open + * transaction. + * + * Writes the row and links it to the recovery generation, and touches nothing + * in memory. Committing the reservation alongside the recovery is the point: + * a crash between the two would otherwise leave a restored workload that no + * run was ever recorded against. Arming the timer is the caller's separate + * step after its transaction commits. + * + * Idempotent on the recovery generation's `health_gate_id`, so a replayed + * recovery reuses its run rather than opening a second one. + */ + public reserveRecoveryRun(args: { + recoveryRef: string; + nodeId: number; + stackName: string; + deployedGenerationId: string; + actor: string | null; + }): { outcome: 'reserved' | 'replayed' | 'disabled'; runId: string | null } { + const db = DatabaseService.getInstance(); + if (!this.readSettings().enabled) return { outcome: 'disabled', runId: null }; + + const linked = db.getStackUpdateRecoveryGeneration(args.recoveryRef)?.health_gate_id ?? null; + if (linked) return { outcome: 'replayed', runId: linked }; + + const runId = randomUUID(); + db.insertHealthGateRun({ + id: runId, + node_id: args.nodeId, + stack_name: args.stackName, + trigger_action: 'recovery', + status: 'observing', + reason: null, + window_seconds: this.readSettings().windowSeconds, + containers_json: '[]', + started_at: Date.now(), + ended_at: null, + created_by: args.actor, + target_scope: 'stack', + service_name: null, + failure_source: null, + deployed_generation_id: args.deployedGenerationId, + }); + db.updateStackUpdateRecoveryGeneration(args.recoveryRef, { health_gate_id: runId }); + return { outcome: 'reserved', runId }; + } + + /** + * Start observing a run that was reserved in a committed transaction. + * + * Inserts nothing: the row already exists, and creating a second one would + * give the same recovery two verdicts. Throws on anything unexpected so the + * caller can finalize the reservation unknown rather than leave an observing + * row with no timer behind it. + * + * Same-process only. A reservation that outlived its process is finalized by + * `start`, never armed here. + */ + public armReservedRun(runId: string, nodeId: number, stackName: string): void { + if (!this.started) throw new Error('health gate service is not started'); + + const run = DatabaseService.getInstance().getHealthGateRun(nodeId, stackName, runId); + if (!run) throw new Error(`reserved health run ${runId} was not found`); + if (run.status !== 'observing' || run.trigger_action !== 'recovery' || run.target_scope !== 'stack') { + throw new Error(`health run ${runId} is not a reserved stack recovery observation`); + } + + const key = this.gateKey(nodeId, stackName, 'stack', null); + if (this.active.get(key)?.runId === runId) return; + this.supersedeGatesForStack(nodeId, stackName); + if (this.active.size >= MAX_CONCURRENT_GATES) { + throw new Error('too many concurrent observations'); + } + + const gate: ActiveGate = { + runId, + nodeId, + stackName, + windowSeconds: run.window_seconds, + startedAt: run.started_at, + timer: null, + expected: null, + consecutivePollErrors: 0, + missingLastPoll: new Set(), + restartingLastPoll: new Set(), + finalized: false, + targetScope: 'stack', + trigger: 'recovery', + serviceName: null, + expectedImageId: null, + expectedReplicas: 0, + collateralEligibleNames: new Set(), + collateralBaselineByName: new Map(), + roleByName: new Map(), + declaredRestartByService: null, + }; + this.active.set(key, gate); + this.scheduleNextPoll(gate); + } + + /** + * Write off a reservation this process could not arm. + * + * Public because the reservation is made inside the recovery transaction and + * armed after it commits, so the window where arming can fail belongs to the + * caller, not to this service. + */ + public abandonReservedRun(runId: string, nodeId: number, stackName: string, reason: string): void { + try { + const run = DatabaseService.getInstance().getHealthGateRun(nodeId, stackName, runId); + if (!run || run.status !== 'observing') return; + this.finalizePersistedRun(run, reason); + } catch (error) { + console.error( + '[HealthGate] Could not finalize unarmed reservation %s for %s:', + runId, sanitizeForLog(stackName), getErrorMessage(error, 'unknown'), + ); + } + } + /** @deprecated Prefer beginStack; retained as a one-PR alias for callers under migration. */ public begin( nodeId: number, stackName: string, trigger: 'update' | 'deploy', actor: string | null, + binding: { deployedGenerationId: string | null }, ): string | null { - return this.beginStack(nodeId, stackName, trigger, actor); + return this.beginStack(nodeId, stackName, trigger, actor, binding); } /** @@ -1018,6 +1176,60 @@ export class HealthGateService { }); } + /** + * Hand a finalized verdict to the GitOps state model. + * + * The generation is read back from the persisted run row rather than taken + * from memory, so the verdict is attributed to what this run was recorded as + * observing. The transition decides whether that is still promotable; this + * method only reports. + * + * Never throws: a health gate is an observer, and a bookkeeping failure must + * not change the verdict that was just written. + */ + private recordGitOpsHealthVerdict( + nodeId: number, + stackName: string, + runId: string, + status: 'passed' | 'failed' | 'unknown', + ): void { + try { + const run = DatabaseService.getInstance().getHealthGateRun(nodeId, stackName, runId); + if (!run) return; + const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName); + if (!app || app.lifecycle_status !== 'active') return; + if (!GitOpsStore.getInstance().getTarget(app.id, nodeId)) return; + GitOpsTransitions.getInstance().healthFinalized({ + applicationId: app.id, + nodeId, + healthRunId: runId, + healthStatus: status, + deployedGenerationId: run.deployed_generation_id ?? null, + targetScope: run.target_scope, + envelope: { operationId: runId, actor: 'system:health-gate', trigger: 'health', at: Date.now() }, + }); + } catch (error) { + console.error( + '[GitOps] Could not record the health verdict for %s:', + sanitizeForLog(stackName), getErrorMessage(error, 'unknown'), + ); + } + } + + /** + * Write an unknown verdict for a run this process is not observing. + * + * Covers a row a previous process left behind and a reservation this process + * could not arm. Both are the same situation: an observing row with no timer + * behind it, which would otherwise sit unresolved for ever. + */ + private finalizePersistedRun(run: HealthGateRunRow, reason: string): void { + DatabaseService.getInstance().finalizeHealthGateRun( + run.id, 'unknown', reason, Date.now(), run.containers_json ?? '[]', null, + ); + this.recordGitOpsHealthVerdict(run.node_id, run.stack_name, run.id, 'unknown'); + } + private finalize( gate: ActiveGate, status: 'passed' | 'failed' | 'unknown', @@ -1047,6 +1259,7 @@ export class HealthGateService { DatabaseService.getInstance().finalizeHealthGateRun( gate.runId, status, reason, Date.now(), JSON.stringify(containers), failureSource, ); + this.recordGitOpsHealthVerdict(gate.nodeId, gate.stackName, gate.runId, status); } catch (error) { // The verdict is lost from the DB (the startup sweep will later rewrite // the row as unknown), so log everything needed to reconstruct it. diff --git a/backend/src/services/SchedulerService.ts b/backend/src/services/SchedulerService.ts index a412a7b0..0d42651a 100644 --- a/backend/src/services/SchedulerService.ts +++ b/backend/src/services/SchedulerService.ts @@ -1342,8 +1342,8 @@ export class SchedulerService { // Health observation starts immediately after Compose; registry recheck is // isolated so a verification failure cannot turn Compose success into a failure. - const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:scheduler'); const orchResult = lock.result; + const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:scheduler', { deployedGenerationId: orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.deployedGenerationId : null }); const recoveryId = orchResult && orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null; if (recoveryId) { const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); diff --git a/backend/src/services/StackUpdateOrchestrator.ts b/backend/src/services/StackUpdateOrchestrator.ts index 0e9af3eb..f753833e 100644 --- a/backend/src/services/StackUpdateOrchestrator.ts +++ b/backend/src/services/StackUpdateOrchestrator.ts @@ -47,7 +47,7 @@ export interface ServiceUpdateOptions { } export type OrchestratorResult = - | { kind: 'stack_compose_done'; recoveryId: string | null } + | { kind: 'stack_compose_done'; recoveryId: string | null; deployedGenerationId: string | null } | { kind: 'service_done'; serviceName: string; @@ -186,7 +186,11 @@ export class StackUpdateOrchestrator { ctx.stackName, options.terminalWs ?? undefined, options.atomic, ); const recoveryId = updateResult?.recoveryId ?? null; - return { kind: 'stack_compose_done', recoveryId }; + return { + kind: 'stack_compose_done', + recoveryId, + deployedGenerationId: updateResult?.deployedGenerationId ?? null, + }; } private async executeServiceUpdate( diff --git a/backend/src/services/StackUpdateRecoveryService.ts b/backend/src/services/StackUpdateRecoveryService.ts index 015609c1..caefa528 100644 --- a/backend/src/services/StackUpdateRecoveryService.ts +++ b/backend/src/services/StackUpdateRecoveryService.ts @@ -34,10 +34,20 @@ import { type StackRecoveryServiceCapture, } from './recoveryServicesJson'; import { getComposeCommandTimeoutMs } from './ComposeService'; +import type { ComposeMutationResult } from './ComposeService'; +import { HealthGateService } from './HealthGateService'; +import type { HealthRunReservation } from './gitops/transitions'; import { assessGenerationEligibility } from './rollbackEligibility'; import { enforcePolicyForImageRefs, type PolicyEnforcementOptions } from './PolicyEnforcement'; import { describePolicyBlock } from '../helpers/policyGate'; import type { GitSourceAppliedSpec } from './DatabaseService'; +import { + captureGitOpsRecoveryBinding, + EMPTY_GITOPS_RECOVERY_CAPTURE, + type GitOpsRecoveryCapture, +} from './gitops/recoveryCapture'; +import { GitOpsStore } from './gitops/store'; +import { GitOpsTransitions } from './gitops/transitions'; import type { GitSourceManifestState } from '../types/gitProjectManifest'; import type { RollbackGenerationManifest, @@ -568,6 +578,7 @@ export class StackUpdateRecoveryService { artifacts_retired: 0, released_at: null, released_by: null, + ...this.captureGitOpsBindingOrEmpty(stackName, nodeId), }; DatabaseService.getInstance().insertStackUpdateRecoveryGeneration(row); return row; @@ -592,6 +603,29 @@ export class StackUpdateRecoveryService { } } + /** + * Read the GitOps binding for this recovery point, degrading to no binding + * if that lookup fails. + * + * These three columns are advisory: they let a later restore rebind the + * generation, artifact, and acceptance pointers instead of guessing. Rollback + * protection itself does not depend on them, so a failure here must not abort + * the capture and block the update it protects. The failure is logged rather + * than swallowed, and the degraded shape is the same one legacy rows carry. + */ + private captureGitOpsBindingOrEmpty(stackName: string, nodeId: number): GitOpsRecoveryCapture { + try { + return captureGitOpsRecoveryBinding(stackName, nodeId); + } catch (error) { + console.warn( + '[StackUpdateRecovery] Could not capture the GitOps binding for %s; recovery point stored without it: %s', + sanitizeForLog(stackName), + sanitizeForLog(getErrorMessage(error, 'unknown')), + ); + return { ...EMPTY_GITOPS_RECOVERY_CAPTURE }; + } + } + /** * Capture the live authored project as the current recovery generation. * Shares captureCandidate with deploy/update (files, holds, override), then @@ -892,12 +926,148 @@ export class StackUpdateRecoveryService { * Post-handoff compensation: restore files + pinned up, then probe before * reporting restored_current / immediate_verified. */ + /** + * Open a GitOps recovery for this restore, or nothing when the stack is not + * modelled. + * + * Returns closures rather than ids so a terminal event cannot be recorded for + * an operation that was never opened. Recording never fails the restore: the + * files and containers are the real work, and the store describes it. + * + * The proof rule is deliberately strict. Pointers move only when the recovery + * point named a generation, that generation still exists, and the manifest + * that was actually restored carries the same commit and manifest version. A + * restore we cannot tie to a generation is still a real recovery, it just has + * nothing to bind, and the transition records it as unproven rather than + * guessing. + */ + /** + * Start observing a run the recovery transaction reserved. + * + * Runs after that transaction commits, because arming is in-memory work that + * a rollback could not undo. Anything that stops the timer starting writes + * the run off immediately: an observing row with nothing behind it would + * otherwise report a restore as still being watched for ever. + */ + private armReservedRecoveryRun( + reservation: HealthRunReservation | null, + row: StackUpdateRecoveryGenerationRow, + ): void { + if (!reservation?.runId || reservation.outcome === 'disabled') return; + const gate = HealthGateService.getInstance(); + try { + gate.armReservedRun(reservation.runId, row.node_id, row.stack_name); + } catch (error) { + const reason = getErrorMessage(error, 'unknown'); + console.warn( + '[HealthGate] Could not arm the reserved recovery run %s for %s: %s', + reservation.runId, sanitizeForLog(row.stack_name), sanitizeForLog(reason), + ); + gate.abandonReservedRun(reservation.runId, row.node_id, row.stack_name, `could not arm: ${reason}`); + } + } + + private beginGitOpsRecovery(row: StackUpdateRecoveryGenerationRow): { + succeeded: ( + restored: RollbackGenerationManifest | null, + binding: 'bound' | 'unbound', + ) => HealthRunReservation | null; + failed: (failureClass: 'pre_mutation' | 'post_mutation') => void; + } | null { + // Returns what the write produced, or null when it could not be recorded. + // Recording never fails the restore: the store describes what happened, it + // does not make it happen. + const record = (what: string, write: () => T): T | null => { + try { + return write(); + } catch (error) { + console.error( + '[GitOps] Could not record recovery %s for %s (recovery %s):', + what, sanitizeForLog(row.stack_name), row.id, + error instanceof Error ? error.stack ?? error.message : String(error), + ); + return null; + } + }; + try { + const store = GitOpsStore.getInstance(); + const app = store.getLiveDirectApplication(row.stack_name); + if (!app || app.lifecycle_status !== 'active') return null; + const target = store.getTarget(app.id, row.node_id); + if (!target || target.target_status !== 'active') return null; + + const tx = GitOpsTransitions.getInstance(); + const envelope = { + operationId: row.id, + actor: 'system:recovery', + trigger: 'recovery', + at: Date.now(), + }; + const capturedGenerationId = row.gitops_generation_id ?? null; + record('start', () => tx.recoveryStarted({ + applicationId: app.id, + nodeId: row.node_id, + recoveryRef: row.id, + recoveryGenerationId: capturedGenerationId, + envelope, + })); + + return { + succeeded: (restored, binding) => record('success', () => { + const generation = capturedGenerationId + ? store.getGeneration(capturedGenerationId) + : undefined; + const proven = !!generation + && !!restored + && restored.git?.commitSha === generation.commit_sha + && restored.git?.manifestVersion === generation.manifest_version; + const result = tx.recoverySucceeded({ + applicationId: app.id, + nodeId: row.node_id, + recoveryRef: row.id, + recoveryGenerationId: capturedGenerationId, + proven, + gitopsBinding: binding, + capturedArtifactSetId: row.gitops_artifact_set_id ?? null, + capturedSourceAcceptanceRef: row.gitops_source_acceptance_ref ?? null, + envelope: { ...envelope, at: Date.now() }, + // Claimed inside the recovery transaction so the run and the + // pointers it describes commit together. Arming it is a separate + // step once that transaction has landed. + reserveHealthRun: (deployedGenerationId) => HealthGateService.getInstance().reserveRecoveryRun({ + recoveryRef: row.id, + nodeId: row.node_id, + stackName: row.stack_name, + deployedGenerationId, + actor: envelope.actor, + }), + }); + return result.healthReservation; + }), + failed: (failureClass) => record('failure', () => tx.recoveryFailed({ + applicationId: app.id, + nodeId: row.node_id, + recoveryRef: row.id, + failureClass, + envelope: { ...envelope, at: Date.now() }, + })), + }; + } catch (error) { + console.error( + '[GitOps] Could not open a recovery for %s:', + sanitizeForLog(row.stack_name), + error instanceof Error ? error.stack ?? error.message : String(error), + ); + return null; + } + } + public async compensateWithCandidate( generationId: string, composeUp: ( overridePath: string, invocation: RollbackInvocationRecord | null, - ) => Promise, + ) => Promise, policyOptions?: PolicyEnforcementOptions, ): Promise { const row = this.get(generationId); @@ -931,6 +1101,11 @@ export class StackUpdateRecoveryService { const generationContentPath = expectsGenerationContent(row) && row.content_path ? row.content_path : null; + // Open the recovery in the model before any file moves, so a crash mid + // restore leaves a target that says what it was doing rather than one that + // merely looks broken. + const gitopsRecovery = this.beginGitOpsRecovery(row); + let filesRestored = false; try { // Eligibility (integrity + held images + security posture) before mutation. @@ -985,7 +1160,10 @@ export class StackUpdateRecoveryService { if (!row.override_path) { throw new Error('Recovery generation has no override path'); } - await composeUp(row.override_path, restoredInvocation); + // Only a callback that reports a Compose mutation licenses the deployed + // pointer. A restore driven some other way resolves the same, and binding + // on that would claim a workload nobody launched. + const composeResult = await composeUp(row.override_path, restoredInvocation); const probeOk = await this.probeRecoveredStack( row.node_id, row.stack_name, @@ -1023,9 +1201,17 @@ export class StackUpdateRecoveryService { is_current: 1, artifact_expires_at: null, }); + const reservation = gitopsRecovery?.succeeded( + restoredManifest ?? null, + composeResult?.mutatedByCompose ? 'bound' : 'unbound', + ) ?? null; + this.armReservedRecoveryRun(reservation, row); return true; } catch (error) { const code = (error as { code?: string }).code; + // Classified by whether the files had already moved. Only a failure + // before that leaves the previous workload provably intact. + gitopsRecovery?.failed(filesRestored ? 'post_mutation' : 'pre_mutation'); if (filesRestored && generationContentPath) { try { await RollbackGenerationStore.reconcileInterruptedRestore( diff --git a/backend/src/services/WebhookService.ts b/backend/src/services/WebhookService.ts index 8f310de5..43c0ed56 100644 --- a/backend/src/services/WebhookService.ts +++ b/backend/src/services/WebhookService.ts @@ -163,7 +163,7 @@ export class WebhookService { atomic, { source: 'webhook', actor: 'system:webhook' }, ); - const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'deploy', 'system:webhook'); + const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'deploy', 'system:webhook', { deployedGenerationId: deployResult.deployedGenerationId }); if (deployResult.recoveryId) { const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); StackUpdateRecoveryService.getInstance().linkGateOrRetain(deployResult.recoveryId, healthGateId); @@ -189,7 +189,7 @@ export class WebhookService { { nodeId, stackName, target: { scope: 'stack' }, trigger: 'webhook', actor: 'system:webhook' }, { atomic: atomic ?? false, terminalWs: null }, ); - const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:webhook'); + const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:webhook', { deployedGenerationId: orchResult.kind === 'stack_compose_done' ? orchResult.deployedGenerationId : null }); const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null; if (recoveryId) { const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService'); diff --git a/backend/src/services/blueprintPreviewProjection.ts b/backend/src/services/blueprintPreviewProjection.ts index d4991406..fb308c1b 100644 --- a/backend/src/services/blueprintPreviewProjection.ts +++ b/backend/src/services/blueprintPreviewProjection.ts @@ -337,6 +337,14 @@ function projectActions( seen.add(node.id); }; + // A severed canonical target is invisible to every automatic action. The + // decision arrays already refuse it, and marking it seen here keeps the + // deployment-status fallbacks below from resurrecting a retry behind + // the model's back. + for (const nodeId of decision.severedNodeIds) { + seen.add(nodeId); + } + // Status-precedence pass: in-flight, name conflict, and clear_* must win over // decision.withdraw / evictBlocked so Confirm never authorizes mid-flight mutates // and never-deployed guards stay on the remove-only clear_stale path. diff --git a/backend/src/services/gitops/blueprintDeploymentProducers.ts b/backend/src/services/gitops/blueprintDeploymentProducers.ts new file mode 100644 index 00000000..0a782c91 --- /dev/null +++ b/backend/src/services/gitops/blueprintDeploymentProducers.ts @@ -0,0 +1,245 @@ +/** + * Blueprint deployment-state writes, recorded by what caused them. + * + * Every production write to a Blueprint deployment row comes through here, so + * the revision state hears one event per real change rather than one per call. + * The cause is passed in rather than inferred from the resulting status, + * because several causes land on the same status: a deploy that failed and a + * withdraw that failed both read `failed`, and telling them apart afterwards is + * impossible. + * + * Preview cleanup deliberately does not come through here. It reverses a + * projection nobody deployed, so recording it would report removals that never + * happened. + */ +import { DatabaseService, type BlueprintDeployment } from '../DatabaseService'; +import { GitOpsStore, emptyTargetRow } from './store'; +import { GitOpsTransitions, GitOpsTransitionError } from './transitions'; +import { envelopeFor, recordableApplication } from './blueprintProducers'; + +/** Why a deployment row moved. */ +export type BlueprintDeploymentCause = + | 'deploy_start' + | 'deploy_ack' + | 'deploy_fail' + | 'name_conflict' + | 'withdraw_start' + | 'withdraw_success' + | 'withdraw_fail' + | 'withdraw_name_conflict' + | 'await_state_review' + | 'await_evict_confirm' + | 'drift_observed' + | 'drift_enforce_start'; + +/** Causes that only observe, and must never acknowledge or mint anything. */ +const OBSERVATION_STAGE = { + await_state_review: 'blueprint_state_review', + await_evict_confirm: 'blueprint_evict_blocked', + drift_observed: 'blueprint_drifted', + drift_enforce_start: 'blueprint_correcting', +} as const; + +type ObservationCause = keyof typeof OBSERVATION_STAGE; + +/** + * Narrows to the observation causes, so the stage lookup below reads as a fact + * the compiler derives rather than one an assertion claims. + */ +function isObservation(cause: BlueprintDeploymentCause): cause is ObservationCause { + return cause in OBSERVATION_STAGE; +} + +type DeploymentFields = Omit[0], 'blueprint_id' | 'node_id'>; + +/** + * Write a deployment row and record what caused it. + * + * The write happens either way. Recording is skipped when the effective status + * did not move, so a reconciler tick that re-asserts a state it already + * reported does not append a second event describing the same fact. + */ +export function commitBlueprintDeploymentCause( + cause: BlueprintDeploymentCause, + blueprintId: number, + nodeId: number, + fields: DeploymentFields, + actor: string | null, +): BlueprintDeployment { + const db = DatabaseService.getInstance(); + + return db.getDb().transaction(() => { + const previous = db.getDeployment(blueprintId, nodeId); + const deployment = db.upsertDeployment({ blueprint_id: blueprintId, node_id: nodeId, ...fields }); + const statusMoved = previous?.status !== deployment.status; + + try { + record(cause, blueprintId, nodeId, statusMoved, actor); + } catch (error) { + // The deployment happened whatever the record says. Failing the write + // here would turn a bookkeeping problem into a stuck rollout. + // + // A rejection is louder than an infrastructure error on purpose: it means + // the model refused this as invalid, and a target that keeps refusing + // holds its active slot and stops recording anything further. + const rejected = error instanceof GitOpsTransitionError; + console.error( + '[GitOps] %s recording blueprint %s for blueprint %d on node %d:', + rejected ? 'Rejected' : 'Could not record', cause, blueprintId, nodeId, + error instanceof Error ? error.stack ?? error.message : String(error), + ); + } + return deployment; + })(); +} + +function record( + cause: BlueprintDeploymentCause, + blueprintId: number, + nodeId: number, + statusMoved: boolean, + actor: string | null, +): void { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + const app = store.getLiveBlueprintApplication(blueprintId); + // A Blueprint that predates the model has nothing to record against. + if (!recordableApplication(app)) return; + + const envelope = envelopeFor(actor, `blueprint_${cause}`); + + if (isObservation(cause)) { + // Observations are the only causes the status guard applies to. A start + // writes the identity terminals are matched against, so suppressing one + // because the row already read `deploying` would let a later + // acknowledgement answer a request that had been superseded. + if (!statusMoved) return; + // A stateful first placement is held for review before anything deploys, + // so there is no target yet and nothing to observe against. Creating it + // here is the same first-contact write `deploy_start` does below: the + // node has been asked to hold this Blueprint, which is exactly what the + // observation is about. Any other cause arriving without a target is + // dropped, which is also what happens to a drift or evict report for a + // Blueprint that migration brought in: migration records the application, + // its intent and its candidate, but no targets, so a fleet that predates + // this model reports nothing here until its next deploy creates one. + const firstPlacement = !store.getTarget(app.id, nodeId); + if (firstPlacement && cause !== 'await_state_review') return; + const stage = OBSERVATION_STAGE[cause]; + // Both writes in one transaction so they succeed or fail together. The + // observation refuses a tombstoned target, and it runs in its own + // savepoint, so creating the target outside this would leave an active + // target with no generation, no stage and no history behind a refusal: a + // placement relationship the model never established, which the delete + // path would later tombstone as if it were real. + DatabaseService.getInstance().getDb().transaction(() => { + if (firstPlacement) store.upsertTarget(emptyTargetRow(app.id, nodeId, envelope.at)); + tx.blueprintObservation({ applicationId: app.id, nodeId, stage, envelope }); + })(); + return; + } + + if (cause === 'deploy_start') { + // First deploy to this node: the target is created here, because a + // Blueprint application has no targets until something is sent somewhere. + if (!store.getTarget(app.id, nodeId)) { + store.upsertTarget(emptyTargetRow(app.id, nodeId, envelope.at)); + } + if (!app.intent_revision_id) return; + tx.blueprintDeployStarted({ + applicationId: app.id, + nodeId, + intentRevisionId: app.intent_revision_id, + rolloutCandidateId: app.rollout_candidate_id, + envelope, + }); + return; + } + + const target = store.getTarget(app.id, nodeId); + if (!target) return; + + // Terminals answer the request the target says it was given, not whatever the + // Blueprint currently wants. An ack matched against the current intent would + // accept work for a revision this node was never sent. + const requested = target.active_operation_stage !== null + ? target.active_intent_revision_id + : target.interruption_intent_revision_id; + + switch (cause) { + case 'deploy_ack': + if (!requested) return; + tx.blueprintAckRecorded({ + applicationId: app.id, + nodeId, + intentRevisionId: requested, + rolloutCandidateId: target.active_operation_stage !== null + ? target.active_rollout_candidate_id + : target.interruption_rollout_candidate_id, + legacyAppliedRevision: null, + envelope, + }); + return; + case 'deploy_fail': + case 'name_conflict': + tx.blueprintDeployFailed({ + applicationId: app.id, + nodeId, + failureClass: cause === 'name_conflict' ? 'name_conflict' : 'post_mutation', + envelope, + }); + return; + case 'withdraw_start': + if (!target.intent_revision_id) return; + tx.blueprintWithdrawStarted({ + applicationId: app.id, + nodeId, + // The intent being removed is the one this node acknowledged, never a + // later replacement. + intentRevisionId: target.intent_revision_id, + envelope, + }); + return; + case 'withdraw_success': + if (!requested) return; + tx.blueprintWithdrawn({ applicationId: app.id, nodeId, intentRevisionId: requested, envelope }); + return; + case 'withdraw_fail': + case 'withdraw_name_conflict': + tx.blueprintWithdrawFailed({ + applicationId: app.id, + nodeId, + failureClass: cause === 'withdraw_name_conflict' ? 'name_conflict' : 'post_mutation', + envelope, + }); + return; + } +} + +/** + * Record a withdraw that removed the deployment row entirely. + * + * Split from the cause above because the row is deleted rather than updated, so + * there is no status to compare. + */ +export function commitBlueprintDeploymentRemoved( + blueprintId: number, + nodeId: number, + actor: string | null, +): void { + const db = DatabaseService.getInstance(); + db.getDb().transaction(() => { + const existed = db.getDeployment(blueprintId, nodeId) !== undefined; + db.deleteDeployment(blueprintId, nodeId); + if (!existed) return; + try { + record('withdraw_success', blueprintId, nodeId, true, actor); + } catch (error) { + console.error( + '[GitOps] Could not record blueprint withdrawal for blueprint %d on node %d:', + blueprintId, nodeId, + error instanceof Error ? error.stack ?? error.message : String(error), + ); + } + })(); +} diff --git a/backend/src/services/gitops/blueprintProducers.ts b/backend/src/services/gitops/blueprintProducers.ts new file mode 100644 index 00000000..d5c67c7a --- /dev/null +++ b/backend/src/services/gitops/blueprintProducers.ts @@ -0,0 +1,401 @@ +/** + * The Blueprint operations that write to the revision state. + * + * Each one wraps the Blueprint source write and its GitOps rows in a single + * transaction, so an operator never sees a Blueprint that exists with nothing + * describing what it means, or an intent for a Blueprint that failed to save. + * + * Desired node ids arrive as an argument rather than being computed here. The + * reconciler that knows how to compute them reaches this layer, so importing it + * back would close a module cycle, and a cycle in this package has already + * produced one silent defect on this branch. + */ +import { createHash, randomUUID } from 'crypto'; +import { DatabaseService, type Blueprint, type BlueprintSelector } from '../DatabaseService'; +import { GitOpsStore } from './store'; +import { GitOpsTransitions, type EventEnvelope } from './transitions'; +import type { GitOpsApplicationRow, GitOpsIntentRevisionRow, GitOpsRolloutCandidateRow } from './types'; + +/** What an operator changed, which decides whether a new intent is minted. */ +export type BlueprintChangeKind = 'operational' | 'metadata_only' | 'none'; + +function sha256(value: string): string { + return createHash('sha256').update(value).digest('hex'); +} + +export function envelopeFor(actor: string | null, trigger: string): EventEnvelope { + return { operationId: randomUUID(), actor: actor ?? 'system:blueprint', trigger, at: Date.now() }; +} + +/** + * Whether an application can be recorded against. + * + * `getLiveBlueprintApplication` answers with `active` or `creating`, because + * its other callers ask "does this Blueprint already hold the live slot", + * where a half-built row counts. The transitions that mint intents and + * candidates accept only `active` and reject anything else by throwing, and + * they run inside the caller's transaction, so a `creating` row reaching one + * would fail the operator's edit and roll back the Blueprint write with it. + * + * **No current path produces that row.** Every Blueprint-mode application is + * inserted through `blankInlineApplication`, which hardcodes `active`, and + * `creating` is reachable only in `direct` mode. So this narrowing excludes + * nothing today and is deliberately defensive: it exists because the getter's + * slot-check semantics and the transitions' stricter requirement have already + * drifted apart once, and the Git-backed `blueprint` mode is what will insert + * a Blueprint application that is still being created. Narrowing here rather + * than in the getter keeps the slot check correct for its own callers. + */ +export function recordableApplication( + app: GitOpsApplicationRow | undefined, +): app is GitOpsApplicationRow { + return !!app && app.lifecycle_status === 'active'; +} + +export type BlueprintUpdates = Parameters[1]; + +/** + * Which fields changed, in the only terms that matter here. + * + * Operational fields describe what gets deployed and where, so changing one + * makes every existing acknowledgement stale. Description and classification + * describe the Blueprint to a reader and change nothing a node runs, so they + * must not mint an intent: a fresh identity would invalidate acknowledgements + * that are still accurate. + */ +export function classifyBlueprintChange( + before: Blueprint, + updates: BlueprintUpdates, +): BlueprintChangeKind { + const changedKeys = changedBlueprintKeys(before, updates); + if (OPERATIONAL_KEYS.some((key) => changedKeys.has(key))) return 'operational'; + return changedKeys.size > 0 ? 'metadata_only' : 'none'; +} + +const OPERATIONAL_KEYS = ['name', 'compose_content', 'selector', 'drift_mode', 'enabled'] as const; + +/** + * A selector compared by value rather than by how it was written. + * + * The lists inside carry request order, so a multi-select that emits click + * order would otherwise read as a placement change and invalidate every + * acknowledgement over a reorder that selects the same nodes. + */ +function canonicalSelector(selector: BlueprintSelector): string { + return selector.type === 'nodes' + ? JSON.stringify({ type: 'nodes', ids: [...selector.ids].sort((a, b) => a - b) }) + : JSON.stringify({ + type: 'labels', + any: [...selector.any].sort(), + all: [...selector.all].sort(), + }); +} + +/** + * The keys whose submitted value actually differs from what is stored. + * + * The editor submits every field on every save, and the source layer decides + * what to invalidate from which keys are *present*. Comparing values here and + * handing that layer the untouched payload made the two disagree: a + * description edit bumped the revision and cleared the approval while this + * layer classified it as metadata and minted nothing, leaving the current + * intent describing a revision that no longer existed. + */ +function changedBlueprintKeys( + before: Blueprint, + updates: BlueprintUpdates, +): Set { + const changed = new Set(); + const differs = (key: keyof BlueprintUpdates): boolean => { + const next = updates[key]; + if (next === undefined) return false; + if (key === 'selector') { + return canonicalSelector(next as BlueprintSelector) !== canonicalSelector(before.selector); + } + if (key === 'classification_reasons') { + return JSON.stringify(next) !== JSON.stringify(before.classification_reasons); + } + return next !== before[key as keyof Blueprint]; + }; + for (const key of ['name', 'compose_content', 'selector', 'drift_mode', 'enabled', 'description', 'classification', 'classification_reasons'] as const) { + if (differs(key)) changed.add(key); + } + return changed; +} + +/** Only the keys that changed, so presence and difference mean the same thing. */ +function prunedUpdates(before: Blueprint, updates: BlueprintUpdates): BlueprintUpdates { + const changedKeys = changedBlueprintKeys(before, updates); + const pruned: BlueprintUpdates = {}; + for (const key of changedKeys) { + Object.assign(pruned, { [key]: updates[key] }); + } + // The revision bump rides on the compose content. Pruned out, it would still + // advance a revision no intent describes. + if (changedKeys.has('compose_content') && updates.bumpRevision) pruned.bumpRevision = true; + return pruned; +} + +export function intentRowFor( + applicationId: string, + blueprint: Blueprint, + operationId: string, + actor: string | null, + at: number, +): GitOpsIntentRevisionRow { + return { + id: randomUUID(), + application_id: applicationId, + blueprint_id: blueprint.id, + compose_content_sha256: sha256(blueprint.compose_content), + blueprint_revision: blueprint.revision, + deploy_stack_name: blueprint.name, + selector_json: JSON.stringify(blueprint.selector), + pinned_node_id: blueprint.pinned_node_id, + cordon_implications_json: JSON.stringify({ pinnedOverridesCordon: blueprint.pinned_node_id !== null }), + rollout_strategy_json: JSON.stringify({ driftMode: blueprint.drift_mode, enabled: blueprint.enabled }), + runtime_drift_policy: blueprint.drift_mode, + stateful_policy_json: null, + health_failure_rollback_policy_json: null, + operation_id: operationId, + actor, + created_at: at, + }; +} + +export function candidateRowFor( + applicationId: string, + intent: GitOpsIntentRevisionRow, + desiredNodeIds: number[], + provenance: GitOpsRolloutCandidateRow['provenance'], + operationId: string, + at: number, +): GitOpsRolloutCandidateRow { + return { + id: randomUUID(), + application_id: applicationId, + intent_revision_id: intent.id, + compose_content_sha256: intent.compose_content_sha256, + accepted_generation_id: null, + artifact_set_id: null, + // Canonical: the required set is compared across revisions, so an order + // change must not read as a placement change. + required_targets_json: JSON.stringify({ nodeIds: [...desiredNodeIds].sort((a, b) => a - b) }), + authoritative: 1, + provenance, + operation_id: operationId, + created_at: at, + }; +} + +/** + * Record a new Blueprint: the source row, its application, and the first + * intent and candidate describing what it currently asks for. + */ +export function commitBlueprintCreate( + input: Parameters[0], + desiredNodeIdsFor: (blueprint: Blueprint) => number[], +): Blueprint { + const db = DatabaseService.getInstance(); + const tx = GitOpsTransitions.getInstance(); + + return db.getDb().transaction(() => { + const blueprint = db.createBlueprint(input); + const envelope = envelopeFor(input.created_by, 'blueprint_create'); + const applicationId = randomUUID(); + + tx.activateInlineBlueprint({ + application: blankInlineApplication(applicationId, blueprint.id, envelope.at), + envelope, + }); + + const intent = intentRowFor(applicationId, blueprint, envelope.operationId, input.created_by, envelope.at); + tx.intentRevised({ applicationId, intent, envelope }); + tx.rolloutCandidateOpened({ + applicationId, + candidate: candidateRowFor( + applicationId, intent, desiredNodeIdsFor(blueprint), 'intent_change', envelope.operationId, envelope.at, + ), + envelope, + }); + return blueprint; + })(); +} + +/** + * Record an edit to a Blueprint. + * + * A change that alters nothing writes nothing at all, and a change that only + * alters how the Blueprint reads updates the source row alone. Only an + * operational change mints a new intent, because only that makes what the + * fleet already acknowledged out of date. + */ +export function commitBlueprintUpdate( + blueprintId: number, + updates: BlueprintUpdates, + actor: string | null, + desiredNodeIdsFor: (blueprint: Blueprint) => number[], +): { blueprint: Blueprint | undefined; change: BlueprintChangeKind } { + const db = DatabaseService.getInstance(); + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + + return db.getDb().transaction(() => { + const before = db.getBlueprint(blueprintId); + if (!before) return { blueprint: undefined, change: 'none' as BlueprintChangeKind }; + + const change = classifyBlueprintChange(before, updates); + if (change === 'none') return { blueprint: before, change }; + + const blueprint = db.updateBlueprint(blueprintId, prunedUpdates(before, updates)); + if (!blueprint || change === 'metadata_only') return { blueprint, change }; + + const app = store.getLiveBlueprintApplication(blueprintId); + // A Blueprint that predates the model has no application yet. Migration + // brings it in; inventing one here would claim a first intent for a + // Blueprint whose deployments nobody has reconciled. + if (!recordableApplication(app)) return { blueprint, change }; + + const envelope = envelopeFor(actor, 'blueprint_update'); + const intent = intentRowFor(app.id, blueprint, envelope.operationId, actor, envelope.at); + tx.intentRevised({ applicationId: app.id, intent, envelope }); + tx.rolloutCandidateOpened({ + applicationId: app.id, + candidate: candidateRowFor( + app.id, intent, desiredNodeIdsFor(blueprint), 'intent_change', envelope.operationId, envelope.at, + ), + envelope, + }); + return { blueprint, change }; + })(); +} + +/** + * Record a pin change. + * + * Pinning moves where a Blueprint is allowed to run, so it revises placement + * the same way a selector edit does. Re-pinning to the node already pinned + * changes nothing and writes nothing. + */ +export function commitBlueprintPin( + blueprintId: number, + nodeId: number | null, + actor: string | null, + desiredNodeIdsFor: (blueprint: Blueprint) => number[], +): { blueprint: Blueprint | undefined; changed: boolean } { + const db = DatabaseService.getInstance(); + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + + return db.getDb().transaction(() => { + const before = db.getBlueprint(blueprintId); + if (!before) return { blueprint: undefined, changed: false }; + if (before.pinned_node_id === nodeId) return { blueprint: before, changed: false }; + + const blueprint = db.setBlueprintPinnedNode(blueprintId, nodeId); + if (!blueprint) return { blueprint: undefined, changed: false }; + + const app = store.getLiveBlueprintApplication(blueprintId); + if (!recordableApplication(app)) return { blueprint, changed: true }; + + const envelope = envelopeFor(actor, 'blueprint_pin'); + const intent = intentRowFor(app.id, blueprint, envelope.operationId, actor, envelope.at); + tx.intentRevised({ applicationId: app.id, intent, envelope }); + tx.rolloutCandidateOpened({ + applicationId: app.id, + candidate: candidateRowFor( + app.id, intent, desiredNodeIdsFor(blueprint), 'roster_change', envelope.operationId, envelope.at, + ), + envelope, + }); + return { blueprint, changed: true }; + })(); +} + +/** + * Retire a Blueprint's application after its deployments have been withdrawn. + * + * Tombstones only. A deleted Blueprint must stop claiming its live-application + * slot, or the name cannot be used again, but nothing here withdraws anything: + * the caller has already done that, and doing it twice would report removals + * that never happened. + */ +export function commitBlueprintDelete(blueprintId: number, actor: string | null): boolean { + const db = DatabaseService.getInstance(); + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + + return db.getDb().transaction(() => { + const app = store.getLiveBlueprintApplication(blueprintId); + const removed = db.deleteBlueprint(blueprintId); + if (!removed || !app) return removed; + + const envelope = envelopeFor(actor, 'blueprint_delete'); + for (const target of store.listTargets(app.id)) { + if (target.target_status !== 'active') continue; + tx.targetTombstoned(app.id, target.node_id, envelope); + } + tx.applicationTombstoned(app.id, 'deleted', envelope); + return removed; + })(); +} + +/** A Blueprint application before anything has been asked of it. */ +export function blankInlineApplication(id: string, blueprintId: number, at: number) { + return { + id, + lifecycle_key: `blueprint:${blueprintId}`, + lifecycle_status: 'active' as const, + target_mode: 'inline_blueprint' as const, + stack_name: null, + blueprint_id: blueprintId, + configured_repo_url: null, + repo_identity_json: null, + configured_ref: null, + compose_paths_json: null, + context_dir: null, + sync_env: 0, + env_path: null, + materialization_fingerprint: null, + desired_commit_sha: null, + fetched_commit_sha: null, + candidate_generation_id: null, + accepted_generation_id: null, + candidate_plan_blocked: 0, + review_required: 0, + artifact_set_id: null, + latest_artifact_set_id: null, + intent_revision_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + rollout_authorization_ref: null, + legacy_combined_approval_ref: null, + preflight_fingerprint: null, + latest_operation_id: null, + active_operation_id: null, + active_operation_stage: null, + active_operation_at: null, + active_generation_id: null, + pause_at: null, + pause_reason: null, + partial_json: null, + failure_stage: null, + failure_class: null, + failure_at: null, + retry_at: null, + retry_count: 0, + suspended_at: null, + recovery_ref: null, + recovery_phase: null, + interruption_stage: null, + interruption_at: null, + interruption_operation_id: null, + interruption_generation_id: null, + evidence_fresh_at: null, + evidence_limitations_json: null, + created_at: at, + updated_at: at, + }; +} diff --git a/backend/src/services/gitops/createCleanup.ts b/backend/src/services/gitops/createCleanup.ts new file mode 100644 index 00000000..29abddf0 --- /dev/null +++ b/backend/src/services/gitops/createCleanup.ts @@ -0,0 +1,171 @@ +import fs from 'fs/promises'; +import path from 'path'; +import { isPathWithinBase } from '../../utils/validation'; +import { sanitizeForLog } from '../../utils/safeLog'; +import { deleteStagingMarker, validateCandidateRelPath } from './createStagingMarker'; +import { isRealPathAtManagedLocation, managedAreaBase } from './managedPaths'; + +/** + * Appended to positional-containment refusals so an operator staring at one + * knows there is no override to flip: something under the managed area is not + * where its own name says it is, and the fix is repairing or removing that + * directory, not relocating the data directory (a whole-area move resolves + * cleanly and never trips this). + */ +const RELOCATION_REMEDIATION = 'repair or remove the redirected directory under DATA_DIR/git-managed, then retry'; + +export type OperationOwnedCleanup = { + /** Absolute path of the stack's managed root. */ + stackManagedRoot: string; + /** Candidate directory this operation staged, relative to the managed root. */ + candidateRelPath: string | null; + /** Applied directory this operation promoted, relative to the managed root. */ + appliedRelPath?: string | null; + /** + * True only when this operation created the managed root itself. Nothing + * else authorizes deleting the whole root, because a root that predated the + * operation may hold another generation's retained content. + */ + ownsManagedRoot: boolean; +}; + +/** + * Remove only what one create operation put on disk. + * + * The rule this enforces is that a failed create must never cost an unrelated + * generation its files. When the operation created the managed root, the whole + * root is ours and goes. Otherwise the blast radius is exactly the directories + * the operation staged, resolved and containment-checked against the root + * before anything is removed. + * + * Throws on the first failed *directory* removal rather than continuing, because + * the caller uses success here as the precondition for tombstoning: a partially + * cleaned area must keep its checkpoint so the next boot can retry. + * + * The staging marker is reported rather than thrown. Once the directories are + * gone the create is torn down, and a marker file nobody could delete is the + * same condition the settled path already treats as non-fatal. Throwing here + * would make one unlink failure the difference between an instance that boots + * and one that does not. + */ +export async function removeOperationOwnedPaths( + input: OperationOwnedCleanup, +): Promise<'cleared' | 'marker_retained'> { + const base = path.resolve(input.stackManagedRoot); + // Inline containment barrier at the removal sink (see `managedAreaBase`). + const areaBase = managedAreaBase(); + if (!base.startsWith(areaBase + path.sep)) { + throw new Error('refusing to remove a managed root outside the managed area'); + } + + if (input.ownsManagedRoot) { + if (!await isRealPathAtManagedLocation(base)) { + throw new Error( + 'refusing to remove a managed root that links outside its managed location. ' + + RELOCATION_REMEDIATION, + ); + } + await fs.rm(base, { recursive: true, force: true }); + return 'cleared'; + } + + for (const relPath of [input.candidateRelPath, input.appliedRelPath ?? null]) { + if (!relPath) continue; + const resolved = path.resolve(base, relPath); + // A strict descendant, not merely "within": `.` and `./` resolve to the + // base itself, and containment alone would let them wipe the whole managed + // root on the branch whose entire purpose is to protect it. + if (resolved === base || !isPathWithinBase(resolved, base)) { + throw new Error('refusing to remove a path outside the managed root'); + } + // Paths that arrive from a persisted row get the same shape check as + // marker paths, so a malformed generation row cannot widen the blast + // radius to a whole generations directory. + if (!/^(generations)[\\/](candidate|applied)-/.test(relPath)) { + throw new Error(`refusing to remove a path that is not a generation directory: ${relPath}`); + } + // Redundant as a security check: `resolved` is already a strict descendant + // of `base`, and `base` of `areaBase`. Present because it is this variable + // that reaches the removal below, and the barrier has to sit at the call. + if (!resolved.startsWith(areaBase + path.sep)) { + throw new Error('refusing to remove a path outside the managed area'); + } + // The checks above are lexical, so a link above this path would still pass + // them while the delete below followed it somewhere else, including into + // another stack's generations directory inside this same managed area. + if (!await isRealPathAtManagedLocation(resolved)) { + throw new Error( + 'refusing to remove a path that links outside its managed location. ' + + RELOCATION_REMEDIATION, + ); + } + await fs.rm(resolved, { recursive: true, force: true }); + } + + try { + await deleteStagingMarker(base); + } catch (error) { + console.warn( + '[GitOps] Removed the staged directories under %s but could not clear its staging marker: %s', + sanitizeForLog(base), + error instanceof Error ? error.message : String(error), + ); + return 'marker_retained'; + } + return 'cleared'; +} + +/** + * Cleanup for a managed root found at startup with no checkpoint and no live + * application, driven entirely by its staging marker. + * + * Returns what was done so the caller can log it. A corrupt or missing marker + * is not an error: it means nothing proves who owns this directory, so the + * only safe action is to leave it alone. + */ +export async function cleanupUnclaimedManagedRoot( + stackManagedRoot: string, + marker: { operationId: string; rootPreexisted: boolean; candidateRelPath: string } | null, +): Promise<'removed_root' | 'removed_candidate' | 'preserved'> { + if (!marker) return 'preserved'; + const reason = validateCandidateRelPath(marker.candidateRelPath, stackManagedRoot); + if (reason) { + // Said out loud, because the caller only logs the outcome. A directory + // that survives every boot with no stated reason is indistinguishable + // from one nothing has looked at. + console.warn( + '[GitOps] Preserving unclaimed managed area %s: %s', + sanitizeForLog(stackManagedRoot), reason, + ); + return 'preserved'; + } + + if (!marker.rootPreexisted) { + // Same inline containment barrier as the sinks above: this one removes a + // whole managed root, so it gets the check even though the analyzer has not + // reported it. Reported as `preserved` rather than thrown, because every + // other unprovable case here answers that way and the caller reads the + // outcome; the warning is what says this one is anomalous rather than + // merely unproven. + const root = path.resolve(stackManagedRoot); + if (!root.startsWith(managedAreaBase() + path.sep) || !await isRealPathAtManagedLocation(root)) { + console.warn( + '[GitOps] Refusing to reap a managed root that links outside its managed location: %s', + sanitizeForLog(stackManagedRoot), + ); + return 'preserved'; + } + await fs.rm(root, { recursive: true, force: true }); + return 'removed_root'; + } + + // The marker outcome is not reported onward: this sweep has no checkpoint to + // keep, so a marker it could not clear is already said out loud by the + // warning inside the call and there is nothing further for a caller to do. + await removeOperationOwnedPaths({ + stackManagedRoot, + candidateRelPath: marker.candidateRelPath, + ownsManagedRoot: false, + }); + return 'removed_candidate'; +} diff --git a/backend/src/services/gitops/createRecovery.ts b/backend/src/services/gitops/createRecovery.ts new file mode 100644 index 00000000..33c01717 --- /dev/null +++ b/backend/src/services/gitops/createRecovery.ts @@ -0,0 +1,355 @@ +import fs from 'fs/promises'; +import path from 'path'; +import { DatabaseService, type GitSourceAppliedSpec } from '../DatabaseService'; +import { FileSystemService } from '../FileSystemService'; +import { GitProjectManifestService } from '../GitProjectManifestService'; +import { sanitizeForLog } from '../../utils/safeLog'; +import { removeOperationOwnedPaths } from './createCleanup'; +import { deleteStagingMarker } from './createStagingMarker'; +import { newGitOpsId, stackManagedRoot } from './directApplication'; +import { GitOpsStore } from './store'; +import { GitOpsTransitions } from './transitions'; +import type { GitOpsCreateCheckpointRow } from './types'; + +/** What the sweep decided about one interrupted create. */ +export type CreateRecoveryOutcome = + | 'completed' + | 'tombstoned' + | 'checkpoint_cleared' + | 'source_preserved' + | 'retained' + /** + * The create itself is settled; only clearing its staging marker failed. + * Distinct from `retained` so the boot log does not send an operator looking + * for an unfinished create that finished. + */ + | 'marker_retained'; + +export type CreateRecoveryResult = { + stackName: string; + applicationId: string; + outcome: CreateRecoveryOutcome; +}; + +/** + * Refuse to continue while any create is still unresolved. + * + * A create that could not be settled leaves a stack directory the deploy path + * cannot tell apart from a finished one, so the alternative to stopping is + * letting a scheduler, webhook or operator act on a half-built stack. Startup + * calls this before the background mutators and the HTTP bind. + * + * Only `retained` counts. `marker_retained` means the create itself is settled + * and a leftover marker file is all that survived, which decides nothing about + * ownership and must not cost an operator their instance. + */ +export function assertCreatesSettled(settled: readonly CreateRecoveryResult[]): void { + const unresolved = settled.filter((entry) => entry.outcome === 'retained'); + if (unresolved.length === 0) return; + const named = unresolved.map((entry) => sanitizeForLog(entry.stackName || entry.applicationId)).join(', '); + throw new Error( + `${unresolved.length} interrupted create(s) could not be settled: ${named}. ` + + 'Sencho does not start while a create is unresolved, because a half-built stack ' + + 'is indistinguishable from a finished one. The cause is logged above; clearing it ' + + 'lets the next start finish the recovery.', + ); +} + +function envelopeFor(checkpoint: GitOpsCreateCheckpointRow) { + return { + operationId: checkpoint.operation_id, + actor: 'system:startup', + trigger: 'create_recovery', + at: Date.now(), + }; +} + +/** + * Three states, because the two callers need opposite fail-safe directions. + * + * Teardown must not treat "cannot tell" as absent, or it would skip a directory + * that is really there. Completion must not treat it as present, or it would + * mark a create live on the strength of a failed stat. + */ +/** + * Clear a settled create's staging marker, reporting rather than throwing. + * + * The checkpoint is only dropped once this succeeds, because a marker left + * behind with no checkpoint has nothing to retry it and refuses every later + * create for that stack name. The cost is that a marker this keeps failing on + * also keeps the checkpoint row, and its encrypted token, alive across boots. + * That is the lesser harm: the row is encrypted at rest with the same key as + * the source it came from, and it is what makes the retry possible at all. + */ +async function clearSettledMarker(stackName: string, managedRoot: string): Promise { + try { + await deleteStagingMarker(managedRoot); + return true; + } catch (error) { + console.warn( + `[GitOps] Settled the create for ${sanitizeForLog(stackName)} but could not clear its staging marker:`, + error instanceof Error ? error.message : String(error), + ); + return false; + } +} + +async function stackDirState(stackName: string): Promise<'present' | 'absent' | 'unknown'> { + try { + const base = FileSystemService.getInstance().getBaseDir(); + await fs.stat(path.join(base, stackName)); + return 'present'; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return 'absent'; + console.warn( + `[GitOps] Cannot determine whether stack ${sanitizeForLog(stackName)} exists on disk:`, + error instanceof Error ? error.message : String(error), + ); + return 'unknown'; + } +} + +/** + * Settle every create that a previous process left in flight. + * + * Runs at boot, before any mutation service, and is idempotent: each row is + * decided from the durable checkpoint phase plus what is actually on disk, so + * a crash during recovery itself just replays on the next boot. + * + * The two rules that shape every branch: a create is only finished when its + * manifest is already committed on disk, and it is only torn down after its + * files are gone. A create whose files cannot be removed keeps its checkpoint + * rather than being recorded as cleanly failed. + */ +export async function resolveInterruptedCreates(): Promise { + const store = GitOpsStore.getInstance(); + const db = DatabaseService.getInstance(); + const results: CreateRecoveryResult[] = []; + + for (const checkpoint of store.listCreateCheckpoints()) { + try { + results.push(await resolveOne(checkpoint)); + } catch (error) { + console.error( + `[GitOps] Could not settle the interrupted create for ${sanitizeForLog(checkpoint.stack_name)}; retrying next boot:`, + error instanceof Error ? error.stack ?? error.message : String(error), + ); + results.push({ + stackName: checkpoint.stack_name, + applicationId: checkpoint.application_id, + outcome: 'retained', + }); + } + } + + // A creating application with no checkpoint at all cannot be finished: the + // facts needed to complete it are gone. Tombstone it so the stack name is + // usable again, and never touch a source row that outlived it, because the + // conservative migration can still build a live application from that row. + for (const app of store.listCreatingDirectApplications()) { + if (store.getCreateCheckpoint(app.id)) continue; + const stackName = app.stack_name ?? ''; + // Guarded per row for the same reason as the loop above: one application + // that cannot be tombstoned must not strand the rest. A creating row that + // survives keeps matching the live-application lookup, so every later + // create for that name would fail until it is cleared. + try { + const sourceRow = stackName ? db.getGitSource(stackName) : null; + GitOpsTransitions.getInstance().createFailed(app.id, 'create_checkpoint_missing', { + operationId: app.latest_operation_id ?? newGitOpsId(), + actor: 'system:startup', + trigger: 'create_recovery', + at: Date.now(), + }); + results.push({ + stackName, + applicationId: app.id, + outcome: sourceRow ? 'source_preserved' : 'tombstoned', + }); + } catch (error) { + console.error( + `[GitOps] Could not tombstone the checkpointless create for ${sanitizeForLog(stackName)}; retrying next boot:`, + error instanceof Error ? error.stack ?? error.message : String(error), + ); + results.push({ stackName, applicationId: app.id, outcome: 'retained' }); + } + } + + return results; +} + +/** + * Reclassify every operation the previous process left open. + * + * An operation that started and never terminated keeps reporting as in flight, + * and the deriver offers no actions while it does, so without this a single + * interrupted fetch or apply strands a stack until someone notices. The + * interruption is recorded as unknown rather than failed: we genuinely do not + * know whether the work completed. + * + * Runs at boot, after create recovery and before any mutation service, and is + * guarded per application so one bad row cannot strand the rest. + */ +export function reclassifyInterruptedOperations(): number { + const store = GitOpsStore.getInstance(); + let reclassified = 0; + for (const app of store.listApplicationsWithOpenOperations()) { + try { + GitOpsTransitions.getInstance().interruptActiveOperations(app.id, { + operationId: app.latest_operation_id ?? newGitOpsId(), + actor: 'system:startup', + trigger: 'startup_reconcile', + at: Date.now(), + }); + reclassified += 1; + } catch (error) { + console.error( + `[GitOps] Could not reclassify the interrupted operation for ${sanitizeForLog(app.stack_name ?? app.id)}:`, + error instanceof Error ? error.stack ?? error.message : String(error), + ); + } + } + return reclassified; +} + +async function resolveOne(checkpoint: GitOpsCreateCheckpointRow): Promise { + const store = GitOpsStore.getInstance(); + const db = DatabaseService.getInstance(); + const app = store.getApplication(checkpoint.application_id); + const stackName = checkpoint.stack_name; + const managedRoot = stackManagedRoot(stackName); + + // Nothing left to decide: the checkpoint outlived its application, the create + // already reached its success boundary, or the application has since moved out + // of `creating` on some other path. Every one of them means this row is stale + // bookkeeping, so they settle identically and share one exit, which is what + // keeps the marker ordering below true of all of them rather than of whichever + // branch last remembered it. + if (!app || checkpoint.phase === 'pointers_committed' || app.lifecycle_status !== 'creating') { + // Marker first: clearing it can fail, and dropping the checkpoint before + // that would leave a marker that makes the stack name uncreatable with + // nothing left to retry it. + if (!await clearSettledMarker(stackName, managedRoot)) { + return { stackName, applicationId: checkpoint.application_id, outcome: 'marker_retained' }; + } + store.deleteCreateCheckpoint(checkpoint.application_id); + return { stackName, applicationId: checkpoint.application_id, outcome: 'checkpoint_cleared' }; + } + + // The manifest is committed on disk, so the authored project the operator + // asked for exists. Finish the create rather than destroying it: this is the + // same source-row plus acceptance commit the live path performs. + if ( + checkpoint.phase === 'manifest_committed' + && checkpoint.generation_id + && await stackDirState(stackName) === 'present' + ) { + const generationId = checkpoint.generation_id; + const appliedSpec = checkpoint.applied_spec_json + ? JSON.parse(checkpoint.applied_spec_json) as GitSourceAppliedSpec + : null; + const manifest = await GitProjectManifestService.getInstance().readManifest( + stackName, checkpoint.repo_url, checkpoint.branch, + ); + db.getDb().transaction(() => { + if (!db.getGitSource(stackName)) { + db.upsertGitSource({ + stack_name: stackName, + repo_url: checkpoint.repo_url, + branch: checkpoint.branch, + compose_path: checkpoint.compose_path, + compose_paths: JSON.parse(checkpoint.compose_paths_json) as string[], + context_dir: checkpoint.context_dir, + sync_env: checkpoint.sync_env === 1, + env_path: checkpoint.env_path, + auth_type: checkpoint.auth_type as 'none' | 'token', + encrypted_token: checkpoint.encrypted_token, + auto_apply_on_webhook: checkpoint.auto_apply_on_webhook === 1, + auto_deploy_on_apply: checkpoint.auto_deploy_on_apply === 1, + last_applied_commit_sha: checkpoint.commit_sha, + last_applied_content_hash: null, + pending_commit_sha: null, + pending_compose_content: null, + pending_env_content: null, + pending_fetched_at: null, + last_debounce_at: null, + }); + } + // The insert does not carry the applied pointers, so stamp them the way + // the live create path does. The content hash is left empty because the + // bytes that produced it belonged to the process that crashed; the first + // pull after recovery re-establishes it. + db.markGitSourceApplied(stackName, checkpoint.commit_sha ?? '', ''); + // The live path also stamps the deploy spec and the manifest cache. Both + // are load-bearing: a null spec silently reverts a multi-file stack to + // single-file auto-discovery, and an unset manifest state makes a managed + // stack render as unmanaged and suppresses its rollback disclosure. + if (appliedSpec) db.setGitSourceAppliedSpec(stackName, appliedSpec); + if (manifest && 'manifestVersion' in manifest) { + db.setGitSourceManifestState( + stackName, + manifest.manifestVersion, + manifest.state, + manifest.generation.appliedDir, + ); + } + GitOpsTransitions.getInstance().applied({ + applicationId: checkpoint.application_id, + generationId, + artifactSetId: newGitOpsId(), + sourceAcceptanceId: newGitOpsId(), + authority: 'operator', + envelope: envelopeFor(checkpoint), + activateCreating: true, + }); + store.updateCreateCheckpoint(checkpoint.application_id, { phase: 'pointers_committed' }, Date.now()); + })(); + // Marker before checkpoint, for the reason given on the branch above. The + // create is live either way by this point: the transaction above committed. + if (!await clearSettledMarker(stackName, managedRoot)) { + return { stackName, applicationId: checkpoint.application_id, outcome: 'marker_retained' }; + } + store.deleteCreateCheckpoint(checkpoint.application_id); + return { stackName, applicationId: checkpoint.application_id, outcome: 'completed' }; + } + + // Everything else stopped before the project was durable. Remove exactly what + // this operation put on disk, then record the failure. Filesystem first: if it + // throws, the checkpoint survives and the next boot retries. + const generation = checkpoint.generation_id ? store.getGeneration(checkpoint.generation_id) : undefined; + // `pre_stack` is durable proof that createStack had not returned, so a + // directory present now was not necessarily made by this operation. It could + // be the operator's own stack that appeared while the create was fetching. + // Removing it on that evidence is the one mistake this path cannot take back, + // so an orphaned directory is left behind instead. + const stackDir = await stackDirState(stackName); + if (checkpoint.phase === 'pre_stack') { + if (stackDir === 'present') { + console.warn( + `[GitOps] Leaving the directory for ${sanitizeForLog(stackName)} in place: the interrupted create never recorded creating it.`, + ); + } + } else if (stackDir === 'present') { + await FileSystemService.getInstance().deleteStack(stackName); + } + const cleanup = await removeOperationOwnedPaths({ + stackManagedRoot: managedRoot, + candidateRelPath: generation?.candidate_dir ?? null, + appliedRelPath: generation?.applied_dir ?? null, + ownsManagedRoot: checkpoint.created_managed_root === 1, + }); + // The staged directories are gone, so nothing deployable survives, but the + // marker still claims the name. Keep the checkpoint and report the same + // non-fatal outcome the settled path uses, rather than letting one unlink + // failure read as an unresolved create and stop the instance booting. + if (cleanup === 'marker_retained') { + return { stackName, applicationId: checkpoint.application_id, outcome: 'marker_retained' }; + } + + GitOpsTransitions.getInstance().createFailed( + checkpoint.application_id, + 'interrupted_create', + envelopeFor(checkpoint), + ); + return { stackName, applicationId: checkpoint.application_id, outcome: 'tombstoned' }; +} diff --git a/backend/src/services/gitops/createStagingMarker.ts b/backend/src/services/gitops/createStagingMarker.ts new file mode 100644 index 00000000..a7c4b110 --- /dev/null +++ b/backend/src/services/gitops/createStagingMarker.ts @@ -0,0 +1,201 @@ +import fs from 'fs/promises'; +import path from 'path'; +import { isPathWithinBase } from '../../utils/validation'; +import { GENERATIONS_DIR, isRealPathAtManagedLocation, managedAreaBase } from './managedPaths'; + +export const CREATE_STAGING_MARKER_FILENAME = '.create-staging.v1.json'; + +const CANDIDATE_PREFIX = `${GENERATIONS_DIR}/candidate-`; + +export type CreateStagingMarker = { + schemaVersion: 1; + operationId: string; + /** True when the managed root already existed before this operation ran. */ + rootPreexisted: boolean; + /** Never null: the candidate path is computed before any managed-root mutation. */ + candidateRelPath: string; + /** Diagnostic only. Never deletion authority. */ + createdAt: number; +}; + +export type ReadStagingMarkerResult = + | { state: 'valid'; marker: CreateStagingMarker } + | { state: 'missing' } + | { state: 'corrupt'; reason: string }; + +/** + * The candidate directory this operation will stage, computed the same way the + * manifest service computes it, before anything touches the managed root. + * + * Recording the path up front is what makes crash cleanup exact: a process that + * dies part-way through building the candidate still left a marker naming the + * one directory it owned. + */ +export function candidateRelPathForSha(commitSha: string): string { + return `${CANDIDATE_PREFIX}${commitSha}`; +} + +/** + * The applied directory promotion will move this candidate into. + * + * Computed here rather than read from the manifest because the generation row + * is written before promotion runs, and the manifest carries an empty applied + * path until then. Storing that empty value would make "not promoted yet" + * indistinguishable from "nothing to clean up" during teardown. + * + * Must stay in step with the promotion path in GitProjectManifestService. + */ +export function appliedRelPathFor(commitSha: string, manifestVersion: number): string { + return `${GENERATIONS_DIR}/applied-${commitSha}-${manifestVersion}`; +} + +function isSafeRelPath(value: unknown): value is string { + if (typeof value !== 'string' || value.length === 0) return false; + if (path.isAbsolute(value)) return false; + const segments = value.split(/[\\/]/); + return !segments.some((seg) => seg === '..' || seg === '.' || seg === ''); +} + +/** + * Validate a candidate path against the stack's own managed root. + * + * Every read re-runs this, not just the write, because the marker is a + * filesystem artifact an operator or a bug could have rewritten between the + * crash and the sweep. A path that fails any check makes the marker corrupt, + * and a corrupt marker preserves the root rather than authorizing a delete. + */ +export function validateCandidateRelPath(candidateRelPath: unknown, stackManagedRoot: string): string | null { + if (!isSafeRelPath(candidateRelPath)) return 'candidateRelPath is not a safe relative path'; + if (!candidateRelPath.startsWith(CANDIDATE_PREFIX)) { + return `candidateRelPath must start with ${CANDIDATE_PREFIX}`; + } + const base = path.resolve(stackManagedRoot); + const resolved = path.resolve(base, candidateRelPath); + if (!isPathWithinBase(resolved, base)) return 'candidateRelPath escapes the managed root'; + return null; +} + +/** Test-only. Production call sites resolve this path inline at their own sink. */ +export function stagingMarkerPath(stackManagedRoot: string): string { + return path.join(stackManagedRoot, CREATE_STAGING_MARKER_FILENAME); +} + +export async function readStagingMarker(stackManagedRoot: string): Promise { + // Inline containment barrier at the read sink (see `managedAreaBase`). A root + // outside the managed area is treated as corrupt rather than thrown, matching + // this module's rule that an unreadable claim preserves rather than deletes. + const areaBase = managedAreaBase(); + const markerPath = path.resolve(stackManagedRoot, CREATE_STAGING_MARKER_FILENAME); + if (!markerPath.startsWith(areaBase + path.sep)) { + return { state: 'corrupt', reason: 'managed root escapes the managed area' }; + } + let raw: string; + try { + raw = await fs.readFile(markerPath, 'utf8'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return { state: 'missing' }; + return { state: 'corrupt', reason: (error as Error).message }; + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return { state: 'corrupt', reason: 'marker is not valid JSON' }; + } + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + return { state: 'corrupt', reason: 'marker is not an object' }; + } + const marker = parsed as Record; + if (marker.schemaVersion !== 1) return { state: 'corrupt', reason: 'unsupported marker schemaVersion' }; + if (typeof marker.operationId !== 'string' || marker.operationId.length === 0) { + return { state: 'corrupt', reason: 'marker operationId is missing' }; + } + if (typeof marker.rootPreexisted !== 'boolean') { + return { state: 'corrupt', reason: 'marker rootPreexisted is missing' }; + } + const pathReason = validateCandidateRelPath(marker.candidateRelPath, stackManagedRoot); + if (pathReason) return { state: 'corrupt', reason: pathReason }; + const createdAt = typeof marker.createdAt === 'number' && Number.isFinite(marker.createdAt) + ? marker.createdAt + : 0; + return { + state: 'valid', + marker: { + schemaVersion: 1, + operationId: marker.operationId, + rootPreexisted: marker.rootPreexisted, + candidateRelPath: marker.candidateRelPath as string, + createdAt, + }, + }; +} + +export class CreateStagingMarkerError extends Error { + constructor(message: string) { + super(message); + this.name = 'CreateStagingMarkerError'; + } +} + +/** + * Write the marker atomically, refusing to trample another operation's. + * + * A live marker for a different operation means a concurrent or abandoned + * create still owns this managed root. Overwriting it would hand our operation + * id deletion authority over their staged directory, so we refuse instead. + */ +export async function writeStagingMarker( + stackManagedRoot: string, + marker: CreateStagingMarker, +): Promise { + const reason = validateCandidateRelPath(marker.candidateRelPath, stackManagedRoot); + if (reason) throw new CreateStagingMarkerError(reason); + + const existing = await readStagingMarker(stackManagedRoot); + // A marker that exists but cannot be read is still a claim. Overwriting it + // would hand this operation deletion authority over whatever the last one + // staged, which is the opposite of what the rest of this module does with a + // corrupt marker, and it would do so with nothing said about why. + if (existing.state === 'corrupt') { + throw new CreateStagingMarkerError( + `this managed area has an unreadable staging marker (${existing.reason}); refusing to claim it`, + ); + } + if (existing.state === 'valid' && existing.marker.operationId !== marker.operationId) { + throw new CreateStagingMarkerError( + 'another create operation already owns this managed area', + ); + } + + // Inline containment barrier at each write sink (see `managedAreaBase`). + const areaBase = managedAreaBase(); + const root = path.resolve(stackManagedRoot); + const target = path.resolve(stackManagedRoot, CREATE_STAGING_MARKER_FILENAME); + const temp = path.resolve(stackManagedRoot, `${CREATE_STAGING_MARKER_FILENAME}.${marker.operationId}.tmp`); + if ( + !root.startsWith(areaBase + path.sep) + || !target.startsWith(areaBase + path.sep) + || !temp.startsWith(areaBase + path.sep) + // Checked on the write and not only on the delete, so a link cannot be + // written through and then refused by the hardened delete below. That + // asymmetry would wedge the stack name: a valid marker naming another + // operation refuses every later create, and nothing could remove it. + || !await isRealPathAtManagedLocation(root) + ) { + throw new CreateStagingMarkerError('managed root links outside its managed location'); + } + + await fs.mkdir(root, { recursive: true }); + await fs.writeFile(temp, JSON.stringify(marker), 'utf8'); + await fs.rename(temp, target); +} + +export async function deleteStagingMarker(stackManagedRoot: string): Promise { + // Inline containment barrier at the removal sink (see `managedAreaBase`). + const areaBase = managedAreaBase(); + const markerPath = path.resolve(stackManagedRoot, CREATE_STAGING_MARKER_FILENAME); + if (!markerPath.startsWith(areaBase + path.sep) || !await isRealPathAtManagedLocation(markerPath)) { + throw new CreateStagingMarkerError('managed root links outside its managed location'); + } + await fs.rm(markerPath, { force: true }); +} diff --git a/backend/src/services/gitops/derive.ts b/backend/src/services/gitops/derive.ts new file mode 100644 index 00000000..1ecb37b7 --- /dev/null +++ b/backend/src/services/gitops/derive.ts @@ -0,0 +1,914 @@ +import { + decodeArtifactEvidenceJson, + decodeGitOpsEvidenceLimitations, + decodeObservedArtifactIdentity, + GitOpsJsonError, +} from './json'; +import { GitOpsStore } from './store'; +import type { BlueprintObservationStage } from './transitions'; +import type { + ArtifactExpectedIdentity, + ArtifactFacet, + ArtifactLatestEvidence, + ArtifactQualification, + FutureGitOpsEvidence, + GitOpsApplicationRow, + GitOpsAvailableAction, + GitOpsLimitation, + GitOpsRevisionProjection, + GitOpsDriftItem, + GitOpsTargetCurrentRow, + GitOpsTargetProjection, + HealthFacet, + LkgFacet, + PlacementFacet, + RolloutFacet, + RuntimeFacet, + SourceFacet, + SourceIdentityFields, +} from './types'; + +/** + * The projection for anything that carries no GitOps application. + * + * One shared instance, so its collections are frozen alongside it: a caller + * that pushed a limitation onto this projection would otherwise corrupt every + * later response in the process. The separate declaration is what gives the + * literal its contextual type; freezing it inline widens the empty tuples to + * `never[]` and fails to typecheck. + */ +const NOT_APPLICABLE: GitOpsRevisionProjection = { + schemaVersion: 1, + targetMode: 'not_applicable', + applicationId: null, + facets: null, + targets: [], + drift: [], + limitations: [], + availableActions: [], + approvals: null, +}; +// Frozen after construction rather than inline: the collections stay mutable +// types so the projection union still matches, while the shared instance +// refuses writes at runtime. +for (const collection of [ + NOT_APPLICABLE.targets, + NOT_APPLICABLE.drift, + NOT_APPLICABLE.limitations, + NOT_APPLICABLE.availableActions, +]) { + Object.freeze(collection); +} +export const NOT_APPLICABLE_REVISION: GitOpsRevisionProjection = Object.freeze(NOT_APPLICABLE); + +export type DeriveFacts = { + application: GitOpsApplicationRow | null; + targets: GitOpsTargetCurrentRow[]; + healthDisabled: boolean; +}; + +export function deriveGitOpsRevision( + facts: DeriveFacts, + futureEvidence: FutureGitOpsEvidence | null, +): GitOpsRevisionProjection { + // Future-only facets are typed here so callers share one deriver; this slice + // projects only persisted current evidence. + void futureEvidence; + const app = facts.application; + if (!app) return NOT_APPLICABLE_REVISION; + const limitations: GitOpsLimitation[] = []; + mergePersistedLimitations(app.evidence_limitations_json, limitations); + const source = deriveSource(app, limitations); + const artifact = deriveArtifact(app, app.accepted_generation_id, app.artifact_set_id, app.latest_artifact_set_id, limitations); + const placement = derivePlacement(app); + const targets = facts.targets + .slice() + .sort((a, b) => a.node_id - b.node_id) + .map((target) => deriveTarget(app, target, facts.healthDisabled, limitations)); + const rollout = deriveRollout(app, targets); + const availableActions = deriveActions(app, source, placement, targets); + return { + schemaVersion: 1, + targetMode: app.target_mode, + applicationId: app.id, + lifecycleStatus: app.lifecycle_status, + stackName: app.stack_name, + blueprintId: app.blueprint_id, + rolloutGenerationId: app.rollout_generation_id, + approvals: { + sourceAcceptanceRef: app.source_acceptance_ref, + placementApprovalRef: app.placement_approval_ref, + rolloutAuthorizationRef: app.rollout_authorization_ref, + legacyCombinedApprovalRef: app.legacy_combined_approval_ref, + }, + facets: { source, artifact, placement, rollout }, + targets, + drift: collectRuntimeDrift(app, targets), + limitations, + availableActions, + }; +} + +/** + * The two drift classes current evidence can confirm on its own. + * + * Most of the seven classes need producers this model deliberately defers, but + * a comparable runtime artifact mismatch and a desired-versus-deployed + * generation mismatch rest entirely on rows that exist now. Leaving `drift` + * empty while a facet says `runtime_artifact_drift` would report one fault + * twice with only one copy readable; the same holds whenever the known + * pointers disagree, whatever presentation status outranks them. + * + * The generation-mismatch item carries the desired generation as expected and + * the deployed generation as observed, owned by ComposeService. It clears + * once the desired generation is deployed. + * + * The artifact item is emitted only for an exact or qualified expectation + * against an exact or qualified observation whose identity strings differ. + * Every other observation kind stays `artifact_verification_pending`, and + * equal identities emit nothing. Policy composition has no producer yet, so + * items carry null rather than a policy nothing wrote. + */ +function collectRuntimeDrift( + app: GitOpsApplicationRow, + targets: GitOpsTargetProjection[], +): GitOpsDriftItem[] { + const items: GitOpsDriftItem[] = []; + for (const target of targets) { + // Generation mismatch: the target's contract is its desired generation, + // and a different generation is running. Judged from the pointers + // themselves, not from any one runtime status: paused, failed, recovering, + // interrupted, and in-flight states all outrank the pointer comparison in + // deriveRuntime, so keying the item to `applied_not_deployed` would drop + // the report exactly when a failed deploy leaves the old workload serving. + // A retired target is excluded: its pointers survive retirement on + // purpose, but nothing can ever rebind it, so its mismatch would be a + // permanently unresolvable item rather than a live divergence. + if ( + !target.tombstoned + && target.desiredGenerationId !== null + && target.deployedGenerationId !== null + && target.desiredGenerationId !== target.deployedGenerationId + ) { + items.push({ + class: 'runtime', + expected: { kind: 'generation', id: target.desiredGenerationId }, + observed: { kind: 'generation', id: target.deployedGenerationId }, + freshnessAt: null, + owner: 'ComposeService', + reason: 'the target is running a different generation than the one it was asked to run', + configuredPolicy: null, + affectedTargets: [{ nodeId: target.nodeId, stackName: app.stack_name }], + // The action answers for this affected target alone, using the same + // legality predicate that puts deploy into availableActions, so a + // sibling's clean convergence can never recommend deploying this one + // while it is paused, failed, or otherwise unable to act. + action: targetDeployLegal(app, target) ? 'deploy' : 'none', + }); + } + // Artifact mismatch: comparable exact/qualified expectation vs observation. + if (target.runtime.status !== 'runtime_artifact_drift') continue; + const expected = target.artifact.status !== 'not_applicable' && 'expected' in target.artifact + ? target.artifact.expected + : null; + if (!expected || expected.identity === null) continue; + const observed = target.observedArtifactIdentity; + if ((observed.kind !== 'exact' && observed.kind !== 'qualified') || observed.identity === expected.identity) { + continue; + } + items.push({ + class: 'runtime', + expected: { + kind: 'artifact_set', + id: expected.artifactSetId, + qualification: expected.qualification, + evidenceVersion: expected.evidenceVersion, + }, + observed: { kind: 'runtime_artifact', identity: observed.identity, observedAt: observed.observedAt }, + freshnessAt: observed.observedAt, + owner: 'observed_artifact_identity', + reason: 'the running workload reports an artifact identity other than the expected artifact set', + configuredPolicy: null, + affectedTargets: [{ nodeId: target.nodeId, stackName: app.stack_name }], + action: 'none', + }); + } + return items; +} + +function deriveSource(app: GitOpsApplicationRow, limitations: GitOpsLimitation[]): SourceFacet { + if (app.target_mode === 'inline_blueprint') return { status: 'not_applicable' }; + const identity = sourceIdentity(app, limitations); + if (app.lifecycle_status === 'detached' || app.lifecycle_status === 'deleted') { + return { ...identity, status: 'not_live', lifecycleStatus: app.lifecycle_status }; + } + if (app.recovery_phase === 'restoring' || app.recovery_phase === 'compensating') { + return { ...identity, status: 'recovery_required', recoveryRef: app.recovery_ref, recoveryGenerationId: null }; + } + if (app.recovery_phase === 'failed' || app.failure_stage === 'recovery') { + return { + ...identity, + status: 'recovery_failed', + recoveryRef: app.recovery_ref, + recoveryGenerationId: null, + failureClass: app.failure_class ?? 'unknown', + failureAt: app.failure_at ?? 0, + }; + } + if (app.active_operation_stage === 'fetch_started') return { ...identity, status: 'checking_fetching' }; + if (app.active_operation_stage === 'apply_started') { + return { + ...identity, + status: 'applying', + activeOperationId: app.active_operation_id ?? '', + activeGenerationId: app.active_generation_id ?? '', + }; + } + if (app.interruption_stage === 'fetch_started' || app.interruption_stage === 'apply_started') { + return { + ...identity, + status: 'source_unknown', + interruptedStage: app.interruption_stage, + interruptedAt: app.interruption_at ?? 0, + interruptedOperationId: app.interruption_operation_id, + interruptedGenerationId: app.interruption_generation_id, + }; + } + if (app.suspended_at) return { ...identity, status: 'source_suspended', suspendedAt: app.suspended_at }; + if (app.failure_stage === 'fetch' || app.failure_stage === 'validation' || app.failure_stage === 'apply' || app.failure_stage === 'create') { + return { + ...identity, + status: 'source_failed', + failureStage: app.failure_stage, + failureClass: app.failure_class ?? app.failure_stage, + failureAt: app.failure_at ?? 0, + retryAt: app.retry_at, + retryCount: app.retry_count, + }; + } + if (app.retry_at) { + return { ...identity, status: 'source_retry_scheduled', retryAt: app.retry_at, retryCount: app.retry_count }; + } + const store = GitOpsStore.getInstance(); + if (app.candidate_generation_id) { + const generation = store.getGeneration(app.candidate_generation_id); + // A candidate only counts as ready when its generation row exists under + // this application and the application's materialization fingerprint is + // still the one the generation was built from; applyStarted refuses + // anything else, so a dangling or foreign row is a reconcile problem, + // not a ready one. + if (!generation || generation.application_id !== app.id) { + limitations.push({ + code: 'candidate_generation_invalid', + message: 'candidate generation row is missing or belongs to another application', + evidence: app.candidate_generation_id, + }); + return { ...identity, status: 'source_reconcile_required' }; + } + if (generation.materialization_fingerprint !== app.materialization_fingerprint) { + return { ...identity, status: 'source_reconcile_required' }; + } + if (app.candidate_plan_blocked === 1) return { ...identity, status: 'source_conflict_blocker' }; + if (app.review_required === 1) return { ...identity, status: 'source_review_pending' }; + return { ...identity, status: 'candidate_ready' }; + } + if (app.accepted_generation_id) { + const accepted = store.getGeneration(app.accepted_generation_id); + // An acceptance only counts when its generation row exists under this + // application; missing evidence cannot establish the fingerprint and sha + // agreement (the latter when a desired commit is configured) that the + // success claim rests on, so a dangling or foreign pointer is a reconcile + // problem, not an accepted one. + if (!accepted || accepted.application_id !== app.id) { + limitations.push({ + code: 'accepted_generation_invalid', + message: 'accepted generation row is missing or belongs to another application', + evidence: app.accepted_generation_id, + }); + return { ...identity, status: 'source_reconcile_required' }; + } + if ( + !app.desired_commit_sha + || accepted.materialization_fingerprint !== app.materialization_fingerprint + || accepted.commit_sha !== app.desired_commit_sha + ) { + return { ...identity, status: 'source_reconcile_required' }; + } + return { ...identity, status: 'application_generation_accepted' }; + } + return { ...identity, status: 'never_reconciled' }; +} + +function sourceIdentity(app: GitOpsApplicationRow, limitations: GitOpsLimitation[]): SourceIdentityFields { + let repoIdentity = { host: '', pathname: '' }; + if (app.repo_identity_json) { + try { + const parsed = JSON.parse(app.repo_identity_json) as { host?: unknown; pathname?: unknown }; + if (typeof parsed.host === 'string' && typeof parsed.pathname === 'string') { + repoIdentity = { host: parsed.host, pathname: parsed.pathname }; + } else { + limitations.push({ code: 'repo_identity_invalid', message: 'repo identity json is invalid', evidence: null }); + } + } catch { + limitations.push({ code: 'repo_identity_invalid', message: 'repo identity json is invalid', evidence: null }); + } + } + return { + configuredRepoUrl: app.configured_repo_url ?? '', + repoIdentity, + configuredRef: app.configured_ref ?? '', + desiredCommitSha: app.desired_commit_sha, + fetchedCommitSha: app.fetched_commit_sha, + candidateGenerationId: app.candidate_generation_id, + acceptedGenerationId: app.accepted_generation_id, + }; +} + +function deriveArtifact( + app: GitOpsApplicationRow, + generationId: string | null, + expectedId: string | null, + latestId: string | null, + limitations: GitOpsLimitation[], +): ArtifactFacet { + if (app.target_mode === 'inline_blueprint' || !generationId) return { status: 'not_applicable' }; + const store = GitOpsStore.getInstance(); + const expected = expectedId ? toExpected(store, expectedId, limitations) : null; + if (!latestId) { + return { + status: 'artifact_unresolved', + generationId, + expected, + latestEvidence: null, + limitation: 'artifact_pointer_missing', + }; + } + const latestRow = store.getArtifactSet(latestId); + if (!latestRow) { + limitations.push({ code: 'artifact_pointer_missing', message: 'latest artifact row is missing', evidence: latestId }); + return { + status: 'artifact_unresolved', + generationId, + expected, + latestEvidence: null, + limitation: 'artifact_pointer_missing', + }; + } + let latestEvidence: ArtifactLatestEvidence; + try { + const decoded = decodeArtifactEvidenceJson(latestRow.evidence_json); + latestEvidence = { + artifactSetId: latestRow.id, + evidenceVersion: latestRow.evidence_version, + qualification: latestRow.qualification, + identity: 'identity' in decoded ? decoded.identity : null, + }; + } catch { + limitations.push({ code: 'artifact_evidence_json_invalid', message: 'latest artifact evidence is invalid', evidence: latestId }); + latestEvidence = { + artifactSetId: latestRow.id, + evidenceVersion: latestRow.evidence_version, + qualification: latestRow.qualification, + identity: null, + }; + return { + status: 'artifact_unresolved', + artifactSetId: latestRow.id, + generationId, + evidenceVersion: latestRow.evidence_version, + qualification: latestRow.qualification, + freshnessAt: latestRow.created_at, + expected, + latestEvidence, + }; + } + const status = artifactStatus(latestRow.qualification, expected, latestEvidence); + return { + status, + artifactSetId: latestRow.id, + generationId, + evidenceVersion: latestRow.evidence_version, + qualification: latestRow.qualification, + freshnessAt: latestRow.created_at, + expected, + latestEvidence, + }; +} + +function artifactStatus( + qualification: ArtifactQualification, + expected: ArtifactExpectedIdentity | null, + latest: ArtifactLatestEvidence, +): Exclude['status'] { + if (qualification === 'unresolved') return expected ? 'artifact_resolution_pending' : 'artifact_unresolved'; + if (qualification === 'stale') return 'artifact_stale'; + if (qualification === 'unavailable') return 'artifact_unavailable'; + if (qualification === 'local_build_unverified') return 'artifact_local_build_unverified'; + if ( + expected + && (expected.qualification === 'exact' || expected.qualification === 'qualified') + && latest.identity + && expected.identity + && latest.identity !== expected.identity + ) { + return 'artifact_identity_changed'; + } + return qualification === 'qualified' ? 'artifact_qualified' : 'artifact_exact'; +} + +function toExpected( + store: GitOpsStore, + id: string, + limitations: GitOpsLimitation[], +): ArtifactExpectedIdentity | null { + const row = store.getArtifactSet(id); + if (!row) { + limitations.push({ code: 'artifact_pointer_missing', message: 'expected artifact row is missing', evidence: id }); + return null; + } + try { + const decoded = decodeArtifactEvidenceJson(row.evidence_json); + return { + artifactSetId: row.id, + evidenceVersion: row.evidence_version, + qualification: row.qualification, + identity: 'identity' in decoded ? decoded.identity : null, + }; + } catch { + limitations.push({ code: 'artifact_evidence_json_invalid', message: 'expected artifact evidence is invalid', evidence: id }); + return { + artifactSetId: row.id, + evidenceVersion: row.evidence_version, + qualification: row.qualification, + identity: null, + }; + } +} + +function derivePlacement(app: GitOpsApplicationRow): PlacementFacet { + if (app.target_mode === 'direct') return { status: 'unbound_direct' }; + if (!app.intent_revision_id) return { status: 'unknown', limitation: 'missing_intent' }; + if (app.legacy_combined_approval_ref && !app.placement_approval_ref) { + return { status: 'placement_review_pending' }; + } + return { status: 'blueprint_bound', completion: 'unknown' }; +} + +function deriveRollout(app: GitOpsApplicationRow, targets: GitOpsTargetProjection[]): RolloutFacet { + if (app.recovery_phase === 'restoring' || app.recovery_phase === 'compensating') { + return { status: 'rollback_in_progress', recoveryRef: app.recovery_ref ?? '', recoveryGenerationId: null }; + } + const failed = targets.find((target) => target.runtime.status === 'recovery_failed'); + if (failed && failed.runtime.status === 'recovery_failed') { + return { + status: 'rollback_partial_failed', + recoveryRef: failed.runtime.recoveryRef ?? app.recovery_ref ?? '', + recoveryGenerationId: failed.runtime.recoveryGenerationId, + failureClass: failed.runtime.failureClass, + failureAt: failed.runtime.failureAt, + }; + } + if (targets.some((target) => target.connectivity === 'unreachable')) return { status: 'target_unreachable' }; + if (targets.some((target) => target.connectivity === 'stale')) return { status: 'target_stale' }; + if (app.pause_at) return { status: 'rollout_paused', pauseAt: app.pause_at, pauseReason: app.pause_reason }; + if (app.partial_json) return { status: 'partially_rolled_out', partial: app.partial_json }; + if (app.target_mode === 'direct') return { status: 'not_applicable' }; + if (app.rollout_candidate_id) return { status: 'rollout_not_executable', rolloutCandidateId: app.rollout_candidate_id }; + return { status: 'not_applicable' }; +} + +function deriveTarget( + app: GitOpsApplicationRow, + target: GitOpsTargetCurrentRow, + healthDisabled: boolean, + limitations: GitOpsLimitation[], +): GitOpsTargetProjection { + let connectivity: GitOpsTargetProjection['connectivity'] = 'unknown'; + if ( + target.connectivity === 'unknown' + || target.connectivity === 'reachable' + || target.connectivity === 'unreachable' + || target.connectivity === 'stale' + ) { + connectivity = target.connectivity; + } else if (target.connectivity) { + limitations.push({ code: 'connectivity_invalid', message: 'stored connectivity is illegal', evidence: target.connectivity }); + } + mergePersistedLimitations(target.evidence_limitations_json, limitations); + const observed = decodeObservedSafe(target.observed_artifact_identity_json, limitations); + const artifact = deriveArtifact(app, target.desired_generation_id, target.expected_artifact_set_id, target.latest_artifact_set_id, limitations); + const runtime = deriveRuntime(target, artifact, observed, healthDisabled); + return { + nodeId: target.node_id, + stackName: app.stack_name, + desiredGenerationId: target.desired_generation_id, + candidateGenerationId: target.candidate_generation_id, + appliedGenerationId: target.applied_generation_id, + deployedGenerationId: target.deployed_generation_id, + healthyGenerationId: target.healthy_generation_id, + lkgGenerationId: target.lkg_generation_id, + lkgArtifactSetId: target.lkg_artifact_set_id, + lkgUnavailableAt: target.lkg_unavailable_at, + lkgUnavailableReason: target.lkg_unavailable_reason, + expectedArtifactSetId: target.expected_artifact_set_id, + latestArtifactSetId: target.latest_artifact_set_id, + artifact, + observedArtifactIdentity: observed, + intentRevisionId: target.intent_revision_id, + rolloutCandidateId: target.rollout_candidate_id, + rolloutGenerationId: target.rollout_generation_id, + approvals: { + sourceAcceptanceRef: target.source_acceptance_ref, + placementApprovalRef: target.placement_approval_ref, + rolloutAuthorizationRef: target.rollout_authorization_ref, + legacyCombinedApprovalRef: target.legacy_combined_approval_ref, + }, + connectivity, + legacyAppliedRevision: target.legacy_applied_revision, + runtime, + health: deriveHealth(target, healthDisabled), + lkg: deriveLkg(target, limitations), + tombstoned: target.target_status === 'tombstoned', + }; +} + +/** + * The runtime status each Blueprint observation stage projects as. + * + * The reconciler records what it saw against the target rather than acting on + * it, so this is the only route those observations have into a derived status. + * + * Two type obligations, and they pull in opposite directions. The declared type + * is keyed on an open string because the value looked up is `latest_stage`, + * which holds whichever stage was recorded last: anything that is not an + * observation must be absent here and fall through to the states below, which + * is exactly how a later transition supersedes an earlier observation. The + * `satisfies` closes the other side, making the map total over the stages the + * reconciler can actually record, so a new observation stage that nothing + * projects fails this build instead of silently reading as never applied. + * + * The `| undefined` is load-bearing: this project does not set + * `noUncheckedIndexedAccess`, so without it a miss would type as a status and + * the guard at the call site would look like dead code. + */ +type ObservationRuntimeStatus = 'pending_state_review' | 'evict_blocked' | 'drifted' | 'correcting'; + +const BLUEPRINT_OBSERVATION_STATUS: Record = { + blueprint_state_review: 'pending_state_review', + blueprint_evict_blocked: 'evict_blocked', + blueprint_drifted: 'drifted', + blueprint_correcting: 'correcting', +} satisfies Record; + +function deriveRuntime( + target: GitOpsTargetCurrentRow, + artifact: ArtifactFacet, + observed: ReturnType, + healthDisabled: boolean, +): RuntimeFacet { + if (target.target_status === 'tombstoned') return { status: 'tombstoned' }; + if (target.recovery_phase === 'restoring' || target.recovery_phase === 'compensating') { + return { status: 'recovery_required' }; + } + if (target.recovery_phase === 'failed' || target.failure_stage === 'recovery') { + return { + status: 'recovery_failed', + recoveryRef: target.recovery_ref, + recoveryGenerationId: target.recovery_generation_id, + failureClass: target.failure_class ?? 'unknown', + failureAt: target.failure_at ?? 0, + }; + } + if (target.active_operation_stage === 'deploy_started') return { status: 'deploying' }; + if ( + target.interruption_stage === 'deploy_started' + || target.interruption_stage === 'blueprint_deploy_started' + || target.interruption_stage === 'blueprint_withdraw_started' + ) { + return { + status: 'completion_unknown', + interruptedStage: target.interruption_stage, + interruptedAt: target.interruption_at ?? 0, + interruptedOperationId: target.interruption_operation_id, + interruptedGenerationId: target.interruption_generation_id, + interruptedIntentRevisionId: target.interruption_intent_revision_id, + interruptedRolloutCandidateId: target.interruption_rollout_candidate_id, + }; + } + if (target.pause_at) return { status: 'paused', pauseAt: target.pause_at, pauseReason: target.pause_reason }; + if (target.partial_json) return { status: 'partially_rolled_out' }; + if (target.failure_stage === 'deploy' && (target.failure_class === 'pre_mutation' || target.failure_class === 'unbound')) { + return { status: 'failed_previous_workload_intact' }; + } + if (target.failure_stage === 'deploy' && target.failure_class === 'post_mutation') { + return { status: 'failed_after_mutation' }; + } + // Placed after every state a live, interrupted or failed mutation puts the + // target in, and before the applied and deployed pointer checks. So an + // observation cannot mask an in-flight deploy or a failure, but does outrank + // pointers that predate it. The case that is easy to miss is the last one: a + // target with no applied generation that has been observed now reports what + // was seen rather than `never_applied`, which is what a deployed Blueprint + // that drifted used to report. + const blueprintStage = BLUEPRINT_OBSERVATION_STATUS[target.latest_stage ?? '']; + if (blueprintStage) return { status: blueprintStage }; + if (!target.applied_generation_id) return { status: 'never_applied' }; + if (!target.deployed_generation_id) return { status: 'applied_not_deployed' }; + // The target's contract is its desired generation, so a populated deployed + // pointer alone proves nothing: a newer applied generation with the old one + // still running stays deploy-pending, or a stack awaiting its deploy would + // read as synced and healthy off the previous workload's pointers. A null + // desired id is the unknown case (legacy rows, recovered targets), where the + // deployed pointer remains the only basis to judge. + if ( + target.desired_generation_id !== null + && target.deployed_generation_id !== target.desired_generation_id + ) { + return { status: 'applied_not_deployed' }; + } + if (artifact.status !== 'not_applicable' && 'expected' in artifact && artifact.expected + && (artifact.expected.qualification === 'exact' || artifact.expected.qualification === 'qualified')) { + if ( + observed.kind === 'unknown' + || observed.kind === 'missing' + || observed.kind === 'unavailable' + || observed.kind === 'stale' + || observed.kind === 'local_build_unverified' + ) { + return { status: 'artifact_verification_pending' }; + } + if ( + (observed.kind === 'exact' || observed.kind === 'qualified') + && artifact.expected.identity + && observed.identity !== artifact.expected.identity + ) { + return { status: 'runtime_artifact_drift' }; + } + } + if (target.retry_at) return { status: 'retry_scheduled' }; + if (healthDisabled) return { status: 'synced_and_healthy' }; + if (target.healthy_generation_id === target.deployed_generation_id) return { status: 'synced_and_healthy' }; + return { status: 'fully_deployed_health_pending' }; +} + +function deriveHealth(target: GitOpsTargetCurrentRow, healthDisabled: boolean): HealthFacet { + if (healthDisabled) return { status: 'not_applicable' }; + if (!target.deployed_generation_id) return { status: 'unbound' }; + // A passing run answers for the generation the target was asked to run, so + // it is judged against the desired id and only falls back to the deployed + // pointer when no desired id is recorded. Judging against whatever is + // deployed would let the previous workload's green run vouch for a newer + // generation nobody has watched. + const expectedGeneration = target.desired_generation_id ?? target.deployed_generation_id; + if (target.healthy_generation_id === expectedGeneration) { + return { status: 'passed', runId: '', deployedGenerationId: target.deployed_generation_id }; + } + return { status: 'pending', runId: null }; +} + +function deriveLkg(target: GitOpsTargetCurrentRow, limitations: GitOpsLimitation[]): LkgFacet { + if (!target.lkg_generation_id && !target.lkg_unavailable_at) return { status: 'none' }; + if (target.lkg_unavailable_at) return { status: 'unavailable' }; + const generation = target.lkg_generation_id + ? GitOpsStore.getInstance().getGeneration(target.lkg_generation_id) + : undefined; + if (target.lkg_generation_id && !generation) { + limitations.push({ code: 'lkg_generation_missing', message: 'LKG generation row is gone', evidence: target.lkg_generation_id }); + return { status: 'unavailable' }; + } + if (target.lkg_artifact_set_id) { + const artifact = GitOpsStore.getInstance().getArtifactSet(target.lkg_artifact_set_id); + if (!artifact || artifact.generation_id !== target.lkg_generation_id) { + limitations.push({ code: 'lkg_artifact_invalid', message: 'captured LKG artifact is invalid', evidence: target.lkg_artifact_set_id }); + return { status: 'available', generationId: target.lkg_generation_id!, artifactSetId: target.lkg_artifact_set_id }; + } + if (artifact.qualification === 'qualified') { + return { status: 'qualified', generationId: target.lkg_generation_id!, artifactSetId: artifact.id }; + } + return { status: 'available', generationId: target.lkg_generation_id!, artifactSetId: artifact.id }; + } + return { status: 'available', generationId: target.lkg_generation_id!, artifactSetId: null }; +} + +/** + * Application-level conditions under which deploying is withheld outright: an + * operation or recovery already owns the stack, so nothing may start a deploy + * even where a target's own state would make one legal. + */ +function appDeployWithheld(app: GitOpsApplicationRow): boolean { + return app.active_operation_stage === 'fetch_started' + || app.active_operation_stage === 'apply_started' + || app.recovery_phase === 'restoring' + || app.recovery_phase === 'compensating' + || app.recovery_phase === 'failed' + || app.failure_stage === 'recovery'; +} + +/** + * Whether deploying this exact target is legal right now, per the revision-state + * plan's available-action rules. Deliberately per target: the application-wide + * action list is a union across targets, so keying an item to it would tell a + * paused or failed sibling to deploy because a healthy sibling diverged. + * + * Direct targets converge a known divergence outright; an interrupted deploy is + * retried only against the generation still applied, exactly what deployStarted + * will demand. Writers keep applied and desired equal today, so keying on + * applied can never advertise an action the transition would refuse. + * + * Targets of Blueprint modes have no Direct deploy at all: their sole retry + * repeats an interrupted deploy or withdraw, legal only while both persisted + * identities equal what the application currently requires. An absent pair + * counts as matching: rollout candidates come from a later-phase producer, so + * inline Blueprints carry no candidate id on either side yet, and demanding one + * here would leave every interrupted inline deploy permanently unactionable. A + * superseded value on either side fails the comparison. + * + * disk_invocation_drift is deliberately absent: no producer reaches that + * status in this slice, and until one lands the predicate fails safe to a + * reported mismatch with no deploy recommendation. + */ +function targetDeployLegal(app: GitOpsApplicationRow, target: GitOpsTargetProjection): boolean { + if (appDeployWithheld(app) || target.tombstoned) return false; + if (app.target_mode !== 'direct') { + if (target.runtime.status !== 'completion_unknown') return false; + const stage = target.runtime.interruptedStage; + if (stage !== 'blueprint_deploy_started' && stage !== 'blueprint_withdraw_started') return false; + return ( + target.runtime.interruptedIntentRevisionId === app.intent_revision_id + && target.runtime.interruptedRolloutCandidateId === app.rollout_candidate_id + ); + } + if (target.runtime.status === 'applied_not_deployed') return true; + return ( + target.runtime.status === 'completion_unknown' + && target.runtime.interruptedStage === 'deploy_started' + && target.runtime.interruptedGenerationId !== null + && target.runtime.interruptedGenerationId === target.appliedGenerationId + ); +} + +function deriveActions( + app: GitOpsApplicationRow, + source: SourceFacet, + placement: PlacementFacet, + targets: GitOpsTargetProjection[], +): GitOpsAvailableAction[] { + if (source.status === 'applying' || source.status === 'checking_fetching') return ['none']; + if (source.status === 'recovery_required' || source.status === 'recovery_failed') return ['none']; + const actions = new Set(); + // Fetch is offered only to live Direct applications: Blueprint Git-source + // integration ships later, until then no Blueprint mode advertises fetch. + if ( + app.target_mode === 'direct' + && ( + source.status === 'never_reconciled' + || source.status === 'source_reconcile_required' + || source.status === 'source_retry_scheduled' + || (source.status === 'source_unknown' && source.interruptedStage === 'fetch_started') + || (source.status === 'source_failed' && (source.failureStage === 'fetch' || source.failureStage === 'validation')) + ) + ) { + actions.add('fetch'); + } + if (source.status === 'candidate_ready') actions.add('apply'); + // An interrupted apply may be finished only while its recorded generation is + // still the current candidate and nothing has since suspended the source or + // blocked that candidate, all of which applyStarted would refuse. No shipped + // producer pairs blockage with a matching interruption today, because + // blocking always mints a fresh candidate id; these clauses hold the gate to + // the transition table's contract regardless of what future producers do. + if ( + source.status === 'source_unknown' + && source.interruptedStage === 'apply_started' + && source.interruptedGenerationId !== null + && source.interruptedGenerationId === app.candidate_generation_id + && !app.suspended_at + && app.candidate_plan_blocked !== 1 + ) { + // applyStarted also demands the candidate generation exist under this + // application with an unchanged materialization fingerprint; prove all of + // it here rather than recommend a transition that would refuse. + const candidate = GitOpsStore.getInstance().getGeneration(app.candidate_generation_id); + if ( + candidate !== undefined + && candidate.application_id === app.id + && candidate.materialization_fingerprint === app.materialization_fingerprint + ) { + actions.add('apply'); + } + } + if (app.candidate_generation_id && !app.active_operation_stage) actions.add('dismiss'); + if (targets.some((target) => targetDeployLegal(app, target))) actions.add('deploy'); + if (placement.status === 'placement_review_pending') actions.add('approve_legacy'); + if (actions.size === 0) return ['none']; + return Array.from(actions); +} + +/** + * Fold the limitations a writer recorded into the ones derived here. + * + * These cannot be re-derived: they describe evidence that was dropped because + * it could not be proven, and once dropped the row looks the same as one that + * never had it. Decoded fail-closed, so a corrupt record surfaces as its own + * limitation rather than disappearing. + */ +function mergePersistedLimitations(raw: string | null, limitations: GitOpsLimitation[]): void { + if (!raw) return; + try { + for (const item of decodeGitOpsEvidenceLimitations(raw)) { + limitations.push({ + code: item.code, + message: 'evidence recorded at write time could not be proven', + evidence: item.detail, + }); + } + } catch (err) { + limitations.push({ + code: 'evidence_limitations_invalid', + message: err instanceof Error ? err.message : String(err), + evidence: raw, + }); + } +} + +function decodeObservedSafe( + raw: string | null, + limitations: GitOpsLimitation[], +): ReturnType { + try { + return decodeObservedArtifactIdentity(raw); + } catch (err) { + // Any failure here means the runtime observation is unusable, so it must + // surface as a limitation. Returning a clean 'unknown' without one would + // read as "nothing observed yet" and quietly downgrade a real artifact + // drift to a pending check. + limitations.push({ + code: err instanceof GitOpsJsonError ? 'artifact_observation_invalid' : 'artifact_observation_decode_failed', + message: err instanceof Error ? err.message : String(err), + evidence: raw, + }); + return { kind: 'unknown' }; + } +} + +/** + * The not-applicable shape, carrying why an application we expected was absent. + * + * Distinct from `NOT_APPLICABLE_REVISION` on purpose. That one means "nothing + * here", which is the honest answer for a stack or Blueprint the model was + * never asked about. This one means "something should have been here and was + * not", which is a fault. Returning the shared sentinel for both would make a + * vanished row indistinguishable from one that never existed, and the reader + * has no third source to tell them apart. + */ +function unreachableApplicationRevision(limitation: GitOpsLimitation): GitOpsRevisionProjection { + return { + schemaVersion: 1, + targetMode: 'not_applicable', + applicationId: null, + facets: null, + targets: [], + drift: [], + limitations: [limitation], + availableActions: [], + approvals: null, + }; +} + +function missingApplicationRevision(applicationId: string): GitOpsRevisionProjection { + return unreachableApplicationRevision({ + code: 'application_row_missing', + message: 'The application this projection was resolved from is no longer present.', + evidence: { applicationId }, + }); +} + +/** + * A Blueprint proven to manage a stack directory, with no application row. + * + * Its own code, not `application_row_missing`, because the evidence differs: + * there is no application id to name, only the Blueprint and the stack whose + * deployment row proved the ownership. + */ +export function missingBlueprintApplicationRevision(blueprintId: number, stackName: string): GitOpsRevisionProjection { + return unreachableApplicationRevision({ + code: 'blueprint_application_missing', + message: 'A Blueprint deployed this stack but has no live application to describe it.', + evidence: { blueprintId, stackName }, + }); +} + +export function projectApplication(applicationId: string, healthDisabled: boolean): GitOpsRevisionProjection { + const store = GitOpsStore.getInstance(); + const application = store.getApplication(applicationId); + // The caller resolved this id from a row it had just read, so a miss here is + // not "no application": it is a row that went away between the two reads, + // which are deliberately not in one transaction. Say so rather than reporting + // the same answer an unmodelled stack gets. + if (!application) return missingApplicationRevision(applicationId); + return deriveGitOpsRevision({ + application, + targets: store.listTargets(applicationId), + healthDisabled, + }, null); +} diff --git a/backend/src/services/gitops/directApplication.ts b/backend/src/services/gitops/directApplication.ts new file mode 100644 index 00000000..5f1c35f3 --- /dev/null +++ b/backend/src/services/gitops/directApplication.ts @@ -0,0 +1,248 @@ +import { randomUUID } from 'crypto'; +import path from 'path'; +import { NodeRegistry } from '../NodeRegistry'; +import { MANAGED_ROOT_NAME } from './managedPaths'; +import { encodeGitOpsJson } from './json'; +import { materializationFingerprint } from './fingerprint'; +import { parseHttpsRepoUrl, parseLegacyRepoUrl, secretFreeRepoUrl, serializeRepoIdentity, type RepoIdentity } from './repoIdentity'; +import type { + GitOpsApplicationRow, + GitOpsCreateCheckpointRow, + GitOpsGenerationRow, +} from './types'; + +/** The material source configuration a Direct application is bound to. */ +export type DirectSourceConfig = { + repoUrl: string; + branch: string; + composePaths: readonly string[]; + contextDir: string | null; + syncEnv: boolean; + envPath: string | null; +}; + +export type DirectSourceIdentity = { + /** Secret-free `https://host/pathname`, safe to persist and to project. */ + repoUrl: string; + identity: RepoIdentity; + fingerprint: string; +}; + +export class GitOpsIdentityError extends Error { + constructor(message: string) { + super(message); + this.name = 'GitOpsIdentityError'; + } +} + +/** + * Derive the storable identity and materialization fingerprint for a source. + * + * The fingerprint is what later decides whether a staged candidate still + * matches the configuration it was built from, so it is computed from the same + * secret-free identity that gets persisted, never from the raw operational URL. + */ +export function directSourceIdentity(config: DirectSourceConfig): DirectSourceIdentity { + const parsed = parseHttpsRepoUrl(config.repoUrl); + if (!parsed.ok) throw new GitOpsIdentityError(`repository URL is not storable: ${parsed.reason}`); + return directSourceIdentityFromUrl(config, parsed.url); +} + +/** + * The same derivation, for URLs that predate strict ingress. + * + * Migration is its only caller. A legacy operational row may still carry + * userinfo or a query string that fetch needs, so the storable identity strips + * them instead of refusing the stack; the strict helper above stays the gate + * for every path a user can drive. + */ +export function migrationDirectSourceIdentity(config: DirectSourceConfig): DirectSourceIdentity { + const parsed = parseLegacyRepoUrl(config.repoUrl); + if (!parsed.ok) throw new GitOpsIdentityError(`repository URL is not storable: ${parsed.reason}`); + return directSourceIdentityFromUrl(config, parsed.url); +} + +function directSourceIdentityFromUrl(config: DirectSourceConfig, url: URL): DirectSourceIdentity { + const identity = serializeRepoIdentity(url); + const material = { + repoIdentity: identity, + configuredRef: config.branch, + composePaths: config.composePaths, + contextDir: config.contextDir, + syncEnv: config.syncEnv, + envPath: config.envPath, + }; + return { + repoUrl: secretFreeRepoUrl(identity), + identity, + fingerprint: materializationFingerprint(material), + }; +} + +/** Absolute managed root for one stack on the local node. */ +export function stackManagedRoot(stackName: string): string { + const dataDir = process.env.DATA_DIR || path.join(process.cwd(), 'data'); + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + return path.join(dataDir, MANAGED_ROOT_NAME, String(nodeId), stackName); +} + +export function newGitOpsId(): string { + return randomUUID(); +} + +/** + * Build a Direct application row. + * + * `creating` is for create-from-Git, where the stack does not exist yet and the + * checkpoint decides what happens if the process dies. `active` is for linking + * a stack that already exists: there is nothing to recover, so it is live from + * the moment the source row commits. + */ +export function buildDirectApplicationRow(args: { + id: string; + stackName: string; + config: DirectSourceConfig; + identity: DirectSourceIdentity; + lifecycleStatus: 'creating' | 'active'; + at: number; +}): GitOpsApplicationRow { + return { + id: args.id, + lifecycle_key: `direct:${args.stackName}`, + lifecycle_status: args.lifecycleStatus, + target_mode: 'direct', + stack_name: args.stackName, + blueprint_id: null, + configured_repo_url: args.identity.repoUrl, + repo_identity_json: encodeGitOpsJson(args.identity.identity), + configured_ref: args.config.branch, + compose_paths_json: encodeGitOpsJson([...args.config.composePaths]), + context_dir: args.config.contextDir, + sync_env: args.config.syncEnv ? 1 : 0, + env_path: args.config.syncEnv ? args.config.envPath : null, + materialization_fingerprint: args.identity.fingerprint, + desired_commit_sha: null, + fetched_commit_sha: null, + candidate_generation_id: null, + accepted_generation_id: null, + candidate_plan_blocked: 0, + review_required: 0, + artifact_set_id: null, + latest_artifact_set_id: null, + intent_revision_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + rollout_authorization_ref: null, + legacy_combined_approval_ref: null, + preflight_fingerprint: null, + latest_operation_id: null, + active_operation_id: null, + active_operation_stage: null, + active_operation_at: null, + active_generation_id: null, + pause_at: null, + pause_reason: null, + partial_json: null, + failure_stage: null, + failure_class: null, + failure_at: null, + retry_at: null, + retry_count: 0, + suspended_at: null, + recovery_ref: null, + recovery_phase: null, + interruption_stage: null, + interruption_at: null, + interruption_operation_id: null, + interruption_generation_id: null, + evidence_fresh_at: null, + evidence_limitations_json: null, + created_at: args.at, + updated_at: args.at, + }; +} + +export function buildGenerationRow(args: { + id: string; + applicationId: string; + commitSha: string; + identity: DirectSourceIdentity; + configuredRef: string; + candidateRelPath: string; + appliedRelPath: string; + manifestVersion: number; + expectedInvocation: unknown; + changePlanFingerprint: string | null; + operationId: string; + trigger: string; + actor: string | null; + at: number; + /** A blocked change plan is recorded, but such a generation can never apply. */ + planBlocked?: boolean; +}): GitOpsGenerationRow { + return { + id: args.id, + application_id: args.applicationId, + commit_sha: args.commitSha, + repo_url: args.identity.repoUrl, + configured_ref: args.configuredRef, + repo_identity_json: encodeGitOpsJson(args.identity.identity), + manifest_version: args.manifestVersion, + candidate_dir: args.candidateRelPath, + applied_dir: args.appliedRelPath, + expected_invocation_json: encodeGitOpsJson(args.expectedInvocation), + materialization_fingerprint: args.identity.fingerprint, + validation_ok: 1, + plan_blocked: args.planBlocked ? 1 : 0, + change_plan_fingerprint: args.changePlanFingerprint, + operation_id: args.operationId, + trigger: args.trigger, + actor: args.actor, + previous_generation_id: null, + redacted_limitations_json: '[]', + created_at: args.at, + }; +} + +export function buildCreateCheckpointRow(args: { + applicationId: string; + stackName: string; + operationId: string; + config: DirectSourceConfig; + identity: DirectSourceIdentity; + authType: string; + encryptedToken: string | null; + autoApplyOnWebhook: boolean; + autoDeployOnApply: boolean; + commitSha: string; + createdManagedRoot: boolean; + at: number; +}): GitOpsCreateCheckpointRow { + return { + application_id: args.applicationId, + stack_name: args.stackName, + phase: 'pre_stack', + generation_id: null, + operation_id: args.operationId, + // Operational URL for fetch compatibility during create. Copies into + // generations and history always go through the secret-free identity. + repo_url: args.config.repoUrl, + branch: args.config.branch, + compose_path: args.config.composePaths[0] ?? '', + compose_paths_json: encodeGitOpsJson([...args.config.composePaths]), + context_dir: args.config.contextDir, + sync_env: args.config.syncEnv ? 1 : 0, + env_path: args.config.syncEnv ? args.config.envPath : null, + auth_type: args.authType, + encrypted_token: args.encryptedToken, + auto_apply_on_webhook: args.autoApplyOnWebhook ? 1 : 0, + auto_deploy_on_apply: args.autoDeployOnApply ? 1 : 0, + commit_sha: args.commitSha, + applied_spec_json: null, + created_managed_root: args.createdManagedRoot ? 1 : 0, + created_at: args.at, + updated_at: args.at, + }; +} diff --git a/backend/src/services/gitops/fingerprint.ts b/backend/src/services/gitops/fingerprint.ts new file mode 100644 index 00000000..35d3513c --- /dev/null +++ b/backend/src/services/gitops/fingerprint.ts @@ -0,0 +1,39 @@ +import { createHash } from 'crypto'; +import { encodeGitOpsJson } from './json'; +import type { RepoIdentity } from './repoIdentity'; + +export type MaterialConfigInput = { + repoIdentity: RepoIdentity; + configuredRef: string; + composePaths: readonly string[]; + contextDir: string | null; + syncEnv: boolean; + envPath: string | null; +}; + +function emptyToNull(value: string | null | undefined): string | null { + if (value === null || value === undefined) return null; + const trimmed = value.trim(); + return trimmed.length === 0 ? null : trimmed; +} + +export function canonicalMaterialConfigJson(input: MaterialConfigInput): string { + const contextDir = emptyToNull(input.contextDir); + const syncEnv = input.syncEnv === true; + const envPath = syncEnv ? emptyToNull(input.envPath) : null; + return encodeGitOpsJson({ + composePaths: [...input.composePaths], + contextDir, + syncEnv, + envPath, + repoIdentity: { + host: input.repoIdentity.host, + pathname: input.repoIdentity.pathname, + }, + configuredRef: input.configuredRef, + }); +} + +export function materializationFingerprint(input: MaterialConfigInput): string { + return createHash('sha256').update(canonicalMaterialConfigJson(input)).digest('hex'); +} diff --git a/backend/src/services/gitops/history.ts b/backend/src/services/gitops/history.ts new file mode 100644 index 00000000..2695d73f --- /dev/null +++ b/backend/src/services/gitops/history.ts @@ -0,0 +1,430 @@ +import { randomUUID } from 'crypto'; +import type Database from 'better-sqlite3'; +import { decodeGitOpsJson, encodeGitOpsJson, isRecord, GitOpsJsonError } from './json'; +import { enqueueHistoryPublication } from './publish'; +import { sanitizeForLog } from '../../utils/safeLog'; +import type { + GitOpsApplicationRow, + GitOpsApprovalRefs, + GitOpsHistoryRow, + GitOpsLimitation, + GitOpsTargetMode, +} from './types'; + +export type HistoryOutcome = GitOpsHistoryRow['outcome']; + +/** + * Every stage a transition can record. + * + * The column itself is open TEXT, because a stage is an audit label rather than + * a state and a future producer must be able to add one without a migration. + * This union constrains the *writers*: it is what makes the set finite at + * compile time, so the metrics keyspace is bounded by the type rather than by + * whatever strings happen to reach the insert. Adding a producer stage without + * adding it here fails the build, which is the point. + */ +export type GitOpsHistoryStage = + | 'application_activated' + | 'application_tombstoned' + | 'applied' + | 'apply_failed' + | 'apply_started' + | 'artifact_evidence_recorded' + | 'artifact_expectation_accepted' + | 'blueprint_ack_recorded' + | 'blueprint_correcting' + | 'blueprint_deploy_failed' + | 'blueprint_deploy_started' + | 'blueprint_drifted' + | 'blueprint_evict_blocked' + | 'blueprint_state_review' + | 'blueprint_withdraw_failed' + | 'blueprint_withdraw_started' + | 'blueprint_withdrawn' + | 'candidate_ready' + | 'candidate_superseded' + | 'config_changed_pending_cleared' + | 'create_failed' + | 'deploy_bound' + | 'deploy_failed' + | 'deploy_started' + | 'deploy_unbound' + | 'dismissed' + | 'fetch_failed' + | 'fetch_started' + | 'fetched' + | 'fetched_invalid' + | 'health_finalized' + | 'intent_revised' + | 'operation_interrupted' + | 'partial_cleared' + | 'partially_rolled_out' + | 'recovery_failed' + | 'recovery_started' + | 'recovery_succeeded' + | 'rollback_completed' + | 'rollback_in_progress' + | 'rollback_partial_failed' + | 'rollout_candidate_opened' + | 'rollout_paused' + | 'rollout_unpaused' + | 'source_conflict_blocker' + | 'source_retry_scheduled' + | 'source_suspended' + | 'source_unsuspended' + | 'target_tombstoned'; + +export type HistoryInsert = { + application: GitOpsApplicationRow; + nodeId: number | null; + dedupeTarget: string; + operationId: string; + stage: GitOpsHistoryStage; + outcome: HistoryOutcome; + trigger: string; + actor: string | null; + // Objects, not `unknown`: the read path reports a non-object payload as an + // unreadable audit record, so a producer must not be able to author one. + before: Record; + after: Record; + generationId?: string | null; + artifactSetId?: string | null; + commitSha?: string | null; + intentRevisionId?: string | null; + rolloutCandidateId?: string | null; + sourceAcceptanceRef?: string | null; + placementApprovalRef?: string | null; + rolloutAuthorizationRef?: string | null; + legacyCombinedApprovalRef?: string | null; + requiredTargetsJson?: string | null; + recoveryRef?: string | null; + redactedReasonClass?: string | null; + at: number; +}; + +/** + * Append one history row, returning null when this exact operation already + * wrote its row (a replay). + * + * The conflict clause names the dedupe index deliberately rather than using + * `INSERT OR IGNORE`: `OR IGNORE` also swallows NOT NULL and CHECK violations, + * which would drop an audit row while the state change committed and report it + * to the caller as a harmless replay. Only a duplicate of the dedupe tuple is + * tolerated here; every other constraint failure throws and rolls the + * transaction back. Callers turn a null return into `replayed: true`, so the + * dedupe index is load-bearing for idempotency, not just for storage hygiene. + * + * An inserted row is also queued for announcement here rather than at the + * transition call sites, because this is the only place that can tell an + * insert from a replay: the callers see a null and turn it into `replayed`, + * by which point the distinction has already been made once. + */ +export function insertHistory(db: Database.Database, row: HistoryInsert): string | null { + const id = randomUUID(); + const result = db.prepare( + `INSERT INTO gitops_history ( + id, created_at, application_id, target_mode, lifecycle_key, stack_name, blueprint_id, + node_id, dedupe_target, repo_url, configured_ref, repo_identity_json, commit_sha, + generation_id, artifact_set_id, intent_revision_id, rollout_candidate_id, rollout_generation_id, + source_acceptance_ref, placement_approval_ref, rollout_authorization_ref, + legacy_combined_approval_ref, operation_id, stage, outcome, trigger, actor, + before_json, after_json, required_targets_json, validation_json, per_target_results_json, + health_run_id, health_snapshot_json, invocation_observed_json, recovery_ref, redacted_reason_class + ) VALUES (${Array(37).fill('?').join(', ')}) + ON CONFLICT(application_id, operation_id, stage, dedupe_target) DO NOTHING`, + ).run( + id, + row.at, + row.application.id, + row.application.target_mode as GitOpsTargetMode, + row.application.lifecycle_key, + row.application.stack_name, + row.application.blueprint_id, + row.nodeId, + row.dedupeTarget, + row.application.configured_repo_url, + row.application.configured_ref, + row.application.repo_identity_json, + row.commitSha ?? row.application.desired_commit_sha, + row.generationId ?? null, + row.artifactSetId ?? null, + row.intentRevisionId ?? row.application.intent_revision_id, + row.rolloutCandidateId ?? row.application.rollout_candidate_id, + row.application.rollout_generation_id, + row.sourceAcceptanceRef ?? row.application.source_acceptance_ref, + row.placementApprovalRef ?? row.application.placement_approval_ref, + row.rolloutAuthorizationRef ?? row.application.rollout_authorization_ref, + row.legacyCombinedApprovalRef ?? row.application.legacy_combined_approval_ref, + row.operationId, + row.stage, + row.outcome, + row.trigger, + row.actor, + encodeGitOpsJson(row.before), + encodeGitOpsJson(row.after), + row.requiredTargetsJson ?? null, + null, + null, + null, + null, + null, + row.recoveryRef ?? null, + row.redactedReasonClass ?? null, + ); + if (result.changes !== 1) return null; + enqueueHistoryPublication({ + db, + id, + stage: row.stage, + outcome: row.outcome, + applicationId: row.application.id, + targetMode: row.application.target_mode, + stackName: row.application.stack_name, + blueprintId: row.application.blueprint_id, + nodeId: row.nodeId, + at: row.at, + }); + return id; +} + +/** Page size when the caller does not ask for one. */ +export const HISTORY_DEFAULT_LIMIT = 50; +/** Hard ceiling on page size, whatever the caller asks for. */ +export const HISTORY_MAX_LIMIT = 100; +/** + * Rows examined per request before the page is cut short. + * + * Authorization runs per row after the query, so a caller with narrow grants + * could otherwise walk the whole table looking for rows they may read. This + * bounds that per-row loop, and the cursor still advances past every examined + * row so the next request resumes rather than rescanning. It is not a bound on + * database work: the query itself is bounded by the index over + * `(created_at, id)`, not by this constant. + */ +export const HISTORY_SCAN_CAP = 1000; + +export type GitOpsHistoryFilters = { + applicationId?: string; + stackName?: string; + /** Secret-free `repo_url`. Credentials never reach this column. */ + repoIdentity?: string; + configuredRef?: string; + commitSha?: string; + generationId?: string; + artifactSetId?: string; + blueprintId?: number; + rolloutCandidateId?: string; + rolloutGenerationId?: string; + nodeId?: number; + trigger?: string; + actor?: string; + outcome?: HistoryOutcome; +}; + +/** + * One history row as the API returns it. + * + * `before` and `after` are the producer's delta for that transition, not a full + * revision projection: each transition records only the fields it moved. They + * are display evidence. Authorization never reads them, so a row whose JSON is + * unreadable still returns its identity, stage, and outcome alongside a + * `history_json_invalid` limitation rather than vanishing from the audit trail. + */ +export type GitOpsHistoryItem = { + id: string; + createdAt: number; + applicationId: string; + targetMode: GitOpsTargetMode; + stackName: string | null; + /** Secret-free repository URL as recorded with the event, matching the `repoIdentity` filter. */ + repoIdentity: string | null; + /** Ref as configured when the event ran, matching the `configuredRef` filter. */ + configuredRef: string | null; + blueprintId: number | null; + nodeId: number | null; + commitSha: string | null; + generationId: string | null; + artifactSetId: string | null; + intentRevisionId: string | null; + rolloutCandidateId: string | null; + rolloutGenerationId: string | null; + approvals: GitOpsApprovalRefs; + operationId: string; + stage: string; + outcome: HistoryOutcome; + trigger: string; + actor: string | null; + before: Record | null; + after: Record | null; + limitations: GitOpsLimitation[]; +}; + +/** + * Page cursor over the `(created_at, id)` ordering. + * + * Both halves are needed: rows written by one transaction share a single + * timestamp by construction, so paginating on the timestamp alone would drop or + * repeat rows at a page boundary. + * + * Encoded in plain text, so treat it as readable and forgeable rather than + * opaque. That is tolerable because it only picks a start position in a scan + * whose rows are authorized individually afterwards, but it does mean the + * cursor discloses one row's timestamp and id to a caller who may not read that + * row. + */ +export type GitOpsHistoryCursor = { createdAt: number; id: string }; + +export function encodeHistoryCursor(cursor: GitOpsHistoryCursor): string { + return `${cursor.createdAt}.${cursor.id}`; +} + +/** History ids are minted with `randomUUID()`, so anything else is not one. */ +const HISTORY_ID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; + +/** + * Parse a caller-supplied cursor, returning null for anything malformed. + * + * Both halves are validated. The id half matters as much as the timestamp: it + * is compared as a string in the page query, so an unvalidated one would not + * fail, it would silently include or exclude an arbitrary slice of the rows + * sharing that millisecond. Callers turn null into a 400 rather than starting + * over, because quietly serving page one to someone who asked to resume is the + * kind of wrong answer an audit reader cannot detect. + */ +export function decodeHistoryCursor(raw: string): GitOpsHistoryCursor | null { + const separator = raw.indexOf('.'); + if (separator <= 0 || separator === raw.length - 1) return null; + const createdAt = Number(raw.slice(0, separator)); + if (!Number.isSafeInteger(createdAt) || createdAt < 0) return null; + const id = raw.slice(separator + 1); + if (!HISTORY_ID_RE.test(id)) return null; + return { createdAt, id }; +} + +/** + * Decode one history JSON column into its delta object. + * + * `null` means the column could not be read as an object. The caller turns that + * into the row's `history_json_invalid` limitation so the entry survives, and + * the reason is logged here because a corrupt audit column is a storage problem + * an operator needs to see rather than a routine response variation. + */ +function decodeHistoryDelta(rowId: string, column: string, raw: string): Record | null { + let value: unknown; + try { + value = decodeGitOpsJson(raw); + } catch (error) { + if (!(error instanceof GitOpsJsonError)) throw error; + console.error(`[GitOps] history ${sanitizeForLog(rowId)}.${column} is not decodable JSON: ${error.message}`); + return null; + } + if (!isRecord(value)) { + console.error(`[GitOps] history ${sanitizeForLog(rowId)}.${column} decoded to ${typeof value}, expected an object`); + return null; + } + return value; +} + +export function toHistoryItem(row: GitOpsHistoryRow): GitOpsHistoryItem { + const before = decodeHistoryDelta(row.id, 'before', row.before_json); + const after = decodeHistoryDelta(row.id, 'after', row.after_json); + const limitations: GitOpsLimitation[] = []; + if (before === null || after === null) { + limitations.push({ + code: 'history_json_invalid', + message: 'Recorded change detail for this entry could not be read.', + evidence: { before: before === null, after: after === null }, + }); + } + return { + id: row.id, + createdAt: row.created_at, + applicationId: row.application_id, + targetMode: row.target_mode, + stackName: row.stack_name, + repoIdentity: row.repo_url, + configuredRef: row.configured_ref, + blueprintId: row.blueprint_id, + nodeId: row.node_id, + commitSha: row.commit_sha, + generationId: row.generation_id, + artifactSetId: row.artifact_set_id, + intentRevisionId: row.intent_revision_id, + rolloutCandidateId: row.rollout_candidate_id, + rolloutGenerationId: row.rollout_generation_id, + approvals: { + sourceAcceptanceRef: row.source_acceptance_ref, + placementApprovalRef: row.placement_approval_ref, + rolloutAuthorizationRef: row.rollout_authorization_ref, + legacyCombinedApprovalRef: row.legacy_combined_approval_ref, + }, + operationId: row.operation_id, + stage: row.stage, + outcome: row.outcome, + trigger: row.trigger, + actor: row.actor, + before, + after, + limitations, + }; +} + +/** + * Read one scan window of history rows, newest first. + * + * Returns raw rows rather than a finished page because authorization is decided + * per row by the caller, which owns the permission context. The caller stops + * once its page is full and uses the last row it *examined* (not the last it + * kept) as the next cursor, so skipped rows are never revisited. + */ +export function queryHistoryRows( + db: Database.Database, + filters: GitOpsHistoryFilters, + cursor: GitOpsHistoryCursor | null, + scanLimit: number, +): GitOpsHistoryRow[] { + const clauses: string[] = []; + const params: Array = []; + const eq = (column: string, value: string | number | undefined): void => { + if (value === undefined) return; + clauses.push(`${column} = ?`); + params.push(value); + }; + + eq('application_id', filters.applicationId); + eq('stack_name', filters.stackName); + eq('repo_url', filters.repoIdentity); + eq('configured_ref', filters.configuredRef); + eq('commit_sha', filters.commitSha); + eq('generation_id', filters.generationId); + eq('artifact_set_id', filters.artifactSetId); + eq('blueprint_id', filters.blueprintId); + eq('rollout_candidate_id', filters.rolloutCandidateId); + // Distinct columns on purpose: a candidate is a proposal, a generation is a + // rollout that ran. Answering one filter from the other's column would report + // a proposal as executed. + eq('rollout_generation_id', filters.rolloutGenerationId); + // A node-scoped page keeps application-level rows: activation and similar + // stages carry no node, and a plain `node_id = ?` would make a proxied hub + // view read as if the application never came into being. + if (filters.nodeId !== undefined) { + clauses.push('(node_id = ? OR node_id IS NULL)'); + params.push(filters.nodeId); + } + eq('trigger', filters.trigger); + eq('actor', filters.actor); + eq('outcome', filters.outcome); + + if (cursor) { + clauses.push('(created_at < ? OR (created_at = ? AND id < ?))'); + params.push(cursor.createdAt, cursor.createdAt, cursor.id); + } + + const where = clauses.length ? `WHERE ${clauses.join(' AND ')}` : ''; + params.push(scanLimit); + return db.prepare( + `SELECT * FROM gitops_history ${where} + ORDER BY created_at DESC, id DESC + LIMIT ?`, + ).all(...params) as GitOpsHistoryRow[]; +} diff --git a/backend/src/services/gitops/json.ts b/backend/src/services/gitops/json.ts new file mode 100644 index 00000000..5dc9c9c8 --- /dev/null +++ b/backend/src/services/gitops/json.ts @@ -0,0 +1,303 @@ +export class GitOpsJsonError extends Error { + constructor(message: string) { + super(message); + this.name = 'GitOpsJsonError'; + } +} + +export function encodeGitOpsJson(value: unknown): string { + let encoded: string | undefined; + try { + encoded = JSON.stringify(value); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + throw new GitOpsJsonError(`gitops json encode failed: ${message}`); + } + // JSON.stringify returns undefined (it does not throw) for undefined, a + // function, or a symbol. Every JSON column is NOT NULL, so letting that + // through would bind SQL NULL and lose the row. + if (typeof encoded !== 'string') { + throw new GitOpsJsonError('gitops json encode produced no output'); + } + return encoded; +} + +export function decodeGitOpsJson(raw: string): unknown { + try { + return JSON.parse(raw); + } catch { + throw new GitOpsJsonError('gitops json decode failed'); + } +} + +export function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +export function isFiniteInteger(value: unknown): value is number { + return typeof value === 'number' && Number.isInteger(value) && Number.isFinite(value); +} + +export function isPositiveInteger(value: unknown): value is number { + return isFiniteInteger(value) && value > 0; +} + +export const PREFLIGHT_FINGERPRINT_RE = /^[0-9a-f]{64}$/; + +export function isPreflightFingerprint(value: unknown): value is string { + return typeof value === 'string' && PREFLIGHT_FINGERPRINT_RE.test(value); +} + +export type GitOpsRequiredTargetsJson = { nodeIds: number[] }; + +export function canonicalizeNodeIds(nodeIds: readonly number[]): number[] { + return Array.from(new Set(nodeIds)).sort((a, b) => a - b); +} + +export function decodeGitOpsRequiredTargetsJson(raw: string): GitOpsRequiredTargetsJson { + const decoded = decodeGitOpsJson(raw); + if (!isRecord(decoded)) { + throw new GitOpsJsonError('required_targets_json must be an object'); + } + const keys = Object.keys(decoded); + if (keys.length !== 1 || keys[0] !== 'nodeIds') { + throw new GitOpsJsonError('required_targets_json must have only nodeIds'); + } + if (!Array.isArray(decoded.nodeIds)) { + throw new GitOpsJsonError('required_targets_json.nodeIds must be an array'); + } + const nodeIds: number[] = []; + for (const item of decoded.nodeIds) { + if (!isFiniteInteger(item)) { + throw new GitOpsJsonError('required_targets_json.nodeIds must be integers'); + } + nodeIds.push(item); + } + const canonical = canonicalizeNodeIds(nodeIds); + if (canonical.length !== nodeIds.length) { + throw new GitOpsJsonError('required_targets_json.nodeIds must be unique'); + } + for (let i = 0; i < nodeIds.length; i += 1) { + if (nodeIds[i] !== canonical[i]) { + throw new GitOpsJsonError('required_targets_json.nodeIds must be sorted unique'); + } + } + return { nodeIds }; +} + +export function encodeGitOpsRequiredTargetsJson(nodeIds: readonly number[]): string { + const canonical = canonicalizeNodeIds(nodeIds); + if (canonical.length !== nodeIds.length) { + throw new GitOpsJsonError('required_targets_json.nodeIds must be unique'); + } + for (let i = 0; i < nodeIds.length; i += 1) { + if (nodeIds[i] !== canonical[i]) { + throw new GitOpsJsonError('required_targets_json.nodeIds must be sorted unique'); + } + if (!isFiniteInteger(nodeIds[i])) { + throw new GitOpsJsonError('required_targets_json.nodeIds must be integers'); + } + } + return encodeGitOpsJson({ nodeIds: [...nodeIds] }); +} + +/** + * Why a writer could not prove something, recorded on the row it affected. + * + * Distinct from the limitations the deriver computes at read time: those are + * re-derivable from current rows, these are facts only the transition that + * dropped a pointer knew. Without them a pointer that was cleared because it + * could not be proven is indistinguishable from one that never existed. + */ +export type GitOpsEvidenceLimitation = { code: string; detail: string | null }; + +export function decodeGitOpsEvidenceLimitations(raw: string | null): GitOpsEvidenceLimitation[] { + if (raw === null) return []; + const decoded = decodeGitOpsJson(raw); + if (!Array.isArray(decoded)) { + throw new GitOpsJsonError('evidence_limitations_json must be an array'); + } + return decoded.map((item) => { + if (!isRecord(item)) throw new GitOpsJsonError('evidence limitation must be an object'); + const keys = Object.keys(item); + if (keys.length !== 2 || !('code' in item) || !('detail' in item)) { + throw new GitOpsJsonError('evidence limitation must have exactly code and detail'); + } + if (typeof item.code !== 'string' || item.code.length === 0) { + throw new GitOpsJsonError('evidence limitation code must be a non-empty string'); + } + if (item.detail !== null && typeof item.detail !== 'string') { + throw new GitOpsJsonError('evidence limitation detail must be a string or null'); + } + return { code: item.code, detail: item.detail }; + }); +} + +/** + * Replace the limitations for one code, keeping every other code intact. + * + * Returns null when nothing remains, so a row that has recovered its evidence + * stores NULL rather than an empty array. + */ +export function encodeGitOpsEvidenceLimitations( + existing: GitOpsEvidenceLimitation[], + code: string, + next: GitOpsEvidenceLimitation | null, +): string | null { + const kept = existing.filter((item) => item.code !== code); + if (next) kept.push(next); + if (kept.length === 0) return null; + const encoded = encodeGitOpsJson(kept); + decodeGitOpsEvidenceLimitations(encoded); + return encoded; +} + +export type GitOpsApprovedTargetEffectJson = Array<{ nodeId: number; outcome: 'place' | 'remove' }>; + +export function decodeGitOpsApprovedTargetEffectJson(raw: string): GitOpsApprovedTargetEffectJson { + const decoded = decodeGitOpsJson(raw); + if (!Array.isArray(decoded)) { + throw new GitOpsJsonError('blast_json must be an array'); + } + const out: GitOpsApprovedTargetEffectJson = []; + const seen = new Set(); + let lastNodeId = Number.NEGATIVE_INFINITY; + for (const item of decoded) { + if (!isRecord(item)) { + throw new GitOpsJsonError('blast_json entries must be objects'); + } + const keys = Object.keys(item); + if (keys.length !== 2 || !('nodeId' in item) || !('outcome' in item)) { + throw new GitOpsJsonError('blast_json entries must have exactly nodeId and outcome'); + } + if (!isPositiveInteger(item.nodeId)) { + throw new GitOpsJsonError('blast_json.nodeId must be a positive integer'); + } + if (item.outcome !== 'place' && item.outcome !== 'remove') { + throw new GitOpsJsonError('blast_json.outcome must be place or remove'); + } + if (seen.has(item.nodeId)) { + throw new GitOpsJsonError('blast_json node ids must be unique'); + } + if (item.nodeId <= lastNodeId) { + throw new GitOpsJsonError('blast_json must be strictly increasing by nodeId'); + } + seen.add(item.nodeId); + lastNodeId = item.nodeId; + out.push({ nodeId: item.nodeId, outcome: item.outcome }); + } + return out; +} + +export function encodeGitOpsApprovedTargetEffectJson( + effect: GitOpsApprovedTargetEffectJson, +): string { + const encoded = encodeGitOpsJson(effect); + // Round-trip through the decoder so an invalid shape throws here rather than + // reaching SQLite. The decoded value is deliberately discarded. + decodeGitOpsApprovedTargetEffectJson(encoded); + return encoded; +} + +export type ArtifactEvidenceJson = + | { kind: 'unresolved' } + | { kind: 'exact'; identity: string } + | { kind: 'qualified'; identity: string } + | { kind: 'stale'; identity: string | null } + | { kind: 'unavailable' } + | { kind: 'local_build_unverified'; identity: string | null }; + +function requireNonEmptyIdentity(value: unknown): string { + if (typeof value !== 'string' || value.length === 0) { + throw new GitOpsJsonError('artifact identity must be a non-empty string'); + } + return value; +} + +export function decodeArtifactEvidenceJson(raw: string): ArtifactEvidenceJson { + const decoded = decodeGitOpsJson(raw); + if (!isRecord(decoded) || typeof decoded.kind !== 'string') { + throw new GitOpsJsonError('evidence_json must have a kind'); + } + const keys = Object.keys(decoded); + switch (decoded.kind) { + case 'unresolved': + case 'unavailable': + if (keys.length !== 1 || 'identity' in decoded) { + throw new GitOpsJsonError(`${decoded.kind} evidence forbids identity`); + } + return { kind: decoded.kind }; + case 'exact': + case 'qualified': + if (keys.length !== 2) { + throw new GitOpsJsonError(`${decoded.kind} evidence requires identity only`); + } + return { kind: decoded.kind, identity: requireNonEmptyIdentity(decoded.identity) }; + case 'stale': + case 'local_build_unverified': + if (keys.length !== 2 || !('identity' in decoded)) { + throw new GitOpsJsonError(`${decoded.kind} evidence requires identity`); + } + if (decoded.identity !== null && typeof decoded.identity !== 'string') { + throw new GitOpsJsonError(`${decoded.kind} identity must be string or null`); + } + return { kind: decoded.kind, identity: decoded.identity }; + default: + throw new GitOpsJsonError('unknown artifact evidence kind'); + } +} + +export function encodeArtifactEvidenceJson(value: ArtifactEvidenceJson): string { + const encoded = encodeGitOpsJson(value); + // Round-trip through the decoder so an invalid shape throws here rather than + // reaching SQLite. The decoded value is deliberately discarded. + decodeArtifactEvidenceJson(encoded); + return encoded; +} + +export type ObservedArtifactIdentity = + | { kind: 'unknown' } + | { kind: 'missing' } + | { kind: 'unavailable' } + | { kind: 'exact'; identity: string; observedAt: number } + | { kind: 'qualified'; identity: string; observedAt: number } + | { kind: 'stale'; identity: string; observedAt: number } + | { kind: 'local_build_unverified'; identity: string; observedAt: number }; + +export function decodeObservedArtifactIdentity(raw: string | null): ObservedArtifactIdentity { + if (raw === null) return { kind: 'unknown' }; + const decoded = decodeGitOpsJson(raw); + if (!isRecord(decoded) || typeof decoded.kind !== 'string') { + throw new GitOpsJsonError('observed artifact identity must have a kind'); + } + const keys = Object.keys(decoded); + switch (decoded.kind) { + case 'unknown': + if (keys.length !== 1) { + throw new GitOpsJsonError('unknown observation forbids extra fields'); + } + return { kind: 'unknown' }; + case 'missing': + case 'unavailable': + if (keys.length !== 1 || 'identity' in decoded) { + throw new GitOpsJsonError(`${decoded.kind} observation forbids identity`); + } + return { kind: decoded.kind }; + case 'exact': + case 'qualified': + case 'stale': + case 'local_build_unverified': + if (keys.length !== 3 || !('identity' in decoded) || !('observedAt' in decoded)) { + throw new GitOpsJsonError(`${decoded.kind} observation requires identity and observedAt`); + } + if (typeof decoded.identity !== 'string' || decoded.identity.length === 0) { + throw new GitOpsJsonError(`${decoded.kind} observation identity must be a non-empty string`); + } + if (typeof decoded.observedAt !== 'number' || !Number.isFinite(decoded.observedAt)) { + throw new GitOpsJsonError(`${decoded.kind} observation observedAt must be a finite number`); + } + return { kind: decoded.kind, identity: decoded.identity, observedAt: decoded.observedAt }; + default: + throw new GitOpsJsonError('unknown observed artifact identity kind'); + } +} diff --git a/backend/src/services/gitops/managedPaths.ts b/backend/src/services/gitops/managedPaths.ts new file mode 100644 index 00000000..2c7510c8 --- /dev/null +++ b/backend/src/services/gitops/managedPaths.ts @@ -0,0 +1,164 @@ +/** + * Managed-area path vocabulary, defined here rather than imported from + * GitProjectManifestService. + * + * The manifest service reaches these modules through GitSourceService, so + * importing its constants back into the gitops layer forms a cycle. Under that + * cycle the binding can still be uninitialized when these modules evaluate, + * which silently yields paths like `undefined/candidate-`: they pass a + * containment check against the managed root and name a directory that does + * not exist, so cleanup removes nothing and leaves the real one behind. + * + * These values must stay in step with the manifest service's layout. + */ +import fs from 'fs/promises'; +import path from 'path'; +import { sanitizeForLog } from '../../utils/safeLog'; + +export const MANAGED_ROOT_NAME = 'git-managed'; +export const GENERATIONS_DIR = 'generations'; + +/** + * The directory every stack's managed area lives under. + * + * Exists so each filesystem call on a managed path can resolve its target and + * check containment in its own scope. CodeQL does not credit the wrapped + * `isPathWithinBase` helper as a barrier, so `js/path-injection` reports the + * call even when the path was already validated; the check has to be inline at + * the call to be recognised. The duplication is deliberate. + * + * The base is node-agnostic, since callers here are given a root rather than a + * node id. On its own it proves only that a path names somewhere inside the + * managed area, which is why the real-path check below pins the path to its own + * position under this base rather than to the base itself. + */ +export function managedAreaBase(): string { + const dataDir = process.env.DATA_DIR || path.join(process.cwd(), 'data'); + return path.resolve(dataDir, MANAGED_ROOT_NAME); +} + +/** + * Resolve one path, keeping "is not there" apart from "could not be read". + * + * The distinction is the whole safety property. Collapsing both to "absent" + * would let the walk below climb past a path it could not resolve and infer + * containment from an ancestor, which is how a junction that throws `EPERM` or + * `ELOOP` instead of resolving would be treated as though it were not there at + * all. Only `ENOENT` means absent; everything else is a failure to establish + * what a path points at, and a containment check that cannot see a path must + * not pass it. + */ +async function resolveRealPath(target: string): Promise< + | { kind: 'resolved'; real: string } + | { kind: 'absent' } + | { kind: 'unreadable'; code: string } +> { + try { + return { kind: 'resolved', real: await fs.realpath(target) }; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === 'ENOENT') return { kind: 'absent' }; + return { kind: 'unreadable', code: code ?? 'UNKNOWN' }; + } +} + +/** + * Whether a path resolves to its own place in the managed area once links are + * followed. + * + * `path.resolve` never touches the filesystem, so lexical containment says + * nothing about a symlink or Windows junction sitting above the target: the + * string stays inside the managed area while a recursive delete walks straight + * out of it. Every sink that creates or removes a managed path asks this before + * it acts: the create teardown and staging-marker sinks, and the manifest + * service's generation pruning, boot-sweep orphan reaping, detach staging, + * staged-area finalization, and whole-area deletion. + * + * The property is positional, not membership. Asking only whether the resolved + * path lands somewhere under the managed area is satisfied by every other node + * and every other stack in it, so a junction from one stack's `generations` into + * another's passes while the delete takes a generation that belongs to someone + * else. What is checked instead is that the path resolves to exactly the + * location its own name claims: the managed area is resolved once, and the + * segments below it must be reached without redirection. + * + * Resolving the area separately is what keeps an operator's relocation working. + * Pointing the data directory at another volume moves the whole area and stays + * legal; a link *inside* the area does not, because nothing under it has any + * reason to live somewhere other than where it is named. + * + * The target itself is resolved when it exists, so a managed root or generation + * directory that turns out to be a link elsewhere is rejected rather than + * deleted through. Only when it is genuinely missing does the walk climb to the + * nearest ancestor that exists, appending the absent segments lexically: + * nothing can be linked at a path that is not there. A path that cannot be read + * at all stops the walk and refuses, because climbing past it would infer a + * location for the one path whose link status could not be established. + * + * Callers still run their own lexical containment check at their sink, because + * the analyzer only credits a barrier it can see at the call. This does not rely + * on them: it establishes the target's own position before resolving anything. + * + * What this does not close is the window between answering and acting. Node + * exposes no directory-relative remove, so a link swapped into an intermediate + * segment after this returns is still followed by the caller's `fs.rm`. Whoever + * could do that already has write access inside the managed area, which is the + * same position they would need to plant the link this rejects, so the check is + * worth having and the window is accepted rather than overlooked. + */ +export async function isRealPathAtManagedLocation(target: string): Promise { + const areaLexical = managedAreaBase(); + const resolved = path.resolve(target); + // The position the target claims for itself, read off the lexical path before + // any link is followed. This is what the real path has to agree with. + const relative = path.relative(areaLexical, resolved); + // The area root itself has no position under the area, and a relative path + // that climbs out never had one. `path.relative` normalizes, so `..` can only + // lead; it never appears in the middle of what this returns. + if (relative === '' || path.isAbsolute(relative) + || relative === '..' || relative.startsWith(`..${path.sep}`)) { + return false; + } + + const area = await resolveRealPath(areaLexical); + if (area.kind === 'unreadable') { + console.warn('[GitOps] Cannot resolve the managed area (%s); refusing to act on anything under it', area.code); + return false; + } + // No managed area on disk means nothing under it exists either, so there is + // no link anywhere beneath it to be misled by, and the check above already + // proved the target names a path inside it. A removal is then a no-op, and + // the one non-removal sink creates the area as it goes. + if (area.kind === 'absent') return true; + + const expected = path.resolve(area.real, relative); + const trailing: string[] = []; + let probe = resolved; + for (;;) { + const real = await resolveRealPath(probe); + if (real.kind === 'resolved') { + const actual = path.resolve(real.real, ...trailing); + if (actual === expected) return true; + // The single most useful fact for an operator staring at a refusal, and + // the only place it exists: the thrown message names neither path, and a + // refusal here can hold the boot gate. + console.warn( + '[GitOps] %s resolves to %s, not to %s; refusing to act on it', + sanitizeForLog(resolved), sanitizeForLog(actual), sanitizeForLog(expected), + ); + return false; + } + if (real.kind === 'unreadable') { + console.warn( + '[GitOps] Cannot resolve %s (%s); refusing to act on it rather than assuming where it points', + sanitizeForLog(probe), real.code, + ); + return false; + } + const parent = path.dirname(probe); + // Reached the filesystem root without finding anything that exists. + if (parent === probe) return false; + trailing.unshift(path.basename(probe)); + probe = parent; + } +} diff --git a/backend/src/services/gitops/migrate.ts b/backend/src/services/gitops/migrate.ts new file mode 100644 index 00000000..c110f175 --- /dev/null +++ b/backend/src/services/gitops/migrate.ts @@ -0,0 +1,508 @@ +import { createHash } from 'crypto'; +import fs from 'fs'; +import path from 'path'; +import { DatabaseService, type Blueprint, type StackGitSource } from '../DatabaseService'; +import { FileSystemService } from '../FileSystemService'; +import { GitProjectManifestService } from '../GitProjectManifestService'; +import { NodeRegistry } from '../NodeRegistry'; +import { sanitizeForLog } from '../../utils/safeLog'; +import { encodeGitOpsEvidenceLimitations, type GitOpsEvidenceLimitation } from './json'; +import { buildDirectApplicationRow, migrationDirectSourceIdentity, newGitOpsId } from './directApplication'; +import { emptyTargetRow, GitOpsStore } from './store'; +import { GitOpsTransitions, type EventEnvelope } from './transitions'; +import { blankInlineApplication } from './blueprintProducers'; +import { evaluateEffectiveApproval, intentFingerprint } from '../blueprintApproval'; +import type { GitOpsApplicationRow, GitOpsGenerationRow, GitOpsTargetCurrentRow } from './types'; + +/** Schema version this migration writes. Bumping it replays every scope. */ +const MIGRATION_SCHEMA_VERSION = 1; + +/** + * What the on-disk manifest proves about a stack. + * + * `trusted` is the only classification that licenses a canonical commit + * pointer, and it means the manifest parsed, validated, carries an identity + * stamp matching the repository and ref the source row configures *now*, and + * names the same commit the source row records as applied. The five failure + * kinds are kept apart because they tell an operator different things, and + * each implies a different next step: nothing was ever written, something was + * written and is unreadable, something was written for a different repository, + * the manifest records no commit at all, or the two records name different + * commits. + * + * The last two are deliberately separate. A manifest adopted from an existing + * directory is written with an empty commit and `state: 'migrated'`, which the + * validator permits, so "no commit yet" is an ordinary state for a stack that + * has never been fetched. Reporting it as a disagreement would name a commit + * the manifest does not contain. + */ +type ManifestTrust = + | { kind: 'trusted'; commitSha: string; manifestVersion: number; appliedDir: string } + | { kind: 'absent' } + | { kind: 'corrupt'; reason: string } + | { kind: 'identity_invalid'; reason: string } + | { kind: 'commit_unresolved' } + | { kind: 'commit_mismatch'; manifestCommitSha: string }; + +export type MigrationOutcome = + | 'skipped_current' + | 'skipped_live_application' + | 'migrated_accepted' + | 'migrated_unreconciled' + | 'tombstoned_missing_stack' + | 'migrated_inline' + | 'failed'; + +export type MigrationResult = { stackName: string; outcome: MigrationOutcome }; + +/** + * Bring Git stacks that predate the revision state model into it. + * + * The governing rule is that a pointer is written only when the evidence proves + * that exact generation under the repository and ref configured now. A legacy + * applied commit is not that proof on its own: the manifest may be gone, may be + * unreadable, may be stamped for a repository the stack no longer points at, or + * may name a different commit than the one the source row records as applied. + * In every one of those cases the canonical pointers stay null and the legacy + * commit survives as recorded limitation evidence, so the projection asks for a + * fetch instead of asserting a state nobody verified. + * + * Idempotent by checkpoint. Replay after a configuration change re-runs the + * matrix but never upgrades an already-justified pointer to a stronger claim. + */ +export function migrateDirectGitStacks(): MigrationResult[] { + const db = DatabaseService.getInstance(); + const results: MigrationResult[] = []; + for (const source of db.getGitSources()) { + try { + results.push(migrateOne(source)); + } catch (error) { + console.error( + `[GitOps] Could not migrate the Git stack ${sanitizeForLog(source.stack_name)}; retrying next boot:`, + error instanceof Error ? error.stack ?? error.message : String(error), + ); + results.push({ stackName: source.stack_name, outcome: 'failed' }); + } + } + return results; +} + +function migrateOne(source: StackGitSource): MigrationResult { + const store = GitOpsStore.getInstance(); + const stackName = source.stack_name; + // Migration-only identity derivation: a legacy operational URL may still + // carry userinfo or a query string that fetch needs, so the storable + // identity strips them and the source row is never rewritten. + const identity = migrationDirectSourceIdentity({ + repoUrl: source.repo_url, + branch: source.branch, + composePaths: source.compose_paths, + contextDir: source.context_dir, + syncEnv: source.sync_env, + envPath: source.env_path, + }); + + const scope = `direct:${stackName}`; + const checkpoint = store.getMigrationCheckpoint(scope); + if ( + checkpoint + && checkpoint.schema_version === MIGRATION_SCHEMA_VERSION + && checkpoint.fingerprint === identity.fingerprint + ) { + return { stackName, outcome: 'skipped_current' }; + } + + // A stack created through the new path already describes itself. Migration + // never touches it: its pointers were written with proof this pass does not + // have. + if (store.getLiveDirectApplication(stackName)) { + store.upsertMigrationCheckpoint(scope, MIGRATION_SCHEMA_VERSION, identity.fingerprint, Date.now()); + return { stackName, outcome: 'skipped_live_application' }; + } + + const trust = classifyManifest(stackName, source); + const limitations = collectLimitations(source, trust); + const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); + const stackPresent = stackDirectoryPresent(stackName, nodeId); + const at = Date.now(); + const envelope: EventEnvelope = { + operationId: newGitOpsId(), + actor: 'system:migration', + trigger: 'migrate', + at, + }; + + const application = buildDirectApplicationRow({ + id: newGitOpsId(), + stackName, + config: { + repoUrl: source.repo_url, + branch: source.branch, + composePaths: source.compose_paths, + contextDir: source.context_dir, + syncEnv: source.sync_env, + envPath: source.env_path, + }, + identity, + // A stack whose directory is gone is recorded as detached rather than live: + // it describes something that no longer exists, and a live application + // would go on claiming the name. + lifecycleStatus: 'active', + at, + }); + + return DatabaseService.getInstance().getDb().transaction((): MigrationResult => { + if (trust.kind === 'trusted' && stackPresent) { + migrateAccepted(application, source, trust, identity.fingerprint, limitations, envelope, nodeId); + store.upsertMigrationCheckpoint(scope, MIGRATION_SCHEMA_VERSION, identity.fingerprint, at); + return { stackName, outcome: 'migrated_accepted' }; + } + + application.evidence_limitations_json = encodeLimitations(limitations); + GitOpsTransitions.getInstance().activateDirect({ application, nodeId, envelope }); + if (!stackPresent) { + GitOpsTransitions.getInstance().targetTombstoned(application.id, nodeId, envelope); + GitOpsTransitions.getInstance().applicationTombstoned(application.id, 'deleted', envelope); + store.upsertMigrationCheckpoint(scope, MIGRATION_SCHEMA_VERSION, identity.fingerprint, at); + return { stackName, outcome: 'tombstoned_missing_stack' }; + } + store.upsertMigrationCheckpoint(scope, MIGRATION_SCHEMA_VERSION, identity.fingerprint, at); + return { stackName, outcome: 'migrated_unreconciled' }; + })(); +} + +/** + * The one path that writes canonical pointers, because the manifest proves the + * applied commit under the configuration in force now. + * + * Deployed, healthy, and last-known-good stay null regardless: a manifest + * proves what was materialized, not what is running, and inventing those is how + * a migration would claim a health record nobody observed. No source acceptance + * is written either, because nobody approved this generation through the model. + */ +function migrateAccepted( + application: GitOpsApplicationRow, + source: StackGitSource, + trust: Extract, + fingerprint: string, + limitations: GitOpsEvidenceLimitation[], + envelope: EventEnvelope, + nodeId: number, +): void { + const store = GitOpsStore.getInstance(); + const generationId = newGitOpsId(); + const artifactSetId = newGitOpsId(); + + application.desired_commit_sha = trust.commitSha; + application.fetched_commit_sha = trust.commitSha; + application.accepted_generation_id = generationId; + application.artifact_set_id = artifactSetId; + application.latest_artifact_set_id = artifactSetId; + application.evidence_limitations_json = encodeLimitations(limitations); + + GitOpsTransitions.getInstance().activateDirect({ application, nodeId, envelope }); + + const generation: GitOpsGenerationRow = { + id: generationId, + application_id: application.id, + commit_sha: trust.commitSha, + repo_url: application.configured_repo_url ?? '', + configured_ref: source.branch, + repo_identity_json: application.repo_identity_json ?? '{}', + manifest_version: trust.manifestVersion, + candidate_dir: `generations/candidate-${trust.commitSha}`, + applied_dir: trust.appliedDir, + expected_invocation_json: '{"composeFileOrder":[],"projectName":null,"projectDirectory":null,"envFileOrder":[]}', + // Equal to the application's, so the accepted generation is not immediately + // reported as stale against its own configuration. + materialization_fingerprint: fingerprint, + validation_ok: 1, + plan_blocked: 0, + change_plan_fingerprint: null, + operation_id: envelope.operationId, + trigger: envelope.trigger, + actor: envelope.actor, + previous_generation_id: null, + redacted_limitations_json: '[]', + created_at: envelope.at, + }; + store.insertGeneration(generation); + store.insertArtifactSet({ + id: artifactSetId, + generation_id: generationId, + evidence_version: 1, + authoritative: 0, + qualification: 'unresolved', + evidence_json: '{"kind":"unresolved"}', + created_at: envelope.at, + }); + + const target: GitOpsTargetCurrentRow = { + ...emptyTargetRow(application.id, nodeId, envelope.at), + desired_generation_id: generationId, + applied_generation_id: generationId, + expected_artifact_set_id: artifactSetId, + latest_artifact_set_id: artifactSetId, + }; + store.upsertTarget(target); + store.writeApplicationPointers(application); +} + +/** Whether the manifest licenses a canonical commit pointer. */ +function classifyManifest(stackName: string, source: StackGitSource): ManifestTrust { + if (!source.last_applied_commit_sha) return { kind: 'absent' }; + const read = readManifestSync(stackName, source); + if (read === null) return { kind: 'absent' }; + if ('corrupt' in read) { + // An identity mismatch is not the same fault as unreadable content: the + // file is fine, it just belongs to a different repository or ref. + return read.corrupt.toLowerCase().includes('identity') || read.corrupt.toLowerCase().includes('mismatch') + ? { kind: 'identity_invalid', reason: read.corrupt } + : { kind: 'corrupt', reason: read.corrupt }; + } + // The two records must name the same commit. The applied directory comes from + // the manifest and the commit from the source row, so trusting them together + // while they disagree would mint a generation that claims one commit and + // points at another's files, which is the exact false proof this migration + // exists to avoid. An adopted manifest carries no commit at all, which is a + // different fact about a different situation and gets its own answer. + if (read.resolvedRevision.commitSha.length === 0) return { kind: 'commit_unresolved' }; + if (read.resolvedRevision.commitSha !== source.last_applied_commit_sha) { + return { kind: 'commit_mismatch', manifestCommitSha: read.resolvedRevision.commitSha }; + } + return { + kind: 'trusted', + commitSha: source.last_applied_commit_sha, + manifestVersion: read.manifestVersion, + appliedDir: read.generation.appliedDir, + }; +} + +/** + * The manifest read, resolved synchronously. + * + * Migration runs inside one transaction per stack, and better-sqlite3 + * transactions cannot await, so the read is performed before the transaction + * opens and passed in. + */ +let manifestReader: (stackName: string, source: StackGitSource) => ManifestReadResult = () => null; +type ManifestReadResult = + | { manifestVersion: number; generation: { appliedDir: string }; resolvedRevision: { commitSha: string } } + | { corrupt: string } + | null; + +export function primeMigrationManifests( + read: (stackName: string, source: StackGitSource) => ManifestReadResult, +): void { + manifestReader = read; +} + +function readManifestSync(stackName: string, source: StackGitSource): ManifestReadResult { + return manifestReader(stackName, source); +} + +/** Read every manifest up front, so the per-stack transaction stays synchronous. */ +export async function loadMigrationManifests(): Promise { + const manifestSvc = GitProjectManifestService.getInstance(); + const cache = new Map(); + for (const source of DatabaseService.getInstance().getGitSources()) { + try { + cache.set(source.stack_name, await manifestSvc.readManifest(source.stack_name, source.repo_url, source.branch)); + } catch (error) { + cache.set(source.stack_name, { corrupt: error instanceof Error ? error.message : String(error) }); + } + } + primeMigrationManifests((stackName) => cache.get(stackName) ?? null); +} + +/** + * Every reason this stack could not be fully described, as recorded evidence. + * + * A legacy applied commit that could not be proven appears here and nowhere + * else. Putting it on a canonical pointer would assert that the stack is at + * that commit under the current configuration, which is exactly what could not + * be established. + */ +function collectLimitations(source: StackGitSource, trust: ManifestTrust): GitOpsEvidenceLimitation[] { + const limitations: GitOpsEvidenceLimitation[] = []; + const legacySha = source.last_applied_commit_sha; + if (legacySha) { + if (trust.kind === 'absent') limitations.push({ code: 'manifest_absent', detail: legacySha }); + if (trust.kind === 'corrupt') limitations.push({ code: 'manifest_corrupt', detail: legacySha }); + if (trust.kind === 'identity_invalid') limitations.push({ code: 'manifest_identity_invalid', detail: legacySha }); + if (trust.kind === 'commit_unresolved') { + limitations.push({ code: 'manifest_commit_unresolved', detail: legacySha }); + } + // Both commits are named: which record is right cannot be decided here, and + // an operator reading one of them alone has no way to see the disagreement. + if (trust.kind === 'commit_mismatch') { + limitations.push({ + code: 'manifest_commit_mismatch', + detail: `${legacySha} (recorded) vs ${trust.manifestCommitSha} (manifest)`, + }); + } + } + // A pending pull proves nothing about the current repository or ref: the blob + // predates any configuration change and carries no identity stamp. + if (source.pending_commit_sha && source.pending_commit_sha !== legacySha) { + limitations.push({ code: 'legacy_pending', detail: source.pending_commit_sha }); + } + return limitations; +} + +function encodeLimitations(limitations: GitOpsEvidenceLimitation[]): string | null { + let encoded: string | null = null; + for (const limitation of limitations) { + encoded = encodeGitOpsEvidenceLimitations( + encoded ? JSON.parse(encoded) as GitOpsEvidenceLimitation[] : [], + limitation.code, + limitation, + ); + } + return encoded; +} + +function stackDirectoryPresent(stackName: string, nodeId: number): boolean { + try { + const base = path.resolve(FileSystemService.getInstance(nodeId).getBaseDir()); + const resolved = path.resolve(base, stackName); + if (!resolved.startsWith(base + path.sep)) return false; + return fs.statSync(resolved).isDirectory(); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; + // Cannot prove it is gone, so treat it as present: tombstoning a stack that + // is actually there is the unrecoverable mistake. + return true; + } +} + +/** + * Bring Blueprints that predate the revision state model into it. + * + * Every Blueprint gets an application, an intent describing what it currently + * asks for, and a candidate marked as coming from the legacy inline record. + * None of that is an acknowledgement. The Blueprint revision and the + * deployment's applied revision are carried as display only, because neither + * proves a node is running the intent this pass just minted, and recording them + * as agreement would report convergence nobody verified. + */ +export function migrateInlineBlueprints(): MigrationResult[] { + const db = DatabaseService.getInstance(); + const results: MigrationResult[] = []; + for (const blueprint of db.listBlueprints()) { + try { + results.push(migrateOneBlueprint(blueprint)); + } catch (error) { + console.error( + `[GitOps] Could not migrate the blueprint ${sanitizeForLog(blueprint.name)}; retrying next boot:`, + error instanceof Error ? error.stack ?? error.message : String(error), + ); + results.push({ stackName: blueprint.name, outcome: 'failed' }); + } + } + return results; +} + +function migrateOneBlueprint(blueprint: Blueprint): MigrationResult { + const store = GitOpsStore.getInstance(); + const fingerprint = intentFingerprint(blueprint); + const scope = `inline_blueprint:${blueprint.id}`; + + const checkpoint = store.getMigrationCheckpoint(scope); + if ( + checkpoint + && checkpoint.schema_version === MIGRATION_SCHEMA_VERSION + && checkpoint.fingerprint === fingerprint + ) { + return { stackName: blueprint.name, outcome: 'skipped_current' }; + } + + // A Blueprint created through the new path already describes itself, and its + // rows were written with proof this pass does not have. + if (store.getLiveBlueprintApplication(blueprint.id)) { + store.upsertMigrationCheckpoint(scope, MIGRATION_SCHEMA_VERSION, fingerprint, Date.now()); + return { stackName: blueprint.name, outcome: 'skipped_live_application' }; + } + + const at = Date.now(); + const envelope: EventEnvelope = { + operationId: newGitOpsId(), + actor: 'system:migration', + trigger: 'migrate', + at, + }; + const applicationId = newGitOpsId(); + const intentId = newGitOpsId(); + + return DatabaseService.getInstance().getDb().transaction((): MigrationResult => { + const tx = GitOpsTransitions.getInstance(); + tx.activateInlineBlueprint({ + application: { + ...blankInlineApplication(applicationId, blueprint.id, at), + evidence_limitations_json: encodeLimitations(inlineApprovalLimitations(blueprint)), + }, + envelope, + }); + + tx.intentRevised({ + applicationId, + intent: { + id: intentId, + application_id: applicationId, + blueprint_id: blueprint.id, + compose_content_sha256: createHash('sha256').update(blueprint.compose_content, 'utf8').digest('hex'), + // Display only. A revision is not an acknowledgement: nothing here + // proves a node is running what this intent describes. + blueprint_revision: blueprint.revision, + deploy_stack_name: blueprint.name, + selector_json: JSON.stringify(blueprint.selector), + pinned_node_id: blueprint.pinned_node_id, + cordon_implications_json: JSON.stringify({ pinnedOverridesCordon: blueprint.pinned_node_id !== null }), + rollout_strategy_json: JSON.stringify({ driftMode: blueprint.drift_mode, enabled: blueprint.enabled }), + runtime_drift_policy: blueprint.drift_mode, + stateful_policy_json: null, + health_failure_rollback_policy_json: null, + operation_id: envelope.operationId, + actor: envelope.actor, + created_at: at, + }, + envelope, + }); + + tx.rolloutCandidateOpened({ + applicationId, + candidate: { + id: newGitOpsId(), + application_id: applicationId, + intent_revision_id: intentId, + compose_content_sha256: createHash('sha256').update(blueprint.compose_content, 'utf8').digest('hex'), + accepted_generation_id: null, + artifact_set_id: null, + // Placement is not resolved here. Migration records what the Blueprint + // asks for, never which nodes currently satisfy it. + required_targets_json: JSON.stringify({ nodeIds: [] }), + authoritative: 1, + provenance: 'legacy_inline', + operation_id: envelope.operationId, + created_at: at, + }, + envelope, + }); + + store.upsertMigrationCheckpoint(scope, MIGRATION_SCHEMA_VERSION, fingerprint, at); + return { stackName: blueprint.name, outcome: 'migrated_inline' }; + })(); +} + +/** + * Why an approval could not be carried across. + * + * An approval authorizes the intent it was given for. A Blueprint edited since + * then, or never approved, has nothing this pass can record as authority, and + * saying so is what stops the gap reading as an approval that is simply absent. + */ +function inlineApprovalLimitations(blueprint: Blueprint): GitOpsEvidenceLimitation[] { + const { effectiveApproval } = evaluateEffectiveApproval(blueprint, []); + if (effectiveApproval === 'approved') return []; + return [{ code: 'blueprint_reapproval_required', detail: String(blueprint.id) }]; +} diff --git a/backend/src/services/gitops/nodePlacementProducers.ts b/backend/src/services/gitops/nodePlacementProducers.ts new file mode 100644 index 00000000..1c543df1 --- /dev/null +++ b/backend/src/services/gitops/nodePlacementProducers.ts @@ -0,0 +1,101 @@ +/** + * Node-side changes that move where Blueprints are allowed to run. + * + * A label is not a statement about any one Blueprint, but it changes which + * nodes a selector matches, so it revises placement for whichever Blueprints + * the change actually moved. That set is computed by comparing the desired + * nodes before and after: a label nothing selects on moves nothing and records + * nothing. + * + * A cordon goes through the same comparison and, as things stand, never moves + * anything. It governs whether new placements may be made, not what a Blueprint + * asks for, and the desired-node computation deliberately ignores it. The + * comparison is still the right shape for it, so the caller gets a truthful + * empty answer instead of a special case. + * + * As in the Blueprint producers, the desired-node computation is supplied by + * the caller. The reconciler that knows how to do it reaches this layer, and + * importing it back would close a module cycle. + */ +import { DatabaseService, type Blueprint } from '../DatabaseService'; +import { sanitizeForLog } from '../../utils/safeLog'; +import { GitOpsStore } from './store'; +import { GitOpsTransitions } from './transitions'; +import { candidateRowFor, envelopeFor, intentRowFor, recordableApplication } from './blueprintProducers'; + +/** Desired node ids per Blueprint id, as placement currently resolves them. */ +export type PlacementSnapshot = Map; + +/** Takes a snapshot of what every enabled Blueprint currently wants. */ +export type SnapshotPlacement = () => PlacementSnapshot; + +function sameNodeSet(a: number[] | undefined, b: number[] | undefined): boolean { + if (!a || !b) return a === b; + if (a.length !== b.length) return false; + const left = [...a].sort((x, y) => x - y); + const right = [...b].sort((x, y) => x - y); + return left.every((value, index) => value === right[index]); +} + +/** + * Revise placement for every Blueprint whose desired node set actually moved. + * + * Comparing sets rather than reacting to the event is what keeps this honest. + * Labelling a node no selector mentions changes nothing a Blueprint wants, and + * minting an intent for it would invalidate every acknowledgement in the fleet + * over an edit that moved nothing. + */ +export function recordPlacementShift( + before: PlacementSnapshot, + after: PlacementSnapshot, + actor: string | null, + trigger: string, +): number[] { + const db = DatabaseService.getInstance(); + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + const moved: number[] = []; + + for (const [blueprintId, desired] of after) { + if (sameNodeSet(before.get(blueprintId), desired)) continue; + const app = store.getLiveBlueprintApplication(blueprintId); + // A Blueprint that predates the model has no application yet. Migration + // brings it in rather than this path inventing a first intent for it. + if (!recordableApplication(app)) continue; + const blueprint = db.getBlueprint(blueprintId); + if (!blueprint) { + // Unlike the skip above, this one is a fault. A live application exists + // for a Blueprint whose own row is gone, and with cascade off nothing + // else will notice. The placement really did move, no intent or candidate + // is minted for it, and the caller goes on to report that nothing moved. + console.error( + '[GitOps] Placement shift skipped: blueprint %s has a live application but no blueprint row.', + sanitizeForLog(blueprintId), + ); + continue; + } + + const envelope = envelopeFor(actor, trigger); + const intent = intentRowFor(app.id, blueprint, envelope.operationId, actor, envelope.at); + tx.intentRevised({ applicationId: app.id, intent, envelope }); + tx.rolloutCandidateOpened({ + applicationId: app.id, + candidate: candidateRowFor(app.id, intent, desired, 'roster_change', envelope.operationId, envelope.at), + envelope, + }); + moved.push(blueprintId); + } + return moved; +} + +/** Placement as it currently resolves, for every Blueprint. */ +export function snapshotPlacementWith( + desiredNodeIdsFor: (blueprint: Blueprint) => number[], + blueprints: Blueprint[], +): PlacementSnapshot { + const snapshot: PlacementSnapshot = new Map(); + for (const blueprint of blueprints) { + snapshot.set(blueprint.id, desiredNodeIdsFor(blueprint)); + } + return snapshot; +} diff --git a/backend/src/services/gitops/publish.ts b/backend/src/services/gitops/publish.ts new file mode 100644 index 00000000..65c7c748 --- /dev/null +++ b/backend/src/services/gitops/publish.ts @@ -0,0 +1,201 @@ +/** + * Announcing transitions after they commit. + * + * A history row that is inserted and still present when the drain runs produces + * one metric increment and, when a sink is installed, one `state-invalidate` + * event. Both have to happen *after* the transaction that wrote the row, and + * neither may happen for a transaction that rolled back, which is why nothing + * here runs inline. + * + * The mechanism is a buffer drained on `setImmediate`. better-sqlite3 is fully + * synchronous, so by the time a macrotask runs, the transaction that enqueued + * the row has committed or rolled back, and so has any outer transaction + * wrapping it. That matters: several producers wrap a handful of transitions in + * one outer transaction, and a publisher that fired when the innermost one + * returned would announce work that a later statement then discarded. Waiting + * for the macrotask covers both nesting depths without having to detect which + * one it is in. + * + * Rollback needs no detection either. The drain checks that each row is still + * there before announcing it, so a discarded transaction publishes nothing on + * its own. A replay publishes nothing for a different reason: the dedupe index + * means no row was inserted, so nothing was ever enqueued. + * + * The event sink is injected rather than imported. Reaching into + * NotificationService from inside the GitOps layer would close a module cycle + * of exactly the kind that once made an imported constant evaluate as + * `undefined` here, and injection also lets the tests observe events without + * standing up the notification stack. + */ +import type Database from 'better-sqlite3'; +import { GitOpsMetricsService } from '../GitOpsMetricsService'; +import type { GitOpsHistoryStage, HistoryOutcome } from './history'; +import type { GitOpsTargetMode } from './types'; + +/** + * The `state-invalidate` payload one committed transition produces. + * + * A type alias rather than an interface so it satisfies the broadcaster's + * open envelope parameter: TypeScript infers an implicit index signature for + * the former and not the latter. + */ +export type GitOpsInvalidateEvent = { + type: 'state-invalidate'; + scope: 'gitops'; + /** The transition's stage, so a client can tell a fetch from a deploy. */ + action: GitOpsHistoryStage; + applicationId: string; + targetMode: GitOpsTargetMode; + stackName: string | null; + blueprintId: number | null; + nodeId: number | null; + ts: number; +}; + +/** + * Pins the requirement the alias above exists to satisfy. + * + * Without this, switching `type` to `interface` compiles here and fails at the + * startup wiring in another module, as an index-signature complaint that says + * nothing about the cause. The failure belongs at the declaration. + */ +type AssertsOpenEnvelope = + GitOpsInvalidateEvent extends { type: string; [key: string]: unknown } ? true : never; +const _openEnvelope: AssertsOpenEnvelope = true; +void _openEnvelope; + +export type GitOpsEventSink = (event: GitOpsInvalidateEvent) => void; + +/** What the drain needs to know, captured while it is still typed. */ +interface PendingRow { + db: Database.Database; + id: string; + stage: GitOpsHistoryStage; + outcome: HistoryOutcome; + applicationId: string; + targetMode: GitOpsTargetMode; + stackName: string | null; + blueprintId: number | null; + nodeId: number | null; + at: number; +} + +let sink: GitOpsEventSink | null = null; +let pending: PendingRow[] = []; +let scheduled = false; +let warnedUnannounced = false; + +/** + * Say once that transitions are committing with nobody to announce them to. + * + * A server that never installs the sink still counts every transition and + * still writes every history row, so the only symptom is that no client ever + * refreshes: the UI silently goes back to being as stale as it was before any + * of this existed. That is precisely the kind of unwired producer this branch + * has already shipped once, so it says so rather than being inferred from an + * absence. Once, not per row: a boot migration would otherwise fill the log, + * and the second occurrence tells a reader nothing the first did not. + */ +function warnUnannounced(): void { + if (warnedUnannounced) return; + warnedUnannounced = true; + console.warn('[GitOps] Transitions are committing with no event sink installed; no client will be told about them.'); +} + +/** + * Install the broadcaster. Called once at startup, and with null by tests that + * want the metrics side without the event side. + */ +export function setGitOpsEventSink(next: GitOpsEventSink | null): void { + sink = next; +} + +/** + * Queue one inserted history row for announcement. + * + * Called only where a row was genuinely inserted. The stage and outcome are + * carried from the insert rather than read back, because they are already typed + * there and re-reading them would turn a closed union into an unvalidated + * column value. + */ +export function enqueueHistoryPublication(row: PendingRow): void { + pending.push(row); + if (scheduled) return; + scheduled = true; + setImmediate(drain); +} + +function drain(): void { + scheduled = false; + const batch = pending; + pending = []; + const metrics = GitOpsMetricsService.getInstance(); + + for (const row of batch) { + if (!survived(row)) continue; + metrics.record(row.stage, row.outcome); + if (!sink) { + warnUnannounced(); + continue; + } + announce(sink, row); + } +} + +/** + * Hand one committed transition to the broadcaster. + * + * A throw is logged and swallowed: one client's broadcast must not cost the + * rest of the batch their events, and none of this is worth failing a + * committed transition over. + */ +function announce(to: GitOpsEventSink, row: PendingRow): void { + try { + to({ + type: 'state-invalidate', + scope: 'gitops', + action: row.stage, + applicationId: row.applicationId, + targetMode: row.targetMode, + stackName: row.stackName, + blueprintId: row.blueprintId, + nodeId: row.nodeId, + ts: row.at, + }); + } catch (error) { + console.error( + '[GitOps] Could not announce %s for application %s:', + row.stage, row.applicationId, + error instanceof Error ? error.stack ?? error.message : String(error), + ); + } +} + +/** + * Whether the row is still in the table. + * + * A missing row means its transaction rolled back, which is an ordinary + * outcome and not worth logging. A failed *query* is different: it means the + * check itself could not be made, so the row is treated as gone rather than + * announced on the strength of a lookup that did not answer. + */ +function survived(row: PendingRow): boolean { + try { + return row.db.prepare('SELECT 1 FROM gitops_history WHERE id = ?').get(row.id) !== undefined; + } catch (error) { + console.error( + '[GitOps] Could not confirm history row %s before announcing it:', + row.id, + error instanceof Error ? error.stack ?? error.message : String(error), + ); + return false; + } +} + +/** Drop anything queued and uninstall the sink, so one test cannot reach the next. */ +export function resetGitOpsPublicationsForTests(): void { + pending = []; + sink = null; + scheduled = false; + warnedUnannounced = false; +} diff --git a/backend/src/services/gitops/readAuth.ts b/backend/src/services/gitops/readAuth.ts new file mode 100644 index 00000000..96829925 --- /dev/null +++ b/backend/src/services/gitops/readAuth.ts @@ -0,0 +1,179 @@ +import type { Request } from 'express'; +import { checkPermission } from '../../middleware/permissions'; +import { isRecord } from './json'; +import type { GitOpsHistoryEvidenceFields, GitOpsRevisionProjection } from './types'; + +/** + * What a caller must hold to read one GitOps row. + * + * `stack_read` is the narrow answer, used whenever a row can be tied to a stack + * the caller may read. The other two are the fail-closed fallbacks for a row + * whose audience cannot be narrowed, and they differ by what the row *is*: + * + * - `audit` for history entries, which are an audit trail. Auditing is what the + * `system:audit` permission exists for, and the request audit log is already + * gated on it, so an entry nobody can tie to a stack belongs to the same + * audience rather than to Admin alone. + * - `admin` for source rows, which are live Git configuration (repository, + * ref, credentials policy, compose paths) rather than a record of events. An + * auditing mandate does not imply reading the configuration of stacks that + * have been deleted or never finished being created. + */ +export type GitOpsReadRequirement = + | { readonly kind: 'admin' } + | { readonly kind: 'audit' } + | { readonly kind: 'stack_read'; readonly stackName: string }; + +const ADMIN: GitOpsReadRequirement = Object.freeze({ kind: 'admin' }); +const AUDIT: GitOpsReadRequirement = Object.freeze({ kind: 'audit' }); + +/** + * The projection field the source-row classifier probes. + * + * Tied to the live projection variant so renaming that field fails the build + * here. Without the tie, a rename would leave the classifier probing a key that + * no longer exists, and every row would quietly fall to Admin: fail-closed, but + * invisible, since nothing would error and no test that hand-builds a payload + * would notice. + */ +const LIFECYCLE_KEY = 'lifecycleStatus' satisfies keyof Extract< + GitOpsRevisionProjection, + { lifecycleStatus: unknown } +>; + +/** + * The evidence a history entry must carry to be classified. + * + * Values stay `unknown` because they may have crossed an instance boundary and + * carry no shape guarantee, but the *key names* are bound to the item type the + * producer emits. Without that tie, renaming a field on the producer would + * leave this probing keys that no longer exist: every row would degrade to the + * audit bucket, fail-closed but silent, with nothing failing to compile and no + * test noticing. The required (`-?`) mapping also makes the call site fail, not + * just this function. + */ +type Evidence = { [K in keyof T]-?: unknown }; +export type HistoryRowEvidence = Evidence< + Pick +>; + +/** + * Validate an owning instance's resource-existence claim. + * + * Only a real JSON boolean counts. A peer that omits the field, sends null, or + * sends a string is treated as "not present", so an older or malformed instance + * degrades to Admin rather than silently widening who may read its rows. + */ +export function normalizeStackResourcePresent(value: unknown): boolean { + return value === true; +} + +function usableStackName(value: unknown): string | null { + return typeof value === 'string' && value.length > 0 ? value : null; +} + +/** + * Lifecycle states whose source rows a stack grant can authorize. + * + * A deleted application no longer has a stack whose grant could authorize it, + * and a creating one does not yet have a stack that survived. In both cases a + * later application may hold the same stack name, so honouring a grant here + * would let one application's rows be read through another's name. + * + * History rows do not use this. They require `active`, for the reason given on + * `classifyHistoryRow`. + */ +function lifecycleAllowsStackRead(lifecycleStatus: unknown): boolean { + return lifecycleStatus === 'active' || lifecycleStatus === 'detached'; +} + +/** + * Authorization for one `GET /api/git-sources` row. + * + * The projection is typed `unknown` rather than as a projection because the + * hub will classify rows that arrived from another instance as parsed JSON, + * which carries no guarantee of shape. No such caller exists yet; every current + * one passes a locally derived projection. + * + * Only `active` is reachable here in practice, and deliberately so. The + * projection comes from `projectStackRevision`, which resolves live Direct + * applications only. Detach deletes the Git-source row in the same transaction + * that tombstones the application, so a source row beside a detached + * application is not a producible state anyway. The detached case is reported + * through the stack-state surface, which never runs this classifier. + * + * Keep it that way. This function takes `stackName` from the Git-source row but + * lifecycle from whatever the projection resolved, so widening that resolution + * to reach another application silently changes who may read this row. + */ +export function classifySourceRow(input: { + stackName: unknown; + gitopsRevision: unknown; + stackResourcePresent: unknown; +}): GitOpsReadRequirement { + const stackName = usableStackName(input.stackName); + if (!stackName) return ADMIN; + if (!isRecord(input.gitopsRevision)) return ADMIN; + if (!lifecycleAllowsStackRead(input.gitopsRevision[LIFECYCLE_KEY])) return ADMIN; + if (!normalizeStackResourcePresent(input.stackResourcePresent)) return ADMIN; + return { kind: 'stack_read', stackName }; +} + +/** + * Authorization for one history row. + * + * Lifecycle comes from the owning application row rather than the entry's + * `before`/`after`, because those record only the fields a transition moved: + * most entries never mention lifecycle at all, so decoding them would send + * nearly every row to Admin and leave operators unable to read the history of + * their own stacks. Reading the application row also keeps authorization off + * the audit payload entirely, so a corrupt delta cannot influence who may see + * it. + */ +export function classifyHistoryRow(input: HistoryRowEvidence): GitOpsReadRequirement { + const stackName = usableStackName(input.stackName); + if (!stackName) return AUDIT; + // Only a live application, never a `detached` one, even though a detached + // application's files are usually still the stack standing at its name. + // + // A stack grant is a grant on whatever occupies that name today, so the + // allowance is only sound while the detached application is still what + // occupies it, and nothing here can establish that. A later Direct + // application is visible in these tables, but a Blueprint deploying under the + // same name records it as `deploy_stack_name` on its intent revision rather + // than as an application `stack_name`, and a plain Compose stack recreated at + // that name leaves no GitOps trace at all. Since the last case cannot be + // detected in principle, the allowance cannot be made sound by detecting + // harder, and a rule that holds only for the successors we happen to see is + // worse than not having one. + // + // Detach therefore moves a stack's trail to the audit audience, the same + // answer `deleted` and `creating` predecessors already get. A create still in + // flight shows its own history through the scope exemption in + // `helpers/gitopsHistoryPage.ts`, which never reaches this classifier. + if (input.applicationLifecycleStatus !== 'active') return AUDIT; + if (!normalizeStackResourcePresent(input.stackResourcePresent)) return AUDIT; + return { kind: 'stack_read', stackName }; +} + +/** + * Whether this caller satisfies a classifier's requirement. + * + * Exhaustive on purpose: a requirement this function does not recognize is + * denied rather than falling through to the narrower stack check. + */ +export function satisfiesGitOpsRead(req: Request, requirement: GitOpsReadRequirement): boolean { + switch (requirement.kind) { + case 'admin': + return req.user?.role === 'admin'; + case 'audit': + return checkPermission(req, 'system:audit'); + case 'stack_read': + return checkPermission(req, 'stack:read', 'stack', requirement.stackName); + default: { + const unrecognized: never = requirement; + void unrecognized; + return false; + } + } +} diff --git a/backend/src/services/gitops/recoveryCapture.ts b/backend/src/services/gitops/recoveryCapture.ts new file mode 100644 index 00000000..5e659689 --- /dev/null +++ b/backend/src/services/gitops/recoveryCapture.ts @@ -0,0 +1,62 @@ +import { GitOpsStore } from './store'; + +export type GitOpsRecoveryCapture = { + gitops_generation_id: string | null; + gitops_artifact_set_id: string | null; + gitops_source_acceptance_ref: string | null; +}; + +export const EMPTY_GITOPS_RECOVERY_CAPTURE: GitOpsRecoveryCapture = { + gitops_generation_id: null, + gitops_artifact_set_id: null, + gitops_source_acceptance_ref: null, +}; + +/** + * Bind a rollback point to the generation that is actually deployed on this + * target. A generation that was applied but never deployed is not rollback + * identity, so only `deployed_generation_id` is read. + * + * The acceptance reference prefers the one recorded on the target and + * otherwise falls back to the newest acceptance *of that same generation*, so + * an acceptance belonging to a later generation can never be captured. Any + * candidate that does not resolve against the deployed generation is stored as + * null rather than as a reference the restore path would have to trust. + */ +export function captureGitOpsRecoveryBinding(stackName: string, nodeId: number): GitOpsRecoveryCapture { + const store = GitOpsStore.getInstance(); + const application = store.getLiveDirectApplication(stackName); + if (!application) return { ...EMPTY_GITOPS_RECOVERY_CAPTURE }; + const target = store.getTarget(application.id, nodeId); + const generationId = target?.deployed_generation_id ?? null; + if (!generationId) return { ...EMPTY_GITOPS_RECOVERY_CAPTURE }; + + let artifactSetId: string | null = null; + if (target?.expected_artifact_set_id) { + const artifact = store.getArtifactSet(target.expected_artifact_set_id); + if (artifact && artifact.generation_id === generationId) { + artifactSetId = artifact.id; + } + } + + const expected = { + kind: 'source_acceptance' as const, + applicationId: application.id, + generationId, + }; + let sourceAcceptanceRef: string | null = null; + if (target?.source_acceptance_ref && store.resolveApprovalRef(target.source_acceptance_ref, expected)) { + sourceAcceptanceRef = target.source_acceptance_ref; + } else { + const newest = store.newestSourceAcceptanceId(application.id, generationId); + if (newest && store.resolveApprovalRef(newest, expected)) { + sourceAcceptanceRef = newest; + } + } + + return { + gitops_generation_id: generationId, + gitops_artifact_set_id: artifactSetId, + gitops_source_acceptance_ref: sourceAcceptanceRef, + }; +} diff --git a/backend/src/services/gitops/repoIdentity.ts b/backend/src/services/gitops/repoIdentity.ts new file mode 100644 index 00000000..fe1c853e --- /dev/null +++ b/backend/src/services/gitops/repoIdentity.ts @@ -0,0 +1,103 @@ +// Upper bound so a caller cannot flood the service with a huge payload. +// Generous compared to anything a real Git provider emits. +export const MAX_REPO_URL_LENGTH = 2048; + +export type RepoIdentity = { host: string; pathname: string }; + +export type ParseHttpsRepoUrlResult = + | { ok: true; url: URL } + | { ok: false; reason: 'not_https' | 'userinfo' | 'query' | 'fragment' | 'too_long' | 'invalid' }; + +export function parseHttpsRepoUrl(raw: string): ParseHttpsRepoUrlResult { + const trimmed = raw.trim(); + if (trimmed.length === 0 || trimmed.length > MAX_REPO_URL_LENGTH) { + return { ok: false, reason: trimmed.length > MAX_REPO_URL_LENGTH ? 'too_long' : 'invalid' }; + } + let url: URL; + try { + url = new URL(trimmed); + } catch { + return { ok: false, reason: 'invalid' }; + } + if (url.protocol !== 'https:') { + return { ok: false, reason: 'not_https' }; + } + if (url.username !== '' || url.password !== '') { + return { ok: false, reason: 'userinfo' }; + } + if (url.search !== '') { + return { ok: false, reason: 'query' }; + } + if (url.hash !== '') { + return { ok: false, reason: 'fragment' }; + } + return { ok: true, url }; +} + +export function serializeRepoIdentity(url: URL): RepoIdentity { + return { host: url.host, pathname: url.pathname }; +} + +export type ParseLegacyRepoUrlResult = + | { ok: true; url: URL } + | { ok: false; reason: 'not_https' | 'too_long' | 'invalid' }; + +/** + * Parse an operational repository URL that predates strict ingress. + * + * Legacy operational rows retain the original URL (with userinfo, query, + * and fragment) because fetch still needs them. Migration derives the + * storable identity by stripping those components instead of refusing the + * stack. Everything strict ingress refuses for want of a recoverable + * identity (non-HTTPS, unparseable, oversized) is refused here too. + */ +export function parseLegacyRepoUrl(raw: string): ParseLegacyRepoUrlResult { + const trimmed = raw.trim(); + if (trimmed.length === 0 || trimmed.length > MAX_REPO_URL_LENGTH) { + return { ok: false, reason: trimmed.length > MAX_REPO_URL_LENGTH ? 'too_long' : 'invalid' }; + } + let url: URL; + try { + url = new URL(trimmed); + } catch { + return { ok: false, reason: 'invalid' }; + } + if (url.protocol !== 'https:') { + return { ok: false, reason: 'not_https' }; + } + url.username = ''; + url.password = ''; + url.search = ''; + url.hash = ''; + return { ok: true, url }; +} + +/** + * Rebuild a storable repository URL from an identity. + * + * The secret-free guarantee comes from `parseHttpsRepoUrl` having already + * rejected userinfo, query strings, and fragments at every ingress. This + * function only reassembles what that check let through. + */ +export function secretFreeRepoUrl(identity: RepoIdentity): string { + return `https://${identity.host}${identity.pathname}`; +} + +export function repoUrlRejectionMessage(raw: string): string | null { + const parsed = parseHttpsRepoUrl(raw); + if (parsed.ok) return null; + switch (parsed.reason) { + case 'too_long': + return 'repo_url is too long'; + case 'not_https': + return 'Only HTTPS repository URLs are supported'; + case 'userinfo': + return 'Repository URL must not include userinfo'; + case 'query': + return 'Repository URL must not include a query string'; + case 'fragment': + return 'Repository URL must not include a fragment'; + default: + return 'Repository URL is invalid'; + } +} diff --git a/backend/src/services/gitops/schema.ts b/backend/src/services/gitops/schema.ts new file mode 100644 index 00000000..45f657fa --- /dev/null +++ b/backend/src/services/gitops/schema.ts @@ -0,0 +1,450 @@ +/** + * DatabaseService executes this at init, then separately seeds + * gitops_schema_version, adds the gitops_* columns to + * stack_update_recovery_generations, and adds deployed_generation_id to + * health_gate_runs. Those three live outside this string because they alter + * pre-existing tables rather than creating GitOps ones. + */ + +export const GITOPS_SCHEMA_SQL = ` +CREATE TABLE IF NOT EXISTS gitops_migration_checkpoints ( + scope TEXT PRIMARY KEY, + schema_version INTEGER NOT NULL, + fingerprint TEXT NOT NULL, + migrated_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS gitops_create_checkpoints ( + application_id TEXT PRIMARY KEY, + stack_name TEXT NOT NULL, + phase TEXT NOT NULL CHECK (phase IN ( + 'pre_stack','stack_created','promoting','manifest_committed','pointers_committed' + )), + generation_id TEXT NULL, + operation_id TEXT NOT NULL, + repo_url TEXT NOT NULL, + branch TEXT NOT NULL, + compose_path TEXT NOT NULL, + compose_paths_json TEXT NOT NULL, + context_dir TEXT NULL, + sync_env INTEGER NOT NULL DEFAULT 0, + env_path TEXT NULL, + auth_type TEXT NOT NULL, + encrypted_token TEXT NULL, + auto_apply_on_webhook INTEGER NOT NULL DEFAULT 0, + auto_deploy_on_apply INTEGER NOT NULL DEFAULT 0, + commit_sha TEXT NULL, + applied_spec_json TEXT NULL, + created_managed_root INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_gitops_create_ck_stack + ON gitops_create_checkpoints(stack_name); +CREATE INDEX IF NOT EXISTS idx_gitops_create_ck_phase + ON gitops_create_checkpoints(phase); + +CREATE TABLE IF NOT EXISTS gitops_applications ( + id TEXT PRIMARY KEY, + lifecycle_key TEXT NOT NULL, + lifecycle_status TEXT NOT NULL CHECK (lifecycle_status IN ( + 'active','creating','detached','deleted' + )), + target_mode TEXT NOT NULL CHECK (target_mode IN ('direct','inline_blueprint','blueprint')), + stack_name TEXT NULL, + blueprint_id INTEGER NULL, + configured_repo_url TEXT NULL, + repo_identity_json TEXT NULL, + configured_ref TEXT NULL, + compose_paths_json TEXT NULL, + context_dir TEXT NULL, + sync_env INTEGER NULL, + env_path TEXT NULL, + materialization_fingerprint TEXT NULL, + desired_commit_sha TEXT NULL, + fetched_commit_sha TEXT NULL, + candidate_generation_id TEXT NULL, + accepted_generation_id TEXT NULL, + candidate_plan_blocked INTEGER NOT NULL DEFAULT 0, + review_required INTEGER NOT NULL DEFAULT 0, + artifact_set_id TEXT NULL, + latest_artifact_set_id TEXT NULL, + intent_revision_id TEXT NULL, + rollout_candidate_id TEXT NULL, + rollout_generation_id TEXT NULL, + source_acceptance_ref TEXT NULL, + placement_approval_ref TEXT NULL, + rollout_authorization_ref TEXT NULL, + legacy_combined_approval_ref TEXT NULL, + preflight_fingerprint TEXT NULL, + latest_operation_id TEXT NULL, + active_operation_id TEXT NULL, + active_operation_stage TEXT NULL CHECK ( + active_operation_stage IS NULL OR active_operation_stage IN ( + 'fetch_started','apply_started','deploy_started','recovery_started' + ) + ), + active_operation_at INTEGER NULL, + active_generation_id TEXT NULL, + pause_at INTEGER NULL, + pause_reason TEXT NULL, + partial_json TEXT NULL, + failure_stage TEXT NULL CHECK ( + failure_stage IS NULL OR failure_stage IN ( + 'fetch','validation','apply','create','recovery' + ) + ), + failure_class TEXT NULL, + failure_at INTEGER NULL, + retry_at INTEGER NULL, + retry_count INTEGER NOT NULL DEFAULT 0, + suspended_at INTEGER NULL, + recovery_ref TEXT NULL, + recovery_phase TEXT NULL CHECK ( + recovery_phase IS NULL OR recovery_phase IN ( + 'capturing','restoring','compensating','complete','failed' + ) + ), + interruption_stage TEXT NULL CHECK ( + interruption_stage IS NULL OR interruption_stage IN ( + 'fetch_started','apply_started','deploy_started','recovery_started' + ) + ), + interruption_at INTEGER NULL, + interruption_operation_id TEXT NULL, + interruption_generation_id TEXT NULL, + evidence_fresh_at INTEGER NULL, + -- Why this row could not prove something, recorded at write time. Read-time + -- limitations are derived; these are the ones only the writer knows. + evidence_limitations_json TEXT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + CHECK (target_mode != 'direct' OR (stack_name IS NOT NULL AND blueprint_id IS NULL)), + CHECK (target_mode != 'inline_blueprint' OR (blueprint_id IS NOT NULL AND stack_name IS NULL AND configured_repo_url IS NULL)), + CHECK (target_mode != 'blueprint' OR (blueprint_id IS NOT NULL AND stack_name IS NULL AND configured_repo_url IS NOT NULL)) +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_gitops_app_active_direct + ON gitops_applications(stack_name) + WHERE lifecycle_status IN ('active','creating') AND target_mode = 'direct'; +CREATE UNIQUE INDEX IF NOT EXISTS idx_gitops_app_active_blueprint_any + ON gitops_applications(blueprint_id) + WHERE lifecycle_status IN ('active','creating') + AND target_mode IN ('inline_blueprint','blueprint'); +CREATE INDEX IF NOT EXISTS idx_gitops_app_lifecycle_key + ON gitops_applications(lifecycle_key); +CREATE INDEX IF NOT EXISTS idx_gitops_app_status + ON gitops_applications(lifecycle_status); +-- The two unique indexes above are partial on the live rows, so neither serves +-- a lookup for a detached one. Without this the drift route's fallback is a +-- full scan and a sort. Direct only: Blueprint retirement writes 'deleted', +-- never 'detached', so the Blueprint equivalent would index an empty set. +CREATE INDEX IF NOT EXISTS idx_gitops_app_detached_direct + ON gitops_applications(stack_name, updated_at DESC) + WHERE lifecycle_status = 'detached' AND target_mode = 'direct'; + +CREATE TABLE IF NOT EXISTS gitops_generations ( + id TEXT PRIMARY KEY, + application_id TEXT NOT NULL, + commit_sha TEXT NOT NULL, + repo_url TEXT NOT NULL, + configured_ref TEXT NOT NULL, + repo_identity_json TEXT NOT NULL, + manifest_version INTEGER NOT NULL, + candidate_dir TEXT NOT NULL, + applied_dir TEXT NOT NULL, + expected_invocation_json TEXT NOT NULL, + materialization_fingerprint TEXT NOT NULL, + validation_ok INTEGER NOT NULL, + plan_blocked INTEGER NOT NULL DEFAULT 0, + change_plan_fingerprint TEXT NULL, + operation_id TEXT NOT NULL, + trigger TEXT NOT NULL, + actor TEXT NULL, + previous_generation_id TEXT NULL, + redacted_limitations_json TEXT NOT NULL DEFAULT '[]', + created_at INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_gitops_gen_app_created + ON gitops_generations(application_id, created_at); +CREATE INDEX IF NOT EXISTS idx_gitops_gen_sha + ON gitops_generations(commit_sha); +CREATE INDEX IF NOT EXISTS idx_gitops_gen_op + ON gitops_generations(operation_id); +CREATE INDEX IF NOT EXISTS idx_gitops_gen_repo_ref + ON gitops_generations(repo_url, configured_ref); + +CREATE TABLE IF NOT EXISTS gitops_artifact_sets ( + id TEXT PRIMARY KEY, + generation_id TEXT NOT NULL, + evidence_version INTEGER NOT NULL, + authoritative INTEGER NOT NULL DEFAULT 0, + qualification TEXT NOT NULL CHECK (qualification IN ( + 'unresolved','exact','qualified','stale','unavailable','local_build_unverified' + )), + evidence_json TEXT NOT NULL DEFAULT '{}', + created_at INTEGER NOT NULL, + UNIQUE (generation_id, evidence_version) +); +CREATE INDEX IF NOT EXISTS idx_gitops_artifact_gen + ON gitops_artifact_sets(generation_id, evidence_version); + +CREATE TABLE IF NOT EXISTS gitops_intent_revisions ( + id TEXT PRIMARY KEY, + application_id TEXT NOT NULL, + blueprint_id INTEGER NOT NULL, + compose_content_sha256 TEXT NOT NULL, + blueprint_revision INTEGER NOT NULL, + deploy_stack_name TEXT NOT NULL, + selector_json TEXT NOT NULL, + pinned_node_id INTEGER NULL, + cordon_implications_json TEXT NOT NULL DEFAULT '[]', + rollout_strategy_json TEXT NOT NULL DEFAULT '{}', + runtime_drift_policy TEXT NULL, + stateful_policy_json TEXT NULL, + health_failure_rollback_policy_json TEXT NULL, + operation_id TEXT NOT NULL, + actor TEXT NULL, + created_at INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_gitops_intent_app_created + ON gitops_intent_revisions(application_id, created_at); +CREATE INDEX IF NOT EXISTS idx_gitops_intent_blueprint + ON gitops_intent_revisions(blueprint_id); +CREATE INDEX IF NOT EXISTS idx_gitops_intent_content + ON gitops_intent_revisions(compose_content_sha256); + +CREATE TABLE IF NOT EXISTS gitops_rollout_candidates ( + id TEXT PRIMARY KEY, + application_id TEXT NOT NULL, + intent_revision_id TEXT NOT NULL, + compose_content_sha256 TEXT NOT NULL, + accepted_generation_id TEXT NULL, + artifact_set_id TEXT NULL, + required_targets_json TEXT NOT NULL, + authoritative INTEGER NOT NULL DEFAULT 0, + provenance TEXT NOT NULL CHECK (provenance IN ( + 'intent_change','roster_change','legacy_inline' + )), + operation_id TEXT NOT NULL, + created_at INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_gitops_rollout_app + ON gitops_rollout_candidates(application_id, created_at); + +CREATE TABLE IF NOT EXISTS gitops_approvals ( + id TEXT PRIMARY KEY, + kind TEXT NOT NULL CHECK (kind IN ( + 'source_acceptance','placement_approval','rollout_authorization','legacy_combined' + )), + authority TEXT NOT NULL CHECK (authority IN ( + 'operator','configured_policy','legacy_combined' + )), + authoritative INTEGER NOT NULL DEFAULT 0, + application_id TEXT NOT NULL, + generation_id TEXT NULL, + intent_revision_id TEXT NULL, + artifact_set_id TEXT NULL, + rollout_candidate_id TEXT NULL, + rollout_generation_id TEXT NULL, + source_acceptance_ref TEXT NULL, + placement_approval_ref TEXT NULL, + required_targets_json TEXT NULL, + preflight_fingerprint TEXT NULL, + fingerprint TEXT NULL, + blast_json TEXT NULL, + policy_provenance_json TEXT NULL, + actor TEXT NULL, + created_at INTEGER NOT NULL, + CHECK ( + kind != 'source_acceptance' OR ( + authoritative = 1 + AND authority IN ('operator','configured_policy') + AND generation_id IS NOT NULL + ) + ), + CHECK ( + kind != 'placement_approval' OR ( + authoritative = 1 + AND authority IN ('operator','configured_policy') + AND intent_revision_id IS NOT NULL + AND blast_json IS NOT NULL + ) + ), + CHECK ( + kind != 'rollout_authorization' OR ( + authoritative = 1 + AND authority IN ('operator','configured_policy') + AND generation_id IS NOT NULL + AND artifact_set_id IS NOT NULL + AND intent_revision_id IS NOT NULL + AND rollout_candidate_id IS NOT NULL + AND source_acceptance_ref IS NOT NULL + AND placement_approval_ref IS NOT NULL + AND required_targets_json IS NOT NULL + AND preflight_fingerprint IS NOT NULL + ) + ), + CHECK ( + kind != 'legacy_combined' OR ( + authoritative = 0 + AND authority = 'legacy_combined' + ) + ) +); +CREATE INDEX IF NOT EXISTS idx_gitops_approval_app + ON gitops_approvals(application_id, created_at); + +CREATE TABLE IF NOT EXISTS gitops_target_current ( + application_id TEXT NOT NULL, + node_id INTEGER NOT NULL, + target_status TEXT NOT NULL CHECK (target_status IN ('active','tombstoned')), + desired_generation_id TEXT NULL, + candidate_generation_id TEXT NULL, + applied_generation_id TEXT NULL, + deployed_generation_id TEXT NULL, + healthy_generation_id TEXT NULL, + lkg_generation_id TEXT NULL, + lkg_artifact_set_id TEXT NULL, + lkg_unavailable_at INTEGER NULL, + lkg_unavailable_reason TEXT NULL CHECK ( + lkg_unavailable_reason IS NULL OR lkg_unavailable_reason IN ( + 'generation_missing','recovery_unretainable' + ) + ), + expected_artifact_set_id TEXT NULL, + latest_artifact_set_id TEXT NULL, + observed_artifact_identity_json TEXT NULL, + intent_revision_id TEXT NULL, + rollout_candidate_id TEXT NULL, + rollout_generation_id TEXT NULL, + source_acceptance_ref TEXT NULL, + placement_approval_ref TEXT NULL, + rollout_authorization_ref TEXT NULL, + legacy_combined_approval_ref TEXT NULL, + legacy_applied_revision INTEGER NULL, + connectivity TEXT NULL CHECK ( + connectivity IS NULL OR connectivity IN ('unknown','reachable','unreachable','stale') + ), + latest_stage TEXT NULL, + active_operation_id TEXT NULL, + active_operation_stage TEXT NULL CHECK ( + active_operation_stage IS NULL OR active_operation_stage IN ( + 'deploy_started','blueprint_deploy_started','blueprint_withdraw_started','recovery_started' + ) + ), + active_operation_at INTEGER NULL, + active_generation_id TEXT NULL, + active_intent_revision_id TEXT NULL, + active_rollout_candidate_id TEXT NULL, + failure_stage TEXT NULL CHECK ( + failure_stage IS NULL OR failure_stage IN ( + 'deploy','recovery','blueprint_deploy','blueprint_withdraw' + ) + ), + failure_class TEXT NULL, + failure_at INTEGER NULL, + recovery_ref TEXT NULL, + recovery_generation_id TEXT NULL, + recovery_phase TEXT NULL CHECK ( + recovery_phase IS NULL OR recovery_phase IN ( + 'capturing','restoring','compensating','complete','failed' + ) + ), + interruption_stage TEXT NULL CHECK ( + interruption_stage IS NULL OR interruption_stage IN ( + 'deploy_started','blueprint_deploy_started','blueprint_withdraw_started','recovery_started' + ) + ), + interruption_at INTEGER NULL, + interruption_operation_id TEXT NULL, + interruption_generation_id TEXT NULL, + interruption_intent_revision_id TEXT NULL, + interruption_rollout_candidate_id TEXT NULL, + pause_at INTEGER NULL, + pause_reason TEXT NULL, + retry_at INTEGER NULL, + suspended_at INTEGER NULL, + partial_json TEXT NULL, + -- Why this target could not prove something, recorded at write time. + evidence_limitations_json TEXT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (application_id, node_id), + CHECK ( + (lkg_unavailable_at IS NULL AND lkg_unavailable_reason IS NULL) + OR (lkg_unavailable_at IS NOT NULL AND lkg_unavailable_reason IS NOT NULL) + ), + CHECK ( + lkg_unavailable_at IS NULL + OR (lkg_generation_id IS NULL AND lkg_artifact_set_id IS NULL) + ) +); +CREATE INDEX IF NOT EXISTS idx_gitops_target_status + ON gitops_target_current(target_status); +CREATE INDEX IF NOT EXISTS idx_gitops_target_node + ON gitops_target_current(node_id); + +CREATE TABLE IF NOT EXISTS gitops_history ( + id TEXT PRIMARY KEY, + created_at INTEGER NOT NULL, + application_id TEXT NOT NULL, + target_mode TEXT NOT NULL, + lifecycle_key TEXT NOT NULL, + stack_name TEXT NULL, + blueprint_id INTEGER NULL, + node_id INTEGER NULL, + dedupe_target TEXT NOT NULL, + repo_url TEXT NULL, + configured_ref TEXT NULL, + repo_identity_json TEXT NULL, + commit_sha TEXT NULL, + generation_id TEXT NULL, + artifact_set_id TEXT NULL, + intent_revision_id TEXT NULL, + rollout_candidate_id TEXT NULL, + rollout_generation_id TEXT NULL, + source_acceptance_ref TEXT NULL, + placement_approval_ref TEXT NULL, + rollout_authorization_ref TEXT NULL, + legacy_combined_approval_ref TEXT NULL, + operation_id TEXT NOT NULL, + stage TEXT NOT NULL, + outcome TEXT NOT NULL CHECK (outcome IN ( + 'committed','failed','skipped','superseded','recovered','unknown' + )), + trigger TEXT NOT NULL, + actor TEXT NULL, + before_json TEXT NOT NULL, + after_json TEXT NOT NULL, + required_targets_json TEXT NULL, + validation_json TEXT NULL, + per_target_results_json TEXT NULL, + health_run_id TEXT NULL, + health_snapshot_json TEXT NULL, + invocation_observed_json TEXT NULL, + recovery_ref TEXT NULL, + redacted_reason_class TEXT NULL +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_gitops_history_dedupe + ON gitops_history(application_id, operation_id, stage, dedupe_target); +CREATE INDEX IF NOT EXISTS idx_gitops_history_app_created + ON gitops_history(application_id, created_at DESC, id DESC); +CREATE INDEX IF NOT EXISTS idx_gitops_history_sha ON gitops_history(commit_sha); +CREATE INDEX IF NOT EXISTS idx_gitops_history_gen ON gitops_history(generation_id); +CREATE INDEX IF NOT EXISTS idx_gitops_history_artifact ON gitops_history(artifact_set_id); +CREATE INDEX IF NOT EXISTS idx_gitops_history_blueprint ON gitops_history(blueprint_id); +CREATE INDEX IF NOT EXISTS idx_gitops_history_rollout ON gitops_history(rollout_candidate_id); +CREATE INDEX IF NOT EXISTS idx_gitops_history_rollout_gen ON gitops_history(rollout_generation_id); +CREATE INDEX IF NOT EXISTS idx_gitops_history_node ON gitops_history(node_id); +CREATE INDEX IF NOT EXISTS idx_gitops_history_trigger ON gitops_history(trigger); +CREATE INDEX IF NOT EXISTS idx_gitops_history_actor ON gitops_history(actor); +CREATE INDEX IF NOT EXISTS idx_gitops_history_outcome ON gitops_history(outcome); +CREATE INDEX IF NOT EXISTS idx_gitops_history_repo_ref + ON gitops_history(repo_url, configured_ref); +CREATE INDEX IF NOT EXISTS idx_gitops_history_stack_created + ON gitops_history(stack_name, created_at DESC, id DESC); +-- Serves the cross-stack history page, whose ordering and cursor are on +-- (created_at, id) with no other filter. Every other index here leads with a +-- different column, so without this one that route sorts the whole table. +CREATE INDEX IF NOT EXISTS idx_gitops_history_created + ON gitops_history(created_at DESC, id DESC); +`; diff --git a/backend/src/services/gitops/store.ts b/backend/src/services/gitops/store.ts new file mode 100644 index 00000000..c9dd41af --- /dev/null +++ b/backend/src/services/gitops/store.ts @@ -0,0 +1,815 @@ +import type Database from 'better-sqlite3'; +import { DatabaseService } from '../DatabaseService'; +import { + decodeArtifactEvidenceJson, + decodeGitOpsApprovedTargetEffectJson, + decodeGitOpsJson, + decodeGitOpsRequiredTargetsJson, + GitOpsJsonError, + isPreflightFingerprint, +} from './json'; +import type { + FutureRolloutAuthorizationBinding, + GitOpsApplicationRow, + GitOpsApprovalRow, + GitOpsArtifactSetRow, + GitOpsCreateCheckpointRow, + GitOpsCreatePhase, + GitOpsGenerationRow, + GitOpsIntentRevisionRow, + GitOpsRolloutCandidateRow, + GitOpsTargetCurrentRow, + ResolveApprovalExpected, +} from './types'; + +export type LiveBlueprintApplication = { + id: string; + targetMode: GitOpsApplicationRow['target_mode']; + lifecycleStatus: GitOpsApplicationRow['lifecycle_status']; +}; + +export type AssertNoLiveBlueprintResult = + | { ok: true } + | { ok: false; existing: LiveBlueprintApplication }; + +export class GitOpsStore { + private static instance: GitOpsStore | undefined; + + static getInstance(): GitOpsStore { + if (!GitOpsStore.instance) { + GitOpsStore.instance = new GitOpsStore(); + } + return GitOpsStore.instance; + } + + static resetForTests(): void { + GitOpsStore.instance = undefined; + } + + private db(): Database.Database { + return DatabaseService.getInstance().getDb(); + } + + assertNoLiveBlueprintApplication(blueprintId: number): AssertNoLiveBlueprintResult { + const row = this.db().prepare( + `SELECT id, target_mode, lifecycle_status + FROM gitops_applications + WHERE blueprint_id = ? + AND lifecycle_status IN ('active','creating') + AND target_mode IN ('inline_blueprint','blueprint') + LIMIT 1`, + ).get(blueprintId) as { + id: string; + target_mode: GitOpsApplicationRow['target_mode']; + lifecycle_status: GitOpsApplicationRow['lifecycle_status']; + } | undefined; + if (!row) return { ok: true }; + return { + ok: false, + existing: { + id: row.id, + targetMode: row.target_mode, + lifecycleStatus: row.lifecycle_status, + }, + }; + } + + getApplication(id: string): GitOpsApplicationRow | undefined { + return this.db().prepare('SELECT * FROM gitops_applications WHERE id = ?').get(id) as GitOpsApplicationRow | undefined; + } + + getLiveDirectApplication(stackName: string): GitOpsApplicationRow | undefined { + return this.db().prepare( + `SELECT * FROM gitops_applications + WHERE stack_name = ? AND target_mode = 'direct' AND lifecycle_status IN ('active','creating')`, + ).get(stackName) as GitOpsApplicationRow | undefined; + } + + /** + * The live application for a Blueprint, in either Blueprint mode. + * + * One query across both modes because a Blueprint owns at most one live + * application whichever way it is delivered, and the unique live index + * enforces exactly that. + */ + getLiveBlueprintApplication(blueprintId: number): GitOpsApplicationRow | undefined { + return this.db().prepare( + `SELECT * FROM gitops_applications + WHERE blueprint_id = ? + AND target_mode IN ('inline_blueprint','blueprint') + AND lifecycle_status IN ('active','creating')`, + ).get(blueprintId) as GitOpsApplicationRow | undefined; + } + + /** + * The most recently detached Direct application for a stack, if any. + * + * Consulted only after the live lookup misses. `applicationTombstoned` keeps + * the configured identity and SHA pointers as frozen facts precisely so the + * projection can still say what an application was, and `deriveSource` has a + * `not_live` status for it, but neither could be reached while every entry + * point filtered to the live rows. + * + * `detached` only, never `deleted`. A detached application's files are still + * on disk and still describe that stack. A deleted one means the stack is + * gone, so any directory of that name now belongs to something else, and + * `readAuth` refuses stack-grant reads on deleted rows for the same + * name-reuse reason. + * + * Newest first, because a stack name can be detached and reattached + * repeatedly and only the latest detachment describes what was there last. + * `rowid` breaks a tie rather than `id`, which is a random UUID and orders + * arbitrarily; ties are reachable because a transaction stamps every row it + * touches with one `envelope.at`. + */ + getDetachedDirectApplication(stackName: string): GitOpsApplicationRow | undefined { + return this.db().prepare( + `SELECT * FROM gitops_applications + WHERE stack_name = ? AND target_mode = 'direct' AND lifecycle_status = 'detached' + ORDER BY updated_at DESC, rowid DESC + LIMIT 1`, + ).get(stackName) as GitOpsApplicationRow | undefined; + } + + /** Direct applications that never reached their success boundary. */ + listCreatingDirectApplications(): GitOpsApplicationRow[] { + return this.db().prepare( + `SELECT * FROM gitops_applications + WHERE target_mode = 'direct' AND lifecycle_status = 'creating' + ORDER BY created_at ASC`, + ).all() as GitOpsApplicationRow[]; + } + + getGeneration(id: string): GitOpsGenerationRow | undefined { + return this.db().prepare('SELECT * FROM gitops_generations WHERE id = ?').get(id) as GitOpsGenerationRow | undefined; + } + + getArtifactSet(id: string): GitOpsArtifactSetRow | undefined { + return this.db().prepare('SELECT * FROM gitops_artifact_sets WHERE id = ?').get(id) as GitOpsArtifactSetRow | undefined; + } + + getIntentRevision(id: string): GitOpsIntentRevisionRow | undefined { + return this.db().prepare('SELECT * FROM gitops_intent_revisions WHERE id = ?').get(id) as GitOpsIntentRevisionRow | undefined; + } + + getRolloutCandidate(id: string): GitOpsRolloutCandidateRow | undefined { + return this.db().prepare('SELECT * FROM gitops_rollout_candidates WHERE id = ?').get(id) as GitOpsRolloutCandidateRow | undefined; + } + + getApproval(id: string): GitOpsApprovalRow | undefined { + return this.db().prepare('SELECT * FROM gitops_approvals WHERE id = ?').get(id) as GitOpsApprovalRow | undefined; + } + + getTarget(applicationId: string, nodeId: number): GitOpsTargetCurrentRow | undefined { + return this.db().prepare( + 'SELECT * FROM gitops_target_current WHERE application_id = ? AND node_id = ?', + ).get(applicationId, nodeId) as GitOpsTargetCurrentRow | undefined; + } + + listTargets(applicationId: string): GitOpsTargetCurrentRow[] { + return this.db().prepare( + 'SELECT * FROM gitops_target_current WHERE application_id = ? ORDER BY node_id ASC', + ).all(applicationId) as GitOpsTargetCurrentRow[]; + } + + /** + * Live applications that were mid-operation, on the application row or on any + * of their targets. + * + * Read at boot to reclassify work the previous process never finished. An + * operation left open reports as still running forever, and offers no actions + * while it does. + */ + listApplicationsWithOpenOperations(): GitOpsApplicationRow[] { + return this.db().prepare( + `SELECT a.* FROM gitops_applications a + WHERE a.lifecycle_status IN ('active','creating') + AND ( + a.active_operation_stage IS NOT NULL + OR EXISTS ( + SELECT 1 FROM gitops_target_current t + WHERE t.application_id = a.id AND t.active_operation_stage IS NOT NULL + ) + ) + ORDER BY a.created_at ASC`, + ).all() as GitOpsApplicationRow[]; + } + + /** Every live target on one node, across all applications. */ + listActiveTargetsForNode(nodeId: number): GitOpsTargetCurrentRow[] { + return this.db().prepare( + `SELECT * FROM gitops_target_current + WHERE node_id = ? AND target_status = 'active' + ORDER BY application_id ASC`, + ).all(nodeId) as GitOpsTargetCurrentRow[]; + } + + newestSourceAcceptanceId(applicationId: string, generationId: string): string | null { + const row = this.db().prepare( + `SELECT id FROM gitops_approvals + WHERE application_id = ? AND kind = 'source_acceptance' AND authoritative = 1 AND generation_id = ? + ORDER BY created_at DESC, id DESC LIMIT 1`, + ).get(applicationId, generationId) as { id: string } | undefined; + return row?.id ?? null; + } + + getMigrationCheckpoint(scope: string): { scope: string; schema_version: number; fingerprint: string } | undefined { + return this.db().prepare( + 'SELECT scope, schema_version, fingerprint FROM gitops_migration_checkpoints WHERE scope = ?', + ).get(scope) as { scope: string; schema_version: number; fingerprint: string } | undefined; + } + + /** + * Record that this scope has been migrated at this schema version and + * configuration fingerprint. + * + * Replay is decided from the triple: an unchanged fingerprint skips, a + * changed one re-runs the matrix. It never licenses upgrading an already + * justified pointer to a stronger claim. + */ + upsertMigrationCheckpoint(scope: string, schemaVersion: number, fingerprint: string, at: number): void { + this.db().prepare( + `INSERT INTO gitops_migration_checkpoints (scope, schema_version, fingerprint, migrated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(scope) DO UPDATE SET + schema_version=excluded.schema_version, + fingerprint=excluded.fingerprint, + migrated_at=excluded.migrated_at`, + ).run(scope, schemaVersion, fingerprint, at); + } + + /** + * Persist an application's mutable columns without going through a + * transition. + * + * Used only by migration, which builds a whole row from evidence rather than + * moving one pointer at a time. Every other writer goes through the + * transitions so the change lands in history. + */ + writeApplicationPointers(app: GitOpsApplicationRow): void { + this.db().prepare( + `UPDATE gitops_applications SET + desired_commit_sha=?, fetched_commit_sha=?, accepted_generation_id=?, + artifact_set_id=?, latest_artifact_set_id=?, evidence_limitations_json=?, updated_at=? + WHERE id=?`, + ).run( + app.desired_commit_sha, app.fetched_commit_sha, app.accepted_generation_id, + app.artifact_set_id, app.latest_artifact_set_id, app.evidence_limitations_json, + app.updated_at, app.id, + ); + } + + insertCreateCheckpoint(row: GitOpsCreateCheckpointRow): void { + decodeGitOpsJson(row.compose_paths_json); + this.db().prepare( + `INSERT INTO gitops_create_checkpoints ( + application_id, stack_name, phase, generation_id, operation_id, repo_url, branch, + compose_path, compose_paths_json, context_dir, sync_env, env_path, auth_type, + encrypted_token, auto_apply_on_webhook, auto_deploy_on_apply, commit_sha, + applied_spec_json, created_managed_root, created_at, updated_at + ) VALUES (${Array(21).fill('?').join(', ')})`, + ).run( + row.application_id, row.stack_name, row.phase, row.generation_id, row.operation_id, + row.repo_url, row.branch, row.compose_path, row.compose_paths_json, row.context_dir, + row.sync_env, row.env_path, row.auth_type, row.encrypted_token, row.auto_apply_on_webhook, + row.auto_deploy_on_apply, row.commit_sha, row.applied_spec_json, row.created_managed_root, + row.created_at, row.updated_at, + ); + } + + getCreateCheckpoint(applicationId: string): GitOpsCreateCheckpointRow | undefined { + return this.db().prepare( + 'SELECT * FROM gitops_create_checkpoints WHERE application_id = ?', + ).get(applicationId) as GitOpsCreateCheckpointRow | undefined; + } + + listCreateCheckpoints(): GitOpsCreateCheckpointRow[] { + return this.db().prepare( + 'SELECT * FROM gitops_create_checkpoints ORDER BY created_at ASC', + ).all() as GitOpsCreateCheckpointRow[]; + } + + /** Advance the phase, and optionally record facts the phase depends on. */ + updateCreateCheckpoint( + applicationId: string, + patch: { + phase?: GitOpsCreatePhase; + generationId?: string | null; + commitSha?: string | null; + appliedSpecJson?: string | null; + createdManagedRoot?: number; + }, + at: number, + ): void { + const current = this.getCreateCheckpoint(applicationId); + if (!current) throw new Error('create checkpoint not found'); + this.db().prepare( + `UPDATE gitops_create_checkpoints SET + phase=?, generation_id=?, commit_sha=?, applied_spec_json=?, + created_managed_root=?, updated_at=? + WHERE application_id=?`, + ).run( + patch.phase ?? current.phase, + patch.generationId === undefined ? current.generation_id : patch.generationId, + patch.commitSha === undefined ? current.commit_sha : patch.commitSha, + patch.appliedSpecJson === undefined ? current.applied_spec_json : patch.appliedSpecJson, + patch.createdManagedRoot ?? current.created_managed_root, + at, + applicationId, + ); + } + + deleteCreateCheckpoint(applicationId: string): void { + this.db().prepare('DELETE FROM gitops_create_checkpoints WHERE application_id = ?').run(applicationId); + } + + insertApproval(row: GitOpsApprovalRow): void { + // Decode every JSON column the resolver will later read, so a malformed + // payload aborts the write instead of persisting an approval that reads + // back as absent. Rollout-authorization blast is reserved policy payload + // and is never a node set, so it is only checked for well-formedness. + if (row.required_targets_json !== null) decodeGitOpsRequiredTargetsJson(row.required_targets_json); + if (row.blast_json !== null) { + if (row.kind === 'placement_approval') decodeGitOpsApprovedTargetEffectJson(row.blast_json); + else decodeGitOpsJson(row.blast_json); + } + if (row.policy_provenance_json !== null) decodeGitOpsJson(row.policy_provenance_json); + this.db().prepare( + `INSERT INTO gitops_approvals ( + id, kind, authority, authoritative, application_id, generation_id, intent_revision_id, + artifact_set_id, rollout_candidate_id, rollout_generation_id, source_acceptance_ref, + placement_approval_ref, required_targets_json, preflight_fingerprint, fingerprint, + blast_json, policy_provenance_json, actor, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ).run( + row.id, row.kind, row.authority, row.authoritative, row.application_id, row.generation_id, + row.intent_revision_id, row.artifact_set_id, row.rollout_candidate_id, row.rollout_generation_id, + row.source_acceptance_ref, row.placement_approval_ref, row.required_targets_json, + row.preflight_fingerprint, row.fingerprint, row.blast_json, row.policy_provenance_json, + row.actor, row.created_at, + ); + } + + insertApplication(row: GitOpsApplicationRow): void { + this.db().prepare( + `INSERT INTO gitops_applications ( + id, lifecycle_key, lifecycle_status, target_mode, stack_name, blueprint_id, + configured_repo_url, repo_identity_json, configured_ref, compose_paths_json, + context_dir, sync_env, env_path, materialization_fingerprint, desired_commit_sha, + fetched_commit_sha, candidate_generation_id, accepted_generation_id, + candidate_plan_blocked, review_required, artifact_set_id, latest_artifact_set_id, + intent_revision_id, rollout_candidate_id, rollout_generation_id, source_acceptance_ref, + placement_approval_ref, rollout_authorization_ref, legacy_combined_approval_ref, + preflight_fingerprint, latest_operation_id, active_operation_id, active_operation_stage, + active_operation_at, active_generation_id, pause_at, pause_reason, partial_json, + failure_stage, failure_class, failure_at, retry_at, retry_count, suspended_at, + recovery_ref, recovery_phase, interruption_stage, interruption_at, + interruption_operation_id, interruption_generation_id, evidence_fresh_at, + evidence_limitations_json, created_at, updated_at + ) VALUES (${Array(54).fill('?').join(', ')})`, + ).run( + row.id, row.lifecycle_key, row.lifecycle_status, row.target_mode, row.stack_name, row.blueprint_id, + row.configured_repo_url, row.repo_identity_json, row.configured_ref, row.compose_paths_json, + row.context_dir, row.sync_env, row.env_path, row.materialization_fingerprint, row.desired_commit_sha, + row.fetched_commit_sha, row.candidate_generation_id, row.accepted_generation_id, + row.candidate_plan_blocked, row.review_required, row.artifact_set_id, row.latest_artifact_set_id, + row.intent_revision_id, row.rollout_candidate_id, row.rollout_generation_id, row.source_acceptance_ref, + row.placement_approval_ref, row.rollout_authorization_ref, row.legacy_combined_approval_ref, + row.preflight_fingerprint, row.latest_operation_id, row.active_operation_id, row.active_operation_stage, + row.active_operation_at, row.active_generation_id, row.pause_at, row.pause_reason, row.partial_json, + row.failure_stage, row.failure_class, row.failure_at, row.retry_at, row.retry_count, row.suspended_at, + row.recovery_ref, row.recovery_phase, row.interruption_stage, row.interruption_at, + row.interruption_operation_id, row.interruption_generation_id, row.evidence_fresh_at, + row.evidence_limitations_json, row.created_at, row.updated_at, + ); + } + + insertGeneration(row: GitOpsGenerationRow): void { + this.db().prepare( + `INSERT INTO gitops_generations ( + id, application_id, commit_sha, repo_url, configured_ref, repo_identity_json, + manifest_version, candidate_dir, applied_dir, expected_invocation_json, + materialization_fingerprint, validation_ok, plan_blocked, change_plan_fingerprint, + operation_id, trigger, actor, previous_generation_id, redacted_limitations_json, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ).run( + row.id, row.application_id, row.commit_sha, row.repo_url, row.configured_ref, row.repo_identity_json, + row.manifest_version, row.candidate_dir, row.applied_dir, row.expected_invocation_json, + row.materialization_fingerprint, row.validation_ok, row.plan_blocked, row.change_plan_fingerprint, + row.operation_id, row.trigger, row.actor, row.previous_generation_id, row.redacted_limitations_json, + row.created_at, + ); + } + + insertArtifactSet(row: GitOpsArtifactSetRow): void { + const evidence = decodeArtifactEvidenceJson(row.evidence_json); + if (evidence.kind !== row.qualification) { + throw new GitOpsJsonError('artifact qualification must match evidence_json.kind'); + } + if (row.authoritative !== 0) { + throw new Error('artifact rows must be non-authoritative until qualification is accepted'); + } + this.db().prepare( + `INSERT INTO gitops_artifact_sets ( + id, generation_id, evidence_version, authoritative, qualification, evidence_json, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?)`, + ).run( + row.id, row.generation_id, row.evidence_version, row.authoritative, + row.qualification, row.evidence_json, row.created_at, + ); + } + + insertIntentRevision(row: GitOpsIntentRevisionRow): void { + this.db().prepare( + `INSERT INTO gitops_intent_revisions ( + id, application_id, blueprint_id, compose_content_sha256, blueprint_revision, + deploy_stack_name, selector_json, pinned_node_id, cordon_implications_json, + rollout_strategy_json, runtime_drift_policy, stateful_policy_json, + health_failure_rollback_policy_json, operation_id, actor, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ).run( + row.id, row.application_id, row.blueprint_id, row.compose_content_sha256, row.blueprint_revision, + row.deploy_stack_name, row.selector_json, row.pinned_node_id, row.cordon_implications_json, + row.rollout_strategy_json, row.runtime_drift_policy, row.stateful_policy_json, + row.health_failure_rollback_policy_json, row.operation_id, row.actor, row.created_at, + ); + } + + insertRolloutCandidate(row: GitOpsRolloutCandidateRow): void { + decodeGitOpsRequiredTargetsJson(row.required_targets_json); + this.db().prepare( + `INSERT INTO gitops_rollout_candidates ( + id, application_id, intent_revision_id, compose_content_sha256, accepted_generation_id, + artifact_set_id, required_targets_json, authoritative, provenance, operation_id, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ).run( + row.id, row.application_id, row.intent_revision_id, row.compose_content_sha256, + row.accepted_generation_id, row.artifact_set_id, row.required_targets_json, + row.authoritative, row.provenance, row.operation_id, row.created_at, + ); + } + + upsertTarget(row: GitOpsTargetCurrentRow): void { + this.assertTargetInvariants(row); + this.db().prepare( + `INSERT INTO gitops_target_current ( + application_id, node_id, target_status, desired_generation_id, candidate_generation_id, + applied_generation_id, deployed_generation_id, healthy_generation_id, lkg_generation_id, + lkg_artifact_set_id, lkg_unavailable_at, lkg_unavailable_reason, expected_artifact_set_id, + latest_artifact_set_id, observed_artifact_identity_json, intent_revision_id, + rollout_candidate_id, rollout_generation_id, source_acceptance_ref, placement_approval_ref, + rollout_authorization_ref, legacy_combined_approval_ref, legacy_applied_revision, + connectivity, latest_stage, active_operation_id, active_operation_stage, active_operation_at, + active_generation_id, active_intent_revision_id, active_rollout_candidate_id, + failure_stage, failure_class, failure_at, recovery_ref, recovery_generation_id, + recovery_phase, interruption_stage, interruption_at, interruption_operation_id, + interruption_generation_id, interruption_intent_revision_id, interruption_rollout_candidate_id, + pause_at, pause_reason, retry_at, suspended_at, partial_json, evidence_limitations_json, updated_at + ) VALUES (${Array(50).fill('?').join(', ')}) + ON CONFLICT(application_id, node_id) DO UPDATE SET + target_status=excluded.target_status, + desired_generation_id=excluded.desired_generation_id, + candidate_generation_id=excluded.candidate_generation_id, + applied_generation_id=excluded.applied_generation_id, + deployed_generation_id=excluded.deployed_generation_id, + healthy_generation_id=excluded.healthy_generation_id, + lkg_generation_id=excluded.lkg_generation_id, + lkg_artifact_set_id=excluded.lkg_artifact_set_id, + lkg_unavailable_at=excluded.lkg_unavailable_at, + lkg_unavailable_reason=excluded.lkg_unavailable_reason, + expected_artifact_set_id=excluded.expected_artifact_set_id, + latest_artifact_set_id=excluded.latest_artifact_set_id, + observed_artifact_identity_json=excluded.observed_artifact_identity_json, + intent_revision_id=excluded.intent_revision_id, + rollout_candidate_id=excluded.rollout_candidate_id, + rollout_generation_id=excluded.rollout_generation_id, + source_acceptance_ref=excluded.source_acceptance_ref, + placement_approval_ref=excluded.placement_approval_ref, + rollout_authorization_ref=excluded.rollout_authorization_ref, + legacy_combined_approval_ref=excluded.legacy_combined_approval_ref, + legacy_applied_revision=excluded.legacy_applied_revision, + connectivity=excluded.connectivity, + latest_stage=excluded.latest_stage, + active_operation_id=excluded.active_operation_id, + active_operation_stage=excluded.active_operation_stage, + active_operation_at=excluded.active_operation_at, + active_generation_id=excluded.active_generation_id, + active_intent_revision_id=excluded.active_intent_revision_id, + active_rollout_candidate_id=excluded.active_rollout_candidate_id, + failure_stage=excluded.failure_stage, + failure_class=excluded.failure_class, + failure_at=excluded.failure_at, + recovery_ref=excluded.recovery_ref, + recovery_generation_id=excluded.recovery_generation_id, + recovery_phase=excluded.recovery_phase, + interruption_stage=excluded.interruption_stage, + interruption_at=excluded.interruption_at, + interruption_operation_id=excluded.interruption_operation_id, + interruption_generation_id=excluded.interruption_generation_id, + interruption_intent_revision_id=excluded.interruption_intent_revision_id, + interruption_rollout_candidate_id=excluded.interruption_rollout_candidate_id, + pause_at=excluded.pause_at, + pause_reason=excluded.pause_reason, + retry_at=excluded.retry_at, + suspended_at=excluded.suspended_at, + partial_json=excluded.partial_json, + evidence_limitations_json=excluded.evidence_limitations_json, + updated_at=excluded.updated_at`, + ).run( + row.application_id, row.node_id, row.target_status, row.desired_generation_id, row.candidate_generation_id, + row.applied_generation_id, row.deployed_generation_id, row.healthy_generation_id, row.lkg_generation_id, + row.lkg_artifact_set_id, row.lkg_unavailable_at, row.lkg_unavailable_reason, row.expected_artifact_set_id, + row.latest_artifact_set_id, row.observed_artifact_identity_json, row.intent_revision_id, + row.rollout_candidate_id, row.rollout_generation_id, row.source_acceptance_ref, row.placement_approval_ref, + row.rollout_authorization_ref, row.legacy_combined_approval_ref, row.legacy_applied_revision, + row.connectivity, row.latest_stage, row.active_operation_id, row.active_operation_stage, row.active_operation_at, + row.active_generation_id, row.active_intent_revision_id, row.active_rollout_candidate_id, + row.failure_stage, row.failure_class, row.failure_at, row.recovery_ref, row.recovery_generation_id, + row.recovery_phase, row.interruption_stage, row.interruption_at, row.interruption_operation_id, + row.interruption_generation_id, row.interruption_intent_revision_id, row.interruption_rollout_candidate_id, + row.pause_at, row.pause_reason, row.retry_at, row.suspended_at, row.partial_json, + row.evidence_limitations_json, row.updated_at, + ); + } + + resolveApprovalRef(id: string, expected: ResolveApprovalExpected): GitOpsApprovalRow | null { + const row = this.getApproval(id); + if (!row) return null; + if (row.application_id !== expected.applicationId) return null; + if (row.kind !== expected.kind) return null; + // legacy_combined inverts the flag on purpose. It is a migration marker for + // an approval made before source acceptance and placement were separable, + // so it can never stand as proof for either one. Requiring + // authoritative = 0 is what stops it being copied into a decomposed slot. + if (expected.kind === 'legacy_combined') { + if (row.authoritative !== 0 || row.authority !== 'legacy_combined') return null; + return row; + } + if (row.authoritative !== 1) return null; + if (row.authority !== 'operator' && row.authority !== 'configured_policy') return null; + + if (expected.kind === 'source_acceptance') { + if (!row.generation_id) return null; + const generation = this.getGeneration(row.generation_id); + if (!generation || generation.application_id !== expected.applicationId) return null; + if (row.generation_id !== expected.generationId) return null; + return row; + } + + if (expected.kind === 'placement_approval') { + if (!row.intent_revision_id || row.intent_revision_id !== expected.intentRevisionId) return null; + if (!row.blast_json) return null; + let effect; + try { + effect = decodeGitOpsApprovedTargetEffectJson(row.blast_json); + } catch { + return null; + } + if (!placementEffectCompatible(effect, expected.requiredNodeIds)) return null; + return row; + } + + const reconstructed = this.reconstructAuthorizationBinding(row); + if (!reconstructed) return null; + if (!authorizationBindingsEqual(reconstructed, expected.binding)) return null; + const source = this.resolveApprovalRef(reconstructed.sourceAcceptanceRef, { + kind: 'source_acceptance', + applicationId: expected.applicationId, + generationId: expected.binding.acceptedGenerationId, + }); + if (!source) return null; + const placement = this.resolveApprovalRef(reconstructed.placementApprovalRef, { + kind: 'placement_approval', + applicationId: expected.applicationId, + intentRevisionId: expected.binding.intentRevisionId, + requiredNodeIds: expected.binding.requiredNodeIds, + }); + if (!placement) return null; + return row; + } + + reconstructAuthorizationBinding(row: GitOpsApprovalRow): FutureRolloutAuthorizationBinding | null { + if (row.kind !== 'rollout_authorization') return null; + if ( + !row.rollout_candidate_id + || !row.generation_id + || !row.artifact_set_id + || !row.intent_revision_id + || !row.source_acceptance_ref + || !row.placement_approval_ref + || !row.required_targets_json + || !row.preflight_fingerprint + ) { + return null; + } + if (!isPreflightFingerprint(row.preflight_fingerprint)) return null; + let required; + try { + required = decodeGitOpsRequiredTargetsJson(row.required_targets_json); + } catch { + return null; + } + return { + rolloutCandidateId: row.rollout_candidate_id, + acceptedGenerationId: row.generation_id, + artifactSetId: row.artifact_set_id, + intentRevisionId: row.intent_revision_id, + requiredNodeIds: required.nodeIds, + sourceAcceptanceRef: row.source_acceptance_ref, + placementApprovalRef: row.placement_approval_ref, + preflightFingerprint: row.preflight_fingerprint, + }; + } + + currentAuthorizationBinding(app: GitOpsApplicationRow): FutureRolloutAuthorizationBinding | null { + if ( + !app.rollout_candidate_id + || !app.accepted_generation_id + || !app.artifact_set_id + || !app.intent_revision_id + || !app.source_acceptance_ref + || !app.placement_approval_ref + || !app.preflight_fingerprint + ) { + return null; + } + if (!isPreflightFingerprint(app.preflight_fingerprint)) return null; + const candidate = this.getRolloutCandidate(app.rollout_candidate_id); + if (!candidate || candidate.application_id !== app.id) return null; + if (candidate.accepted_generation_id !== app.accepted_generation_id) return null; + if (candidate.artifact_set_id !== app.artifact_set_id) return null; + if (candidate.intent_revision_id !== app.intent_revision_id) return null; + const generation = this.getGeneration(app.accepted_generation_id); + if (!generation || generation.application_id !== app.id) return null; + const artifact = this.getArtifactSet(app.artifact_set_id); + if (!artifact || artifact.generation_id !== app.accepted_generation_id) return null; + let required; + try { + required = decodeGitOpsRequiredTargetsJson(candidate.required_targets_json); + } catch { + return null; + } + const source = this.resolveApprovalRef(app.source_acceptance_ref, { + kind: 'source_acceptance', + applicationId: app.id, + generationId: app.accepted_generation_id, + }); + if (!source) return null; + const placement = this.resolveApprovalRef(app.placement_approval_ref, { + kind: 'placement_approval', + applicationId: app.id, + intentRevisionId: app.intent_revision_id, + requiredNodeIds: required.nodeIds, + }); + if (!placement) return null; + return { + rolloutCandidateId: app.rollout_candidate_id, + acceptedGenerationId: app.accepted_generation_id, + artifactSetId: app.artifact_set_id, + intentRevisionId: app.intent_revision_id, + requiredNodeIds: required.nodeIds, + sourceAcceptanceRef: app.source_acceptance_ref, + placementApprovalRef: app.placement_approval_ref, + preflightFingerprint: app.preflight_fingerprint, + }; + } + + private assertTargetInvariants(row: GitOpsTargetCurrentRow): void { + if (row.lkg_artifact_set_id) { + const artifact = this.getArtifactSet(row.lkg_artifact_set_id); + if (!artifact || artifact.generation_id !== row.lkg_generation_id) { + throw new Error('lkg_artifact_set_id must belong to lkg_generation_id'); + } + } + if (row.lkg_unavailable_at !== null && row.lkg_generation_id !== null) { + throw new Error('lkg unavailability cannot coexist with an LKG generation'); + } + this.assertArtifactPointer(row.expected_artifact_set_id, row.desired_generation_id); + this.assertArtifactPointer(row.latest_artifact_set_id, row.desired_generation_id); + // A pointer to a row that does not exist is rejected, not tolerated: the + // write is where the bad reference is cheap to find. Tolerating it moves + // the symptom to derivation, far from the transition that caused it. + if (row.desired_generation_id) { + const generation = this.getGeneration(row.desired_generation_id); + if (!generation || generation.application_id !== row.application_id) { + throw new Error('target desired generation must belong to the application'); + } + } + if (row.candidate_generation_id) { + const generation = this.getGeneration(row.candidate_generation_id); + if (!generation || generation.application_id !== row.application_id) { + throw new Error('target candidate generation must belong to the application'); + } + } + } + + private assertArtifactPointer(artifactSetId: string | null, desiredGenerationId: string | null): void { + if (!artifactSetId) return; + if (!desiredGenerationId) { + throw new Error('artifact pointer requires desired_generation_id'); + } + const artifact = this.getArtifactSet(artifactSetId); + if (!artifact || artifact.generation_id !== desiredGenerationId) { + throw new Error('target artifact pointer must belong to desired_generation_id'); + } + } +} + +/** + * Whether an approved placement effect can authorize this required target set. + * + * This is a non-contradiction check, not a coverage check. A node the approval + * places must be required, and a node it removes must not be, but a required + * node absent from the effect is fine: it is already converged, so the approved + * action list has nothing to say about it. An empty effect is therefore valid + * against any required set. Do not tighten this into set equality; a converged + * fleet legitimately produces no actions to approve. + */ +export function placementEffectCompatible( + effect: Array<{ nodeId: number; outcome: 'place' | 'remove' }>, + requiredNodeIds: readonly number[], +): boolean { + const required = new Set(requiredNodeIds); + for (const entry of effect) { + if (entry.outcome === 'place' && !required.has(entry.nodeId)) return false; + if (entry.outcome === 'remove' && required.has(entry.nodeId)) return false; + } + return true; +} + +export function authorizationBindingsEqual( + left: FutureRolloutAuthorizationBinding, + right: FutureRolloutAuthorizationBinding, +): boolean { + if (left.rolloutCandidateId !== right.rolloutCandidateId) return false; + if (left.acceptedGenerationId !== right.acceptedGenerationId) return false; + if (left.artifactSetId !== right.artifactSetId) return false; + if (left.intentRevisionId !== right.intentRevisionId) return false; + if (left.sourceAcceptanceRef !== right.sourceAcceptanceRef) return false; + if (left.placementApprovalRef !== right.placementApprovalRef) return false; + if (left.preflightFingerprint !== right.preflightFingerprint) return false; + if (left.requiredNodeIds.length !== right.requiredNodeIds.length) return false; + for (let i = 0; i < left.requiredNodeIds.length; i += 1) { + if (left.requiredNodeIds[i] !== right.requiredNodeIds[i]) return false; + } + return true; +} + +export function emptyTargetRow( + applicationId: string, + nodeId: number, + now: number, +): GitOpsTargetCurrentRow { + return { + application_id: applicationId, + node_id: nodeId, + target_status: 'active', + desired_generation_id: null, + candidate_generation_id: null, + applied_generation_id: null, + deployed_generation_id: null, + healthy_generation_id: null, + lkg_generation_id: null, + lkg_artifact_set_id: null, + lkg_unavailable_at: null, + lkg_unavailable_reason: null, + expected_artifact_set_id: null, + latest_artifact_set_id: null, + observed_artifact_identity_json: null, + intent_revision_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + rollout_authorization_ref: null, + legacy_combined_approval_ref: null, + legacy_applied_revision: null, + connectivity: null, + latest_stage: null, + active_operation_id: null, + active_operation_stage: null, + active_operation_at: null, + active_generation_id: null, + active_intent_revision_id: null, + active_rollout_candidate_id: null, + failure_stage: null, + failure_class: null, + failure_at: null, + recovery_ref: null, + recovery_generation_id: null, + recovery_phase: null, + interruption_stage: null, + interruption_at: null, + interruption_operation_id: null, + interruption_generation_id: null, + interruption_intent_revision_id: null, + interruption_rollout_candidate_id: null, + pause_at: null, + pause_reason: null, + retry_at: null, + suspended_at: null, + partial_json: null, + evidence_limitations_json: null, + updated_at: now, + }; +} diff --git a/backend/src/services/gitops/transitions.ts b/backend/src/services/gitops/transitions.ts new file mode 100644 index 00000000..278ffb2c --- /dev/null +++ b/backend/src/services/gitops/transitions.ts @@ -0,0 +1,2285 @@ +import { DatabaseService } from '../DatabaseService'; +import { + decodeArtifactEvidenceJson, + decodeGitOpsEvidenceLimitations, + decodeGitOpsJson, + encodeArtifactEvidenceJson, + encodeGitOpsEvidenceLimitations, +} from './json'; +import { insertHistory, type GitOpsHistoryStage, type HistoryOutcome } from './history'; +import { emptyTargetRow, GitOpsStore } from './store'; +import type { + ArtifactQualification, + GitOpsApplicationRow, + GitOpsApprovalAuthority, + GitOpsArtifactSetRow, + GitOpsCreateCheckpointRow, + GitOpsGenerationRow, + GitOpsIntentRevisionRow, + GitOpsRolloutCandidateRow, + GitOpsTargetCurrentRow, +} from './types'; + +export type EventEnvelope = { + operationId: string; + actor: string | null; + trigger: string; + at: number; +}; + +export type AppliedArgs = { + applicationId: string; + generationId: string; + artifactSetId: string; + sourceAcceptanceId: string; + authority: Exclude; + envelope: EventEnvelope; + activateCreating?: boolean; +}; + +export type TransitionResult = { + historyIds: string[]; + replayed: boolean; +}; + +/** + * The stages the reconciler may record against a target as an observation. + * + * Exported because the deriver has to project every one of them, and a stage + * added here that nothing projects is exactly the defect that made these + * observations unreadable. `BLUEPRINT_OBSERVATION_STATUS` in `derive.ts` is + * declared total over this union, so widening it fails that build rather than + * silently dropping the new stage back into the pointer-derived states. + */ +export type BlueprintObservationStage = + | 'blueprint_state_review' + | 'blueprint_evict_blocked' + | 'blueprint_drifted' + | 'blueprint_correcting'; + +/** + * What a health run claimed inside a recovery transaction reported back. + * + * `replayed` means the recovery already owned a run, so the caller arms that + * one rather than opening a second observation of the same restore. + */ +export type HealthRunReservation = { + outcome: 'reserved' | 'replayed' | 'disabled'; + runId: string | null; +}; + +export class GitOpsTransitionError extends Error { + constructor(message: string) { + super(message); + this.name = 'GitOpsTransitionError'; + } +} + +export class GitOpsTransitions { + private static instance: GitOpsTransitions | undefined; + + static getInstance(): GitOpsTransitions { + if (!GitOpsTransitions.instance) GitOpsTransitions.instance = new GitOpsTransitions(); + return GitOpsTransitions.instance; + } + + static resetForTests(): void { + GitOpsTransitions.instance = undefined; + } + + private store(): GitOpsStore { + return GitOpsStore.getInstance(); + } + + private raw() { + return DatabaseService.getInstance().getDb(); + } + + activateDirect(args: { + application: GitOpsApplicationRow; + nodeId: number; + envelope: EventEnvelope; + }): TransitionResult { + return this.raw().transaction(() => { + const live = this.store().getLiveDirectApplication(args.application.stack_name ?? ''); + if (live) throw new GitOpsTransitionError('live direct application already exists'); + this.store().insertApplication(args.application); + this.store().upsertTarget(emptyTargetRow(args.application.id, args.nodeId, args.envelope.at)); + const historyId = this.history(args.application, args.envelope, { + stage: 'application_activated', + outcome: 'committed', + before: { lifecycleStatus: null }, + after: { lifecycleStatus: args.application.lifecycle_status, targetMode: 'direct' }, + }); + return { historyIds: historyId ? [historyId] : [], replayed: !historyId }; + })(); + } + + /** + * Bring an Inline Blueprint into the model. + * + * No target is created. A Blueprint application has no targets until + * something is deployed somewhere, unlike a Direct one which always has the + * node its stack lives on. + */ + activateInlineBlueprint(args: { + application: GitOpsApplicationRow; + envelope: EventEnvelope; + }): TransitionResult { + return this.raw().transaction(() => { + const blueprintId = args.application.blueprint_id; + if (blueprintId === null) { + throw new GitOpsTransitionError('an inline blueprint application needs a blueprint id'); + } + if (this.store().getLiveBlueprintApplication(blueprintId)) { + throw new GitOpsTransitionError('live blueprint application already exists'); + } + this.store().insertApplication(args.application); + const historyId = this.history(args.application, args.envelope, { + stage: 'application_activated', + outcome: 'committed', + before: { lifecycleStatus: null }, + after: { lifecycleStatus: args.application.lifecycle_status, targetMode: 'inline_blueprint' }, + }); + return { historyIds: historyId ? [historyId] : [], replayed: !historyId }; + })(); + } + + fetched(applicationId: string, commitSha: string, envelope: EventEnvelope): TransitionResult { + return this.mutateApp(applicationId, envelope, 'fetched', 'committed', (app) => { + this.requireMatchingFetch(app, envelope); + this.clearActive(app); + app.desired_commit_sha = commitSha; + app.fetched_commit_sha = commitSha; + app.retry_count = 0; + this.clearAppFailure(app, ['fetch', 'validation']); + this.clearInterruption(app, 'fetch_started'); + }); + } + + fetchStarted(applicationId: string, envelope: EventEnvelope): TransitionResult { + return this.mutateApp(applicationId, envelope, 'fetch_started', 'committed', (app) => { + if (app.suspended_at) throw new GitOpsTransitionError('source is suspended'); + if (app.active_operation_stage === 'apply_started') { + throw new GitOpsTransitionError('apply is already active'); + } + if (app.active_operation_stage && app.active_operation_id !== envelope.operationId) { + throw new GitOpsTransitionError('conflicting active operation'); + } + app.active_operation_id = envelope.operationId; + app.active_operation_stage = 'fetch_started'; + app.active_operation_at = envelope.at; + app.active_generation_id = null; + app.retry_at = null; + this.clearInterruption(app, 'fetch_started'); + }); + } + + fetchFailed(applicationId: string, envelope: EventEnvelope): TransitionResult { + return this.mutateApp(applicationId, envelope, 'fetch_failed', 'failed', (app) => { + this.requireMatchingFetch(app, envelope); + this.clearActive(app); + app.failure_stage = 'fetch'; + app.failure_class = 'fetch'; + app.failure_at = envelope.at; + this.clearInterruption(app, 'fetch_started'); + }); + } + + /** + * A fetch that resolved a commit whose project does not validate. + * + * The SHA still advances: we know what the remote has, we just cannot build + * from it. Retry count is deliberately not reset, because nothing about this + * outcome suggests the next attempt will differ. + */ + fetchedInvalid(applicationId: string, commitSha: string, envelope: EventEnvelope): TransitionResult { + return this.mutateApp(applicationId, envelope, 'fetched_invalid', 'failed', (app) => { + this.requireMatchingFetch(app, envelope); + this.clearActive(app); + app.desired_commit_sha = commitSha; + app.fetched_commit_sha = commitSha; + app.failure_stage = 'validation'; + app.failure_class = 'validation'; + app.failure_at = envelope.at; + this.clearInterruption(app, 'fetch_started'); + }); + } + + /** + * A candidate whose change plan is blocked. + * + * It becomes the current candidate so the operator can see what is waiting, + * but it is marked blocked and can never be applied. + */ + sourceConflictBlocker(applicationId: string, generationId: string, envelope: EventEnvelope): TransitionResult { + return this.mutateApp(applicationId, envelope, 'source_conflict_blocker', 'committed', (app, extras) => { + const generation = this.requireOwnedGeneration(app.id, generationId); + if (generation.plan_blocked !== 1) { + throw new GitOpsTransitionError('generation is not blocked'); + } + if (app.active_operation_stage === 'apply_started' && app.candidate_generation_id !== generationId) { + throw new GitOpsTransitionError('cannot replace the candidate while an apply is in flight'); + } + this.supersedeCandidate(app, generationId, envelope, extras); + app.candidate_generation_id = generationId; + app.candidate_plan_blocked = 1; + this.forEachLiveDirectTarget(app, (target) => { + target.candidate_generation_id = generationId; + this.store().upsertTarget(target); + }); + }, { generationId }); + } + + /** + * The operator declined the pending candidate. + * + * Only the candidate is cleared. Whatever is accepted, applied, or deployed + * stays exactly where it is, and an operation in flight blocks the dismissal + * rather than pulling the candidate out from under it. + */ + dismissed(applicationId: string, envelope: EventEnvelope): TransitionResult { + return this.mutateApp(applicationId, envelope, 'dismissed', 'skipped', (app) => { + if (!app.candidate_generation_id) throw new GitOpsTransitionError('no candidate to dismiss'); + if (app.active_operation_stage) { + throw new GitOpsTransitionError('cannot dismiss while an operation is in flight'); + } + app.candidate_generation_id = null; + app.candidate_plan_blocked = 0; + app.review_required = 0; + this.forEachLiveDirectTarget(app, (target) => { + target.candidate_generation_id = null; + this.store().upsertTarget(target); + }); + }); + } + + /** + * The material source configuration changed under a staged candidate. + * + * Everything derived from the old configuration is cleared, including the + * candidate, because a candidate built from a different repository, ref, or + * file set can no longer be applied. Accepted, applied, deployed, and healthy + * pointers survive: the workload that is running did not change just because + * the configuration pointing at it did. + */ + configChangedPendingCleared(args: { + applicationId: string; + identity: { repoUrl: string; repoIdentityJson: string; configuredRef: string }; + material: { + composePathsJson: string; + contextDir: string | null; + syncEnv: number; + envPath: string | null; + fingerprint: string; + }; + envelope: EventEnvelope; + }): TransitionResult { + return this.mutateApp(args.applicationId, args.envelope, 'config_changed_pending_cleared', 'committed', (app) => { + if (app.target_mode !== 'direct') { + throw new GitOpsTransitionError('material configuration applies to Direct applications only'); + } + // The same guard dismissal and candidate replacement carry, and for the + // same reason: pulling the candidate out from under a live operation + // makes that operation's completion event unmatchable. + if (app.active_operation_stage) { + throw new GitOpsTransitionError('cannot change material configuration while an operation is in flight'); + } + app.configured_repo_url = args.identity.repoUrl; + app.repo_identity_json = args.identity.repoIdentityJson; + app.configured_ref = args.identity.configuredRef; + app.compose_paths_json = args.material.composePathsJson; + app.context_dir = args.material.contextDir; + app.sync_env = args.material.syncEnv; + app.env_path = args.material.envPath; + app.materialization_fingerprint = args.material.fingerprint; + app.desired_commit_sha = null; + app.fetched_commit_sha = null; + app.candidate_generation_id = null; + app.candidate_plan_blocked = 0; + app.review_required = 0; + this.clearAppFailure(app, ['fetch', 'validation']); + this.forEachLiveDirectTarget(app, (target) => { + target.candidate_generation_id = null; + this.store().upsertTarget(target); + }); + }); + } + + candidateReady(applicationId: string, generationId: string, reviewRequired: boolean, envelope: EventEnvelope): TransitionResult { + return this.mutateApp(applicationId, envelope, 'candidate_ready', 'committed', (app, extras) => { + const generation = this.requireOwnedGeneration(app.id, generationId); + if (generation.validation_ok !== 1 || generation.plan_blocked !== 0) { + throw new GitOpsTransitionError('generation is not a valid candidate'); + } + if (generation.materialization_fingerprint !== app.materialization_fingerprint) { + throw new GitOpsTransitionError('generation fingerprint does not match current configuration'); + } + if (app.active_operation_stage === 'apply_started' && app.candidate_generation_id !== generationId) { + throw new GitOpsTransitionError('cannot replace the candidate while an apply is in flight'); + } + this.supersedeCandidate(app, generationId, envelope, extras); + app.candidate_generation_id = generationId; + app.candidate_plan_blocked = 0; + app.review_required = reviewRequired ? 1 : 0; + this.clearAppFailure(app, ['validation']); + this.forEachLiveDirectTarget(app, (target) => { + target.candidate_generation_id = generationId; + this.store().upsertTarget(target); + }); + }, { generationId }); + } + + applyStarted(applicationId: string, generationId: string, envelope: EventEnvelope): TransitionResult { + return this.mutateApp(applicationId, envelope, 'apply_started', 'committed', (app) => { + if (app.suspended_at) throw new GitOpsTransitionError('source is suspended'); + if (app.candidate_generation_id !== generationId) { + throw new GitOpsTransitionError('apply generation is not the current candidate'); + } + if (app.candidate_plan_blocked === 1) throw new GitOpsTransitionError('candidate is blocked'); + const generation = this.requireOwnedGeneration(app.id, generationId); + if (generation.materialization_fingerprint !== app.materialization_fingerprint) { + throw new GitOpsTransitionError('generation fingerprint does not match current configuration'); + } + if (app.active_operation_stage && app.active_operation_id !== envelope.operationId) { + throw new GitOpsTransitionError('conflicting active operation'); + } + app.active_operation_id = envelope.operationId; + app.active_operation_stage = 'apply_started'; + app.active_operation_at = envelope.at; + app.active_generation_id = generationId; + this.clearInterruption(app, 'apply_started'); + }, { generationId }); + } + + applyFailed(applicationId: string, failureClass: string, envelope: EventEnvelope): TransitionResult { + return this.mutateApp(applicationId, envelope, 'apply_failed', 'failed', (app) => { + if (app.active_operation_stage !== 'apply_started' && app.interruption_stage !== 'apply_started') { + throw new GitOpsTransitionError('no matching apply operation'); + } + this.clearActive(app); + app.failure_stage = 'apply'; + app.failure_class = failureClass; + app.failure_at = envelope.at; + this.clearInterruption(app, 'apply_started'); + }); + } + + applied(args: AppliedArgs): TransitionResult { + return this.mutateApp(args.applicationId, args.envelope, 'applied', 'committed', (app) => { + const targets = this.acceptanceTargets(app, args); + this.insertAcceptanceRecords(app, args); + app.accepted_generation_id = args.generationId; + app.artifact_set_id = args.artifactSetId; + app.latest_artifact_set_id = args.artifactSetId; + app.source_acceptance_ref = args.sourceAcceptanceId; + app.candidate_generation_id = null; + app.candidate_plan_blocked = 0; + app.review_required = 0; + if (args.activateCreating && app.lifecycle_status === 'creating') { + app.lifecycle_status = 'active'; + } + this.clearActive(app); + this.clearAppFailure(app, ['apply', 'fetch', 'validation']); + this.clearInterruption(app, 'apply_started'); + for (const target of targets) { + if (app.target_mode === 'direct') { + target.desired_generation_id = args.generationId; + target.applied_generation_id = args.generationId; + target.expected_artifact_set_id = args.artifactSetId; + target.latest_artifact_set_id = args.artifactSetId; + target.source_acceptance_ref = args.sourceAcceptanceId; + target.candidate_generation_id = null; + } + this.store().upsertTarget(target); + } + }, { + generationId: args.generationId, + artifactSetId: args.artifactSetId, + sourceAcceptanceRef: args.sourceAcceptanceId, + }); + } + + /** + * The single transaction that makes a create-from-Git durable. + * + * Nothing here runs until the fetch, the candidate build, and the change-plan + * classification have all succeeded, so a create that fails early leaves no + * GitOps rows at all. Once this commits, the crash matrix can finish the + * create from the checkpoint instead of guessing what the process intended. + * + * History order is fixed: activation, then the fetch that resolved the SHA, + * then the candidate that fetch produced. Source acceptance is deliberately + * not written here; it belongs to `applied`, which is the success boundary. + */ + activateCreateFromGit(args: { + application: GitOpsApplicationRow; + nodeId: number; + commitSha: string; + generation: GitOpsGenerationRow; + checkpoint: GitOpsCreateCheckpointRow; + envelope: EventEnvelope; + }): TransitionResult { + return this.raw().transaction(() => { + const stackName = args.application.stack_name ?? ''; + if (this.store().getLiveDirectApplication(stackName)) { + throw new GitOpsTransitionError('live direct application already exists'); + } + if (args.application.lifecycle_status !== 'creating') { + throw new GitOpsTransitionError('create activation requires a creating application'); + } + if (args.generation.application_id !== args.application.id) { + throw new GitOpsTransitionError('generation does not belong to the application'); + } + if (args.generation.materialization_fingerprint !== args.application.materialization_fingerprint) { + throw new GitOpsTransitionError('generation fingerprint does not match the application'); + } + if (args.generation.validation_ok !== 1 || args.generation.plan_blocked !== 0) { + throw new GitOpsTransitionError('create cannot persist an invalid or blocked candidate'); + } + + const app = { ...args.application }; + this.store().insertApplication(app); + const target = emptyTargetRow(app.id, args.nodeId, args.envelope.at); + this.store().upsertTarget(target); + const historyIds: string[] = []; + const pushHistory = (id: string | null): void => { + if (id) historyIds.push(id); + }; + + pushHistory(this.history(app, args.envelope, { + stage: 'application_activated', + outcome: 'committed', + before: { lifecycleStatus: null }, + after: { lifecycleStatus: 'creating', targetMode: app.target_mode }, + })); + + app.desired_commit_sha = args.commitSha; + app.fetched_commit_sha = args.commitSha; + app.retry_count = 0; + pushHistory(this.history(app, args.envelope, { + stage: 'fetched', + outcome: 'committed', + before: { desiredCommitSha: null }, + after: { desiredCommitSha: args.commitSha }, + })); + + this.store().insertGeneration(args.generation); + this.store().insertCreateCheckpoint({ ...args.checkpoint, generation_id: args.generation.id }); + + app.candidate_generation_id = args.generation.id; + app.candidate_plan_blocked = 0; + app.review_required = 0; + app.latest_operation_id = args.envelope.operationId; + app.updated_at = args.envelope.at; + this.writeApplication(app); + target.candidate_generation_id = args.generation.id; + target.updated_at = args.envelope.at; + this.store().upsertTarget(target); + pushHistory(this.history(app, args.envelope, { + stage: 'candidate_ready', + outcome: 'committed', + generationId: args.generation.id, + before: { candidateGenerationId: null }, + after: { candidateGenerationId: args.generation.id, reviewRequired: false }, + })); + + return { historyIds, replayed: historyIds.length === 0 }; + })(); + } + + /** + * Tear down a create that never reached `applied`. + * + * Callers must have already removed this operation's files. Filesystem work + * cannot join a SQLite transaction, so ordering it first is what keeps the + * two consistent: if cleanup fails, the caller leaves the checkpoint in place + * and never calls this, and the next boot retries from the crash matrix. + */ + createFailed(applicationId: string, failureClass: string, envelope: EventEnvelope): TransitionResult { + return this.raw().transaction(() => { + const app = this.requireApp(applicationId); + if (app.lifecycle_status !== 'creating') { + throw new GitOpsTransitionError('create_failed requires a creating application'); + } + app.failure_stage = 'create'; + app.failure_class = failureClass; + app.failure_at = envelope.at; + app.lifecycle_status = 'deleted'; + this.clearActive(app); + app.latest_operation_id = envelope.operationId; + app.updated_at = envelope.at; + this.writeApplication(app); + + for (const target of this.store().listTargets(app.id)) { + target.target_status = 'tombstoned'; + this.clearTargetActive(target); + target.failure_stage = null; + target.failure_class = null; + target.failure_at = null; + target.lkg_generation_id = null; + target.lkg_artifact_set_id = null; + target.lkg_unavailable_at = null; + target.lkg_unavailable_reason = null; + target.updated_at = envelope.at; + this.store().upsertTarget(target); + } + + this.store().deleteCreateCheckpoint(app.id); + const id = this.history(app, envelope, { + stage: 'create_failed', + outcome: 'failed', + before: { lifecycleStatus: 'creating' }, + after: { lifecycleStatus: 'deleted', failureClass }, + }); + return { historyIds: id ? [id] : [], replayed: !id }; + })(); + } + + recordArtifactEvidence(args: { + applicationId: string; + generationId: string; + artifactSetId: string; + evidenceVersion: number; + qualification: ArtifactQualification; + evidenceJson: string; + authoritative: number; + envelope: EventEnvelope; + }): TransitionResult { + return this.mutateApp(args.applicationId, args.envelope, 'artifact_evidence_recorded', 'committed', (app) => { + if (args.authoritative !== 0) { + throw new GitOpsTransitionError('artifact rows must be non-authoritative until qualification is accepted'); + } + const generation = this.requireOwnedGeneration(app.id, args.generationId); + const evidence = decodeArtifactEvidenceJson(args.evidenceJson); + if (evidence.kind !== args.qualification) { + throw new GitOpsTransitionError('qualification must match evidence_json.kind'); + } + const maxRow = this.raw().prepare( + 'SELECT MAX(evidence_version) AS max FROM gitops_artifact_sets WHERE generation_id = ?', + ).get(args.generationId) as { max: number | null }; + const expectedVersion = (maxRow.max ?? 0) + 1; + if (args.evidenceVersion !== expectedVersion) { + throw new GitOpsTransitionError('evidenceVersion must be max+1'); + } + const loadedAppExpected = app.artifact_set_id; + this.store().insertArtifactSet({ + id: args.artifactSetId, + generation_id: generation.id, + evidence_version: args.evidenceVersion, + authoritative: 0, + qualification: args.qualification, + evidence_json: args.evidenceJson, + created_at: args.envelope.at, + }); + // Pointer advancement is decided from the values loaded at the top of + // this transaction, never from evidenceVersion-1: intervening unaccepted + // rows must not block the first real resolution. Write serialization + // comes from the enclosing synchronous SQLite transaction. + if (app.accepted_generation_id === args.generationId) { + app.latest_artifact_set_id = args.artifactSetId; + if (this.allowedExpectedAdvance(loadedAppExpected, args.qualification)) { + app.artifact_set_id = args.artifactSetId; + } + } + this.forEachLiveDirectTarget(app, (target) => { + if (target.desired_generation_id !== args.generationId) return; + const loadedExpected = target.expected_artifact_set_id; + target.latest_artifact_set_id = args.artifactSetId; + if (this.allowedExpectedAdvance(loadedExpected, args.qualification)) { + target.expected_artifact_set_id = args.artifactSetId; + } + this.store().upsertTarget(target); + }); + }, { generationId: args.generationId, artifactSetId: args.artifactSetId }); + } + + deployStarted(applicationId: string, nodeId: number, generationId: string, envelope: EventEnvelope): TransitionResult { + return this.mutateTarget(applicationId, nodeId, envelope, 'deploy_started', generationId, (target) => { + if (target.target_status !== 'active') throw new GitOpsTransitionError('target not found'); + if (target.applied_generation_id !== generationId) { + throw new GitOpsTransitionError('deploy generation is not applied'); + } + if (target.active_operation_stage && target.active_operation_id !== envelope.operationId) { + throw new GitOpsTransitionError('conflicting target operation'); + } + const before = { deployedGenerationId: target.deployed_generation_id }; + target.active_operation_id = envelope.operationId; + target.active_operation_stage = 'deploy_started'; + target.active_operation_at = envelope.at; + target.active_generation_id = generationId; + this.clearTargetInterruption(target, 'deploy_started'); + return { before, after: { activeGenerationId: generationId } }; + }); + } + + deployBound(applicationId: string, nodeId: number, generationId: string, envelope: EventEnvelope): TransitionResult { + return this.mutateTarget(applicationId, nodeId, envelope, 'deploy_bound', generationId, (target) => { + if (target.target_status !== 'active') { + throw new GitOpsTransitionError('cannot bind a deploy on a tombstoned target'); + } + this.requireMatchingDeploy(target, generationId, envelope); + target.deployed_generation_id = generationId; + this.clearTargetActive(target); + if (target.failure_stage === 'deploy') { + target.failure_stage = null; + target.failure_class = null; + target.failure_at = null; + } + this.clearTargetInterruption(target, 'deploy_started'); + return { before: {}, after: { deployedGenerationId: generationId } }; + }); + } + + /** + * The deploy ran but Compose did not bind the generation to the workload. + * + * Deployed does not move: the previous workload is what is still running, and + * claiming otherwise would make health and rollback reason about a generation + * that was never live. + */ + deployUnbound(applicationId: string, nodeId: number, generationId: string, envelope: EventEnvelope): TransitionResult { + return this.mutateTarget(applicationId, nodeId, envelope, 'deploy_unbound', generationId, (target) => { + this.requireMatchingDeploy(target, generationId, envelope); + this.clearTargetActive(target); + target.failure_stage = 'deploy'; + target.failure_class = 'unbound'; + target.failure_at = envelope.at; + this.clearTargetInterruption(target, 'deploy_started'); + return { + before: { deployedGenerationId: target.deployed_generation_id }, + after: { failureClass: 'unbound' }, + }; + }, 'failed'); + } + + /** + * The deploy failed outright. + * + * `pre_mutation` means the workload was never touched, `post_mutation` means + * it was. The deriver reports those differently because only one of them + * leaves the previous workload intact, so the caller must classify honestly + * from whether the mutation was handed off. + */ + deployFailed( + applicationId: string, + nodeId: number, + failureClass: 'pre_mutation' | 'post_mutation', + envelope: EventEnvelope, + ): TransitionResult { + return this.mutateTarget(applicationId, nodeId, envelope, 'deploy_failed', null, (target) => { + this.clearTargetActive(target); + target.failure_stage = 'deploy'; + target.failure_class = failureClass; + target.failure_at = envelope.at; + this.clearTargetInterruption(target, 'deploy_started'); + return { + before: { deployedGenerationId: target.deployed_generation_id }, + after: { failureClass }, + }; + }, 'failed'); + } + + /** + * A health gate reached a verdict on a deployed generation. + * + * Promotion is deliberately narrow. Healthy and last-known-good move only + * when the run passed, it observed the whole stack, and the generation it + * watched is still the one deployed. A run that observed generation A cannot + * vouch for B, so a stale verdict records history and moves nothing. + * + * Last-known-good keeps the artifact expectation only when that expectation + * belongs to the generation being promoted. Otherwise the generation is still + * good, its executable identity just is not proven, so the artifact pointer is + * left null rather than borrowed from a different generation. + */ + healthFinalized(args: { + applicationId: string; + nodeId: number; + healthRunId: string; + healthStatus: 'passed' | 'failed' | 'unknown'; + deployedGenerationId: string | null; + targetScope: 'stack' | 'service'; + envelope: EventEnvelope; + }): TransitionResult { + return this.mutateTarget( + args.applicationId, + args.nodeId, + args.envelope, + 'health_finalized', + args.deployedGenerationId, + (target) => { + const before = { + healthyGenerationId: target.healthy_generation_id, + lkgGenerationId: target.lkg_generation_id, + }; + const promotable = args.healthStatus === 'passed' + && target.target_status === 'active' + && args.targetScope === 'stack' + && !!args.deployedGenerationId + && target.deployed_generation_id === args.deployedGenerationId; + if (!promotable) { + return { before, after: { ...before, promoted: false, healthStatus: args.healthStatus } }; + } + + const generationId = args.deployedGenerationId as string; + target.healthy_generation_id = generationId; + target.lkg_generation_id = generationId; + + const expected = target.expected_artifact_set_id + ? this.store().getArtifactSet(target.expected_artifact_set_id) + : undefined; + target.lkg_artifact_set_id = expected && expected.generation_id === generationId + ? expected.id + : null; + // A generation that just passed is available again, whatever made the + // previous one unavailable. + target.lkg_unavailable_at = null; + target.lkg_unavailable_reason = null; + + return { + before, + after: { + healthyGenerationId: generationId, + lkgGenerationId: generationId, + lkgArtifactSetId: target.lkg_artifact_set_id, + promoted: true, + }, + }; + }, + args.healthStatus === 'unknown' ? 'unknown' : 'committed', + ); + } + + /** + * Retire an application. Tombstones never reactivate. + * + * Configured identity and SHA pointers are kept as frozen facts so the + * projection can still say what this application was, and a reattach must + * mint a new application id rather than reviving this row. + */ + applicationTombstoned( + applicationId: string, + lifecycleStatus: 'deleted' | 'detached', + envelope: EventEnvelope, + ): TransitionResult { + return this.mutateApp(applicationId, envelope, 'application_tombstoned', 'committed', (app) => { + if (app.lifecycle_status === 'deleted' || app.lifecycle_status === 'detached') { + throw new GitOpsTransitionError('application is already tombstoned'); + } + app.lifecycle_status = lifecycleStatus; + this.clearActive(app); + app.failure_stage = null; + app.failure_class = null; + app.failure_at = null; + }); + } + + /** Retire one target, clearing its last-known-good along with its operation state. */ + targetTombstoned(applicationId: string, nodeId: number, envelope: EventEnvelope): TransitionResult { + return this.mutateTarget(applicationId, nodeId, envelope, 'target_tombstoned', null, (target) => { + const before = { targetStatus: target.target_status }; + target.target_status = 'tombstoned'; + this.clearTargetActive(target); + target.failure_stage = null; + target.failure_class = null; + target.failure_at = null; + target.lkg_generation_id = null; + target.lkg_artifact_set_id = null; + target.lkg_unavailable_at = null; + target.lkg_unavailable_reason = null; + return { before, after: { targetStatus: 'tombstoned' } }; + }); + } + + /** + * Schedule a retry after a failed fetch. + * + * The failure stays visible: a scheduled retry is a plan, not a resolution, + * and hiding the failure behind it would make a stack that keeps failing look + * merely busy. `fetch_started` clears the schedule when the retry runs. + */ + sourceRetryScheduled( + applicationId: string, + retryAt: number, + retryCount: number, + envelope: EventEnvelope, + ): TransitionResult { + return this.mutateApp(applicationId, envelope, 'source_retry_scheduled', 'committed', (app) => { + if (app.suspended_at) throw new GitOpsTransitionError('source is suspended'); + if (app.active_operation_stage) { + throw new GitOpsTransitionError('cannot schedule a retry while an operation is in flight'); + } + app.retry_at = retryAt; + app.retry_count = retryCount; + }); + } + + /** + * Stop acting on a source without forgetting anything about it. + * + * Suspension is a decision about future work, so every success pointer stays + * exactly where it is. An operation in flight is interrupted rather than + * abandoned, so it does not report as running for ever. + */ + sourceSuspended(applicationId: string, reason: string, envelope: EventEnvelope): TransitionResult { + return this.mutateApp(applicationId, envelope, 'source_suspended', 'committed', (app, extras) => { + if (app.lifecycle_status !== 'active') throw new GitOpsTransitionError('source is not live'); + if (app.active_operation_stage) { + const interrupted = this.interruptActiveOperations(applicationId, envelope); + extras.historyIds.push(...interrupted.historyIds); + const reloaded = this.requireApp(applicationId); + app.interruption_stage = reloaded.interruption_stage; + app.interruption_at = reloaded.interruption_at; + app.interruption_operation_id = reloaded.interruption_operation_id; + app.interruption_generation_id = reloaded.interruption_generation_id; + this.clearActive(app); + } + app.suspended_at = envelope.at; + app.pause_reason = reason; + }); + } + + /** Resume acting on a source. Does not fetch: the operator decides when. */ + sourceUnsuspended(applicationId: string, envelope: EventEnvelope): TransitionResult { + return this.mutateApp(applicationId, envelope, 'source_unsuspended', 'committed', (app) => { + if (!app.suspended_at) throw new GitOpsTransitionError('source is not suspended'); + app.suspended_at = null; + app.pause_reason = null; + }); + } + + /** + * Pause a rollout, application-wide or on one target. + * + * A pause says nothing about health: whatever was deployed is still deployed, + * and treating a paused rollout as converged is the misreading this guards. + */ + rolloutPaused( + applicationId: string, + nodeId: number | null, + reason: string, + envelope: EventEnvelope, + ): TransitionResult { + if (nodeId === null) { + return this.mutateApp(applicationId, envelope, 'rollout_paused', 'committed', (app) => { + if (app.lifecycle_status !== 'active') throw new GitOpsTransitionError('application is not live'); + app.pause_at = envelope.at; + app.pause_reason = reason; + }); + } + return this.mutateTarget(applicationId, nodeId, envelope, 'rollout_paused', null, (target) => { + if (target.target_status !== 'active') throw new GitOpsTransitionError('target is tombstoned'); + const before = { pauseAt: target.pause_at }; + target.pause_at = envelope.at; + target.pause_reason = reason; + return { before, after: { pauseAt: envelope.at, pauseReason: reason } }; + }); + } + + rolloutUnpaused(applicationId: string, nodeId: number | null, envelope: EventEnvelope): TransitionResult { + if (nodeId === null) { + return this.mutateApp(applicationId, envelope, 'rollout_unpaused', 'committed', (app) => { + if (!app.pause_at) throw new GitOpsTransitionError('application is not paused'); + app.pause_at = null; + app.pause_reason = null; + }); + } + return this.mutateTarget(applicationId, nodeId, envelope, 'rollout_unpaused', null, (target) => { + if (!target.pause_at) throw new GitOpsTransitionError('target is not paused'); + const before = { pauseAt: target.pause_at }; + target.pause_at = null; + target.pause_reason = null; + return { before, after: { pauseAt: null } }; + }); + } + + /** + * Record that a rollout reached some targets and not others. + * + * The partial record is descriptive only. It never stands in for a deployed + * pointer: those move per target, from the deploy events, or not at all. + */ + partiallyRolledOut( + applicationId: string, + nodeId: number | null, + partialJson: string, + envelope: EventEnvelope, + ): TransitionResult { + decodeGitOpsJson(partialJson); + if (nodeId === null) { + return this.mutateApp(applicationId, envelope, 'partially_rolled_out', 'committed', (app) => { + app.partial_json = partialJson; + }); + } + return this.mutateTarget(applicationId, nodeId, envelope, 'partially_rolled_out', null, (target) => { + const before = { partial: target.partial_json !== null }; + target.partial_json = partialJson; + return { before, after: { partial: true } }; + }); + } + + /** + * A Blueprint's effective source state changed, so a new intent describes it. + * + * Minting is the caller's decision, not this transition's: a no-op edit must + * mint nothing, because every later comparison is against the intent that is + * current, and a fresh id for an unchanged Blueprint would invalidate + * acknowledgements that are still accurate. + */ + intentRevised(args: { + applicationId: string; + intent: GitOpsIntentRevisionRow; + envelope: EventEnvelope; + }): TransitionResult { + return this.mutateApp(args.applicationId, args.envelope, 'intent_revised', 'committed', (app) => { + if (app.lifecycle_status !== 'active') throw new GitOpsTransitionError('application is not live'); + if (args.intent.application_id !== args.applicationId) { + throw new GitOpsTransitionError('intent belongs to another application'); + } + this.store().insertIntentRevision(args.intent); + app.intent_revision_id = args.intent.id; + }); + } + + /** + * Open a rollout candidate for an intent and the nodes it must reach. + * + * Carries candidate-time facts only. Approval and preflight identities are + * composed later from their own rows, so a candidate can be opened before + * anything has authorized it without implying that anything has. + */ + rolloutCandidateOpened(args: { + applicationId: string; + candidate: GitOpsRolloutCandidateRow; + envelope: EventEnvelope; + }): TransitionResult { + return this.mutateApp(args.applicationId, args.envelope, 'rollout_candidate_opened', 'committed', (app) => { + if (app.lifecycle_status !== 'active') throw new GitOpsTransitionError('application is not live'); + if (args.candidate.application_id !== args.applicationId) { + throw new GitOpsTransitionError('candidate belongs to another application'); + } + if (args.candidate.intent_revision_id !== app.intent_revision_id) { + throw new GitOpsTransitionError('candidate does not name the current intent'); + } + this.store().insertRolloutCandidate(args.candidate); + app.rollout_candidate_id = args.candidate.id; + }); + } + + /** + * A Blueprint deploy was handed to one node. + * + * The intent and candidate it was launched for are recorded on the target, so + * the acknowledgement can be matched against what was actually requested. A + * later intent supersedes this one, and an ack that arrives for the older + * request is then ignored rather than accepted as current. + */ + blueprintDeployStarted(args: { + applicationId: string; + nodeId: number; + intentRevisionId: string; + rolloutCandidateId: string | null; + envelope: EventEnvelope; + }): TransitionResult { + return this.mutateTarget( + args.applicationId, + args.nodeId, + args.envelope, + 'blueprint_deploy_started', + null, + (target) => { + // An explicit deploy re-opens a placement the model had severed. The + // reconciler never starts one of these (it skips severed nodes), so a + // start arriving here for a tombstoned target is a deliberate request + // for this node, and the revival rides the same recorded event rather + // than surfacing as a refusal the physical deploy would ignore. + const before = { + activeStage: target.active_operation_stage, + targetStatus: target.target_status, + }; + target.target_status = 'active'; + // A newer deploy supersedes an older one, which is how a redeploy of a + // stuck request takes over. Anything else in flight is a different + // operation, and displacing it would abandon it with no terminal event. + if ( + target.active_operation_stage + && target.active_operation_stage !== 'blueprint_deploy_started' + && target.active_operation_id !== args.envelope.operationId + ) { + throw new GitOpsTransitionError('conflicting target operation'); + } + target.active_operation_id = args.envelope.operationId; + target.active_operation_stage = 'blueprint_deploy_started'; + target.active_operation_at = args.envelope.at; + target.active_intent_revision_id = args.intentRevisionId; + target.active_rollout_candidate_id = args.rolloutCandidateId; + // This start supersedes an interrupted one, which would otherwise keep + // matching terminals and report completion_unknown for ever. + this.clearTargetInterruption(target, 'blueprint_deploy_started'); + return { + before, + after: { + activeStage: 'blueprint_deploy_started', + intentRevisionId: args.intentRevisionId, + targetStatus: 'active', + }, + }; + }, + ); + } + + /** + * A node acknowledged the Blueprint deploy it was given. + * + * Accepted only for the identity the deploy was launched with, live or + * interrupted. An ack naming a superseded intent is a statement about work + * that no longer describes what is wanted, and recording it would report the + * node as converged on something it is not running. + */ + blueprintAckRecorded(args: { + applicationId: string; + nodeId: number; + intentRevisionId: string; + rolloutCandidateId: string | null; + legacyAppliedRevision: number | null; + envelope: EventEnvelope; + }): TransitionResult { + return this.mutateTarget( + args.applicationId, + args.nodeId, + args.envelope, + 'blueprint_ack_recorded', + null, + (target) => { + const request = this.matchBlueprintRequest( + target, 'blueprint_deploy_started', args.intentRevisionId, 'acknowledge', + ); + if (args.rolloutCandidateId !== request.rolloutCandidateId) { + throw new GitOpsTransitionError('acknowledged candidate is not the one deployed'); + } + const before = { intentRevisionId: target.intent_revision_id }; + this.clearTargetActive(target); + this.clearTargetInterruption(target, 'blueprint_deploy_started'); + target.intent_revision_id = args.intentRevisionId; + // From the matched request, never from the payload: the two can name + // different sides, and pairing one request's intent with another's + // candidate is the exact misattribution this guards. + target.rollout_candidate_id = request.rolloutCandidateId; + // Display only. The acknowledged identity is the intent, never this. + target.legacy_applied_revision = args.legacyAppliedRevision; + if (target.failure_stage === 'blueprint_deploy') { + target.failure_stage = null; + target.failure_class = null; + target.failure_at = null; + } + return { before, after: { intentRevisionId: args.intentRevisionId } }; + }, + ); + } + + /** A Blueprint deploy failed. Acknowledgement pointers stay where they were. */ + blueprintDeployFailed(args: { + applicationId: string; + nodeId: number; + failureClass: string; + envelope: EventEnvelope; + }): TransitionResult { + return this.mutateTarget( + args.applicationId, + args.nodeId, + args.envelope, + 'blueprint_deploy_failed', + null, + (target) => { + const before = { failureStage: target.failure_stage }; + this.clearTargetActive(target); + this.clearTargetInterruption(target, 'blueprint_deploy_started'); + target.failure_stage = 'blueprint_deploy'; + target.failure_class = args.failureClass; + target.failure_at = args.envelope.at; + return { before, after: { failureStage: 'blueprint_deploy', failureClass: args.failureClass } }; + }, + 'failed', + ); + } + + /** + * A Blueprint deployment is being removed from one node. + * + * The intent recorded here is the one currently acknowledged, the thing being + * taken away, not a later replacement. Withdrawing is finished only against + * that same id. + */ + blueprintWithdrawStarted(args: { + applicationId: string; + nodeId: number; + intentRevisionId: string; + envelope: EventEnvelope; + }): TransitionResult { + return this.mutateTarget( + args.applicationId, + args.nodeId, + args.envelope, + 'blueprint_withdraw_started', + null, + (target) => { + if (target.target_status !== 'active') { + throw new GitOpsTransitionError('cannot withdraw from a tombstoned target'); + } + if (target.active_operation_stage && target.active_operation_id !== args.envelope.operationId) { + throw new GitOpsTransitionError('conflicting target operation'); + } + const before = { activeStage: target.active_operation_stage }; + target.active_operation_id = args.envelope.operationId; + target.active_operation_stage = 'blueprint_withdraw_started'; + target.active_operation_at = args.envelope.at; + target.active_intent_revision_id = args.intentRevisionId; + this.clearTargetInterruption(target, 'blueprint_withdraw_started'); + return { before, after: { activeStage: 'blueprint_withdraw_started' } }; + }, + ); + } + + /** The deployment is gone from this node, so the target stops claiming it. */ + blueprintWithdrawn(args: { + applicationId: string; + nodeId: number; + intentRevisionId: string; + envelope: EventEnvelope; + }): TransitionResult { + return this.mutateTarget( + args.applicationId, + args.nodeId, + args.envelope, + 'blueprint_withdrawn', + null, + (target) => { + this.matchBlueprintRequest( + target, 'blueprint_withdraw_started', args.intentRevisionId, 'withdraw', + ); + const before = { targetStatus: target.target_status }; + this.clearTargetActive(target); + this.clearTargetInterruption(target, 'blueprint_withdraw_started'); + target.target_status = 'tombstoned'; + target.failure_stage = null; + target.failure_class = null; + target.failure_at = null; + return { before, after: { targetStatus: 'tombstoned' } }; + }, + ); + } + + /** A withdraw failed, which is a different state from a deploy that failed. */ + blueprintWithdrawFailed(args: { + applicationId: string; + nodeId: number; + failureClass: string; + envelope: EventEnvelope; + }): TransitionResult { + return this.mutateTarget( + args.applicationId, + args.nodeId, + args.envelope, + 'blueprint_withdraw_failed', + null, + (target) => { + const before = { failureStage: target.failure_stage }; + this.clearTargetActive(target); + this.clearTargetInterruption(target, 'blueprint_withdraw_started'); + target.failure_stage = 'blueprint_withdraw'; + target.failure_class = args.failureClass; + target.failure_at = args.envelope.at; + return { before, after: { failureStage: 'blueprint_withdraw', failureClass: args.failureClass } }; + }, + 'failed', + ); + } + + /** + * The reconciler observed something worth recording against a target. + * + * History, plus the runtime status derived from the recorded stage. None of + * these mints an intent or a candidate, and none acknowledges anything: they + * say what was seen, not what was decided. + */ + blueprintObservation(args: { + applicationId: string; + nodeId: number; + stage: BlueprintObservationStage; + envelope: EventEnvelope; + }): TransitionResult { + return this.mutateTarget( + args.applicationId, + args.nodeId, + args.envelope, + args.stage, + null, + (target) => { + if (target.target_status !== 'active') { + throw new GitOpsTransitionError('cannot observe a tombstoned target'); + } + // The runtime facet projects these four stages, which is the only route + // an observation has into the derived status: the reconciler records + // what it saw rather than moving any pointer. The history row written + // alongside is the separate, unprojected record. `mutateTarget` stamps + // the stage. + const before = { latestStage: target.latest_stage }; + return { before, after: { observed: args.stage } }; + }, + ); + } + + /** + * The intent a Blueprint request was launched against, live or interrupted. + * + * A terminal event has to match what was actually requested. Accepting one + * that names a superseded intent would report a node as converged on an + * intent nobody asked it for. + */ + private matchBlueprintRequest( + target: GitOpsTargetCurrentRow, + expectedStage: 'blueprint_deploy_started' | 'blueprint_withdraw_started', + intentRevisionId: string, + what: string, + ): { rolloutCandidateId: string | null } { + // Stage and identity have to come from the same side. Matching an id + // against one request while a different one is in flight is what would let + // a deploy be acknowledged out of a withdraw, or one request's intent be + // paired with another's candidate. + if ( + target.active_operation_stage === expectedStage + && target.active_intent_revision_id === intentRevisionId + ) { + return { rolloutCandidateId: target.active_rollout_candidate_id }; + } + if ( + target.interruption_stage === expectedStage + && target.interruption_intent_revision_id === intentRevisionId + ) { + return { rolloutCandidateId: target.interruption_rollout_candidate_id }; + } + throw new GitOpsTransitionError( + `cannot ${what} an intent this target was not asked to run`, + ); + } + + /** + * A rollout-scoped rollback started. + * + * The same columns and the same rules as `recovery_started`; only the trigger + * differs. Direct Git recovery emits the `recovery_*` names, and a later + * rollout producer emits these. Nothing in this PR writes them. + */ + rollbackInProgress(args: { + applicationId: string; + nodeId: number | null; + recoveryRef: string; + recoveryGenerationId: string | null; + envelope: EventEnvelope; + }): TransitionResult { + if (args.nodeId === null) { + return this.mutateApp(args.applicationId, args.envelope, 'rollback_in_progress', 'committed', (app) => { + if (app.lifecycle_status !== 'active') throw new GitOpsTransitionError('application is not live'); + app.recovery_phase = 'restoring'; + app.recovery_ref = args.recoveryRef; + }); + } + return this.mutateTarget( + args.applicationId, + args.nodeId, + args.envelope, + 'rollback_in_progress', + args.recoveryGenerationId, + (target) => { + if (target.target_status !== 'active') { + throw new GitOpsTransitionError('cannot roll back a tombstoned target'); + } + const before = { recoveryPhase: target.recovery_phase }; + target.recovery_phase = 'restoring'; + target.recovery_ref = args.recoveryRef; + target.recovery_generation_id = args.recoveryGenerationId; + this.withApplication(args.applicationId, args.envelope, (app) => { + app.recovery_phase = 'restoring'; + app.recovery_ref = args.recoveryRef; + }); + return { before, after: { recoveryPhase: 'restoring', recoveryRef: args.recoveryRef } }; + }, + ); + } + + /** + * A rollout-scoped rollback failed, wholly or on some targets. + * + * Persists the failure class it was given rather than deriving one, because + * the projection reports these columns verbatim. `partial` is the class this + * alias adds over `recovery_failed`: some targets came back and some did not, + * which is neither of the recovery classes. + */ + rollbackPartialFailed(args: { + applicationId: string; + nodeId: number | null; + recoveryRef: string; + failureClass: 'pre_mutation' | 'post_mutation' | 'partial'; + envelope: EventEnvelope; + }): TransitionResult { + const applyFailure = (row: { + recovery_phase: string | null; + recovery_ref: string | null; + failure_stage: string | null; + failure_class: string | null; + failure_at: number | null; + }): void => { + row.recovery_phase = 'failed'; + row.recovery_ref = args.recoveryRef; + row.failure_stage = 'recovery'; + row.failure_class = args.failureClass; + row.failure_at = args.envelope.at; + }; + + if (args.nodeId === null) { + return this.mutateApp(args.applicationId, args.envelope, 'rollback_partial_failed', 'failed', (app) => { + applyFailure(app); + }); + } + return this.mutateTarget( + args.applicationId, + args.nodeId, + args.envelope, + 'rollback_partial_failed', + null, + (target) => { + const before = { recoveryPhase: target.recovery_phase, failureStage: target.failure_stage }; + applyFailure(target); + return { before, after: { recoveryPhase: 'failed', failureClass: args.failureClass } }; + }, + 'failed', + ); + } + + /** + * A rollout-scoped rollback finished and the target is back on its generation. + * + * Refuses without a generation it can prove, on the same rule as + * `recovery_succeeded`: the target's own `recovery_generation_id` must name a + * generation that still exists and belongs to this application. There is no + * unproven variant, because a rollback nobody can bind to a generation has + * nothing to complete against. + */ + rollbackCompleted(args: { + applicationId: string; + nodeId: number; + recoveryRef: string; + capturedArtifactSetId: string | null; + capturedSourceAcceptanceRef: string | null; + envelope: EventEnvelope; + }): TransitionResult { + return this.mutateTarget( + args.applicationId, + args.nodeId, + args.envelope, + 'rollback_completed', + null, + (target) => { + const restored = target.recovery_generation_id; + if (!restored) { + throw new GitOpsTransitionError('rollback_completed requires a bound recovery generation'); + } + const generation = this.store().getGeneration(restored); + if (!generation || generation.application_id !== args.applicationId) { + throw new GitOpsTransitionError('rollback_completed names a generation this application does not own'); + } + + const before = { + desiredGenerationId: target.desired_generation_id, + deployedGenerationId: target.deployed_generation_id, + healthyGenerationId: target.healthy_generation_id, + }; + this.clearTargetActive(target); + target.recovery_phase = 'complete'; + target.recovery_ref = args.recoveryRef; + if (target.failure_stage === 'recovery') { + target.failure_stage = null; + target.failure_class = null; + target.failure_at = null; + } + this.withApplication(args.applicationId, args.envelope, (app) => { + app.recovery_phase = 'complete'; + this.clearAppFailure(app, ['recovery']); + }); + + target.desired_generation_id = restored; + target.applied_generation_id = restored; + // A restored workload has not been observed healthy yet, whatever the + // previous generation proved. + target.healthy_generation_id = null; + + this.restoreArtifactPointers(target, restored, args.capturedArtifactSetId); + this.restoreLastKnownGood(target, args.applicationId, args.envelope.at); + this.restoreSourceAcceptance(target, args.applicationId, restored, args.capturedSourceAcceptanceRef); + + return { + before, + after: { desiredGenerationId: restored, deployedGenerationId: target.deployed_generation_id, healthyGenerationId: null }, + }; + }, + 'recovered', + ); + } + + partialCleared(applicationId: string, nodeId: number | null, envelope: EventEnvelope): TransitionResult { + if (nodeId === null) { + return this.mutateApp(applicationId, envelope, 'partial_cleared', 'committed', (app) => { + if (!app.partial_json) throw new GitOpsTransitionError('application has no partial state'); + app.partial_json = null; + }); + } + return this.mutateTarget(applicationId, nodeId, envelope, 'partial_cleared', null, (target) => { + if (!target.partial_json) throw new GitOpsTransitionError('target has no partial state'); + const before = { partial: true }; + target.partial_json = null; + return { before, after: { partial: false } }; + }); + } + + /** + * A restore is about to touch the filesystem. + * + * The recovery reference and the generation it intends to restore are made + * durable before anything moves, so a crash mid-restore leaves a target that + * says what it was doing rather than one that looks merely broken. Success + * pointers are untouched here: nothing has been restored yet. + */ + recoveryStarted(args: { + applicationId: string; + nodeId: number; + recoveryRef: string; + recoveryGenerationId: string | null; + envelope: EventEnvelope; + }): TransitionResult { + return this.mutateTarget( + args.applicationId, + args.nodeId, + args.envelope, + 'recovery_started', + args.recoveryGenerationId, + (target) => { + if (target.target_status !== 'active') { + throw new GitOpsTransitionError('cannot recover a tombstoned target'); + } + const before = { recoveryPhase: target.recovery_phase }; + target.recovery_phase = 'restoring'; + target.recovery_ref = args.recoveryRef; + target.recovery_generation_id = args.recoveryGenerationId; + // The application carries the same phase so the source facet reports a + // recovery in flight instead of whatever the source last did. + this.withApplication(args.applicationId, args.envelope, (app) => { + app.recovery_phase = 'restoring'; + app.recovery_ref = args.recoveryRef; + }); + target.active_operation_id = args.envelope.operationId; + target.active_operation_stage = 'recovery_started'; + target.active_operation_at = args.envelope.at; + target.active_generation_id = args.recoveryGenerationId; + return { before, after: { recoveryPhase: 'restoring', recoveryRef: args.recoveryRef } }; + }, + ); + } + + /** + * A restore finished and the workload is back. + * + * `proven` is the whole question. It means the recovery row named a + * generation, that generation still exists and belongs to this application, + * and the restored files match it. Only then do pointers move, and they move + * to the restored generation rather than to whatever the application has + * since accepted: the target is running the old thing again, and saying + * otherwise would make every later comparison wrong. + * + * An unproven restore is still a real operational recovery. It is recorded as + * one and moves nothing, because there is no evidence to move pointers to. + */ + recoverySucceeded(args: { + applicationId: string; + nodeId: number; + recoveryRef: string; + recoveryGenerationId: string | null; + proven: boolean; + gitopsBinding: 'bound' | 'unbound' | 'not_applicable' | 'service_only'; + capturedArtifactSetId: string | null; + capturedSourceAcceptanceRef: string | null; + envelope: EventEnvelope; + /** + * Claim a health run for this recovery, inside this transaction. + * + * Injected rather than reached for directly: the health gate reports its + * verdicts back through this store, so importing it here would close a + * module cycle. Called only for a recovery that both proved its generation + * and bound the deployed pointer, because a run against an unproven restore + * would be observing a generation nobody can name. + */ + reserveHealthRun?: (deployedGenerationId: string) => HealthRunReservation; + }): TransitionResult & { healthReservation: HealthRunReservation | null } { + let healthReservation: HealthRunReservation | null = null; + const result = this.mutateTarget( + args.applicationId, + args.nodeId, + args.envelope, + 'recovery_succeeded', + args.recoveryGenerationId, + (target) => { + const before = { + desiredGenerationId: target.desired_generation_id, + deployedGenerationId: target.deployed_generation_id, + healthyGenerationId: target.healthy_generation_id, + }; + this.clearTargetActive(target); + target.recovery_phase = 'complete'; + if (target.failure_stage === 'recovery') { + target.failure_stage = null; + target.failure_class = null; + target.failure_at = null; + } + this.withApplication(args.applicationId, args.envelope, (app) => { + app.recovery_phase = 'complete'; + this.clearAppFailure(app, ['recovery']); + }); + + const generationId = args.recoveryGenerationId; + const generation = generationId ? this.store().getGeneration(generationId) : undefined; + const provable = args.proven + && !!generationId + && !!generation + && generation.application_id === args.applicationId; + if (!provable) { + // Nothing moved, so every pointer still agrees with itself and the + // target would otherwise read as healthy after a restore we could not + // prove restored anything. + this.noteTargetLimitation( + target, + 'recovery_unproven', + args.recoveryGenerationId ?? args.recoveryRef, + ); + return { before, after: { ...before, proven: false } }; + } + this.noteTargetLimitation(target, 'recovery_unproven', null); + + const restored = generationId as string; + target.desired_generation_id = restored; + target.applied_generation_id = restored; + if (args.gitopsBinding === 'bound') target.deployed_generation_id = restored; + // A restored workload has not been observed healthy yet, whatever the + // previous generation proved. + target.healthy_generation_id = null; + + this.restoreArtifactPointers(target, restored, args.capturedArtifactSetId); + this.restoreLastKnownGood(target, args.applicationId, args.envelope.at); + this.restoreSourceAcceptance(target, args.applicationId, restored, args.capturedSourceAcceptanceRef); + + if (args.gitopsBinding === 'bound' && args.reserveHealthRun) { + healthReservation = args.reserveHealthRun(restored); + } + + return { + before, + after: { + desiredGenerationId: restored, + deployedGenerationId: target.deployed_generation_id, + healthyGenerationId: null, + proven: true, + }, + }; + }, + 'recovered', + ); + return { ...result, healthReservation }; + } + + /** A restore failed. Success pointers stay exactly where they were. */ + recoveryFailed(args: { + applicationId: string; + nodeId: number; + recoveryRef: string; + failureClass: 'pre_mutation' | 'post_mutation'; + envelope: EventEnvelope; + }): TransitionResult { + return this.mutateTarget( + args.applicationId, + args.nodeId, + args.envelope, + 'recovery_failed', + null, + (target) => { + const before = { recoveryPhase: target.recovery_phase }; + this.clearTargetActive(target); + target.recovery_phase = 'failed'; + target.recovery_ref = args.recoveryRef; + target.failure_stage = 'recovery'; + target.failure_class = args.failureClass; + target.failure_at = args.envelope.at; + this.withApplication(args.applicationId, args.envelope, (app) => { + app.recovery_phase = 'failed'; + app.recovery_ref = args.recoveryRef; + app.failure_stage = 'recovery'; + app.failure_class = args.failureClass; + app.failure_at = args.envelope.at; + }); + return { before, after: { recoveryPhase: 'failed', failureClass: args.failureClass } }; + }, + 'failed', + ); + } + + /** + * Apply a change to the owning application inside the current transaction. + * + * Recovery state lives on both rows: the target says which node is being + * restored, the application says the stack is in recovery at all. Writing + * only one leaves the projection contradicting itself. + */ + private withApplication( + applicationId: string, + envelope: EventEnvelope, + mutate: (app: GitOpsApplicationRow) => void, + ): void { + const app = this.requireApp(applicationId); + mutate(app); + app.updated_at = envelope.at; + this.writeApplication(app); + } + + + /** + * Record, on the row itself, that a pointer was dropped because it could not + * be proven. + * + * The deriver computes limitations it can re-derive from current rows. This + * is for the ones only the writer knows: after the pointer is gone, "could + * not prove it" and "there was never one" look identical. Passing null clears + * the code, so a row that recovers its evidence stops reporting the old + * limitation. + */ + private noteTargetLimitation( + target: GitOpsTargetCurrentRow, + code: string, + detail: string | null, + ): void { + const existing = decodeGitOpsEvidenceLimitations(target.evidence_limitations_json); + target.evidence_limitations_json = encodeGitOpsEvidenceLimitations( + existing, + code, + detail === null ? null : { code, detail }, + ); + } + + /** + * Rebind the artifact expectation to the restored generation. + * + * The expectation comes from what the recovery point captured, never from + * what the application expects now: those describe different generations + * after a restore. Latest becomes the newest evidence for the restored + * generation, which is a statement about what has been seen, not an + * acceptance of it. + */ + private restoreArtifactPointers( + target: GitOpsTargetCurrentRow, + generationId: string, + capturedArtifactSetId: string | null, + ): void { + const captured = capturedArtifactSetId ? this.store().getArtifactSet(capturedArtifactSetId) : undefined; + const usable = captured && captured.generation_id === generationId; + target.expected_artifact_set_id = usable ? captured.id : null; + // Dropping this silently would also disable the runtime artifact-drift + // check for this target, so the projection would read healthier, not less + // certain, than before the restore. + this.noteTargetLimitation( + target, + 'artifact_expectation_unprovable', + capturedArtifactSetId && !usable ? capturedArtifactSetId : null, + ); + + const newest = this.raw().prepare( + `SELECT id FROM gitops_artifact_sets + WHERE generation_id = ? + ORDER BY evidence_version DESC LIMIT 1`, + ).get(generationId) as { id: string } | undefined; + target.latest_artifact_set_id = newest?.id ?? target.expected_artifact_set_id; + } + + /** + * Decide what survives of the last-known-good after a restore. + * + * A last-known-good that still exists and still belongs to this application + * is kept: restoring an older generation does not invalidate the knowledge + * that some generation once passed. Only when that generation is gone, or + * turns out to belong elsewhere, is the pointer cleared, and then the reason + * is recorded so the projection can say "unavailable" rather than "none". + */ + private restoreLastKnownGood( + target: GitOpsTargetCurrentRow, + applicationId: string, + at: number, + ): void { + if (!target.lkg_generation_id) return; + const lkg = this.store().getGeneration(target.lkg_generation_id); + const reason = !lkg + ? 'generation_missing' + : lkg.application_id !== applicationId ? 'recovery_unretainable' : null; + if (reason) { + target.lkg_generation_id = null; + target.lkg_artifact_set_id = null; + target.lkg_unavailable_at = at; + target.lkg_unavailable_reason = reason; + return; + } + // The generation stands. Its captured artifact only stands with it. + if (!target.lkg_artifact_set_id) return; + const artifact = this.store().getArtifactSet(target.lkg_artifact_set_id); + if (!artifact || artifact.generation_id !== target.lkg_generation_id) { + // Without this the last-known-good silently drops from qualified to + // merely available, which reads as "it never had qualifying evidence". + this.noteTargetLimitation(target, 'lkg_artifact_unprovable', target.lkg_artifact_set_id); + target.lkg_artifact_set_id = null; + } + } + + /** + * Restore the acceptance that authorized the generation now running. + * + * Only a captured reference that still proves this exact generation is kept. + * The application's current reference is never borrowed: it authorizes a + * newer generation, and pointing it at this one would fabricate approval. + */ + private restoreSourceAcceptance( + target: GitOpsTargetCurrentRow, + applicationId: string, + generationId: string, + capturedRef: string | null, + ): void { + if (!capturedRef) { + target.source_acceptance_ref = null; + this.noteTargetLimitation(target, 'source_acceptance_unprovable', null); + return; + } + const resolved = this.store().resolveApprovalRef(capturedRef, { + kind: 'source_acceptance', + applicationId, + generationId, + }); + target.source_acceptance_ref = resolved ? capturedRef : null; + // A workload running under an approval we can no longer prove is not the + // same as one that was never approved, and the second reads better. + this.noteTargetLimitation( + target, + 'source_acceptance_unprovable', + resolved ? null : capturedRef, + ); + } + + /** + * Retire every live target on a node that is going away. + * + * Must run before the node's rows are deleted, so the tombstones and their + * history are written while the target rows still exist. Applications are + * left alone: a Direct application whose only target was on this node still + * describes a real stack, and a Blueprint application may have targets + * elsewhere. + */ + tombstoneNodeTargets(nodeId: number, envelope: EventEnvelope): TransitionResult { + return this.raw().transaction(() => { + const historyIds: string[] = []; + for (const target of this.store().listActiveTargetsForNode(nodeId)) { + const result = this.targetTombstoned(target.application_id, nodeId, envelope); + historyIds.push(...result.historyIds); + } + return { historyIds, replayed: historyIds.length === 0 }; + })(); + } + + interruptActiveOperations(applicationId: string, envelope: EventEnvelope): TransitionResult { + return this.raw().transaction(() => { + const app = this.requireApp(applicationId); + const historyIds: string[] = []; + // A restore that never finished is not still running. Left alone, the + // phase reports a live recovery for ever, because only the terminal + // recovery events clear it and neither one is coming. + const interruptedRecovery = app.recovery_phase === 'restoring' || app.recovery_phase === 'compensating'; + if (interruptedRecovery) { + app.recovery_phase = 'failed'; + app.failure_stage = 'recovery'; + app.failure_class = 'interrupted'; + app.failure_at = envelope.at; + } + if (app.active_operation_stage || interruptedRecovery) { + app.interruption_stage = app.active_operation_stage; + app.interruption_at = envelope.at; + app.interruption_operation_id = app.active_operation_id; + app.interruption_generation_id = app.active_generation_id; + this.clearActive(app); + app.updated_at = envelope.at; + this.writeApplication(app); + const id = this.history(app, envelope, { + stage: 'operation_interrupted', + outcome: 'unknown', + before: { interruptedStage: app.interruption_stage }, + after: { activeOperationStage: null }, + }); + if (id) historyIds.push(id); + } + for (const target of this.store().listTargets(applicationId)) { + const targetRecoveryInterrupted = target.recovery_phase === 'restoring' + || target.recovery_phase === 'compensating'; + if (!target.active_operation_stage && !targetRecoveryInterrupted) continue; + if (targetRecoveryInterrupted) { + target.recovery_phase = 'failed'; + target.failure_stage = 'recovery'; + target.failure_class = 'interrupted'; + target.failure_at = envelope.at; + } + target.interruption_stage = target.active_operation_stage; + target.interruption_at = envelope.at; + target.interruption_operation_id = target.active_operation_id; + target.interruption_generation_id = target.active_generation_id; + target.interruption_intent_revision_id = target.active_intent_revision_id; + target.interruption_rollout_candidate_id = target.active_rollout_candidate_id; + this.clearTargetActive(target); + target.updated_at = envelope.at; + this.store().upsertTarget(target); + const id = this.history(app, envelope, { + nodeId: target.node_id, + stage: 'operation_interrupted', + outcome: 'unknown', + before: { interruptedStage: target.interruption_stage }, + after: { activeOperationStage: null }, + }); + if (id) historyIds.push(id); + } + return { historyIds, replayed: historyIds.length === 0 }; + })(); + } + + acceptArtifactExpectation(args: { + applicationId: string; + generationId: string; + artifactSetId: string; + envelope: EventEnvelope; + }): TransitionResult { + return this.mutateApp(args.applicationId, args.envelope, 'artifact_expectation_accepted', 'committed', (app) => { + const artifact = this.store().getArtifactSet(args.artifactSetId); + if (!artifact || artifact.generation_id !== args.generationId) { + throw new GitOpsTransitionError('artifact set is not owned by the generation'); + } + if (artifact.qualification !== 'exact' && artifact.qualification !== 'qualified') { + throw new GitOpsTransitionError('only exact or qualified evidence can be accepted'); + } + if (app.accepted_generation_id !== args.generationId) { + throw new GitOpsTransitionError('application accepted generation does not match'); + } + app.artifact_set_id = args.artifactSetId; + this.forEachLiveDirectTarget(app, (target) => { + if (target.desired_generation_id !== args.generationId) return; + target.expected_artifact_set_id = args.artifactSetId; + this.store().upsertTarget(target); + }); + }, { generationId: args.generationId, artifactSetId: args.artifactSetId }); + } + + /** + * Guard every precondition of `applied` and return the active targets the + * acceptance has to bind. + */ + private acceptanceTargets(app: GitOpsApplicationRow, args: AppliedArgs): GitOpsTargetCurrentRow[] { + if (app.candidate_generation_id !== args.generationId) { + throw new GitOpsTransitionError('applied generation is not the current candidate'); + } + // The seed artifact row is always evidence_version 1, so re-accepting a + // generation that is already accepted would collide on the version + // uniqueness constraint. Reject it here as a domain error instead of + // letting a raw driver error escape. + if (app.accepted_generation_id === args.generationId) { + throw new GitOpsTransitionError('generation is already accepted'); + } + this.requireOwnedGeneration(app.id, args.generationId); + if (app.active_operation_stage === 'apply_started') { + if (app.active_generation_id !== args.generationId) { + throw new GitOpsTransitionError('live apply is bound to a different generation'); + } + if (app.active_operation_id && app.active_operation_id !== args.envelope.operationId) { + throw new GitOpsTransitionError('live apply belongs to a different operation'); + } + } + const targets = this.store().listTargets(app.id).filter((row) => row.target_status === 'active'); + if (app.target_mode === 'direct') { + for (const target of targets) { + if (target.candidate_generation_id !== args.generationId) { + throw new GitOpsTransitionError('direct target candidate does not match applied generation'); + } + } + } + return targets; + } + + /** Seed the unresolved artifact row and the source acceptance this apply proves. */ + private insertAcceptanceRecords(app: GitOpsApplicationRow, args: AppliedArgs): void { + const artifact: GitOpsArtifactSetRow = { + id: args.artifactSetId, + generation_id: args.generationId, + evidence_version: 1, + authoritative: 0, + qualification: 'unresolved', + evidence_json: encodeArtifactEvidenceJson({ kind: 'unresolved' }), + created_at: args.envelope.at, + }; + this.store().insertArtifactSet(artifact); + this.store().insertApproval({ + id: args.sourceAcceptanceId, + kind: 'source_acceptance', + authority: args.authority, + authoritative: 1, + application_id: app.id, + generation_id: args.generationId, + intent_revision_id: null, + artifact_set_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + required_targets_json: null, + preflight_fingerprint: null, + fingerprint: null, + blast_json: null, + policy_provenance_json: null, + actor: args.envelope.actor, + created_at: args.envelope.at, + }); + } + + /** + * A terminal deploy event must name the operation it is settling. + * + * It matches either the live operation or, after a restart, the interruption + * that operation left behind. A late result for a superseded operation is + * rejected rather than allowed to move pointers a newer deploy now owns. + */ + private requireMatchingDeploy( + target: GitOpsTargetCurrentRow, + generationId: string, + envelope: EventEnvelope, + ): void { + const live = target.active_operation_stage === 'deploy_started' + && target.active_generation_id === generationId + && (!target.active_operation_id || target.active_operation_id === envelope.operationId); + const interrupted = target.interruption_stage === 'deploy_started' + && target.interruption_generation_id === generationId + && (!target.interruption_operation_id || target.interruption_operation_id === envelope.operationId); + if (!live && !interrupted) throw new GitOpsTransitionError('no matching deploy operation'); + } + + /** Record that a new candidate displaced the one already pending. */ + private supersedeCandidate( + app: GitOpsApplicationRow, + nextGenerationId: string, + envelope: EventEnvelope, + extras: { historyIds: string[] }, + ): void { + if (!app.candidate_generation_id || app.candidate_generation_id === nextGenerationId) return; + const superseded = this.history(app, envelope, { + stage: 'candidate_superseded', + outcome: 'superseded', + generationId: app.candidate_generation_id, + before: { candidateGenerationId: app.candidate_generation_id }, + after: { candidateGenerationId: nextGenerationId }, + }); + if (superseded) extras.historyIds.push(superseded); + } + + private allowedExpectedAdvance( + loadedExpectedId: string | null, + qualification: ArtifactQualification, + ): boolean { + if (qualification !== 'exact' && qualification !== 'qualified') return false; + if (!loadedExpectedId) return true; + const loaded = this.store().getArtifactSet(loadedExpectedId); + if (!loaded) return true; + // An already-resolved expectation never advances here. A changed + // executable identity is an explicit acceptance, not an implicit one, and + // an identical identity is already the expectation. + return loaded.qualification !== 'exact' && loaded.qualification !== 'qualified'; + } + + private mutateApp( + applicationId: string, + envelope: EventEnvelope, + stage: GitOpsHistoryStage, + outcome: HistoryOutcome, + mutate: (app: GitOpsApplicationRow, extras: { historyIds: string[] }) => void, + extraHistory: { + generationId?: string; + artifactSetId?: string; + sourceAcceptanceRef?: string; + } = {}, + ): TransitionResult { + return this.raw().transaction(() => { + const app = this.store().getApplication(applicationId); + if (!app) throw new GitOpsTransitionError('application not found'); + const before = snapshotApp(app); + const extras = { historyIds: [] as string[] }; + mutate(app, extras); + app.latest_operation_id = envelope.operationId; + app.updated_at = envelope.at; + this.writeApplication(app); + const historyId = this.history(app, envelope, { + stage, + outcome, + generationId: extraHistory.generationId, + artifactSetId: extraHistory.artifactSetId, + sourceAcceptanceRef: extraHistory.sourceAcceptanceRef, + before, + after: snapshotApp(app), + }); + if (historyId) extras.historyIds.push(historyId); + return { historyIds: extras.historyIds, replayed: extras.historyIds.length === 0 }; + })(); + } + + /** + * The per-target counterpart of `mutateApp`. The mutator returns its own + * history snapshots because each target stage records a different slice of + * the row. + */ + private mutateTarget( + applicationId: string, + nodeId: number, + envelope: EventEnvelope, + stage: GitOpsHistoryStage, + generationId: string | null, + mutate: (target: GitOpsTargetCurrentRow) => { before: Record; after: Record }, + outcome: HistoryOutcome = 'committed', + ): TransitionResult { + return this.raw().transaction(() => { + const app = this.requireApp(applicationId); + const target = this.store().getTarget(applicationId, nodeId); + if (!target) throw new GitOpsTransitionError('target not found'); + const snapshots = mutate(target); + // Stamped for every target mutation, not just the observations that are + // projected from it. That is what makes an observation stop being the + // latest thing that happened: a deploy, withdraw or tombstone after it + // overwrites the stage, and the projection falls back to the pointers. + // Set after `mutate` so a transition can still snapshot the prior value. + // + // Safe to stamp unconditionally only because a Blueprint target receives + // Blueprint transitions and nothing else, so every write that lands here + // genuinely answers the observation it replaces. The deploy, health and + // recovery producers all resolve their application through + // `getLiveDirectApplication`, which filters `target_mode = 'direct'` and + // can never return a Blueprint one. A producer that reached a Blueprint + // target by some other route would silently erase a drift the reconciler + // will not re-record, because it only records an observation when the + // deployment status moves. + target.latest_stage = stage; + target.updated_at = envelope.at; + this.store().upsertTarget(target); + const historyId = this.history(app, envelope, { + nodeId, + stage, + outcome, + generationId, + before: snapshots.before, + after: snapshots.after, + }); + return { historyIds: historyId ? [historyId] : [], replayed: !historyId }; + })(); + } + + /** + * Append one history row for this application, scoped to a node when + * `nodeId` is given. The dedupe target follows that scope, so an + * application-wide row and a per-node row of the same stage and operation do + * not collide on the dedupe index. + */ + private history( + app: GitOpsApplicationRow, + envelope: EventEnvelope, + fields: { + stage: GitOpsHistoryStage; + outcome: HistoryOutcome; + before: Record; + after: Record; + nodeId?: number; + generationId?: string | null; + artifactSetId?: string | null; + sourceAcceptanceRef?: string | null; + }, + ): string | null { + const nodeId = fields.nodeId ?? null; + return insertHistory(this.raw(), { + application: app, + nodeId, + dedupeTarget: nodeId === null ? 'app' : `node:${nodeId}`, + operationId: envelope.operationId, + stage: fields.stage, + outcome: fields.outcome, + trigger: envelope.trigger, + actor: envelope.actor, + before: fields.before, + after: fields.after, + generationId: fields.generationId, + artifactSetId: fields.artifactSetId, + sourceAcceptanceRef: fields.sourceAcceptanceRef, + at: envelope.at, + }); + } + + private requireMatchingFetch(app: GitOpsApplicationRow, envelope: EventEnvelope): void { + const live = app.active_operation_stage === 'fetch_started' + && (!app.active_operation_id || app.active_operation_id === envelope.operationId); + const interrupted = app.interruption_stage === 'fetch_started' + && (!app.interruption_operation_id || app.interruption_operation_id === envelope.operationId); + if (!live && !interrupted) throw new GitOpsTransitionError('no matching fetch operation'); + } + + private requireApp(applicationId: string): GitOpsApplicationRow { + const app = this.store().getApplication(applicationId); + if (!app) throw new GitOpsTransitionError('application not found'); + return app; + } + + private requireOwnedGeneration(applicationId: string, generationId: string): GitOpsGenerationRow { + const generation = this.store().getGeneration(generationId); + if (!generation || generation.application_id !== applicationId) { + throw new GitOpsTransitionError('generation is not owned by this application'); + } + return generation; + } + + private clearActive(app: GitOpsApplicationRow): void { + app.active_operation_id = null; + app.active_operation_stage = null; + app.active_operation_at = null; + app.active_generation_id = null; + } + + private clearInterruption(app: GitOpsApplicationRow, stage: GitOpsApplicationRow['interruption_stage']): void { + if (app.interruption_stage !== stage) return; + app.interruption_stage = null; + app.interruption_at = null; + app.interruption_operation_id = null; + app.interruption_generation_id = null; + } + + /** + * Release the in-flight operation, identity included. + * + * The identity columns go with the stage. Leaving them behind lets a later + * start that sets only the stage resurrect a superseded intent as though it + * were live, which is how an acknowledgement for work nobody asked for gets + * accepted. + */ + private clearTargetActive(target: GitOpsTargetCurrentRow): void { + target.active_operation_id = null; + target.active_operation_stage = null; + target.active_operation_at = null; + target.active_generation_id = null; + target.active_intent_revision_id = null; + target.active_rollout_candidate_id = null; + } + + /** + * Retire an interruption once its stage has reached a real outcome. + * + * Identity goes with it, for the same reason as above: an interruption that + * outlives its resolution keeps matching, so a late terminal for the + * interrupted request is still accepted after two later ones have succeeded. + */ + private clearTargetInterruption( + target: GitOpsTargetCurrentRow, + stage: GitOpsTargetCurrentRow['interruption_stage'], + ): void { + if (target.interruption_stage !== stage) return; + target.interruption_stage = null; + target.interruption_at = null; + target.interruption_operation_id = null; + target.interruption_generation_id = null; + target.interruption_intent_revision_id = null; + target.interruption_rollout_candidate_id = null; + } + + private clearAppFailure(app: GitOpsApplicationRow, stages: Array>): void { + if (!app.failure_stage || !stages.includes(app.failure_stage)) return; + app.failure_stage = null; + app.failure_class = null; + app.failure_at = null; + } + + private forEachLiveDirectTarget(app: GitOpsApplicationRow, fn: (target: GitOpsTargetCurrentRow) => void): void { + if (app.target_mode !== 'direct') return; + for (const target of this.store().listTargets(app.id)) { + if (target.target_status !== 'active') continue; + fn(target); + } + } + + /** + * Persist the mutable half of an application row. + * + * Every column a transition is allowed to change is written here. The mutator + * callback receives the whole row, so a column missing from this UPDATE would + * compile, appear in the history snapshot, and then be silently dropped at + * commit. Only identity and provenance are excluded, because they are fixed + * at insert: id, lifecycle_key, target_mode, stack_name, blueprint_id, + * created_at. + */ + private writeApplication(app: GitOpsApplicationRow): void { + this.raw().prepare( + `UPDATE gitops_applications SET + lifecycle_status=?, configured_repo_url=?, repo_identity_json=?, configured_ref=?, + compose_paths_json=?, context_dir=?, sync_env=?, env_path=?, + materialization_fingerprint=?, desired_commit_sha=?, fetched_commit_sha=?, + candidate_generation_id=?, accepted_generation_id=?, candidate_plan_blocked=?, + review_required=?, artifact_set_id=?, latest_artifact_set_id=?, + intent_revision_id=?, rollout_candidate_id=?, rollout_generation_id=?, + source_acceptance_ref=?, placement_approval_ref=?, rollout_authorization_ref=?, + legacy_combined_approval_ref=?, preflight_fingerprint=?, + latest_operation_id=?, active_operation_id=?, + active_operation_stage=?, active_operation_at=?, active_generation_id=?, + pause_at=?, pause_reason=?, partial_json=?, + failure_stage=?, failure_class=?, failure_at=?, retry_at=?, retry_count=?, + suspended_at=?, recovery_ref=?, recovery_phase=?, + interruption_stage=?, interruption_at=?, interruption_operation_id=?, + interruption_generation_id=?, evidence_fresh_at=?, evidence_limitations_json=?, updated_at=? + WHERE id=?`, + ).run( + app.lifecycle_status, app.configured_repo_url, app.repo_identity_json, app.configured_ref, + app.compose_paths_json, app.context_dir, app.sync_env, app.env_path, + app.materialization_fingerprint, app.desired_commit_sha, app.fetched_commit_sha, + app.candidate_generation_id, app.accepted_generation_id, app.candidate_plan_blocked, + app.review_required, app.artifact_set_id, app.latest_artifact_set_id, + app.intent_revision_id, app.rollout_candidate_id, app.rollout_generation_id, + app.source_acceptance_ref, app.placement_approval_ref, app.rollout_authorization_ref, + app.legacy_combined_approval_ref, app.preflight_fingerprint, + app.latest_operation_id, app.active_operation_id, + app.active_operation_stage, app.active_operation_at, app.active_generation_id, + app.pause_at, app.pause_reason, app.partial_json, + app.failure_stage, app.failure_class, app.failure_at, app.retry_at, app.retry_count, + app.suspended_at, app.recovery_ref, app.recovery_phase, + app.interruption_stage, app.interruption_at, app.interruption_operation_id, + app.interruption_generation_id, app.evidence_fresh_at, app.evidence_limitations_json, + app.updated_at, app.id, + ); + } +} + +function snapshotApp(app: GitOpsApplicationRow): Record { + return { + lifecycleStatus: app.lifecycle_status, + desiredCommitSha: app.desired_commit_sha, + fetchedCommitSha: app.fetched_commit_sha, + candidateGenerationId: app.candidate_generation_id, + acceptedGenerationId: app.accepted_generation_id, + artifactSetId: app.artifact_set_id, + latestArtifactSetId: app.latest_artifact_set_id, + sourceAcceptanceRef: app.source_acceptance_ref, + activeOperationStage: app.active_operation_stage, + failureStage: app.failure_stage, + }; +} diff --git a/backend/src/services/gitops/types.ts b/backend/src/services/gitops/types.ts new file mode 100644 index 00000000..732d0321 --- /dev/null +++ b/backend/src/services/gitops/types.ts @@ -0,0 +1,818 @@ +import type { ArtifactEvidenceJson, ObservedArtifactIdentity } from './json'; +import type { RepoIdentity } from './repoIdentity'; + +export type GitOpsTargetMode = 'direct' | 'inline_blueprint' | 'blueprint'; +export type GitOpsLifecycleStatus = 'active' | 'creating' | 'detached' | 'deleted'; +export type ArtifactQualification = + | 'unresolved' + | 'exact' + | 'qualified' + | 'stale' + | 'unavailable' + | 'local_build_unverified'; + +export type GitOpsApprovalKind = + | 'source_acceptance' + | 'placement_approval' + | 'rollout_authorization' + | 'legacy_combined'; + +export type GitOpsApprovalAuthority = 'operator' | 'configured_policy' | 'legacy_combined'; + +export type ApplicationActiveStage = 'fetch_started' | 'apply_started' | 'deploy_started' | 'recovery_started'; +export type TargetActiveStage = + | 'deploy_started' + | 'blueprint_deploy_started' + | 'blueprint_withdraw_started' + | 'recovery_started'; +export type RecoveryPhase = 'capturing' | 'restoring' | 'compensating' | 'complete' | 'failed'; +export type ApplicationFailureStage = 'fetch' | 'validation' | 'apply' | 'create' | 'recovery'; +export type TargetFailureStage = 'deploy' | 'recovery' | 'blueprint_deploy' | 'blueprint_withdraw'; +export type Connectivity = 'unknown' | 'reachable' | 'unreachable' | 'stale'; +export type LkgUnavailableReason = 'generation_missing' | 'recovery_unretainable'; + +export type GitOpsApplicationRow = { + id: string; + lifecycle_key: string; + lifecycle_status: GitOpsLifecycleStatus; + target_mode: GitOpsTargetMode; + stack_name: string | null; + blueprint_id: number | null; + configured_repo_url: string | null; + repo_identity_json: string | null; + configured_ref: string | null; + compose_paths_json: string | null; + context_dir: string | null; + sync_env: number | null; + env_path: string | null; + materialization_fingerprint: string | null; + desired_commit_sha: string | null; + fetched_commit_sha: string | null; + candidate_generation_id: string | null; + accepted_generation_id: string | null; + candidate_plan_blocked: number; + review_required: number; + artifact_set_id: string | null; + latest_artifact_set_id: string | null; + intent_revision_id: string | null; + rollout_candidate_id: string | null; + rollout_generation_id: string | null; + source_acceptance_ref: string | null; + placement_approval_ref: string | null; + rollout_authorization_ref: string | null; + legacy_combined_approval_ref: string | null; + preflight_fingerprint: string | null; + latest_operation_id: string | null; + active_operation_id: string | null; + active_operation_stage: ApplicationActiveStage | null; + active_operation_at: number | null; + active_generation_id: string | null; + pause_at: number | null; + pause_reason: string | null; + partial_json: string | null; + failure_stage: ApplicationFailureStage | null; + failure_class: string | null; + failure_at: number | null; + retry_at: number | null; + retry_count: number; + suspended_at: number | null; + recovery_ref: string | null; + recovery_phase: RecoveryPhase | null; + interruption_stage: ApplicationActiveStage | null; + interruption_at: number | null; + interruption_operation_id: string | null; + interruption_generation_id: string | null; + evidence_fresh_at: number | null; + /** Write-time record of what this row could not prove. See json.ts. */ + evidence_limitations_json: string | null; + created_at: number; + updated_at: number; +}; + +/** + * How far a create-from-Git operation got before it stopped. + * + * The phase is what startup uses to decide between finishing the create and + * tearing it down, so it is advanced only after the durable write it names has + * actually committed. + */ +export type GitOpsCreatePhase = + | 'pre_stack' + | 'stack_created' + | 'promoting' + | 'manifest_committed' + | 'pointers_committed'; + +export type GitOpsCreateCheckpointRow = { + application_id: string; + stack_name: string; + phase: GitOpsCreatePhase; + generation_id: string | null; + operation_id: string; + repo_url: string; + branch: string; + compose_path: string; + compose_paths_json: string; + context_dir: string | null; + sync_env: number; + env_path: string | null; + auth_type: string; + encrypted_token: string | null; + auto_apply_on_webhook: number; + auto_deploy_on_apply: number; + commit_sha: string | null; + applied_spec_json: string | null; + /** + * 1 only when this operation observed the managed root as absent and then + * created it. Deleting the whole root during cleanup requires that proof. + */ + created_managed_root: number; + created_at: number; + updated_at: number; +}; + +export type GitOpsGenerationRow = { + id: string; + application_id: string; + commit_sha: string; + repo_url: string; + configured_ref: string; + repo_identity_json: string; + manifest_version: number; + candidate_dir: string; + applied_dir: string; + expected_invocation_json: string; + materialization_fingerprint: string; + validation_ok: number; + plan_blocked: number; + change_plan_fingerprint: string | null; + operation_id: string; + trigger: string; + actor: string | null; + previous_generation_id: string | null; + redacted_limitations_json: string; + created_at: number; +}; + +export type GitOpsArtifactSetRow = { + id: string; + generation_id: string; + evidence_version: number; + authoritative: number; + qualification: ArtifactQualification; + evidence_json: string; + created_at: number; +}; + +export type GitOpsIntentRevisionRow = { + id: string; + application_id: string; + blueprint_id: number; + compose_content_sha256: string; + blueprint_revision: number; + deploy_stack_name: string; + selector_json: string; + pinned_node_id: number | null; + cordon_implications_json: string; + rollout_strategy_json: string; + runtime_drift_policy: string | null; + stateful_policy_json: string | null; + health_failure_rollback_policy_json: string | null; + operation_id: string; + actor: string | null; + created_at: number; +}; + +export type GitOpsRolloutCandidateRow = { + id: string; + application_id: string; + intent_revision_id: string; + compose_content_sha256: string; + accepted_generation_id: string | null; + artifact_set_id: string | null; + required_targets_json: string; + authoritative: number; + provenance: 'intent_change' | 'roster_change' | 'legacy_inline'; + operation_id: string; + created_at: number; +}; + +export type GitOpsApprovalRow = { + id: string; + kind: GitOpsApprovalKind; + authority: GitOpsApprovalAuthority; + authoritative: number; + application_id: string; + generation_id: string | null; + intent_revision_id: string | null; + artifact_set_id: string | null; + rollout_candidate_id: string | null; + rollout_generation_id: string | null; + source_acceptance_ref: string | null; + placement_approval_ref: string | null; + required_targets_json: string | null; + preflight_fingerprint: string | null; + fingerprint: string | null; + blast_json: string | null; + policy_provenance_json: string | null; + actor: string | null; + created_at: number; +}; + +export type GitOpsTargetCurrentRow = { + application_id: string; + node_id: number; + target_status: 'active' | 'tombstoned'; + desired_generation_id: string | null; + candidate_generation_id: string | null; + applied_generation_id: string | null; + deployed_generation_id: string | null; + healthy_generation_id: string | null; + lkg_generation_id: string | null; + lkg_artifact_set_id: string | null; + lkg_unavailable_at: number | null; + lkg_unavailable_reason: LkgUnavailableReason | null; + expected_artifact_set_id: string | null; + latest_artifact_set_id: string | null; + observed_artifact_identity_json: string | null; + intent_revision_id: string | null; + rollout_candidate_id: string | null; + rollout_generation_id: string | null; + source_acceptance_ref: string | null; + placement_approval_ref: string | null; + rollout_authorization_ref: string | null; + legacy_combined_approval_ref: string | null; + legacy_applied_revision: number | null; + connectivity: Connectivity | null; + latest_stage: string | null; + active_operation_id: string | null; + active_operation_stage: TargetActiveStage | null; + active_operation_at: number | null; + active_generation_id: string | null; + active_intent_revision_id: string | null; + active_rollout_candidate_id: string | null; + failure_stage: TargetFailureStage | null; + failure_class: string | null; + failure_at: number | null; + recovery_ref: string | null; + recovery_generation_id: string | null; + recovery_phase: RecoveryPhase | null; + interruption_stage: TargetActiveStage | null; + interruption_at: number | null; + interruption_operation_id: string | null; + interruption_generation_id: string | null; + interruption_intent_revision_id: string | null; + interruption_rollout_candidate_id: string | null; + pause_at: number | null; + pause_reason: string | null; + retry_at: number | null; + suspended_at: number | null; + partial_json: string | null; + /** Write-time record of what this target could not prove. See json.ts. */ + evidence_limitations_json: string | null; + updated_at: number; +}; + +export type GitOpsHistoryRow = { + id: string; + created_at: number; + application_id: string; + target_mode: GitOpsTargetMode; + lifecycle_key: string; + stack_name: string | null; + blueprint_id: number | null; + node_id: number | null; + dedupe_target: string; + repo_url: string | null; + configured_ref: string | null; + repo_identity_json: string | null; + commit_sha: string | null; + generation_id: string | null; + artifact_set_id: string | null; + intent_revision_id: string | null; + rollout_candidate_id: string | null; + rollout_generation_id: string | null; + source_acceptance_ref: string | null; + placement_approval_ref: string | null; + rollout_authorization_ref: string | null; + legacy_combined_approval_ref: string | null; + operation_id: string; + stage: string; + outcome: 'committed' | 'failed' | 'skipped' | 'superseded' | 'recovered' | 'unknown'; + trigger: string; + actor: string | null; + before_json: string; + after_json: string; + required_targets_json: string | null; + validation_json: string | null; + per_target_results_json: string | null; + health_run_id: string | null; + health_snapshot_json: string | null; + invocation_observed_json: string | null; + recovery_ref: string | null; + redacted_reason_class: string | null; +}; + +export type FutureRolloutAuthorizationBinding = { + readonly rolloutCandidateId: string; + readonly acceptedGenerationId: string; + readonly artifactSetId: string; + readonly intentRevisionId: string; + readonly requiredNodeIds: readonly number[]; + readonly sourceAcceptanceRef: string; + readonly placementApprovalRef: string; + readonly preflightFingerprint: string; +}; + +export type ResolveApprovalExpected = + | { kind: 'source_acceptance'; applicationId: string; generationId: string } + | { kind: 'placement_approval'; applicationId: string; intentRevisionId: string; requiredNodeIds: readonly number[] } + | { kind: 'rollout_authorization'; applicationId: string; binding: FutureRolloutAuthorizationBinding } + | { kind: 'legacy_combined'; applicationId: string }; + +export type FutureGitOpsEvidence = { + readonly applicationId: string; + readonly source: Readonly<{ kind: 'source_superseded'; supersededGenerationId: string }> | null; + readonly placement: + | Readonly<{ kind: 'source_acceptance_pending'; candidateGenerationId: string }> + | Readonly<{ kind: 'authorization_pending'; binding: FutureRolloutAuthorizationBinding }> + | Readonly<{ kind: 'authorization_stale'; rolloutAuthorizationRef: string; bound: FutureRolloutAuthorizationBinding }> + | Readonly<{ kind: 'preflight_blocked'; reason: string; binding: FutureRolloutAuthorizationBinding }> + | null; + readonly rollout: Readonly<{ + kind: + | 'queued' + | 'canary' + | 'batch' + | 'superseded' + | 'fully_deployed_health_pending' + | 'configuration_converged_artifact_qualified' + | 'exactly_converged_healthy'; + rolloutGenerationId: string; + }> | null; + readonly targetRuntime: ReadonlyArray>; +}; + +export type GitOpsApprovalRefs = { + sourceAcceptanceRef: string | null; + placementApprovalRef: string | null; + rolloutAuthorizationRef: string | null; + legacyCombinedApprovalRef: string | null; +}; + +export type EvidenceSource = 'current' | 'future' | 'current_or_future' | 'not_applicable'; + +export type SourceIdentityFields = { + configuredRepoUrl: string; + repoIdentity: RepoIdentity; + configuredRef: string; + desiredCommitSha: string | null; + fetchedCommitSha: string | null; + candidateGenerationId: string | null; + acceptedGenerationId: string | null; +}; + +export type SourceFacet = + | { status: 'not_applicable' } + | (SourceIdentityFields & { + status: + | 'never_reconciled' + | 'checking_fetching' + | 'application_generation_accepted' + | 'candidate_ready' + | 'source_review_pending' + | 'source_conflict_blocker' + | 'source_reconcile_required'; + }) + | (SourceIdentityFields & { status: 'source_superseded'; supersededGenerationId: string }) + | (SourceIdentityFields & { status: 'applying'; activeOperationId: string; activeGenerationId: string }) + | (SourceIdentityFields & { status: 'source_retry_scheduled'; retryAt: number; retryCount: number }) + | (SourceIdentityFields & { status: 'source_suspended'; suspendedAt: number }) + | (SourceIdentityFields & { + status: 'source_failed'; + failureStage: 'fetch' | 'validation' | 'apply' | 'create'; + failureClass: string; + failureAt: number; + retryAt: number | null; + retryCount: number; + }) + | (SourceIdentityFields & { + status: 'source_unknown'; + interruptedStage: 'fetch_started' | 'apply_started'; + interruptedAt: number; + interruptedOperationId: string | null; + interruptedGenerationId: string | null; + }) + | (SourceIdentityFields & { + status: 'recovery_required'; + recoveryRef: string | null; + recoveryGenerationId: string | null; + }) + | (SourceIdentityFields & { + status: 'recovery_failed'; + recoveryRef: string | null; + recoveryGenerationId: string | null; + failureClass: string; + failureAt: number; + }) + | (SourceIdentityFields & { status: 'not_live'; lifecycleStatus: 'detached' | 'deleted' }); + +export type ArtifactExpectedIdentity = { + artifactSetId: string; + evidenceVersion: number; + qualification: ArtifactQualification; + identity: string | null; +}; + +export type ArtifactLatestEvidence = { + artifactSetId: string; + evidenceVersion: number; + qualification: ArtifactQualification; + identity: string | null; +}; + +export type ArtifactFacet = + | { status: 'not_applicable' } + | { + status: 'artifact_unresolved'; + generationId: string; + expected: ArtifactExpectedIdentity | null; + latestEvidence: null; + limitation: 'artifact_pointer_missing'; + } + | { + status: + | 'artifact_unresolved' + | 'artifact_resolution_pending' + | 'artifact_exact' + | 'artifact_qualified' + | 'artifact_stale' + | 'artifact_unavailable' + | 'artifact_local_build_unverified' + | 'artifact_identity_changed'; + artifactSetId: string; + generationId: string; + evidenceVersion: number; + qualification: ArtifactQualification; + freshnessAt: number; + expected: ArtifactExpectedIdentity | null; + latestEvidence: ArtifactLatestEvidence; + }; + +export type PlacementFacet = + | { status: 'not_applicable' } + | { status: 'unbound_direct' } + | { status: 'unknown'; limitation: 'missing_intent' } + | { status: 'source_acceptance_pending'; sourceAcceptanceRef: string | null; candidateGenerationId: string } + | { status: 'placement_review_pending' } + | { status: 'rollout_authorization_pending'; rolloutAuthorizationRef: null; binding: FutureRolloutAuthorizationBinding } + | { status: 'rollout_authorization_stale'; rolloutAuthorizationRef: string; bound: FutureRolloutAuthorizationBinding } + | { status: 'stateful_confirmation_required' } + | { status: 'preflight_blocked'; reason: string; binding: FutureRolloutAuthorizationBinding } + | { status: 'blueprint_bound'; completion: 'unknown' }; + +export type RolloutFacet = + | { status: 'not_applicable' } + | { status: 'rollout_not_executable'; rolloutCandidateId: string } + | { status: 'rollout_queued'; rolloutGenerationId: string } + | { status: 'canary_in_progress'; rolloutGenerationId: string } + | { status: 'batch_in_progress'; rolloutGenerationId: string } + | { status: 'rollout_paused'; pauseAt: number; pauseReason: string | null } + | { status: 'partially_rolled_out'; partial: unknown } + | { status: 'fully_deployed_health_pending'; rolloutGenerationId: string } + | { status: 'configuration_converged_artifact_qualified'; rolloutGenerationId: string } + | { status: 'exactly_converged_healthy'; rolloutGenerationId: string } + | { status: 'rollout_superseded'; rolloutGenerationId: string } + | { status: 'target_stale' } + | { status: 'target_unreachable' } + | { status: 'rollback_in_progress'; recoveryRef: string; recoveryGenerationId: string | null } + | { status: 'rollback_partial_failed'; recoveryRef: string; recoveryGenerationId: string | null; failureClass: string; failureAt: number } + | { status: 'recovery_required' } + | { status: 'completion_unknown' }; + +export type RuntimeFacet = + | { + status: + | 'tombstoned' + | 'recovery_required' + | 'deploying' + | 'withdrawing' + | 'failed_previous_workload_intact' + | 'failed_after_mutation' + | 'disk_invocation_drift' + | 'rollout_artifact_drift' + | 'runtime_artifact_drift' + | 'artifact_verification_pending' + | 'never_applied' + | 'applied_not_deployed' + | 'acknowledged_completion_unknown' + | 'stale_acknowledgement' + | 'pending_state_review' + | 'evict_blocked' + | 'drifted' + | 'correcting' + | 'fully_deployed_health_pending' + | 'health_checking' + | 'synced_and_healthy' + | 'health_drift' + | 'partially_rolled_out' + | 'retry_scheduled'; + } + | { status: 'paused'; pauseAt: number; pauseReason: string | null } + | { + status: 'recovery_failed'; + recoveryRef: string | null; + recoveryGenerationId: string | null; + failureClass: string; + failureAt: number; + } + | { + status: 'completion_unknown'; + interruptedStage: 'deploy_started' | 'blueprint_deploy_started' | 'blueprint_withdraw_started'; + interruptedAt: number; + interruptedOperationId: string | null; + interruptedGenerationId: string | null; + interruptedIntentRevisionId: string | null; + interruptedRolloutCandidateId: string | null; + }; + +export type LkgFacet = + | { status: 'none' } + | { status: 'available'; generationId: string; artifactSetId: string | null } + | { status: 'unavailable' } + | { status: 'qualified'; generationId: string; artifactSetId: string }; + +export type HealthFacet = + | { status: 'not_applicable' | 'unbound' } + | { status: 'pending'; runId: string | null } + | { status: 'checking'; runId: string; deployedGenerationId: string | null } + | { status: 'passed'; runId: string; deployedGenerationId: string } + | { status: 'failed'; runId: string; deployedGenerationId: string | null } + | { status: 'unknown'; runId: string | null; limitation: 'health_unknown' }; + +export type FacetEvidenceSource = { + source: Record; + artifact: Record; + placement: Record; + rollout: Record; + runtime: Record; + lkg: Record; + health: Record; +}; + +/** + * What kind of evidence can produce each facet status. + * + * `current` means it derives from persisted rows. `future` means it can only + * come from a rollout-evidence envelope that no producer emits yet, so the + * derivers in this module must never return it. `current_or_future` marks the + * statuses both paths can reach. + * + * The `Record` type makes coverage a compile error in both directions: a new + * facet status without an entry here fails to compile, and so does an entry + * whose status no longer exists. + */ +export const FACET_EVIDENCE_SOURCE: FacetEvidenceSource = { + source: { + not_applicable: 'not_applicable', + never_reconciled: 'current', + checking_fetching: 'current', + applying: 'current', + candidate_ready: 'current', + source_review_pending: 'current', + source_conflict_blocker: 'current', + source_reconcile_required: 'current', + application_generation_accepted: 'current', + source_superseded: 'future', + source_retry_scheduled: 'current', + source_suspended: 'current', + source_failed: 'current', + source_unknown: 'current', + recovery_required: 'current', + recovery_failed: 'current', + not_live: 'current', + }, + artifact: { + not_applicable: 'not_applicable', + artifact_unresolved: 'current', + artifact_resolution_pending: 'current', + artifact_exact: 'current', + artifact_qualified: 'current', + artifact_stale: 'current', + artifact_unavailable: 'current', + artifact_local_build_unverified: 'current', + artifact_identity_changed: 'current', + }, + placement: { + not_applicable: 'not_applicable', + unbound_direct: 'current', + unknown: 'current', + placement_review_pending: 'current', + stateful_confirmation_required: 'current', + blueprint_bound: 'current', + source_acceptance_pending: 'future', + rollout_authorization_pending: 'future', + rollout_authorization_stale: 'future', + preflight_blocked: 'future', + }, + rollout: { + not_applicable: 'not_applicable', + rollout_not_executable: 'current', + rollout_paused: 'current', + partially_rolled_out: 'current', + target_stale: 'current', + target_unreachable: 'current', + rollback_in_progress: 'current', + rollback_partial_failed: 'current', + recovery_required: 'current', + completion_unknown: 'current_or_future', + rollout_queued: 'future', + canary_in_progress: 'future', + batch_in_progress: 'future', + fully_deployed_health_pending: 'future', + configuration_converged_artifact_qualified: 'future', + exactly_converged_healthy: 'future', + rollout_superseded: 'future', + }, + runtime: { + tombstoned: 'current', + recovery_required: 'current', + deploying: 'current', + withdrawing: 'current', + failed_previous_workload_intact: 'current', + failed_after_mutation: 'current', + disk_invocation_drift: 'current', + runtime_artifact_drift: 'current', + artifact_verification_pending: 'current', + never_applied: 'current', + applied_not_deployed: 'current', + acknowledged_completion_unknown: 'current', + stale_acknowledgement: 'current', + pending_state_review: 'current', + evict_blocked: 'current', + drifted: 'current', + correcting: 'current', + fully_deployed_health_pending: 'current', + health_checking: 'current', + synced_and_healthy: 'current', + health_drift: 'current', + partially_rolled_out: 'current', + retry_scheduled: 'current', + paused: 'current', + recovery_failed: 'current', + completion_unknown: 'current', + rollout_artifact_drift: 'future', + }, + lkg: { + none: 'current', + available: 'current', + unavailable: 'current', + qualified: 'current', + }, + health: { + not_applicable: 'not_applicable', + unbound: 'current', + pending: 'current', + checking: 'current', + passed: 'current', + failed: 'current', + unknown: 'current', + }, +}; + +export type GitOpsFacets = { + source: SourceFacet; + artifact: ArtifactFacet; + placement: PlacementFacet; + rollout: RolloutFacet; +}; + +export type AuthoredInvocationIdentity = { + composeFileOrder: string[]; + projectName: string | null; + projectDirectory: string | null; + envFileOrder: string[]; +}; + +export type GitOpsIdentityRef = + | { kind: 'none' } + | { kind: 'unknown' } + | { kind: 'commit'; sha: string; repoUrl: string; ref: string } + | { kind: 'generation'; id: string } + | { kind: 'artifact_set'; id: string; qualification: ArtifactQualification; evidenceVersion: number } + | { kind: 'runtime_artifact'; identity: string; observedAt: number | null } + | { kind: 'intent'; id: string; composeContentSha256: string } + | { kind: 'rollout_candidate'; id: string } + | { kind: 'rollout_generation'; id: string } + | { kind: 'invocation'; authored: AuthoredInvocationIdentity } + | { kind: 'health_run'; runId: string; deployedGenerationId: string | null }; + +/** + * Cross-instance evidence a history entry carries so any reader can decide who + * may see it. Only the owning instance can answer any of these questions, so + * they all travel with the row rather than being inferred by a reader that + * holds none of that instance's state. + */ +export type GitOpsHistoryEvidenceFields = { + stackName: string | null; + applicationLifecycleStatus: GitOpsLifecycleStatus | null; + stackResourcePresent: boolean; +}; + +export type GitOpsLimitation = { code: string; message: string; evidence: unknown }; +export type GitOpsAvailableAction = 'fetch' | 'apply' | 'dismiss' | 'deploy' | 'approve_legacy' | 'none'; + +export type ConfiguredPolicy = + | { kind: 'git_source'; autoApplyOnWebhook: boolean; autoDeployOnApply: boolean } + | { kind: 'blueprint_drift'; driftMode: 'observe' | 'suggest' | 'enforce' } + | null; + +export type GitOpsDriftItem = { + class: 'source' | 'managed_project' | 'invocation' | 'placement' | 'rollout' | 'runtime' | 'health'; + expected: GitOpsIdentityRef; + observed: GitOpsIdentityRef; + freshnessAt: number | null; + owner: string; + reason: string; + configuredPolicy: ConfiguredPolicy; + affectedTargets: Array<{ nodeId: number | null; stackName: string | null }>; + action: GitOpsAvailableAction; +}; + +export type GitOpsTargetProjection = { + nodeId: number; + stackName: string | null; + desiredGenerationId: string | null; + candidateGenerationId: string | null; + appliedGenerationId: string | null; + deployedGenerationId: string | null; + healthyGenerationId: string | null; + lkgGenerationId: string | null; + lkgArtifactSetId: string | null; + lkgUnavailableAt: number | null; + lkgUnavailableReason: LkgUnavailableReason | null; + expectedArtifactSetId: string | null; + latestArtifactSetId: string | null; + artifact: ArtifactFacet; + observedArtifactIdentity: ObservedArtifactIdentity; + intentRevisionId: string | null; + rolloutCandidateId: string | null; + rolloutGenerationId: string | null; + approvals: GitOpsApprovalRefs; + connectivity: Connectivity; + legacyAppliedRevision: number | null; + runtime: RuntimeFacet; + health: HealthFacet; + lkg: LkgFacet; + tombstoned: boolean; +}; + +export type GitOpsRevisionProjection = + | { + schemaVersion: 1; + targetMode: 'not_applicable'; + applicationId: null; + facets: null; + targets: []; + drift: []; + /** + * Why there is nothing to project, when the answer is not simply "no + * application". + * + * Empty for the ordinary case: a stack or Blueprint the model has never + * been asked about. Non-empty when the projection could not reach an + * application it had reason to believe exists, which is a different fact + * and must not read as the ordinary one. + * + * `readonly` because the shared frozen NOT_APPLICABLE_REVISION is this + * variant: a caller pushing onto it would corrupt every later response, + * and this keeps that a compile error rather than a runtime throw. + */ + limitations: readonly GitOpsLimitation[]; + availableActions: []; + approvals: null; + } + | { + schemaVersion: 1; + targetMode: GitOpsTargetMode; + applicationId: string; + lifecycleStatus: GitOpsLifecycleStatus; + stackName: string | null; + blueprintId: number | null; + rolloutGenerationId: string | null; + approvals: GitOpsApprovalRefs; + facets: GitOpsFacets; + targets: GitOpsTargetProjection[]; + drift: GitOpsDriftItem[]; + limitations: GitOpsLimitation[]; + availableActions: GitOpsAvailableAction[]; + }; + +export type { ArtifactEvidenceJson, ObservedArtifactIdentity, RepoIdentity }; diff --git a/backend/src/services/updateGuard/types.ts b/backend/src/services/updateGuard/types.ts index a0ab6dbd..00f2267c 100644 --- a/backend/src/services/updateGuard/types.ts +++ b/backend/src/services/updateGuard/types.ts @@ -94,7 +94,7 @@ export interface HealthGateReport { stack: string; id: string | null; status: HealthGateStatus | 'never-run'; - trigger: 'update' | 'deploy' | 'service_update' | 'service_restore' | null; + trigger: 'update' | 'deploy' | 'service_update' | 'service_restore' | 'recovery' | null; reason: string | null; windowSeconds: number | null; startedAt: number | null; diff --git a/backend/src/types/express.ts b/backend/src/types/express.ts index 2292e2a0..b0b4e206 100644 --- a/backend/src/types/express.ts +++ b/backend/src/types/express.ts @@ -24,6 +24,14 @@ declare global { sessionRemember?: boolean; /** Cached remote-proxy target resolved by `remoteNodeProxy`'s outer gate so the http-proxy router/proxyReq callbacks do not re-resolve. */ proxyTarget?: { apiUrl: string; apiToken: string }; + /** + * What the GitOps identity hop decided before forwarding: the outbound + * query (including whether the remote must filter to its own node) and + * the pre-rewrite route. Both are stashed because `pathRewrite` mutates + * `req.url` before the response callbacks run, so re-deriving either one + * there would silently look at `/api/...` and match nothing. + */ + gitopsIdentity?: { query: string; preRewritePath: string }; /** Trusted deploy provenance from machine auth or gateway overwrite. */ deployContext?: import('../services/network/missingExternalNetworksError').DeployInvocationContext; /** Verified JWT scope for machine credentials (`node_proxy` / `pilot_tunnel`). */ diff --git a/backend/src/utils/gitSourceHttp.ts b/backend/src/utils/gitSourceHttp.ts index ad8d755e..088b288a 100644 --- a/backend/src/utils/gitSourceHttp.ts +++ b/backend/src/utils/gitSourceHttp.ts @@ -28,6 +28,7 @@ export function gitSourceStatus(code: GitSourceErrorCode): number { case 'PLAN_BLOCKED': case 'LEGACY_PENDING': case 'PLAN_UNAVAILABLE': + case 'OPERATION_IN_FLIGHT': return 409; case 'NETWORK_TIMEOUT': return 504; diff --git a/docs/features/blueprint-model.mdx b/docs/features/blueprint-model.mdx index b6af1467..f606b92a 100644 --- a/docs/features/blueprint-model.mdx +++ b/docs/features/blueprint-model.mdx @@ -25,6 +25,10 @@ Three moving parts cooperate per blueprint. The marker is the trust root. If a directory by the blueprint's name already exists on a node and does not carry a matching marker, the reconciler refuses to touch it and surfaces a **Name conflict** on the deployment row. A Blueprint named `nginx` will never overwrite an existing user-authored `nginx` stack on any node. + + The revision number counts compose edits. It is a label on the spec, not a statement about the fleet: two nodes can both sit at revision 7 with one of them still deploying it. What Sencho has established per node is reported separately: the **Deployments** table on the detail sheet for each node's status, and the stack's Drift tab for its GitOps state. Where the two seem to disagree, the established state is the one that has been proven. The **GitOps** section of the detail sheet is narrower than either: it appears only when something about this Blueprint's own state could not be read or could not be proven. + + Statelessness vs statefulness is decided at author time by parsing the compose file. Stateless blueprints (no persistent volumes, or only `tmpfs` mounts) deploy and evict freely. Stateful blueprints (named volumes or bind mounts) get explicit operator-confirmation prompts on the first deploy to a fresh node and on eviction from any node. Blueprints with `external: true` volumes are classified as **unknown** and treated as stateful for safety. Drift detection runs on every tick for every Active deployment regardless of policy. The policy only governs what Sencho does next: surface the drift silently, notify, or auto-redeploy. diff --git a/docs/features/dashboard.mdx b/docs/features/dashboard.mdx index 5e8825df..321ae397 100644 --- a/docs/features/dashboard.mdx +++ b/docs/features/dashboard.mdx @@ -65,7 +65,7 @@ The row itself carries the health tint (see below); there is no separate status- | Column | Description | |--------|-------------| -| **STACK** | Stack name with an orange "Update available" badge when a newer image has been detected. When per-service status is known, the badge narrows to the outdated service name or a count (for example `2 updates`); hover for the full breakdown. The badge appears regardless of the sidebar indicator setting. Sortable. | +| **STACK** | Stack name with an orange "Update available" badge when a newer image has been detected. When per-service status is known, the badge narrows to the outdated service name or a count (for example `2 updates`); hover for the full breakdown. The badge appears regardless of the sidebar indicator setting. Also carries the GitOps source-state chip when Sencho has state for the stack (see below). Sortable. | | **SOURCE** | `Git` when the stack is linked to a [Git source](/features/git-sources), `Local` otherwise | | **PORT** | The stack's main published port, or `--` when it does not publish one | | **UP** | How long the oldest running container has been up, in compact units (`s` / `m` / `h` / `d`); a stopped or never-started stack reads `--`. Sortable. | @@ -77,6 +77,16 @@ By default, rows sort by state (errors → warnings → healthy) and then by 10- The table paginates at eight rows; chevrons appear in the header along with an `N / M` indicator when there is more than one page. When the active node has no stacks at all the card renders an empty state with a layered-disks glyph and the message `No stacks found. Create one from the sidebar.` +### GitOps state chip + +A stack linked to a [Git source](/features/git-sources) carries a small chip beside its name naming its current source state: **pending update**, **review required**, **applying**, **source failed**, and so on. The wording is the same one the Git Source panel and the Drift tab use, so a stack reads the same way wherever you look at it. + +The chip and the row tint answer different questions and are independent. The tint is about containers and load right now. The chip is about what Sencho has reconciled and accepted, so a healthy row can carry **pending update** (containers fine, a new commit waiting) and a red row can carry **accepted** (the current generation is the right one and something has since crashed). + +Hover the chip for the full sentence. A stack with no Git source carries no chip, including one delivered by a [Blueprint](/features/blueprint-model): a Blueprint stack has no Git source of its own, so there is no source state to name. The **SOURCE** column still says where the files come from either way. The chip updates as soon as Sencho records a change, without waiting for the next refresh. + +The phone dashboard shows the same chip on its stack rows, beside the node name. + ## Configuration Status The Configuration Status card is the at-a-glance audit of every toggleable automation and security feature on the active node, so nothing is silently off when you expect it to be on. diff --git a/docs/features/git-sources.mdx b/docs/features/git-sources.mdx index b9eda3b4..cbd94abe 100644 --- a/docs/features/git-sources.mdx +++ b/docs/features/git-sources.mdx @@ -26,11 +26,44 @@ Writes land in the stack's existing directory using the same storage Sencho uses The panel groups four regions: -- **Pending update banner.** Appears at the top when a webhook in **Review only** mode has fetched a new commit. Click **Review** to re-fetch the incoming commit and open the change plan. If local files conflict with the incoming commit, the banner says the update is blocked. +- **Pending update banner.** Appears at the top whenever a fetched commit is staged, however it was fetched. Its heading is the source state, so it says whether the commit is ready to apply, waiting on review, or blocked by local conflicts. Click **Review** to re-fetch the incoming commit and open the change plan. - **Form fields.** Repository URL, branch, the ordered compose-file picker, an optional project directory, optional sibling `.env` sync, authentication toggle, and the apply behavior radio group. -- **Last applied stat strip.** Shows the short SHA of the last commit Sencho applied to disk, plus the timestamp of the most recent successful save or pull. +- **Last applied stat strip.** Shows the short SHA of the last commit Sencho applied to disk, the source state (see below), and the timestamp of the most recent successful save or pull. - **Footer actions.** **Remove** disconnects the source by exporting the effective compose model into a single `compose.yaml` and removing auto-discovered override files; the remaining materialized files are kept. **Pull now** fetches the configured branch's HEAD; **Save** or **Update** persists form changes after a reachability check passes. +## Source state + +A commit SHA tells you which files Sencho wrote. It does not tell you whether that commit was accepted, whether something newer is waiting, or whether an operation was interrupted halfway. The **source state** answers those, in one short phrase that means the same thing everywhere it appears: the Git Source panel, the Drift tab, the stack list, and the stack rows on the dashboard. + +These are the states you will see most often. + +| State | What it means | +| --- | --- | +| **Never reconciled** | No commit from this repository has been accepted yet. | +| **Fetching** | Sencho is reading from the repository right now. | +| **Accepted** | The fetched commit has been accepted as the current generation. | +| **Pending update** | A fetched commit is ready to apply. | +| **Review required** | A fetched commit is waiting for review before it can apply. | +| **Pending update blocked** | The change plan has local conflicts. Apply stays disabled until they are resolved. | +| **Reconcile required** | What was reconciled no longer matches the configuration in force. Pull again to rebuild the plan. | +| **Applying** | A commit is being written to the stack directory. | +| **Retry scheduled** | The last attempt failed and another is queued. | +| **Source failed** | The last operation on this source failed. | +| **Outcome unknown** | An operation was interrupted, so Sencho cannot confirm how it ended. | +| **Recovering** | A recovery is running on this stack. | + +The last two matter most. Sencho reports an interrupted operation as unknown rather than guessing, so a stack whose pull was cut short by a restart says so instead of quietly reading as up to date. + + + Changing the repository, the branch, the compose files, the project directory, or the `.env` sync clears any staged commit, because the plan was built against the settings you just replaced. Pull again to rebuild it. The state moves to **Reconcile required** if a commit had already been accepted, and to **Never reconciled** if none ever was. Changing only the token or the apply behavior leaves a staged commit alone, since neither changes what would be materialized. + + +### When part of the state could not be proven + +The panel sometimes lists one or more things it could not prove, at the top above the form fields. A managed manifest that does not identify this stack on this node from the repository configured now, an approval that could not be restored after a recovery, a record that has gone: each is reported as its own line. + +These are qualifications, not failures. The state above them is real, and the lines tell you which part of it to treat with less confidence. That is deliberate: a stack whose evidence is partly missing looks identical to one with complete evidence unless Sencho says otherwise, and the safer wrong answer is never the quiet one. + ## Create a stack from a Git repository Skip the "empty stack then link later" detour and point at a repo from the start. Click **Create Stack** in the sidebar, switch to the **From Git** tab, and fill in the same fields you would on the Git Source panel. diff --git a/docs/features/stack-drift.mdx b/docs/features/stack-drift.mdx index 246e90e1..9a411027 100644 --- a/docs/features/stack-drift.mdx +++ b/docs/features/stack-drift.mdx @@ -54,6 +54,14 @@ This signal is independent of the runtime status. A stack can be **In sync** wit Sencho records two hashes at deploy time: a raw-file hash (catches any text change, including formatting) and a parsed-model hash (only changes when structure changes, ignoring comments and whitespace). The "source changed" signal uses both: if only the raw hash differs, the edit was formatting or comments. If the model hash also differs, a structural change has been made. +## GitOps state + +For a stack managed by a Git source or a Blueprint, a **gitops** section sits below the deploy signal. For a Git-backed stack it opens with the source state, the same phrase the Git Source panel shows; a Blueprint-delivered stack has no Git source of its own, so it shows only its per-node lines. Those say what Sencho has established about the stack on each node: whether a generation has been applied, whether it is deployed, whether its health verdict is still outstanding, and whether an operation was interrupted without a confirmed outcome. + +This is a different question from the two signals above it. Those compare files and containers as they are right now. The gitops section reports what Sencho has proven and accepted over time, so a stack can be **In sync** at runtime while its GitOps state reads **outcome unknown**, meaning the containers look right but the operation that produced them was never confirmed. + +Anything Sencho could not prove is listed underneath, one line per item, in the same form the Git Source panel uses. A stack with no Git source and no Blueprint shows no gitops section at all. + ## Findings When a stack is drifted, each reason is listed in the **Findings** section against the service it affects. diff --git a/docs/tutorials/connect-a-git-source.mdx b/docs/tutorials/connect-a-git-source.mdx index 6d38aa48..e6258345 100644 --- a/docs/tutorials/connect-a-git-source.mdx +++ b/docs/tutorials/connect-a-git-source.mdx @@ -67,10 +67,10 @@ This tutorial covers linking an existing stack to a Git source and running one m Check from two places, since neither alone proves the pull was actually applied and deployed. -**The Git Source panel.** Reopen it. The **Last applied commit** stat strip at the bottom now shows the short SHA of the commit you just pulled, with an updated timestamp. +**The Git Source panel.** Reopen it. The stat strip at the bottom now shows the short SHA of the commit you just pulled, with an updated timestamp, and a **Source state** of `accepted`, meaning Sencho took that commit as the current generation rather than merely writing its files. Nothing is listed at the top of the panel, which is what a stack with complete evidence looks like. - Git source panel for marketing-site with Apply behavior still on Review only, and a Last applied commit stat strip at the bottom showing a short commit SHA and a just-now timestamp. + Git source panel for marketing-site with Apply behavior still on Review only, and a stat strip at the bottom showing a short commit SHA, a Source state of accepted, and a just-now timestamp. **The stack itself.** Reopen `marketing-site`. The container recreated on the new image: a fresh uptime, and its logs now start with `nginx/1.28.3` instead of `1.27.5`. The compose editor tab reflects the same change; if it still shows the old tag, reload the page, since the editor buffer doesn't refresh itself after a Git-driven write. diff --git a/e2e/git-sources.spec.ts b/e2e/git-sources.spec.ts index 605242e0..d3712c8e 100644 --- a/e2e/git-sources.spec.ts +++ b/e2e/git-sources.spec.ts @@ -582,7 +582,16 @@ test.describe('Git Sources complete-project materialization (local git server)', return { sourceStatus: res.status, body: await res.json() }; }, stackName); expect(after.sourceStatus).toBe(200); - expect(after.body).toEqual({ linked: false }); + // Asserted field by field rather than by whole-object equality: the + // response also carries the additive GitOps revision fields, and a detached + // stack has no application to project while its directory is still on disk. + expect(after.body.linked).toBe(false); + expect(after.body.stackResourcePresent).toBe(true); + expect(after.body.gitopsRevision).toMatchObject({ + schemaVersion: 1, + targetMode: 'not_applicable', + applicationId: null, + }); const exported = await page.evaluate(async (name) => { const res = await fetch(`/api/stacks/${name}/files/content?path=compose.yaml`, { credentials: 'include' }); return res.ok ? await res.text() : ''; diff --git a/frontend/src/__tests__/gitopsFixtures.ts b/frontend/src/__tests__/gitopsFixtures.ts new file mode 100644 index 00000000..8916c032 --- /dev/null +++ b/frontend/src/__tests__/gitopsFixtures.ts @@ -0,0 +1,174 @@ +// Builders for GitOps revision projections in tests. +// +// A full projection is roughly forty fields across four facets and a target, so +// hand-rolling one per suite is both noise and a place for drift. Not a suite +// itself: vitest collects only *.test.* / *.spec.*. + +import type { + GitOpsApprovalRefs, + GitOpsDriftItem, + GitOpsFacets, + GitOpsLimitation, + GitOpsRevisionAbsent, + GitOpsRevisionLive, + GitOpsTargetProjection, + SourceFacet, + SourceIdentityFields, +} from '@/types/gitops'; + +/** + * The candidate-bearing source statuses, which are the ones this slice renders. + * + * Narrower than the set that structurally fits: the identity defaults below + * describe a stack that has fetched and accepted a commit, which is a state + * `never_reconciled` and `checking_fetching` cannot be in. Building those from + * here would produce a fixture no backend could emit. + */ +export type PlainSourceStatus = + | 'application_generation_accepted' + | 'candidate_ready' + | 'source_review_pending' + | 'source_conflict_blocker' + | 'source_reconcile_required'; + +export const noApprovals: GitOpsApprovalRefs = { + sourceAcceptanceRef: null, + placementApprovalRef: null, + rolloutAuthorizationRef: null, + legacyCombinedApprovalRef: null, +}; + +export function sourceIdentity(overrides: Partial = {}): SourceIdentityFields { + return { + configuredRepoUrl: 'https://example.test/acme/infra.git', + repoIdentity: { host: 'example.test', pathname: '/acme/infra' }, + configuredRef: 'main', + desiredCommitSha: 'a1b2c3d4e5f60718293a4b5c6d7e8f9012345678', + fetchedCommitSha: 'a1b2c3d4e5f60718293a4b5c6d7e8f9012345678', + candidateGenerationId: 'gen-candidate', + acceptedGenerationId: 'gen-accepted', + ...overrides, + }; +} + +/** A source facet with a waiting candidate by default. Pass `candidateGenerationId: null` for the accepted case. */ +export function plainSource( + status: PlainSourceStatus, + overrides: Partial = {}, +): SourceFacet { + return { ...sourceIdentity(overrides), status }; +} + +export function target(overrides: Partial = {}): GitOpsTargetProjection { + return { + nodeId: 1, + stackName: 'bookstack', + desiredGenerationId: 'gen-accepted', + candidateGenerationId: null, + appliedGenerationId: 'gen-accepted', + deployedGenerationId: 'gen-accepted', + healthyGenerationId: 'gen-accepted', + lkgGenerationId: null, + lkgArtifactSetId: null, + lkgUnavailableAt: null, + lkgUnavailableReason: null, + expectedArtifactSetId: null, + latestArtifactSetId: null, + artifact: { status: 'not_applicable' }, + observedArtifactIdentity: { kind: 'unknown' }, + intentRevisionId: null, + rolloutCandidateId: null, + rolloutGenerationId: null, + approvals: noApprovals, + connectivity: 'reachable', + legacyAppliedRevision: null, + runtime: { status: 'synced_and_healthy' }, + health: { status: 'not_applicable' }, + lkg: { status: 'none' }, + tombstoned: false, + ...overrides, + }; +} + +/** + * One classified divergence. The backend emits the runtime class from a + * comparable artifact mismatch, so this fixture shapes itself after that item; + * the other classes still have no producer. + */ +export function driftItem(overrides: Partial = {}): GitOpsDriftItem { + return { + class: 'runtime', + expected: { kind: 'artifact_set', id: 'art-accepted', qualification: 'exact', evidenceVersion: 1 }, + observed: { kind: 'runtime_artifact', identity: 'nginx@sha256:abc', observedAt: 1 }, + freshnessAt: 1, + owner: 'observed_artifact_identity', + reason: 'the running workload reports an artifact identity other than the expected artifact set', + configuredPolicy: null, + affectedTargets: [{ nodeId: 1, stackName: 'bookstack' }], + action: 'none', + ...overrides, + }; +} + +export function facets(overrides: Partial = {}): GitOpsFacets { + return { + source: plainSource('candidate_ready'), + artifact: { status: 'not_applicable' }, + placement: { status: 'unbound_direct' }, + rollout: { status: 'not_applicable' }, + ...overrides, + }; +} + +/** A live Direct application with one healthy target and a candidate waiting. */ +export function liveRevision(overrides: Partial = {}): GitOpsRevisionLive { + return { + schemaVersion: 1, + targetMode: 'direct', + applicationId: 'app-1', + lifecycleStatus: 'active', + stackName: 'bookstack', + blueprintId: null, + rolloutGenerationId: null, + approvals: noApprovals, + facets: facets(), + targets: [target()], + drift: [], + limitations: [], + availableActions: ['apply', 'dismiss'], + ...overrides, + }; +} + +/** + * The common case in the consumer suites: a live Direct application whose Git + * source sits in one named state. Pass `candidateGenerationId: null` for the + * variants that must have no candidate waiting behind that state. + */ +export function sourceRevision( + status: PlainSourceStatus, + overrides: Partial = {}, +): GitOpsRevisionLive { + return liveRevision({ facets: facets({ source: plainSource(status, overrides) }) }); +} + +/** Nothing to project. Pass limitations for the fault case; empty is the ordinary one. */ +export function absentRevision(limitations: GitOpsLimitation[] = []): GitOpsRevisionAbsent { + return { + schemaVersion: 1, + targetMode: 'not_applicable', + applicationId: null, + facets: null, + targets: [], + drift: [], + limitations, + availableActions: [], + approvals: null, + }; +} + +export const missingApplicationLimitation: GitOpsLimitation = { + code: 'application_row_missing', + message: 'The application row backing this stack could not be read, so its GitOps state cannot be reported.', + evidence: { applicationId: 'app-1' }, +}; diff --git a/frontend/src/components/EditorLayout.tsx b/frontend/src/components/EditorLayout.tsx index e16397aa..d39bdaa4 100644 --- a/frontend/src/components/EditorLayout.tsx +++ b/frontend/src/components/EditorLayout.tsx @@ -293,6 +293,25 @@ export default function EditorLayout() { .filter((item): item is NonNullable => item !== null); }, [quickLinkIds, navModel.quickLinkCandidates]); + // Coalesce a burst of GitOps transitions into one refetch of the derived + // state. One operation commits several transitions in a row, and a boot-time + // migration commits a great many, so refetching per event would thrash the + // API for a picture that only settles at the end. Same 250ms window the stack + // refresh uses. The ref indirection is needed because stackActions is built + // below this point. + const refreshGitOpsRef = useRef<() => void>(() => {}); + const gitopsRefreshTimerRef = useRef | null>(null); + const scheduleGitOpsRefresh = useCallback(() => { + if (gitopsRefreshTimerRef.current) clearTimeout(gitopsRefreshTimerRef.current); + gitopsRefreshTimerRef.current = setTimeout(() => { + gitopsRefreshTimerRef.current = null; + refreshGitOpsRef.current(); + }, 250); + }, []); + useEffect(() => () => { + if (gitopsRefreshTimerRef.current) clearTimeout(gitopsRefreshTimerRef.current); + }, []); + const { notifications, tickerConnected, @@ -304,6 +323,7 @@ export default function EditorLayout() { nodes, onStateInvalidate: scheduleStateInvalidateRefresh, onImageUpdatesChange: fetchImageUpdates, + onGitOpsChange: scheduleGitOpsRefresh, }); const { stats: containerStats, error: containerStatsError } = useContainerStats( @@ -338,6 +358,10 @@ export default function EditorLayout() { canReapplyCompose, }); + // Close the loop opened above: the debounced GitOps refresh now has something + // to call. Assigned every render so it never holds a stale stackActions. + refreshGitOpsRef.current = () => { void stackActions.refreshGitSourcePending(); }; + // Wire the ref now that stackActions is available resetEditorStateRef.current = stackActions.resetEditorState; diff --git a/frontend/src/components/EditorLayout/EditorView.tsx b/frontend/src/components/EditorLayout/EditorView.tsx index 0801fae3..ce71dd56 100644 --- a/frontend/src/components/EditorLayout/EditorView.tsx +++ b/frontend/src/components/EditorLayout/EditorView.tsx @@ -48,6 +48,7 @@ import type { Node } from '@/context/NodeContext'; import type { useAuth } from '@/context/AuthContext'; import type { useStackMuteActions } from '@/hooks/useMuteRuleActions'; import type { EffectiveServiceSpec } from '@/types/effectiveServices'; +import type { GitSourcePendingMap } from '@/lib/gitopsState'; import type { StackServiceUpdateStatus } from '@/types/imageUpdates'; export interface ContainerInfo { @@ -142,7 +143,7 @@ export interface EditorViewProps { containersSyncStale?: boolean; onRetrySync?: () => void; backupInfo: { exists: boolean; timestamp: number | null }; - gitSourcePendingMap: Record; + gitSourcePendingMap: GitSourcePendingMap; notifications: NotificationItem[]; // Editor mode @@ -733,7 +734,7 @@ export function EditorView(props: EditorViewProps) { content={content} envContent={envContent} selectedEnvFile={selectedEnvFile} - gitSourcePending={Boolean(gitSourcePendingMap[stackName])} + gitSourcePending={gitSourcePendingMap[stackName] ?? null} onEditCompose={openComposeEditor} onOpenFiles={canRead ? () => { setEditingCompose(true); setActiveTab('files'); } : undefined} onOpenGitSource={() => setGitSourceOpen(true)} diff --git a/frontend/src/components/EditorLayout/MobileStackDetail.tsx b/frontend/src/components/EditorLayout/MobileStackDetail.tsx index 4abbecba..2e894e32 100644 --- a/frontend/src/components/EditorLayout/MobileStackDetail.tsx +++ b/frontend/src/components/EditorLayout/MobileStackDetail.tsx @@ -268,7 +268,7 @@ export function MobileStackDetail(props: EditorViewProps) { content={content} envContent={envContent} selectedEnvFile={selectedEnvFile} - gitSourcePending={Boolean(gitSourcePendingMap[stackName])} + gitSourcePending={gitSourcePendingMap[stackName] ?? null} onEditCompose={openComposeEditor} onOpenGitSource={() => setGitSourceOpen(true)} onApplyUpdate={() => { void updateStack(); }} diff --git a/frontend/src/components/EditorLayout/hooks/useEditorViewState.ts b/frontend/src/components/EditorLayout/hooks/useEditorViewState.ts index e2f09f98..f274ff3e 100644 --- a/frontend/src/components/EditorLayout/hooks/useEditorViewState.ts +++ b/frontend/src/components/EditorLayout/hooks/useEditorViewState.ts @@ -1,6 +1,7 @@ import { useEffect, useState } from 'react'; import type { ContainerInfo } from '../EditorView'; import type { EffectiveServiceSpec } from '@/types/effectiveServices'; +import type { GitSourcePendingMap } from '@/lib/gitopsState'; export const LOGS_MODE_STORAGE_KEY = 'sencho.stackView.logsMode'; @@ -51,7 +52,10 @@ export function useEditorViewState() { }, [logsMode]); const [gitSourceOpen, setGitSourceOpen] = useState(false); - const [gitSourcePendingMap, setGitSourcePendingMap] = useState>({}); + // Keyed by the API's stack_name and read by the sidebar's file key, which + // coincide for every stack the sidebar can show. A key being present means a + // Git candidate is waiting; the value names which state it is waiting in. + const [gitSourcePendingMap, setGitSourcePendingMap] = useState({}); const [isFileLoading, setIsFileLoading] = useState(false); const [backupInfo, setBackupInfo] = useState({ exists: false, timestamp: null }); const [isEditing, setIsEditing] = useState(false); diff --git a/frontend/src/components/EditorLayout/hooks/useNotifications.test.ts b/frontend/src/components/EditorLayout/hooks/useNotifications.test.ts index 4791341f..fc0893e9 100644 --- a/frontend/src/components/EditorLayout/hooks/useNotifications.test.ts +++ b/frontend/src/components/EditorLayout/hooks/useNotifications.test.ts @@ -72,7 +72,7 @@ describe('useNotifications', () => { it('starts with empty notifications and disconnected state', () => { const { result } = renderHook(() => - useNotifications({ nodes: [localNode], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + useNotifications({ nodes: [localNode], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn(), onGitOpsChange: vi.fn() }), ); expect(result.current.notifications).toEqual([]); expect(result.current.tickerConnected).toBe(false); @@ -80,7 +80,7 @@ describe('useNotifications', () => { it('opens a local notification WebSocket on mount', () => { renderHook(() => - useNotifications({ nodes: [localNode], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + useNotifications({ nodes: [localNode], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn(), onGitOpsChange: vi.fn() }), ); expect(MockWS.instances.length).toBeGreaterThanOrEqual(1); expect(MockWS.instances[0]).toBeDefined(); @@ -88,7 +88,7 @@ describe('useNotifications', () => { it('sets tickerConnected true when local WS opens', () => { const { result } = renderHook(() => - useNotifications({ nodes: [localNode], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + useNotifications({ nodes: [localNode], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn(), onGitOpsChange: vi.fn() }), ); act(() => { MockWS.instances[0]?.onopen?.(); }); expect(result.current.tickerConnected).toBe(true); @@ -96,7 +96,7 @@ describe('useNotifications', () => { it('adds notification when local WS receives notification message', () => { const { result } = renderHook(() => - useNotifications({ nodes: [localNode], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + useNotifications({ nodes: [localNode], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn(), onGitOpsChange: vi.fn() }), ); act(() => { MockWS.instances[0]?.onopen?.(); }); act(() => { @@ -110,7 +110,7 @@ describe('useNotifications', () => { it('clearAllNotifications empties the local state', async () => { const { result } = renderHook(() => - useNotifications({ nodes: [localNode], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + useNotifications({ nodes: [localNode], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn(), onGitOpsChange: vi.fn() }), ); act(() => { MockWS.instances[0]?.onopen?.(); }); act(() => { @@ -127,7 +127,7 @@ describe('useNotifications', () => { const onStateInvalidate = vi.fn(); const onImageUpdatesChange = vi.fn(); renderHook(() => - useNotifications({ nodes: [localNode], onStateInvalidate, onImageUpdatesChange }), + useNotifications({ nodes: [localNode], onStateInvalidate, onImageUpdatesChange, onGitOpsChange: vi.fn() }), ); act(() => { MockWS.instances[0]?.onopen?.(); }); act(() => { @@ -145,7 +145,7 @@ describe('useNotifications', () => { it('fires onImageUpdatesChange on update-status-reconciled', () => { const onImageUpdatesChange = vi.fn(); renderHook(() => - useNotifications({ nodes: [localNode], onStateInvalidate: vi.fn(), onImageUpdatesChange }), + useNotifications({ nodes: [localNode], onStateInvalidate: vi.fn(), onImageUpdatesChange, onGitOpsChange: vi.fn() }), ); act(() => { MockWS.instances[0]?.onopen?.(); }); act(() => { @@ -162,7 +162,7 @@ describe('useNotifications', () => { it('ignores unrelated image-updates actions for the refresh callback', () => { const onImageUpdatesChange = vi.fn(); renderHook(() => - useNotifications({ nodes: [localNode], onStateInvalidate: vi.fn(), onImageUpdatesChange }), + useNotifications({ nodes: [localNode], onStateInvalidate: vi.fn(), onImageUpdatesChange, onGitOpsChange: vi.fn() }), ); act(() => { MockWS.instances[0]?.onopen?.(); }); act(() => { @@ -180,7 +180,7 @@ describe('useNotifications', () => { const onStateInvalidate = vi.fn(); const onImageUpdatesChange = vi.fn(); renderHook(() => - useNotifications({ nodes: [localNode], onStateInvalidate, onImageUpdatesChange }), + useNotifications({ nodes: [localNode], onStateInvalidate, onImageUpdatesChange, onGitOpsChange: vi.fn() }), ); act(() => { MockWS.instances[0]?.onopen?.(); }); act(() => { @@ -195,9 +195,113 @@ describe('useNotifications', () => { expect(onImageUpdatesChange).not.toHaveBeenCalled(); }); + it('fires onGitOpsChange for any gitops stage on the local socket', () => { + const onGitOpsChange = vi.fn(); + renderHook(() => + useNotifications({ + nodes: [localNode], + onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn(), onGitOpsChange, + }), + ); + act(() => { MockWS.instances[0]?.onopen?.(); }); + // Two unrelated stages. Unlike image updates there is no action filter, so + // both count: any transition can move the state the surfaces derive. + act(() => { + MockWS.instances[0]?.onmessage?.({ + data: JSON.stringify({ + type: 'state-invalidate', scope: 'gitops', action: 'fetch_started', + applicationId: 'app-1', targetMode: 'direct', stackName: 'foo', + blueprintId: null, nodeId: 1, ts: 1000, + }), + }); + MockWS.instances[0]?.onmessage?.({ + data: JSON.stringify({ + type: 'state-invalidate', scope: 'gitops', action: 'applied', + applicationId: 'app-1', targetMode: 'direct', stackName: 'foo', + blueprintId: null, nodeId: 1, ts: 1001, + }), + }); + }); + expect(onGitOpsChange).toHaveBeenCalledTimes(2); + }); + + it('re-dispatches a gitops invalidate as a window event', () => { + // The dashboard badges refetch off the window event, not off the callback + // below, so narrowing this dispatch into one scope branch would leave them + // permanently stale while every callback assertion stayed green. + const seen: Array<{ scope?: string }> = []; + const onWindow = (e: Event) => seen.push((e as CustomEvent<{ scope?: string }>).detail); + window.addEventListener('sencho:state-invalidate', onWindow); + try { + renderHook(() => + useNotifications({ + nodes: [localNode], + onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn(), onGitOpsChange: vi.fn(), + }), + ); + act(() => { MockWS.instances[0]?.onopen?.(); }); + act(() => { + MockWS.instances[0]?.onmessage?.({ + data: JSON.stringify({ + type: 'state-invalidate', scope: 'gitops', action: 'applied', + applicationId: 'app-1', targetMode: 'direct', stackName: 'foo', + blueprintId: null, nodeId: 1, ts: 1000, + }), + }); + }); + expect(seen.filter((d) => d?.scope === 'gitops')).toHaveLength(1); + } finally { + window.removeEventListener('sencho:state-invalidate', onWindow); + } + }); + + it('does not fire onGitOpsChange for another scope', () => { + const onGitOpsChange = vi.fn(); + renderHook(() => + useNotifications({ + nodes: [localNode], + onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn(), onGitOpsChange, + }), + ); + act(() => { MockWS.instances[0]?.onopen?.(); }); + act(() => { + MockWS.instances[0]?.onmessage?.({ + data: JSON.stringify({ + type: 'state-invalidate', scope: 'stack', nodeId: 1, + stackName: 'foo', action: 'start', ts: 1000, + }), + }); + }); + expect(onGitOpsChange).not.toHaveBeenCalled(); + }); + + it('fires onGitOpsChange for a remote node transition', async () => { + const onGitOpsChange = vi.fn(); + const remote = makeRemoteNode('online', { id: 2 }); + renderHook(() => + useNotifications({ + nodes: [localNode, remote], + onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn(), onGitOpsChange, + }), + ); + await waitFor(() => expect(MockWS.instances).toHaveLength(2)); + act(() => { MockWS.instances[1]?.onopen?.(); }); + act(() => { + MockWS.instances[1]?.onmessage?.({ + data: JSON.stringify({ + type: 'state-invalidate', scope: 'gitops', action: 'deploy_started', + applicationId: 'app-2', targetMode: 'direct', stackName: 'bar', + // The remote's own numbering, which the hub never adopts. + blueprintId: null, nodeId: 7, ts: 1000, + }), + }); + }); + expect(onGitOpsChange).toHaveBeenCalledTimes(1); + }); + it('deleteNotification removes the matching item', async () => { const { result } = renderHook(() => - useNotifications({ nodes: [localNode], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + useNotifications({ nodes: [localNode], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn(), onGitOpsChange: vi.fn() }), ); act(() => { MockWS.instances[0]?.onopen?.(); }); const notif = makeNotif({ id: 5, nodeId: localNode.id }); @@ -212,7 +316,7 @@ describe('useNotifications', () => { it('does not open a WS or poll an offline remote node', async () => { renderHook(() => - useNotifications({ nodes: [localNode, makeRemoteNode('offline')], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + useNotifications({ nodes: [localNode, makeRemoteNode('offline')], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn(), onGitOpsChange: vi.fn() }), ); // Only the local notification socket is created; the offline node is skipped. expect(MockWS.instances).toHaveLength(1); @@ -223,7 +327,7 @@ describe('useNotifications', () => { it('opens a WS and polls an online remote node', async () => { renderHook(() => - useNotifications({ nodes: [localNode, makeRemoteNode('online')], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + useNotifications({ nodes: [localNode, makeRemoteNode('online')], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn(), onGitOpsChange: vi.fn() }), ); // Local socket plus a per-node socket for the online remote. expect(MockWS.instances).toHaveLength(2); @@ -241,7 +345,7 @@ describe('useNotifications', () => { }); const { result } = renderHook(() => - useNotifications({ nodes: [localNode, remote], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + useNotifications({ nodes: [localNode, remote], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn(), onGitOpsChange: vi.fn() }), ); await waitFor(() => expect(result.current.notifications).toHaveLength(1)); @@ -255,7 +359,7 @@ describe('useNotifications', () => { renderHook(() => useNotifications({ nodes: [localNode, makeRemoteNode('online', { id: 2 }), makeRemoteNode('offline', { id: 3, name: 'Dead' })], - onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn(), + onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn(), onGitOpsChange: vi.fn(), }), ); // Local + online remote only; the offline node gets no socket. @@ -266,7 +370,7 @@ describe('useNotifications', () => { it('closes the socket when a subscribed node transitions to offline', () => { const { rerender } = renderHook( - ({ nodes }) => useNotifications({ nodes, onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + ({ nodes }) => useNotifications({ nodes, onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn(), onGitOpsChange: vi.fn() }), { initialProps: { nodes: [localNode, makeRemoteNode('online')] } }, ); // instances[0] is the local socket; instances[1] is the online remote's socket. @@ -281,7 +385,7 @@ describe('useNotifications', () => { it('still subscribes to and polls a remote node with unknown status', async () => { renderHook(() => - useNotifications({ nodes: [localNode, makeRemoteNode('unknown')], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + useNotifications({ nodes: [localNode, makeRemoteNode('unknown')], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn(), onGitOpsChange: vi.fn() }), ); // 'unknown' is not yet probed, so it is treated as reachable (not filtered out) // on both the WS and the REST-poll surfaces. @@ -293,7 +397,7 @@ describe('useNotifications', () => { (apiFetch as ReturnType).mockResolvedValue({ ok: true, json: async () => [] }); const remote = makeRemoteNode('online', { id: 2, name: 'Remote-B' }); const { result } = renderHook(() => - useNotifications({ nodes: [localNode, remote], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + useNotifications({ nodes: [localNode, remote], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn(), onGitOpsChange: vi.fn() }), ); await waitFor(() => expect(apiFetch).toHaveBeenCalled()); @@ -348,7 +452,7 @@ describe('useNotifications', () => { const onStateInvalidate = vi.fn(); const onImageUpdatesChange = vi.fn(); renderHook(() => - useNotifications({ nodes, onStateInvalidate, onImageUpdatesChange }), + useNotifications({ nodes, onStateInvalidate, onImageUpdatesChange, onGitOpsChange: vi.fn() }), ); await waitFor(() => expect(apiFetch).toHaveBeenCalled()); const afterMount = (apiFetch as ReturnType).mock.calls.filter( @@ -387,7 +491,7 @@ describe('useNotifications', () => { }); const { result } = renderHook(() => - useNotifications({ nodes, onStateInvalidate, onImageUpdatesChange }), + useNotifications({ nodes, onStateInvalidate, onImageUpdatesChange, onGitOpsChange: vi.fn() }), ); await waitFor(() => expect(fetchForNode).toHaveBeenCalledWith('/notifications', 2)); const afterMount = (fetchForNode as ReturnType).mock.calls.filter( @@ -418,7 +522,7 @@ describe('useNotifications', () => { (apiFetch as ReturnType).mockResolvedValue({ ok: true, json: async () => [] }); const nodes = [localNode]; renderHook(() => - useNotifications({ nodes, onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + useNotifications({ nodes, onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn(), onGitOpsChange: vi.fn() }), ); await waitFor(() => expect(apiFetch).toHaveBeenCalled()); const afterMount = (apiFetch as ReturnType).mock.calls.filter( @@ -449,7 +553,7 @@ describe('useNotifications', () => { const remote = makeRemoteNode('online', { id: 7, name: 'Remote-7' }); const nodes = [localNode, remote]; const { result } = renderHook(() => - useNotifications({ nodes, onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + useNotifications({ nodes, onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn(), onGitOpsChange: vi.fn() }), ); await waitFor(() => expect(fetchForNode).toHaveBeenCalledWith('/notifications', 7)); @@ -512,7 +616,7 @@ describe('useNotifications', () => { }); const { result } = renderHook(() => - useNotifications({ nodes, onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + useNotifications({ nodes, onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn(), onGitOpsChange: vi.fn() }), ); await waitFor(() => expect(result.current.notifications.some((n) => n.message === 'remote-web')).toBe(true)); expect(nodeMessageKeys(result.current.notifications)).toEqual([ @@ -580,7 +684,7 @@ describe('useNotifications', () => { const onStateInvalidate = vi.fn(); const onImageUpdatesChange = vi.fn(); const { result } = renderHook(() => - useNotifications({ nodes, onStateInvalidate, onImageUpdatesChange }), + useNotifications({ nodes, onStateInvalidate, onImageUpdatesChange, onGitOpsChange: vi.fn() }), ); await waitFor(() => expect(apiFetch).toHaveBeenCalled()); const afterMount = call; diff --git a/frontend/src/components/EditorLayout/hooks/useNotifications.ts b/frontend/src/components/EditorLayout/hooks/useNotifications.ts index ec164cdf..20f30882 100644 --- a/frontend/src/components/EditorLayout/hooks/useNotifications.ts +++ b/frontend/src/components/EditorLayout/hooks/useNotifications.ts @@ -9,6 +9,14 @@ interface UseNotificationsOptions { nodes: Node[]; onStateInvalidate: () => void; onImageUpdatesChange: () => void; + /** + * A GitOps transition committed somewhere. Every stage is worth a refetch, + * so unlike image updates there is no action filter: the surfaces that read + * GitOps state derive it from the projection, and any stage can move it. + * The callback is expected to coalesce, since a single operation commits + * several transitions in a row. + */ + onGitOpsChange: () => void; } /** Local stack-updated and preview-reconcile clears both refresh the update map. */ @@ -16,7 +24,7 @@ function isImageUpdatesRefreshAction(action: unknown): boolean { return action === 'stack-updated' || action === 'update-status-reconciled'; } -export function useNotifications({ nodes, onStateInvalidate, onImageUpdatesChange }: UseNotificationsOptions) { +export function useNotifications({ nodes, onStateInvalidate, onImageUpdatesChange, onGitOpsChange }: UseNotificationsOptions) { const [notifications, setNotifications] = useState([]); const [tickerConnected, setTickerConnected] = useState(false); @@ -29,6 +37,8 @@ export function useNotifications({ nodes, onStateInvalidate, onImageUpdatesChang onStateInvalidateRef.current = onStateInvalidate; const onImageUpdatesChangeRef = useRef(onImageUpdatesChange); onImageUpdatesChangeRef.current = onImageUpdatesChange; + const onGitOpsChangeRef = useRef(onGitOpsChange); + onGitOpsChangeRef.current = onGitOpsChange; // One-shot: the notifications_ready milestone reflects the first local settle. // Its spans instrument only that first fetch so later polls do not pollute the // report. @@ -196,6 +206,8 @@ export function useNotifications({ nodes, onStateInvalidate, onImageUpdatesChang reconcileNotificationsInvalidateRef.current(msg); } else if (msg.scope === 'image-updates' && isImageUpdatesRefreshAction(msg.action)) { onImageUpdatesChangeRef.current(); + } else if (msg.scope === 'gitops') { + onGitOpsChangeRef.current(); } } } catch (e) { @@ -278,6 +290,11 @@ export function useNotifications({ nodes, onStateInvalidate, onImageUpdatesChang reconcileNotificationsInvalidateRef.current({ ...msg, nodeId: rn.id }); } else if (msg.scope === 'image-updates' && isImageUpdatesRefreshAction(msg.action)) { onImageUpdatesChangeRef.current(); + } else if (msg.scope === 'gitops') { + // The payload's own nodeId is the remote's numbering, which is + // meaningless here. The refresh is fleet-wide anyway, so it is + // dropped rather than translated. + onGitOpsChangeRef.current(); } } } catch (e) { diff --git a/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts b/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts index cdafa685..dfd71135 100644 --- a/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts +++ b/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts @@ -19,6 +19,11 @@ vi.mock('@/components/ui/toast-store', () => ({ })); import { apiFetch } from '@/lib/api'; +import { + absentRevision, + missingApplicationLimitation, + sourceRevision, +} from '@/__tests__/gitopsFixtures'; import { toast } from '@/components/ui/toast-store'; type EditorState = ReturnType; @@ -2406,3 +2411,86 @@ describe('useStackActions reactive external-network retry ownership', () => { }); }); +describe('useStackActions.refreshGitSourcePending', () => { + beforeEach(() => { + vi.mocked(apiFetch).mockReset(); + }); + + function gitSourceRow(stackName: string, revision: unknown, pendingSha: string | null = null) { + return { stack_name: stackName, pending_commit_sha: pendingSha, gitopsRevision: revision }; + } + + it('records the derived state of each waiting candidate', async () => { + vi.mocked(apiFetch).mockResolvedValue(okJson([ + gitSourceRow('web', sourceRevision('candidate_ready')), + gitSourceRow('api', sourceRevision('source_conflict_blocker')), + ])); + const { result, editorState } = setup(); + await result.current.refreshGitSourcePending(); + expect(editorState.setGitSourcePendingMap).toHaveBeenCalledWith({ + web: 'candidate_ready', + api: 'source_conflict_blocker', + }); + }); + + it('skips a stack whose projection has no candidate waiting', async () => { + // The raw pointer is set, but the model says the candidate is gone. The + // model wins: this is the conflation the derived read exists to remove. + vi.mocked(apiFetch).mockResolvedValue(okJson([ + gitSourceRow('web', sourceRevision('source_reconcile_required', { candidateGenerationId: null }), 'a1b2c3d'), + ])); + const { result, editorState } = setup(); + await result.current.refreshGitSourcePending(); + expect(editorState.setGitSourcePendingMap).toHaveBeenCalledWith({}); + }); + + it('falls back to the raw pointer only when there is no projection to read', async () => { + vi.mocked(apiFetch).mockResolvedValue(okJson([ + gitSourceRow('web', absentRevision(), 'a1b2c3d'), + gitSourceRow('api', absentRevision(), null), + ])); + const { result, editorState } = setup(); + await result.current.refreshGitSourcePending(); + expect(editorState.setGitSourcePendingMap).toHaveBeenCalledWith({ web: 'candidate_ready' }); + }); + + it('keeps reading the rest of the list when a row predates the revision model', async () => { + // /git-sources is proxied, so an older node answers rows with no + // projection at all. Throwing on one row would abandon the whole map. + vi.mocked(apiFetch).mockResolvedValue(okJson([ + { stack_name: 'legacy', pending_commit_sha: 'a1b2c3d' }, + gitSourceRow('web', sourceRevision('candidate_ready')), + ])); + const { result, editorState } = setup(); + await result.current.refreshGitSourcePending(); + expect(editorState.setGitSourcePendingMap).toHaveBeenCalledWith({ + legacy: 'candidate_ready', + web: 'candidate_ready', + }); + }); + + it('does not fabricate a ready candidate when the projection reports a fault', async () => { + // A fault means an application was expected and could not be read, so the + // flat pointer is not evidence that anything is ready to apply. + vi.mocked(apiFetch).mockResolvedValue(okJson([ + gitSourceRow('web', absentRevision([missingApplicationLimitation]), 'a1b2c3d'), + ])); + const { result, editorState } = setup(); + await result.current.refreshGitSourcePending(); + expect(editorState.setGitSourcePendingMap).toHaveBeenCalledWith({}); + }); + + it('leaves the prior map alone when the request fails', async () => { + vi.mocked(apiFetch).mockResolvedValue(new Response('boom', { status: 500 })); + const { result, editorState } = setup(); + await result.current.refreshGitSourcePending(); + expect(editorState.setGitSourcePendingMap).not.toHaveBeenCalled(); + }); + + it('leaves the prior map alone when the request throws', async () => { + vi.mocked(apiFetch).mockRejectedValue(new Error('offline')); + const { result, editorState } = setup(); + await result.current.refreshGitSourcePending(); + expect(editorState.setGitSourcePendingMap).not.toHaveBeenCalled(); + }); +}); diff --git a/frontend/src/components/EditorLayout/hooks/useStackActions.ts b/frontend/src/components/EditorLayout/hooks/useStackActions.ts index daed68e4..7fb49946 100644 --- a/frontend/src/components/EditorLayout/hooks/useStackActions.ts +++ b/frontend/src/components/EditorLayout/hooks/useStackActions.ts @@ -14,6 +14,8 @@ import { toast } from '@/components/ui/toast-store'; import { buildServiceUrl, openServiceUrl } from '@/lib/serviceUrl'; import { requestServiceUpdate as postServiceUpdate, requestServiceRestore as postServiceRestore } from '@/lib/serviceUpdate'; import type { EffectiveServiceModelResult } from '@/types/effectiveServices'; +import { absentFault, pendingSourceStatus, type GitSourcePendingMap } from '@/lib/gitopsState'; +import type { GitOpsRevisionCarrier } from '@/types/gitops'; import type { useEditorViewState } from './useEditorViewState'; import type { useStackListState } from './useStackListState'; import type { useViewNavigationState } from './useViewNavigationState'; @@ -827,11 +829,34 @@ export function useStackActions(options: UseStackActionsOptions) { try { const res = await apiFetch('/git-sources'); if (!res.ok) return; - const sources: Array<{ stack_name: string; pending_commit_sha: string | null }> = - await res.json(); - const map: Record = {}; + // The revision is optional because this route is proxied: a node that + // predates the revision model answers rows without one. + const sources: Array< + { stack_name: string; pending_commit_sha: string | null } & Partial + > = await res.json(); + const map: GitSourcePendingMap = {}; for (const s of sources) { - if (s.pending_commit_sha) map[s.stack_name] = true; + const revision = s.gitopsRevision; + const status = revision ? pendingSourceStatus(revision) : null; + if (status) { + map[s.stack_name] = status; + continue; + } + // Nothing answered. Either the row predates the model, or a GitOps write + // failed and was swallowed while the pending commit still committed. The + // flat pointer is the only thing left that can answer, and going quiet on + // a stack that genuinely has an update waiting would be a regression. + // + // A projection that reports a fault is excluded: it means an application + // was expected and could not be read, so the pointer is not evidence that + // a candidate is ready, and naming a state here would be a guess. The + // panels surface that fault properly; this indicator only ever claims + // that an update is waiting. + const unanswered = !revision + || (revision.targetMode === 'not_applicable' && absentFault(revision).length === 0); + if (unanswered && s.pending_commit_sha) { + map[s.stack_name] = 'candidate_ready'; + } } editorState.setGitSourcePendingMap(map); } catch { diff --git a/frontend/src/components/HomeDashboard.tsx b/frontend/src/components/HomeDashboard.tsx index 6e716a29..8c9cc95d 100644 --- a/frontend/src/components/HomeDashboard.tsx +++ b/frontend/src/components/HomeDashboard.tsx @@ -11,6 +11,7 @@ import { useDashboardData, } from './dashboard'; import { DashboardActivityCard } from './dashboard/DashboardActivityCard'; +import { useGitOpsSourceStates } from './dashboard/useGitOpsSourceStates'; interface HomeDashboardProps { onNavigateToStack?: (stackFile: string) => void; @@ -25,6 +26,7 @@ const NOOP = () => {}; export default function HomeDashboard({ onNavigateToStack, onOpenSettingsSection, notifications, onClearNotifications, stackUpdates = {} }: HomeDashboardProps) { const { activeNode, nodes } = useNodes(); const data = useDashboardData(); + const gitopsSourceStates = useGitOpsSourceStates(); const activeNodeName = activeNode?.name || 'Local'; return ( @@ -55,6 +57,7 @@ export default function HomeDashboard({ onNavigateToStack, onOpenSettingsSection stackCpuSeries={data.stackCpuSeries} onNavigateToStack={onNavigateToStack ?? NOOP} stackUpdates={stackUpdates} + gitopsSourceStates={gitopsSourceStates} />
diff --git a/frontend/src/components/StackAnatomyPanel.doctor.test.tsx b/frontend/src/components/StackAnatomyPanel.doctor.test.tsx index c04219ed..446c9ae7 100644 --- a/frontend/src/components/StackAnatomyPanel.doctor.test.tsx +++ b/frontend/src/components/StackAnatomyPanel.doctor.test.tsx @@ -38,7 +38,7 @@ function panel() { content={'services:\n web:\n image: nginx:1.25\n'} envContent="" selectedEnvFile=".env" - gitSourcePending={false} + gitSourcePending={null} onEditCompose={vi.fn()} onOpenGitSource={vi.fn()} onApplyUpdate={vi.fn()} diff --git a/frontend/src/components/StackAnatomyPanel.test.tsx b/frontend/src/components/StackAnatomyPanel.test.tsx index 0024a857..7d13b882 100644 --- a/frontend/src/components/StackAnatomyPanel.test.tsx +++ b/frontend/src/components/StackAnatomyPanel.test.tsx @@ -20,6 +20,8 @@ vi.mock('@/context/NodeContext', () => ({ useNodes: () => ({ activeNode: nodeSta import { apiFetch } from '@/lib/api'; import StackAnatomyPanel from './StackAnatomyPanel'; +import { SOURCE_STATE } from '@/lib/gitopsState'; +import type { GitOpsSourceStatus } from '@/types/gitops'; const COMPOSE = 'services:\n web:\n image: nginx:1.25\n'; @@ -134,7 +136,7 @@ function panel(applying: boolean, onApplyUpdate: () => void = vi.fn(), stackName content={COMPOSE} envContent="" selectedEnvFile=".env" - gitSourcePending={false} + gitSourcePending={null} onEditCompose={vi.fn()} onOpenGitSource={vi.fn()} onApplyUpdate={onApplyUpdate} @@ -158,7 +160,7 @@ describe('StackAnatomyPanel edit affordance', () => { content={COMPOSE} envContent="" selectedEnvFile=".env" - gitSourcePending={false} + gitSourcePending={null} onEditCompose={vi.fn()} onOpenGitSource={vi.fn()} onApplyUpdate={vi.fn()} @@ -170,6 +172,44 @@ describe('StackAnatomyPanel edit affordance', () => { }); }); +describe('StackAnatomyPanel git source state', () => { + function withPending(gitSourcePending: GitOpsSourceStatus | null) { + return ( + + ); + } + + it('shows only the dot for an ordinary waiting update', () => { + // The dot already means "an update is waiting", so naming that state would + // be saying the same thing twice, and it is the common case. + const { container } = render(withPending('candidate_ready')); + expect(container.querySelector('.animate-pulse')).not.toBeNull(); + expect(screen.queryByText(SOURCE_STATE.candidate_ready.label)).not.toBeInTheDocument(); + }); + + it('names a state the dot cannot express, without losing the dot', () => { + const { container } = render(withPending('source_conflict_blocker')); + expect(screen.getByText(SOURCE_STATE.source_conflict_blocker.label)).toBeInTheDocument(); + expect(container.querySelector('.animate-pulse')).not.toBeNull(); + }); + + it('shows neither when nothing is waiting', () => { + const { container } = render(withPending(null)); + expect(container.querySelector('.animate-pulse')).toBeNull(); + }); +}); + describe('StackAnatomyPanel update banner', () => { it('hides apply when only a newer tag is available', async () => { vi.mocked(apiFetch).mockImplementation(async (input: RequestInfo | URL) => { @@ -482,7 +522,7 @@ describe('StackAnatomyPanel exposed footer', () => { content={content} envContent="" selectedEnvFile=".env" - gitSourcePending={false} + gitSourcePending={null} onEditCompose={vi.fn()} onOpenGitSource={vi.fn()} onApplyUpdate={vi.fn()} @@ -524,7 +564,7 @@ describe('StackAnatomyPanel effective dossier (multi-file Git)', () => { content={content} envContent="" selectedEnvFile=".env" - gitSourcePending={false} + gitSourcePending={null} onEditCompose={vi.fn()} onOpenGitSource={vi.fn()} onApplyUpdate={vi.fn()} diff --git a/frontend/src/components/StackAnatomyPanel.tsx b/frontend/src/components/StackAnatomyPanel.tsx index e51fb7c4..6f6c7983 100644 --- a/frontend/src/components/StackAnatomyPanel.tsx +++ b/frontend/src/components/StackAnatomyPanel.tsx @@ -17,6 +17,8 @@ import { type AnatomyMarkdownInput, type PortRow, type VolumeRow } from '@/lib/a import { usePreflightDismiss } from '@/hooks/usePreflightDismiss'; import { useScanBannerDismiss } from '@/hooks/useScanBannerDismiss'; import { parseAnatomy, parseEnvKeys, formatGitSource, imageName, primaryPublishedHostPort, type GitSourceInfo } from '@/lib/anatomy'; +import { SOURCE_STATE_LOOKUP } from '@/lib/gitopsState'; +import type { GitOpsSourceStatus } from '@/types/gitops'; import { buildServiceUrl } from '@/lib/serviceUrl'; import { StackActivityTimeline } from './stack/StackActivityTimeline'; import StackDossierPanel from './stack/StackDossierPanel'; @@ -35,7 +37,8 @@ interface StackAnatomyPanelProps { content: string; envContent: string; selectedEnvFile: string; - gitSourcePending: boolean; + /** The source state of a waiting Git candidate, or null when none is waiting. */ + gitSourcePending: GitOpsSourceStatus | null; onEditCompose: () => void; onOpenGitSource: () => void; onApplyUpdate: () => void; @@ -610,7 +613,17 @@ export default function StackAnatomyPanel({ local )} {gitSourcePending && ( - + <> + + {/* The dot already says "an update is waiting". Name the state + only when it is something else: blocked, held for review, + stale against the configuration, or mid-apply. */} + {gitSourcePending !== 'candidate_ready' && ( + + {SOURCE_STATE_LOOKUP[gitSourcePending]?.label ?? gitSourcePending} + + )} + )} diff --git a/frontend/src/components/blueprints/BlueprintDetail.test.tsx b/frontend/src/components/blueprints/BlueprintDetail.test.tsx index 942983b6..3da88fc1 100644 --- a/frontend/src/components/blueprints/BlueprintDetail.test.tsx +++ b/frontend/src/components/blueprints/BlueprintDetail.test.tsx @@ -31,8 +31,9 @@ vi.mock('./RolloutPreviewDialog', () => ({ import { getBlueprint } from '@/lib/blueprintsApi'; import { BlueprintDetail } from './BlueprintDetail'; +import { absentRevision, missingApplicationLimitation } from '@/__tests__/gitopsFixtures'; -function summary(): BlueprintSummary { +function summary(overrides: Partial = {}): BlueprintSummary { return { blueprint: { id: 1, @@ -53,6 +54,8 @@ function summary(): BlueprintSummary { deployments: [], statusCounts: {}, effectiveApproval: 'pending', + gitopsRevision: absentRevision(), + ...overrides, }; } @@ -168,3 +171,33 @@ describe('BlueprintDetail action gating', () => { expect(screen.queryByRole('button', { name: /^delete$/i })).not.toBeInTheDocument(); }); }); + +describe('BlueprintDetail GitOps state', () => { + function detail() { + return ( + + ); + } + + it('reports a Blueprint whose application row could not be reached', async () => { + vi.mocked(getBlueprint).mockResolvedValue( + summary({ gitopsRevision: absentRevision([missingApplicationLimitation]) }), + ); + render(detail()); + expect(await screen.findByTestId('gitops-fault')).toHaveTextContent(missingApplicationLimitation.message); + }); + + it('stays silent when there is simply nothing to project', async () => { + vi.mocked(getBlueprint).mockResolvedValue(summary()); + render(detail()); + await screen.findByText('Show compose source'); + expect(screen.queryByTestId('gitops-fault')).not.toBeInTheDocument(); + }); +}); diff --git a/frontend/src/components/blueprints/BlueprintDetail.tsx b/frontend/src/components/blueprints/BlueprintDetail.tsx index 22d58117..5a7950af 100644 --- a/frontend/src/components/blueprints/BlueprintDetail.tsx +++ b/frontend/src/components/blueprints/BlueprintDetail.tsx @@ -1,6 +1,9 @@ import { useEffect, useRef, useState, useCallback } from 'react'; import { Pencil, Pin, Play, Power, Trash2 } from 'lucide-react'; import { SystemSheet, SheetSection } from '@/components/ui/system-sheet'; +import { GitOpsFaultCard } from '@/components/gitops/GitOpsStateCard'; +import GitOpsCaveats from '@/components/gitops/GitOpsCaveats'; +import { absentFault, liveCaveats } from '@/lib/gitopsState'; import { Modal, ModalDestructiveHeader, ModalBody, ModalFooter } from '@/components/ui/modal'; import { Input } from '@/components/ui/input'; import { Button } from '@/components/ui/button'; @@ -82,6 +85,14 @@ export function BlueprintDetail({ blueprintId, open, onOpenChange, onChanged, ca if (!open) return null; const blueprint = summary?.blueprint; + // A Blueprint with no live application row is a fact nothing else in the + // product can express. Everything else a Blueprint projection carries is + // rollout surface, which is not this sheet's job. + const gitopsFaults = summary ? absentFault(summary.gitopsRevision) : []; + // Caveats are the exception to the note above: an approval that no longer + // covers what the Blueprint asks for is a fact about this Blueprint, not + // about a rollout, and nothing else on the sheet says it. + const gitopsCaveats = summary ? liveCaveats(summary.gitopsRevision) : []; const canApply = !!blueprint && (can ? can('stack:create') && can('stack:deploy') : canEdit); const canDeleteBlueprint = !!blueprint && (can ? can('stack:delete') : canEdit); const canDeployOnNode = (nodeId: number) => !!blueprint @@ -300,6 +311,13 @@ export function BlueprintDetail({ blueprintId, open, onOpenChange, onChanged, ca )} + {(gitopsFaults.length > 0 || gitopsCaveats.length > 0) && ( + + {gitopsFaults.length > 0 && } + {summary && } + + )} + ({ + addNodeLabel: vi.fn(), + removeNodeLabel: vi.fn(), + getLabelsForNode: vi.fn(), + listDistinctLabels: vi.fn(), +})); +vi.mock('@/components/ui/toast-store', () => ({ + toast: { success: vi.fn(), error: vi.fn() }, +})); + +import { addNodeLabel, getLabelsForNode, listDistinctLabels } from '@/lib/blueprintsApi'; +import { toast } from '@/components/ui/toast-store'; +import { NodeLabelPicker } from './NodeLabelPicker'; +import { absentRevision } from '@/__tests__/gitopsFixtures'; + +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(getLabelsForNode).mockResolvedValue([]); + vi.mocked(listDistinctLabels).mockResolvedValue([]); +}); + +async function addLabel(gitopsRevisions: ReturnType[]) { + vi.mocked(addNodeLabel).mockResolvedValue({ nodeId: 1, label: 'prod', gitopsRevisions }); + render(); + fireEvent.click(await screen.findByLabelText('Add label')); + fireEvent.change(await screen.findByPlaceholderText('prod'), { target: { value: 'prod' } }); + fireEvent.click(screen.getByRole('button', { name: 'Add' })); + await waitFor(() => expect(addNodeLabel).toHaveBeenCalledWith(1, 'prod')); +} + +describe('NodeLabelPicker', () => { + it('reports how many blueprints the new label re-placed', async () => { + await addLabel([absentRevision(), absentRevision()]); + await waitFor(() => + expect(toast.success).toHaveBeenCalledWith('Label added. 2 blueprints re-placed.'), + ); + }); + + it('uses the singular for one', async () => { + await addLabel([absentRevision()]); + await waitFor(() => + expect(toast.success).toHaveBeenCalledWith('Label added. 1 blueprint re-placed.'), + ); + }); + + it('says nothing when the list is empty', async () => { + // Empty means both "nothing moved" and "the projection faulted after the + // write committed", so it can never be reported as the former. + await addLabel([]); + await waitFor(() => expect(getLabelsForNode).toHaveBeenCalledTimes(2)); + expect(toast.success).not.toHaveBeenCalled(); + }); +}); diff --git a/frontend/src/components/blueprints/NodeLabelPicker.tsx b/frontend/src/components/blueprints/NodeLabelPicker.tsx index 390b6eff..ed5fe529 100644 --- a/frontend/src/components/blueprints/NodeLabelPicker.tsx +++ b/frontend/src/components/blueprints/NodeLabelPicker.tsx @@ -55,7 +55,15 @@ export function NodeLabelPicker({ nodeId, canEdit = true, onChange }: NodeLabelP if (!trimmed) return; setBusy(true); try { - await addNodeLabel(nodeId, trimmed); + const result = await addNodeLabel(nodeId, trimmed); + // A label add is exactly what makes a selector match a new node, so + // this is the common case for a re-placement. Count only, and only + // when non-zero: an empty list also means the projection faulted + // after the write committed. + const moved = result.gitopsRevisions.length; + if (moved > 0) { + toast.success(`Label added. ${moved} blueprint${moved === 1 ? '' : 's'} re-placed.`); + } setInput(''); await refresh(); } catch (err) { diff --git a/frontend/src/components/dashboard/StackHealthTable.tsx b/frontend/src/components/dashboard/StackHealthTable.tsx index d4723691..b9ff648b 100644 --- a/frontend/src/components/dashboard/StackHealthTable.tsx +++ b/frontend/src/components/dashboard/StackHealthTable.tsx @@ -10,6 +10,8 @@ import { isConfirmedImageUpdate, isConfirmedServiceUpdate } from '@/types/imageU import { aggregateCurrentUsage } from './aggregateCurrentUsage'; import { classifyRow, type RowState } from './classifyRow'; import { updateAvailableLabel } from '@/lib/updateAvailableLabel'; +import GitOpsBadge from '@/components/gitops/GitOpsBadge'; +import type { GitOpsSourceStateMap } from './useGitOpsSourceStates'; interface StackHealthTableProps { stackStatuses: Record; @@ -20,6 +22,11 @@ interface StackHealthTableProps { stackCpuSeries: Record; onNavigateToStack: (stackFile: string) => void; stackUpdates?: Record; + /** + * GitOps source state per stack name. A stack the model says nothing about + * is absent, and its SOURCE cell keeps the plain Git or Local label. + */ + gitopsSourceStates?: GitOpsSourceStateMap; } type SortKey = 'stack' | 'up' | 'cpu' | 'mem'; @@ -96,6 +103,7 @@ export function StackHealthTable({ stackCpuSeries, onNavigateToStack, stackUpdates = {}, + gitopsSourceStates = {}, }: StackHealthTableProps) { const [page, setPage] = useState(0); // null = the default health-state ordering (worst first); a SortKey switches @@ -273,6 +281,10 @@ export function StackHealthTable({
    {pagedRows.map((row) => { const updateLabel = row.hasUpdate ? updateAvailableLabel(row.outdatedServices) : null; + // Looked up here rather than folded into the memoized rows: it is + // presentation, and threading it through would make the row memo + // recompute on every parent render for no benefit. + const gitopsSourceState = gitopsSourceStates[row.name]; return (
  • )} + {gitopsSourceState && ( + + )} {row.source === 'git' ? 'Git' : 'Local'} diff --git a/frontend/src/components/dashboard/__tests__/StackHealthTable.gitopsBadge.test.tsx b/frontend/src/components/dashboard/__tests__/StackHealthTable.gitopsBadge.test.tsx new file mode 100644 index 00000000..aa8b1a29 --- /dev/null +++ b/frontend/src/components/dashboard/__tests__/StackHealthTable.gitopsBadge.test.tsx @@ -0,0 +1,83 @@ +import { describe, it, expect, vi } from 'vitest'; +import { render, screen } from '@testing-library/react'; +import { StackHealthTable } from '../StackHealthTable'; +import type { GitOpsSourceStateMap } from '../useGitOpsSourceStates'; +import type { StackStatusEntry } from '../types'; +import type { GitOpsSourceStatus } from '@/types/gitops'; + +const stackStatuses: Record = { + 'app.yml': { status: 'running', source: 'git' }, + 'plain.yml': { status: 'running', source: 'local' }, +}; + +function renderTable(gitopsSourceStates?: GitOpsSourceStateMap) { + return render( + , + ); +} + +describe('StackHealthTable GitOps badge', () => { + it('badges only the stacks the model has state for', () => { + renderTable({ app: 'candidate_ready' }); + + const badges = screen.getAllByTestId('gitops-badge'); + expect(badges).toHaveLength(1); + expect(badges[0]).toHaveAttribute('data-state', 'candidate_ready'); + }); + + it('states the condition in words, not only in colour', () => { + renderTable({ app: 'source_conflict_blocker' }); + + const badge = screen.getByTestId('gitops-badge'); + // Asserted against the visible node and by equality, not containment: + // `toHaveTextContent` also matches sr-only text, and "pending update" is a + // prefix of this label, so a substring match could not tell a blocked plan + // apart from an ordinary one. + const visible = badge.querySelector(':scope > span:not(.sr-only)'); + expect(visible?.textContent).toBe('pending update blocked'); + // The title carries the whole sentence, so the state survives a reader who + // cannot see the tone. + expect(badge).toHaveAttribute( + 'title', + 'The change plan has local conflicts. Apply stays disabled until they are resolved.', + ); + }); + + it('renders nothing for a status this build does not know', () => { + // The value crosses a proxy from a node that may run a newer vocabulary. + // The map is closed at compile time, which says nothing about the wire, so + // an unmapped key must render nothing rather than dereference undefined + // inside a row and take the whole table down with it. + renderTable({ app: 'a_status_from_a_newer_build' as GitOpsSourceStatus }); + expect(screen.queryByTestId('gitops-badge')).toBeNull(); + expect(screen.getByText('app')).toBeInTheDocument(); + }); + + it('renders no badge when the join is empty', () => { + renderTable({}); + expect(screen.queryByTestId('gitops-badge')).toBeNull(); + }); + + it('renders no badge when the caller passes nothing at all', () => { + // The prop is optional so an older caller, or one on a surface with no + // GitOps join, keeps rendering exactly as before. + renderTable(undefined); + expect(screen.queryByTestId('gitops-badge')).toBeNull(); + }); + + it('keeps the source column reading Git or Local either way', () => { + // The badge says what GitOps thinks; the column still says where the files + // come from. Replacing one with the other would lose a fact. + renderTable({ app: 'candidate_ready' }); + expect(screen.getByText('Git')).toBeInTheDocument(); + expect(screen.getByText('Local')).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/components/dashboard/useGitOpsSourceStates.test.ts b/frontend/src/components/dashboard/useGitOpsSourceStates.test.ts new file mode 100644 index 00000000..4e9408e4 --- /dev/null +++ b/frontend/src/components/dashboard/useGitOpsSourceStates.test.ts @@ -0,0 +1,181 @@ +import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest'; +import { act, renderHook, waitFor } from '@testing-library/react'; +import { absentRevision, missingApplicationLimitation, sourceRevision } from '@/__tests__/gitopsFixtures'; +import { useGitOpsSourceStates } from './useGitOpsSourceStates'; + +const apiFetch = vi.hoisted(() => vi.fn()); +const activeNode = vi.hoisted(() => ({ current: { id: 1, name: 'Local' } as { id: number; name: string } | null })); + +vi.mock('@/lib/api', () => ({ apiFetch })); +vi.mock('@/context/NodeContext', () => ({ useNodes: () => ({ activeNode: activeNode.current }) })); + +const ok = (rows: unknown) => ({ ok: true, status: 200, json: async () => rows }); + +/** Fire the invalidate the publisher's event turns into on the client. */ +const announceGitOps = () => { + window.dispatchEvent(new CustomEvent('sencho:state-invalidate', { detail: { scope: 'gitops' } })); +}; + +describe('useGitOpsSourceStates', () => { + beforeEach(() => { + vi.useFakeTimers({ shouldAdvanceTime: true }); + apiFetch.mockReset(); + activeNode.current = { id: 1, name: 'Local' }; + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('maps each row with a live source facet by stack name', async () => { + apiFetch.mockResolvedValue(ok([ + { stack_name: 'bookstack', gitopsRevision: sourceRevision('candidate_ready') }, + { stack_name: 'wiki', gitopsRevision: sourceRevision('source_review_pending') }, + ])); + + const { result } = renderHook(() => useGitOpsSourceStates()); + + await waitFor(() => expect(result.current).toEqual({ + bookstack: 'candidate_ready', + wiki: 'source_review_pending', + })); + }); + + it('omits a row from a node that predates the revision model', async () => { + // /git-sources is proxied, so a row without the field is an ordinary + // answer from an older node, not an error. + apiFetch.mockResolvedValue(ok([ + { stack_name: 'legacy' }, + { stack_name: 'bookstack', gitopsRevision: sourceRevision('candidate_ready') }, + ])); + + const { result } = renderHook(() => useGitOpsSourceStates()); + + await waitFor(() => expect(result.current).toEqual({ bookstack: 'candidate_ready' })); + }); + + it('omits a projection that could not be read', async () => { + // A fault means an application was expected and could not be reached. + // Naming a source state for it would be a guess; the panels report the + // fault properly and this badge stays away. + apiFetch.mockResolvedValue(ok([ + { stack_name: 'broken', gitopsRevision: absentRevision([missingApplicationLimitation]) }, + ])); + + const { result } = renderHook(() => useGitOpsSourceStates()); + + await waitFor(() => expect(apiFetch).toHaveBeenCalled()); + expect(result.current).toEqual({}); + }); + + it('refetches on a gitops announcement, coalescing a burst into one call', async () => { + apiFetch.mockResolvedValue(ok([ + { stack_name: 'bookstack', gitopsRevision: sourceRevision('candidate_ready') }, + ])); + const { result } = renderHook(() => useGitOpsSourceStates()); + await waitFor(() => expect(apiFetch).toHaveBeenCalledTimes(1)); + + // `candidateGenerationId: null` because the deriver returns from the + // candidate branch before it can reach an accepted status, so the pair + // without it is a state no backend can emit. + apiFetch.mockResolvedValue(ok([ + { + stack_name: 'bookstack', + gitopsRevision: sourceRevision('application_generation_accepted', { candidateGenerationId: null }), + }, + ])); + act(() => { + announceGitOps(); + announceGitOps(); + announceGitOps(); + }); + // One operation commits several transitions in a row, so three events must + // not become three fetches. + expect(apiFetch).toHaveBeenCalledTimes(1); + await act(async () => { await vi.advanceTimersByTimeAsync(300); }); + expect(apiFetch).toHaveBeenCalledTimes(2); + // The refetch's answer has to land, not just be requested. + await waitFor(() => expect(result.current).toEqual({ bookstack: 'application_generation_accepted' })); + }); + + it('blanks the map on a node switch before the new node answers', async () => { + apiFetch.mockResolvedValue(ok([ + { stack_name: 'bookstack', gitopsRevision: sourceRevision('candidate_ready') }, + ])); + const { result, rerender } = renderHook(() => useGitOpsSourceStates()); + await waitFor(() => expect(result.current).toEqual({ bookstack: 'candidate_ready' })); + + // Never resolves, so the only thing that can clear the old node's map is + // the blanking itself. + apiFetch.mockImplementation(() => new Promise(() => {})); + activeNode.current = { id: 2, name: 'Remote' }; + rerender(); + + await waitFor(() => expect(result.current).toEqual({})); + }); + + it('skips a row it cannot read and keeps the rest', async () => { + // A proxied node is free to answer with a shape this build does not + // assume. One bad row must not abandon the loop and freeze every badge. + apiFetch.mockResolvedValue(ok([ + { stack_name: 'broken', gitopsRevision: { targetMode: 'direct' } }, + { stack_name: 'bookstack', gitopsRevision: sourceRevision('candidate_ready') }, + ])); + + const { result } = renderHook(() => useGitOpsSourceStates()); + + await waitFor(() => expect(result.current).toEqual({ bookstack: 'candidate_ready' })); + }); + + it('ignores an announcement from another scope', async () => { + apiFetch.mockResolvedValue(ok([])); + renderHook(() => useGitOpsSourceStates()); + await waitFor(() => expect(apiFetch).toHaveBeenCalledTimes(1)); + + act(() => { + window.dispatchEvent(new CustomEvent('sencho:state-invalidate', { detail: { scope: 'image-updates' } })); + }); + await act(async () => { await vi.advanceTimersByTimeAsync(300); }); + + expect(apiFetch).toHaveBeenCalledTimes(1); + }); + + it('keeps the previous map when a refetch fails', async () => { + apiFetch.mockResolvedValue(ok([ + { stack_name: 'bookstack', gitopsRevision: sourceRevision('candidate_ready') }, + ])); + const { result } = renderHook(() => useGitOpsSourceStates()); + await waitFor(() => expect(result.current).toEqual({ bookstack: 'candidate_ready' })); + + apiFetch.mockRejectedValue(new Error('offline')); + act(() => { announceGitOps(); }); + await act(async () => { await vi.advanceTimersByTimeAsync(300); }); + + // Blanking every badge on one failed poll would report "no GitOps here" + // for stacks GitOps is managing. + expect(result.current).toEqual({ bookstack: 'candidate_ready' }); + }); + + it('drops a slow answer for the node the operator has left', async () => { + let releaseFirst: (() => void) | null = null; + apiFetch.mockImplementationOnce(async () => { + await new Promise((resolve) => { releaseFirst = resolve; }); + return ok([{ stack_name: 'bookstack', gitopsRevision: sourceRevision('candidate_ready') }]); + }); + apiFetch.mockResolvedValue(ok([ + { stack_name: 'other', gitopsRevision: sourceRevision('source_review_pending') }, + ])); + + const { result, rerender } = renderHook(() => useGitOpsSourceStates()); + await waitFor(() => expect(apiFetch).toHaveBeenCalledTimes(1)); + + activeNode.current = { id: 2, name: 'Remote' }; + rerender(); + await waitFor(() => expect(result.current).toEqual({ other: 'source_review_pending' })); + + // Stack names repeat across nodes, so node one's answer landing now would + // label node two's list with node one's state. + await act(async () => { releaseFirst?.(); await Promise.resolve(); }); + expect(result.current).toEqual({ other: 'source_review_pending' }); + }); +}); diff --git a/frontend/src/components/dashboard/useGitOpsSourceStates.ts b/frontend/src/components/dashboard/useGitOpsSourceStates.ts new file mode 100644 index 00000000..f7db4043 --- /dev/null +++ b/frontend/src/components/dashboard/useGitOpsSourceStates.ts @@ -0,0 +1,128 @@ +import { useCallback, useEffect, useRef, useState } from 'react'; +import { useNodes } from '@/context/NodeContext'; +import { apiFetch } from '@/lib/api'; +import { liveSourceFacet } from '@/lib/gitopsState'; +import type { GitOpsRevisionCarrier, GitOpsSourceStatus } from '@/types/gitops'; + +/** Trailing-edge window for a burst of GitOps transitions, as elsewhere on this card. */ +const INVALIDATE_DEBOUNCE_MS = 250; + +/** Stack name to the source state of its GitOps application. A miss is a stack with none. */ +export type GitOpsSourceStateMap = Record; + +/** + * A Git source row as this hook needs it. + * + * `gitopsRevision` is optional because `/git-sources` is proxied and a node + * that predates the revision model answers rows without one. Typing it as + * required is what once made a whole map freeze behind a swallowed throw. + */ +type GitSourceRow = { stack_name: string } & Partial; + +/** + * Rows to states, keeping a row this build cannot read out of the map. + * + * Separate from the fetch so the derivation is one readable pass and the + * request handling is another. + */ +function projectSourceStates(rows: GitSourceRow[]): GitOpsSourceStateMap { + const next: GitOpsSourceStateMap = {}; + let unreadable = 0; + for (const row of rows) { + // Per row, because the derivation reaches into a shape this build assumes + // and a proxied node is free to answer with another one. An uncaught throw + // here would abandon the whole loop and land in the caller's catch, + // freezing every badge on the dashboard at its last value with nothing on + // screen to say so: the same swallowed-throw failure that once froze the + // sidebar's pending map. + try { + const source = row.gitopsRevision ? liveSourceFacet(row.gitopsRevision) : null; + if (source) next[row.stack_name] = source.status; + } catch { + unreadable += 1; + } + } + if (unreadable > 0) { + console.error(`[GitOps] ${unreadable} source row(s) could not be read; those stacks show no state.`); + } + return next; +} + +/** + * Source state per stack, for the surfaces that list many stacks at once. + * + * Keyed on stack name because that is what the dashboards have: they are built + * from container status, which knows nothing about application ids. + * + * A stack is absent from the map unless the model has something to say about + * it. A row without a revision, a Blueprint-owned application (whose source + * facet is `not_applicable`, since naming a Git state there would be a claim + * the model never made), and a projection fault all resolve to absent, so the + * badge simply does not render rather than showing a state nobody derived. + */ +export function useGitOpsSourceStates(): GitOpsSourceStateMap { + const { activeNode } = useNodes(); + const nodeId = activeNode?.id; + + const [states, setStates] = useState({}); + /** + * Bumped by anything that invalidates an in-flight answer. Stack names repeat + * across nodes, so a slow response for the node the operator just left would + * otherwise land as this node's state, labelling the wrong stacks. + */ + const generation = useRef(0); + + const fetchStates = useCallback(async () => { + const current = ++generation.current; + try { + const res = await apiFetch('/git-sources'); + if (!res.ok) { + // Says which kind of refusal it was. A 403 here means the badges are + // frozen because this account may no longer read Git sources, which + // looks identical on screen to a fleet where nothing has changed. + console.error(`[GitOps] source-state fetch HTTP ${res.status}`); + return; + } + const rows = await res.json() as GitSourceRow[]; + const next = projectSourceStates(rows); + if (current !== generation.current) return; + setStates(next); + } catch (e) { + // The request itself failed. Keep the previous map rather than blanking + // every badge on one bad poll, but say so: silence here and a quiet + // fleet look identical. + console.error('[GitOps] source-state fetch failed:', e); + } + }, []); + + // Blank on a node switch so no stale badge survives into the new node's list, + // then ask the node that is now active. + useEffect(() => { + generation.current += 1; + setStates({}); + void fetchStates(); + }, [nodeId, fetchStates]); + + // GitOps state changes on transitions, not on a clock, so there is no poll + // here: the announcement is the trigger. Debounced because one operation + // commits several transitions in a row. + useEffect(() => { + let timer: ReturnType | null = null; + const onInvalidate = (e: Event) => { + const detail = (e as CustomEvent<{ scope?: string }>).detail; + if (detail?.scope !== 'gitops') return; + if (timer) clearTimeout(timer); + timer = setTimeout(() => { + timer = null; + void fetchStates(); + }, INVALIDATE_DEBOUNCE_MS); + }; + window.addEventListener('sencho:state-invalidate', onInvalidate); + return () => { + window.removeEventListener('sencho:state-invalidate', onInvalidate); + if (timer) clearTimeout(timer); + }; + }, [fetchStates]); + + return states; +} diff --git a/frontend/src/components/fleet/FederationTab.test.tsx b/frontend/src/components/fleet/FederationTab.test.tsx index d38679f0..692ce606 100644 --- a/frontend/src/components/fleet/FederationTab.test.tsx +++ b/frontend/src/components/fleet/FederationTab.test.tsx @@ -29,6 +29,7 @@ vi.mock('@/components/ui/toast-store', () => ({ import { listBlueprints, pinBlueprint } from '@/lib/blueprintsApi'; import { listNodes } from '@/lib/nodesApi'; import { FederationTab } from './FederationTab'; +import { absentRevision } from '@/__tests__/gitopsFixtures'; function node(id: number, name: string, overrides: Partial = {}): NodeRecord { return { id, name, type: 'local', status: 'online', cordoned: false, cordoned_at: null, cordoned_reason: null, ...overrides }; @@ -52,6 +53,7 @@ function blueprint(overrides: Partial = {}): BlueprintListIte pinned_node_id: null, deploymentCounts: {}, deploymentTotal: 0, + gitopsRevision: absentRevision(), ...overrides, }; } diff --git a/frontend/src/components/gitops/GitOpsBadge.tsx b/frontend/src/components/gitops/GitOpsBadge.tsx new file mode 100644 index 00000000..d6de617c --- /dev/null +++ b/frontend/src/components/gitops/GitOpsBadge.tsx @@ -0,0 +1,72 @@ +import { GITOPS_TONE_CLASS, SOURCE_STATE_LOOKUP, RUNTIME_STATE_LOOKUP } from '@/lib/gitopsState'; +import type { GitOpsRuntimeStatus, GitOpsSourceStatus } from '@/types/gitops'; +import { cn } from '@/lib/utils'; + +/** + * Which vocabulary the status belongs to, paired with a status from it. + * + * A discriminated union rather than two loose fields: the two status unions + * overlap on several names (`recovery_required`, `recovery_failed`) with + * different copy, so pairing them at the type level is what stops a runtime + * status being rendered with source wording. + */ +type GitOpsBadgeFacet = + | { facet: 'source'; status: GitOpsSourceStatus } + | { facet: 'runtime'; status: GitOpsRuntimeStatus }; + +type GitOpsBadgeProps = GitOpsBadgeFacet & { + /** + * Drops the label to an icon and a tooltip title, for rows too narrow to + * carry words. The title still states the whole sentence, so the state is + * never conveyed by colour alone. + */ + compact?: boolean; + className?: string; +}; + +/** + * One GitOps state as a small inline chip. + * + * The chip is a lighter reading of the same state the cards show, for places + * that list many stacks at once: a dashboard row has space for a word, not a + * sentence. Both read from the shared vocabulary, so a stack cannot be + * "pending update" in the sidebar and something else on the dashboard. + * + * Presentation only. Whether a stack has GitOps state worth showing is the + * caller's decision, because the answer differs per surface. + */ +export default function GitOpsBadge(props: GitOpsBadgeProps) { + const { compact = false, className } = props; + // Read through the partial views, because the status arrives over a proxy + // from a node that may run a newer vocabulary than this build knows. The maps + // are total over the closed unions, which says nothing about what is on the + // wire, and an unmapped key would otherwise dereference undefined inside a + // stack row and take the whole list down with it. Going through the views + // rather than annotating the result means the miss is a fact the compiler + // derives, so this guard cannot read as dead code. Rendering nothing matches + // what the join already does for a stack it has no state for. + const state = props.facet === 'source' + ? SOURCE_STATE_LOOKUP[props.status] + : RUNTIME_STATE_LOOKUP[props.status]; + if (!state) return null; + const Icon = state.icon; + + return ( + + + {/* Compact keeps the label for screen readers and drops it visually. */} + {state.label} + + ); +} diff --git a/frontend/src/components/gitops/GitOpsCaveats.test.tsx b/frontend/src/components/gitops/GitOpsCaveats.test.tsx new file mode 100644 index 00000000..42ccce44 --- /dev/null +++ b/frontend/src/components/gitops/GitOpsCaveats.test.tsx @@ -0,0 +1,76 @@ +import { describe, expect, it } from 'vitest'; +import { render, screen } from '@testing-library/react'; +import { absentRevision, liveRevision, missingApplicationLimitation } from '@/__tests__/gitopsFixtures'; +import { GITOPS_LIMITATION_COPY } from '@/lib/gitopsLimitations'; +import GitOpsCaveats from './GitOpsCaveats'; +import type { GitOpsLimitation } from '@/types/gitops'; + +const limitation = (code: string): GitOpsLimitation => ({ code, message: 'log wording', evidence: null }); + +/** + * The operator wording for a code, insisting it exists. + * + * The map is typed with an optional value so a lookup miss is visible to the + * compiler rather than only to a comment. These cases are about codes that do + * have copy, so a miss here means the fixture is wrong and should say so + * loudly instead of comparing against undefined. + */ +function copyFor(code: string): string { + const copy = GITOPS_LIMITATION_COPY[code]; + if (!copy) throw new Error(`no operator copy for ${code}`); + return copy; +} + +describe('GitOpsCaveats', () => { + it('renders nothing when the state has nothing to qualify', () => { + render(); + expect(screen.queryByTestId('gitops-caveats')).toBeNull(); + }); + + it('renders nothing when there is no projection at all', () => { + render(); + expect(screen.queryByTestId('gitops-caveats')).toBeNull(); + }); + + it('shows the operator wording for each caveat', () => { + render(); + + expect(screen.getByText(copyFor('legacy_pending'))).toBeInTheDocument(); + expect(screen.getByText(copyFor('lkg_generation_missing'))).toBeInTheDocument(); + expect(screen.queryByText('log wording')).toBeNull(); + }); + + it('counts the caveats in the heading', () => { + render(); + expect(screen.getByText('one thing could not be proven')).toBeInTheDocument(); + }); + + it('pluralizes the heading', () => { + render(); + expect(screen.getByText('2 things could not be proven')).toBeInTheDocument(); + }); + + it('shows one condition once however many times it was recorded', () => { + // The same condition is recorded per target and again per application, so + // both the heading count and the list keys depend on the dedup happening + // here rather than only in the map builder. + render(); + + expect(screen.getByText('one thing could not be proven')).toBeInTheDocument(); + expect(screen.getAllByText(copyFor('lkg_generation_missing'))).toHaveLength(1); + }); + + it('says nothing for a fault on the absent arm', () => { + // A fault means the state could not be reported at all, so rendering it + // here would present an unavailable projection as a qualified one. The + // fault card owns that case. + render(); + expect(screen.queryByTestId('gitops-caveats')).toBeNull(); + }); +}); diff --git a/frontend/src/components/gitops/GitOpsCaveats.tsx b/frontend/src/components/gitops/GitOpsCaveats.tsx new file mode 100644 index 00000000..83a50759 --- /dev/null +++ b/frontend/src/components/gitops/GitOpsCaveats.tsx @@ -0,0 +1,51 @@ +import { Info } from 'lucide-react'; + +import { limitationCaveats } from '@/lib/gitopsLimitations'; +import { liveCaveats } from '@/lib/gitopsState'; +import type { GitOpsRevisionProjection } from '@/types/gitops'; +import { cn } from '@/lib/utils'; + +interface GitOpsCaveatsProps { + revision: GitOpsRevisionProjection | null; + className?: string; +} + +/** + * What could not be proven about the state shown above. + * + * Deliberately quiet: neutral tone, no icon per line, sitting under the state + * rather than competing with it. A caveat is not a failure. The state is real + * and one piece of evidence behind it is missing, so the reader needs to know + * which part to distrust without being told the whole thing is broken. + * + * Renders nothing when there is nothing to qualify, which is the ordinary case. + * Faults on the absent arm are not shown here; those replace the state entirely + * and belong to the fault card. + */ +export default function GitOpsCaveats({ revision, className }: GitOpsCaveatsProps) { + const caveats = revision ? limitationCaveats(liveCaveats(revision)) : []; + if (caveats.length === 0) return null; + + return ( +
    +
    + +
    + + {caveats.length === 1 ? 'one thing could not be proven' : `${caveats.length} things could not be proven`} + +
      + {caveats.map((caveat) => ( +
    • + {caveat} +
      • + ))} +
    +
    +
    +
    + ); +} diff --git a/frontend/src/components/gitops/GitOpsStateCard.tsx b/frontend/src/components/gitops/GitOpsStateCard.tsx new file mode 100644 index 00000000..10633190 --- /dev/null +++ b/frontend/src/components/gitops/GitOpsStateCard.tsx @@ -0,0 +1,76 @@ +import type { ReactNode } from 'react'; +import { TriangleAlert } from 'lucide-react'; + +import { GITOPS_TONE_CLASS, type GitOpsStateMeta } from '@/lib/gitopsState'; +import { cn } from '@/lib/utils'; + +interface GitOpsStateCardProps { + /** + * The whole state as one concept: label, tone, line and icon together. From + * SOURCE_STATE_LOOKUP or RUNTIME_STATE_LOOKUP, or built inline for the + * projection faults, which are limitations rather than facet statuses. + * + * Optional because a status that crossed the wire may belong to a vocabulary + * this build has never seen, and the lookups answer `undefined` for one. + * Taking it here rather than at each caller means a new surface cannot + * reintroduce the unguarded dereference by rendering a card the usual way. + */ + state: GitOpsStateMeta | undefined; + /** Rendered as data-state so a test can assert the state without matching copy. */ + stateKey: string; + /** Trailing slot on the header row, for a single small action. */ + action?: ReactNode; + /** Detail under the line: a short commit sha, the node this target is on. */ + children?: ReactNode; + 'data-testid'?: string; +} + +/** + * One GitOps state, in the drift status card's shell plus the bevel a card is + * supposed to carry. Presentation only: every decision about which state to + * show, and whether to show one at all, belongs to the caller. + */ +export default function GitOpsStateCard( + { state, stateKey, action, children, 'data-testid': testId }: GitOpsStateCardProps, +) { + if (!state) return null; + const Icon = state.icon; + return ( +
    +
    + +
    + {state.label} +
    {state.line}
    + {children} +
    + {action} +
    +
    + ); +} + +/** + * An application the projection had reason to believe exists and could not + * reach. A fault is a limitation rather than a facet status, so it has no entry + * in the state maps; its copy lives here so the Git source panel, the Drift tab + * and the Blueprint sheet cannot report the same failure three different ways. + */ +export function GitOpsFaultCard({ message }: { message: string }) { + return ( + + ); +} diff --git a/frontend/src/components/mobile/MobileDashboard.tsx b/frontend/src/components/mobile/MobileDashboard.tsx index a73c0855..77f0407e 100644 --- a/frontend/src/components/mobile/MobileDashboard.tsx +++ b/frontend/src/components/mobile/MobileDashboard.tsx @@ -2,6 +2,8 @@ import { useEffect, useMemo, useState, type ReactNode } from 'react'; import { useNodes } from '@/context/NodeContext'; import { useDashboardData } from '@/components/dashboard'; import { deriveHealth } from '@/components/dashboard/deriveHealth'; +import { useGitOpsSourceStates } from '@/components/dashboard/useGitOpsSourceStates'; +import GitOpsBadge from '@/components/gitops/GitOpsBadge'; import type { HealthLevel, NotificationItem, StackCpuSeries, StackStatusEntry } from '@/components/dashboard/types'; import { Bar, Kicker, Masthead, MSparkline, SectionHead, StateDot } from './mobile-ui'; import { NodeSwitcher } from '@/components/NodeSwitcher'; @@ -79,6 +81,7 @@ function StripCell({ label, value, bar }: { label: string; value: string; bar?: export function MobileDashboard({ notifications, headerActions, onNavigateToStack, onViewAllStacks, onManageNodes }: MobileDashboardProps) { const { activeNode } = useNodes(); const data = useDashboardData(); + const gitopsSourceStates = useGitOpsSourceStates(); const activeNodeName = activeNode?.name || 'Local'; // Re-render every few seconds so the "sync Xs" freshness label advances @@ -168,7 +171,9 @@ export function MobileDashboard({ notifications, headerActions, onNavigateToStac } else { stackHealthBody = (
    - {visibleRows.map(row => ( + {visibleRows.map(row => { + const gitopsSourceState = gitopsSourceStates[row.name]; + return ( - ))} + ); + })}
    ); } diff --git a/frontend/src/components/mobile/__tests__/MobileDashboard.gitopsBadge.test.tsx b/frontend/src/components/mobile/__tests__/MobileDashboard.gitopsBadge.test.tsx new file mode 100644 index 00000000..87cae495 --- /dev/null +++ b/frontend/src/components/mobile/__tests__/MobileDashboard.gitopsBadge.test.tsx @@ -0,0 +1,87 @@ +/** + * The phone dashboard joins the same GitOps state the desktop table does, from + * the same hook, so a stack cannot read one way on a laptop and another way on + * a phone. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { render, screen } from '@testing-library/react'; +import type { GitOpsSourceStateMap } from '@/components/dashboard/useGitOpsSourceStates'; +import type { StackStatusEntry } from '@/components/dashboard/types'; + +const sourceStates = vi.hoisted(() => ({ current: {} as GitOpsSourceStateMap })); +const stackStatuses = vi.hoisted(() => ({ + current: { 'app.yml': { status: 'running', source: 'git' } } as Record, +})); + +vi.mock('@/context/NodeContext', () => ({ + useNodes: () => ({ activeNode: { id: 1, name: 'Local' } }), +})); +vi.mock('@/components/NodeSwitcher', () => ({ NodeSwitcher: () => null })); +vi.mock('@/components/dashboard/useGitOpsSourceStates', () => ({ + useGitOpsSourceStates: () => sourceStates.current, +})); +vi.mock('@/components/dashboard', () => ({ + useDashboardData: () => ({ + stats: { active: 1, managed: 1, unmanaged: 0, exited: 0, total: 1 }, + systemStats: null, + stackStatuses: stackStatuses.current, + stackCpuSeries: {}, + stackStatusesLoadStatus: 'success', + stackStatusesLoadError: null, + retryStackStatuses: vi.fn(), + cpuHistory: [], + netHistory: [], + historyEndAt: null, + lastSyncAt: null, + metricsStale: false, + metrics: [], + nodeCount: 1, + }), +})); + +import { MobileDashboard } from '../MobileDashboard'; + +function renderDashboard() { + return render( + , + ); +} + +describe('MobileDashboard GitOps badge', () => { + beforeEach(() => { + sourceStates.current = {}; + }); + + it('badges a stack the model has state for', () => { + sourceStates.current = { app: 'candidate_ready' }; + renderDashboard(); + + const badge = screen.getByTestId('gitops-badge'); + expect(badge).toHaveAttribute('data-state', 'candidate_ready'); + // Touch has no hover, so the word has to be on screen rather than only in + // the title. `toHaveTextContent` also matches sr-only text, so it would + // stay green if the badge were rendered compact, which is the thing this + // is here to rule out: assert against the visible node instead. + const visible = badge.querySelector(':scope > span:not(.sr-only)'); + expect(visible?.textContent).toBe('pending update'); + }); + + it('renders no badge for a stack the model says nothing about', () => { + renderDashboard(); + expect(screen.queryByTestId('gitops-badge')).toBeNull(); + }); + + it('keeps the node name on the row alongside the badge', () => { + sourceStates.current = { app: 'source_review_pending' }; + renderDashboard(); + + expect(screen.getByText('Local')).toBeInTheDocument(); + expect(screen.getByTestId('gitops-badge')).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/components/nodes/useNodeActions.test.tsx b/frontend/src/components/nodes/useNodeActions.test.tsx new file mode 100644 index 00000000..ff86f5ca --- /dev/null +++ b/frontend/src/components/nodes/useNodeActions.test.tsx @@ -0,0 +1,77 @@ +/** + * Covers what deleting a node reports back. Deleting a node re-places every + * Blueprint that was targeting it, and the delete response carries that list. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { render, screen, fireEvent, waitFor } from '@testing-library/react'; + +const nodeCtl = vi.hoisted(() => ({ refreshNodes: vi.fn() })); + +vi.mock('@/lib/api', () => ({ apiFetch: vi.fn() })); +vi.mock('@/context/NodeContext', () => ({ + useNodes: () => ({ nodes: [], refreshNodes: nodeCtl.refreshNodes }), +})); +vi.mock('@/components/ui/toast-store', () => ({ + toast: { success: vi.fn(), error: vi.fn(), info: vi.fn() }, +})); + +import { apiFetch } from '@/lib/api'; +import { toast } from '@/components/ui/toast-store'; +import { useNodeActions } from './useNodeActions'; +import { absentRevision } from '@/__tests__/gitopsFixtures'; +import type { Node } from '@/context/NodeContext'; + +const NODE = { id: 3, name: 'edge-02', type: 'remote' } as Node; + +beforeEach(() => { + vi.clearAllMocks(); +}); + +// The hook owns the modal's open state, so the trigger and the modal have to +// live in one tree; driving it from a separate renderHook leaves the rendered +// modal reading a stale snapshot. +function Harness() { + const { openDelete, NodeActionModals } = useNodeActions(); + return ( + <> + + {NodeActionModals} + + ); +} + +async function deleteNode(gitopsRevisions: ReturnType[]) { + vi.mocked(apiFetch).mockResolvedValue( + new Response(JSON.stringify({ success: true, gitopsRevisions }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }), + ); + render(); + fireEvent.click(screen.getByRole('button', { name: 'open delete' })); + fireEvent.click(await screen.findByRole('button', { name: /^delete$/i })); + await waitFor(() => expect(apiFetch).toHaveBeenCalledWith('/nodes/3', { method: 'DELETE' })); +} + +describe('useNodeActions delete', () => { + it('reports how many blueprints the deletion re-placed', async () => { + await deleteNode([absentRevision(), absentRevision()]); + await waitFor(() => + expect(toast.success).toHaveBeenCalledWith('Node "edge-02" deleted. 2 blueprints re-placed.'), + ); + }); + + it('uses the singular for one', async () => { + await deleteNode([absentRevision()]); + await waitFor(() => + expect(toast.success).toHaveBeenCalledWith('Node "edge-02" deleted. 1 blueprint re-placed.'), + ); + }); + + it('falls back to the plain message when the list is empty', async () => { + // Empty means both "nothing moved" and "the projection faulted after the + // delete committed", so it can never be reported as the former. + await deleteNode([]); + await waitFor(() => expect(toast.success).toHaveBeenCalledWith('Node "edge-02" deleted')); + }); +}); diff --git a/frontend/src/components/nodes/useNodeActions.tsx b/frontend/src/components/nodes/useNodeActions.tsx index fb9eb6b5..b03b25c1 100644 --- a/frontend/src/components/nodes/useNodeActions.tsx +++ b/frontend/src/components/nodes/useNodeActions.tsx @@ -3,6 +3,7 @@ import { Check, Copy, AlertTriangle, Globe, Monitor, RefreshCw } from 'lucide-re import { useNodes, type Node, type NodeMode } from '@/context/NodeContext'; import { apiFetch } from '@/lib/api'; import { copyToClipboard } from '@/lib/clipboard'; +import type { GitOpsRevisionsCarrier } from '@/types/gitops'; import { toast } from '@/components/ui/toast-store'; import { Modal, ModalHeader, ModalBody, ModalFooter, ConfirmModal } from '@/components/ui/modal'; import { Button } from '@/components/ui/button'; @@ -195,7 +196,16 @@ export function useNodeActions(opts: UseNodeActionsOptions = {}): UseNodeActions const err = await res.json(); throw new Error(err.error || 'Failed to delete node'); } - toast.success(`Node "${deletingNode.name}" deleted`); + // The delete has already committed, so nothing about reading this body may + // be able to report it as a failure. Report the count only when there is + // one: an empty list means both "nothing moved" and "the projection + // faulted after the delete committed", so it can never be reported as the + // former. + const body = (await res.json().catch(() => null)) as GitOpsRevisionsCarrier | null; + const moved = body?.gitopsRevisions?.length ?? 0; + toast.success(moved > 0 + ? `Node "${deletingNode.name}" deleted. ${moved} blueprint${moved === 1 ? '' : 's'} re-placed.` + : `Node "${deletingNode.name}" deleted`); await refresh(); } catch (error) { toast.error((error as Error).message || 'Failed to delete node'); diff --git a/frontend/src/components/sidebar/StackList.tsx b/frontend/src/components/sidebar/StackList.tsx index d95e4671..206c291a 100644 --- a/frontend/src/components/sidebar/StackList.tsx +++ b/frontend/src/components/sidebar/StackList.tsx @@ -5,6 +5,7 @@ import { CommandItem, CommandList } from '@/components/ui/command'; import { Skeleton } from '@/components/ui/skeleton'; import type { Label } from '@/components/label-types'; import type { StackUpdateInfo } from '@/types/imageUpdates'; +import type { GitSourcePendingMap } from '@/lib/gitopsState'; import { StackRow } from './StackRow'; import { statusText, statusColor } from './stack-status-utils'; import type { StackRowStatus } from './stack-status-utils'; @@ -41,7 +42,7 @@ export interface StackListProps { stackStatuses: Record; stackCounts: Record; stackUpdates: Record; - gitSourcePendingMap: Record; + gitSourcePendingMap: GitSourcePendingMap; pinnedFiles: string[]; isCollapsed: (groupKey: string) => boolean; toggleCollapse: (groupKey: string) => void; @@ -273,7 +274,7 @@ export function StackList(props: StackListProps & StackListBulkProps) { .map((s) => s.service)} checkStatus={stackUpdates[file]?.checkStatus} lastError={stackUpdates[file]?.lastError ?? undefined} - hasGitPending={!!gitSourcePendingMap[file]} + gitPending={gitSourcePendingMap[file] ?? null} onSelect={onSelectFile} kebabSlot={} bulkMode={bulkMode} diff --git a/frontend/src/components/sidebar/StackRow.tsx b/frontend/src/components/sidebar/StackRow.tsx index 8706f767..1ecd4d85 100644 --- a/frontend/src/components/sidebar/StackRow.tsx +++ b/frontend/src/components/sidebar/StackRow.tsx @@ -10,6 +10,8 @@ import { sidebarRowActive, sidebarRowBase, sidebarRowCheckboxSlot } from './side import { statusText, statusColor } from './stack-status-utils'; import type { StackRowStatus } from './stack-status-utils'; import { updateAvailableLabel } from '@/lib/updateAvailableLabel'; +import { SOURCE_STATE_LOOKUP } from '@/lib/gitopsState'; +import type { GitOpsSourceStatus } from '@/types/gitops'; interface StackRowProps { file: string; @@ -31,7 +33,12 @@ interface StackRowProps { // use a distinct indicator so they are not mistaken for a confirmed update. checkStatus?: CheckStatus; lastError?: string; - hasGitPending: boolean; + /** + * The source state of a waiting Git candidate, or null when none is waiting. + * The indicator itself is identical for every state; only the tooltip differs, + * so a blocked plan reads as blocked instead of as an ordinary update. + */ + gitPending: GitOpsSourceStatus | null; onSelect: (file: string) => void; kebabSlot: ReactNode; bulkMode?: boolean; @@ -85,7 +92,7 @@ function failedCheckTooltip(hasUpdate: boolean, lastError?: string): string { export function StackRow(props: StackRowProps) { const { file, displayName, status, running, total, isBusy, isActive, - hasUpdate, outdatedServices, checkStatus, lastError, hasGitPending, onSelect, kebabSlot, + hasUpdate, outdatedServices, checkStatus, lastError, gitPending, onSelect, kebabSlot, bulkMode = false, isSelected = false, onToggleSelect, hydrationDisplay = 'pending', } = props; @@ -179,10 +186,10 @@ export function StackRow(props: StackRowProps) { trigger={} label={failedCheckTooltip(hasUpdate, lastError)} /> - ) : hasGitPending ? ( + ) : gitPending ? (     } - label="Git source update pending" + label={SOURCE_STATE_LOOKUP[gitPending]?.line ?? 'A Git update is waiting on this stack.'} /> ) : null} diff --git a/frontend/src/components/sidebar/__tests__/StackList.test.tsx b/frontend/src/components/sidebar/__tests__/StackList.test.tsx index 17804826..8b139e60 100644 --- a/frontend/src/components/sidebar/__tests__/StackList.test.tsx +++ b/frontend/src/components/sidebar/__tests__/StackList.test.tsx @@ -1,6 +1,7 @@ import type React from 'react'; import { describe, it, expect, vi } from 'vitest'; -import { render, screen } from '@testing-library/react'; +import { render, screen, fireEvent } from '@testing-library/react'; +import { SOURCE_STATE } from '@/lib/gitopsState'; import { StackList } from '../StackList'; import { Command } from '@/components/ui/command'; import { isStacksListSettled, isStacksListLoading } from '../stacksLoadUi'; @@ -199,4 +200,28 @@ describe('StackList hydration display', () => { const row = screen.getByTestId('stack-row'); expect(row.textContent).toContain('UP'); }); + + it('passes a waiting Git state through to the row it belongs to', async () => { + render( + + minimalCtx as never, + })} + /> + , + ); + const indicators = screen.getAllByTestId('stack-trailing-git-pending'); + expect(indicators).toHaveLength(1); + // The state has to survive the trip, not just the fact that one is waiting. + fireEvent.pointerMove(indicators[0]); + expect( + (await screen.findAllByText(SOURCE_STATE.source_conflict_blocker.line)).length, + ).toBeGreaterThan(0); + }); }); diff --git a/frontend/src/components/sidebar/__tests__/StackRow.test.tsx b/frontend/src/components/sidebar/__tests__/StackRow.test.tsx index a8848f43..eeed245a 100644 --- a/frontend/src/components/sidebar/__tests__/StackRow.test.tsx +++ b/frontend/src/components/sidebar/__tests__/StackRow.test.tsx @@ -3,6 +3,7 @@ import { render, screen, fireEvent } from '@testing-library/react'; import type { ComponentProps } from 'react'; import { StackRow } from '../StackRow'; import type { Label } from '@/components/label-types'; +import { SOURCE_STATE } from '@/lib/gitopsState'; function base(overrides: Partial> = {}) { return { @@ -13,7 +14,7 @@ function base(overrides: Partial> = {}) { isActive: false, labels: [] as Label[], hasUpdate: false, - hasGitPending: false, + gitPending: null, onSelect: vi.fn(), kebabSlot: null, ...overrides, @@ -149,6 +150,45 @@ describe('StackRow', () => { expect(container.querySelector('.bg-update')).toBeNull(); }); + // ── Git source pending indicator ─────────────────────────────────────── + + it('shows no git indicator when no candidate is waiting', () => { + render(); + expect(screen.queryByTestId('stack-trailing-git-pending')).not.toBeInTheDocument(); + }); + + it('names the waiting state in the tooltip instead of a generic one', async () => { + render(); + fireEvent.pointerMove(screen.getByTestId('stack-trailing-git-pending')); + expect( + (await screen.findAllByText(SOURCE_STATE.source_conflict_blocker.line)).length, + ).toBeGreaterThan(0); + }); + + it('names the ordinary waiting state too', async () => { + render(); + fireEvent.pointerMove(screen.getByTestId('stack-trailing-git-pending')); + expect((await screen.findAllByText(SOURCE_STATE.candidate_ready.line)).length).toBeGreaterThan(0); + }); + + it('renders the same indicator whatever the waiting state is', () => { + // The state changes the tooltip, never the pixels. This is what keeps the + // sidebar's rendered output unchanged from before the states existed. + const { container: blocked } = render(); + const { container: ready } = render(); + const markup = (root: HTMLElement) => + root.querySelector('[data-testid="stack-row-trailing"]')?.outerHTML; + // Non-empty first: two missing indicators would otherwise compare equal. + expect(markup(blocked)).toContain('stack-trailing-git-pending'); + expect(markup(blocked)).toBe(markup(ready)); + }); + + it('keeps a confirmed update above the git indicator in the trailing slot', () => { + render(); + expect(screen.getByTestId('stack-trailing-update')).toBeInTheDocument(); + expect(screen.queryByTestId('stack-trailing-git-pending')).not.toBeInTheDocument(); + }); + it('constrains long stack names so trailing indicators stay in the row', () => { const longName = 'tick-grafana-docker-observability-stack'; render(); diff --git a/frontend/src/components/stack/DriftPanel.test.tsx b/frontend/src/components/stack/DriftPanel.test.tsx index fe7ab0f4..b27ccceb 100644 --- a/frontend/src/components/stack/DriftPanel.test.tsx +++ b/frontend/src/components/stack/DriftPanel.test.tsx @@ -8,11 +8,22 @@ import { render, screen, fireEvent, waitFor } from '@testing-library/react'; vi.mock('@/lib/api', () => ({ apiFetch: vi.fn() })); vi.mock('@/components/ui/toast-store', () => ({ toast: { success: vi.fn(), error: vi.fn() } })); -vi.mock('@/context/NodeContext', () => ({ useNodes: () => ({ activeNode: { id: 1 } }) })); +vi.mock('@/context/NodeContext', () => ({ + useNodes: () => ({ activeNode: { id: 1 }, nodes: [{ id: 1, name: 'local' }, { id: 2, name: 'edge-02' }] }), +})); import { apiFetch } from '@/lib/api'; import { toast } from '@/components/ui/toast-store'; import DriftPanel from './DriftPanel'; +import { + absentRevision, + driftItem, + facets, + liveRevision, + missingApplicationLimitation, + plainSource, + target, +} from '@/__tests__/gitopsFixtures'; interface DriftReport { stack: string; @@ -24,6 +35,7 @@ interface DriftReport { temporal?: { hasBaseline: boolean; sourceChanged: boolean; renderedChanged: boolean }; ledger?: Array<{ service: string; kind: string; message: string; detectedAt: number; resolvedAt: number | null }>; lastCheckedAt?: number | null; + gitopsRevision?: unknown; } function report(partial: Partial): DriftReport { @@ -242,3 +254,124 @@ describe('DriftPanel', () => { expect(screen.getByText('compose-primary local-modified')).toBeInTheDocument(); }); }); + +describe('DriftPanel GitOps state', () => { + it('renders the source state and one card per target for a Direct stack', async () => { + vi.mocked(apiFetch).mockResolvedValue(jsonRes(report({ + gitopsRevision: liveRevision({ + facets: facets({ source: plainSource('candidate_ready') }), + targets: [target({ nodeId: 1, runtime: { status: 'applied_not_deployed' } })], + }), + }))); + render(); + + const source = await screen.findByTestId('gitops-source'); + expect(source).toHaveAttribute('data-state', 'candidate_ready'); + const targets = screen.getAllByTestId('gitops-target'); + expect(targets).toHaveLength(1); + expect(targets[0]).toHaveAttribute('data-state', 'applied_not_deployed'); + expect(targets[0]).toHaveTextContent('local'); + }); + + it('shows no source card for a Blueprint-owned stack, only its targets', async () => { + // The drift route resolves through whatever manages the directory. A + // Blueprint application has no Git source, and inventing one would be a + // claim the model never made. + vi.mocked(apiFetch).mockResolvedValue(jsonRes(report({ + gitopsRevision: liveRevision({ + targetMode: 'inline_blueprint', + blueprintId: 7, + facets: facets({ + source: { status: 'not_applicable' }, + placement: { status: 'blueprint_bound', completion: 'unknown' }, + }), + targets: [ + target({ nodeId: 1, runtime: { status: 'synced_and_healthy' } }), + target({ nodeId: 2, runtime: { status: 'drifted' } }), + ], + }), + }))); + render(); + + await waitFor(() => expect(screen.getAllByTestId('gitops-target')).toHaveLength(2)); + expect(screen.queryByTestId('gitops-source')).not.toBeInTheDocument(); + expect(screen.getAllByTestId('gitops-target')[1]).toHaveTextContent('edge-02'); + }); + + it('reports an application the projection could not reach', async () => { + vi.mocked(apiFetch).mockResolvedValue(jsonRes(report({ + gitopsRevision: absentRevision([missingApplicationLimitation]), + }))); + render(); + + expect(await screen.findByTestId('gitops-fault')).toHaveTextContent(missingApplicationLimitation.message); + }); + + it('renders nothing new for a stack the model was never asked about', async () => { + // The common case by far: no Git source, no Blueprint. A section header over + // an empty block would be worse than silence. + vi.mocked(apiFetch).mockResolvedValue(jsonRes(report({ gitopsRevision: absentRevision() }))); + render(); + + await screen.findByTestId('drift-status'); + expect(screen.queryByTestId('gitops-fault')).not.toBeInTheDocument(); + expect(screen.queryByTestId('gitops-source')).not.toBeInTheDocument(); + expect(screen.queryByText('gitops')).not.toBeInTheDocument(); + }); + + it('renders exactly today output for a report from a node that predates the model', async () => { + vi.mocked(apiFetch).mockResolvedValue(jsonRes(report({ status: 'drifted' }))); + render(); + + expect(await screen.findByTestId('drift-status')).toHaveAttribute('data-status', 'drifted'); + expect(screen.queryByTestId('gitops-source')).not.toBeInTheDocument(); + expect(screen.queryByTestId('gitops-fault')).not.toBeInTheDocument(); + }); + + it('does not treat a live application caveat as a fault', async () => { + // Live-arm limitations are caveats on state that is being reported. Reading + // them as faults would recreate the conflation in the opposite direction. + vi.mocked(apiFetch).mockResolvedValue(jsonRes(report({ + gitopsRevision: liveRevision({ + limitations: [{ code: 'repo_identity_invalid', message: 'Repository identity could not be read.', evidence: null }], + }), + }))); + render(); + + await screen.findByTestId('gitops-source'); + expect(screen.queryByTestId('gitops-fault')).not.toBeInTheDocument(); + }); + + it('renders a drift item as expected against observed', async () => { + vi.mocked(apiFetch).mockResolvedValue(jsonRes(report({ + gitopsRevision: liveRevision({ drift: [driftItem()] }), + }))); + render(); + + expect(await screen.findByText('gitops drift')).toBeInTheDocument(); + // The backend now emits the runtime artifact drift item with artifact_set + // expected identity and the reason describes the artifact mismatch. + expect(screen.getByText('the running workload reports an artifact identity other than the expected artifact set')).toBeInTheDocument(); + // identityRefLabel formats artifact_set as "artifact · " + expect(screen.getByText('artifact art-acce · exact')).toBeInTheDocument(); + // runtime_artifact identity is rendered as-is + expect(screen.getByText('nginx@sha256:abc')).toBeInTheDocument(); + }); + + it('names a target on a node this client has no record of', async () => { + vi.mocked(apiFetch).mockResolvedValue(jsonRes(report({ + gitopsRevision: liveRevision({ targets: [target({ nodeId: 9 })] }), + }))); + render(); + + expect(await screen.findByTestId('gitops-target')).toHaveTextContent('node 9'); + }); + + it('renders no drift section while the backend derives no items', async () => { + vi.mocked(apiFetch).mockResolvedValue(jsonRes(report({ gitopsRevision: liveRevision({ drift: [] }) }))); + render(); + + await screen.findByTestId('gitops-source'); + expect(screen.queryByText('gitops drift')).not.toBeInTheDocument(); + }); +}); diff --git a/frontend/src/components/stack/DriftPanel.tsx b/frontend/src/components/stack/DriftPanel.tsx index 61aa38b3..bc4d3e49 100644 --- a/frontend/src/components/stack/DriftPanel.tsx +++ b/frontend/src/components/stack/DriftPanel.tsx @@ -8,6 +8,10 @@ import { cn } from '@/lib/utils'; import { toast } from '@/components/ui/toast-store'; import { formatTimeAgo } from '@/lib/relativeTime'; import { useNodes } from '@/context/NodeContext'; +import GitOpsStateCard, { GitOpsFaultCard } from '@/components/gitops/GitOpsStateCard'; +import GitOpsCaveats from '@/components/gitops/GitOpsCaveats'; +import { RUNTIME_STATE_LOOKUP, SOURCE_STATE_LOOKUP, absentFault, identityRefLabel, liveSourceFacet } from '@/lib/gitopsState'; +import type { GitOpsDriftItem, GitOpsRevisionProjection } from '@/types/gitops'; // Mirrors the backend payload shape (the frontend never imports backend). type StackDriftStatus = 'in-sync' | 'drifted' | 'missing-runtime' | 'unreachable'; @@ -50,6 +54,11 @@ interface StackDriftReport { // When the ledger was last reconciled (re-check, deploy, or background scan); null // if never. The history is "as of" this time, not the live status above it. lastCheckedAt?: number | null; + // Optional for the same reason as temporal and ledger above: a report proxied + // from an older remote node predates the revision model and omits it. That is + // rendered the same as an answer of "nothing here", because the alternative is + // telling an operator their node is out of date on a tab about drift. + gitopsRevision?: GitOpsRevisionProjection; } const LABEL_CLASS = 'font-mono text-[10px] uppercase tracking-[0.18em] text-stat-subtitle'; @@ -171,8 +180,30 @@ function LedgerRow({ entry }: { entry: DriftLedgerEntry }) { ); } +/** + * One classified divergence between intent and observation, in the same + * expected-to-observed idiom the compose findings above it use. + */ +function GitOpsDriftRow({ item }: { item: GitOpsDriftItem }) { + return ( +
    +
    + {item.class} + {item.owner} +
    +
    {item.reason}
    +
    + expected + {identityRefLabel(item.expected)} + → observed + {identityRefLabel(item.observed)} +
    +
    + ); +} + export default function DriftPanel({ stackName }: { stackName: string }) { - const { activeNode } = useNodes(); + const { activeNode, nodes } = useNodes(); const nodeId = activeNode?.id; const [report, setReport] = useState(null); const [loading, setLoading] = useState(true); @@ -245,6 +276,18 @@ export default function DriftPanel({ stackName }: { stackName: string }) { const lastChecked = report?.lastCheckedAt != null ? formatTimeAgo(report.lastCheckedAt) : null; const busy = loading || rechecking; + const revision = report?.gitopsRevision ?? null; + const gitopsFaults = revision ? absentFault(revision) : []; + const gitopsLive = revision && revision.targetMode !== 'not_applicable' ? revision : null; + // Null for a Blueprint-owned stack: this route resolves through whatever + // manages the directory, and a Blueprint application has no Git source facet. + const gitopsSource = liveSourceFacet(revision); + const gitopsTargets = gitopsLive?.targets ?? []; + const gitopsDrift = gitopsLive?.drift ?? []; + // A target can name a node this client has no record of, so fall back to the + // id rather than rendering an empty cell. + const nodeLabel = (id: number) => nodes.find(n => n.id === id)?.name ?? `node ${id}`; + return (
    @@ -301,6 +344,47 @@ export default function DriftPanel({ stackName }: { stackName: string }) {
    )} + {gitopsFaults.length > 0 && } + + {(gitopsSource || gitopsTargets.length > 0) && ( +
    +
    gitops
    +
    + {gitopsSource && ( + + )} + {gitopsTargets.map(t => ( + +
    + {nodeLabel(t.nodeId)}{t.stackName ? ` · ${t.stackName}` : ''} +
    +     + ))} + +
    +
    + )} + + {gitopsDrift.length > 0 && ( +
    +
    gitops drift
    +
    + {gitopsDrift.map((d, i) => ( + + ))} +
    +
    + )} + {report.parseError && (
    {report.parseError} diff --git a/frontend/src/components/stack/GitSourcePanel.test.tsx b/frontend/src/components/stack/GitSourcePanel.test.tsx index e3d30542..f2c57539 100644 --- a/frontend/src/components/stack/GitSourcePanel.test.tsx +++ b/frontend/src/components/stack/GitSourcePanel.test.tsx @@ -5,7 +5,7 @@ * treating the sentinel as a configured source. */ import { describe, it, expect, vi, beforeEach } from 'vitest'; -import { render, screen, waitFor, fireEvent } from '@testing-library/react'; +import { render, screen, waitFor, fireEvent, within } from '@testing-library/react'; // Mutable controls so a deploy-mode test can set the active node and capture the // runWithLog params, while the load tests keep the default (no active node). @@ -33,12 +33,16 @@ vi.mock('@/context/NodeContext', () => ({ // diff UI; the panel passes applyPull as onApply. vi.mock('./GitSourceDiffDialog', () => ({ GitSourceDiffDialog: ({ + open, onApply, + onDismiss, pull, }: { + open: boolean; onApply: (sha: string, deploy: boolean, fp: string) => void; + onDismiss: () => void; pull: PullResult | null; - }) => ( + }) => open ? (
    {pull?.planFingerprint ?? ''} +
    - ), + ) : null, })); vi.mock('@/components/ui/toast-store', () => ({ toast: { @@ -70,6 +77,14 @@ import { apiFetch } from '@/lib/api'; import { GitSourcePanel } from './GitSourcePanel'; import { toast } from '@/components/ui/toast-store'; import type { PullResult } from './GitSourceDiffDialog'; +import { + absentRevision, + facets, + liveRevision, + missingApplicationLimitation, + sourceRevision, +} from '@/__tests__/gitopsFixtures'; +import { SOURCE_STATE } from '@/lib/gitopsState'; function jsonRes(body: unknown, ok = true, status = 200) { return { ok, status, json: async () => body, text: async () => '' } as unknown as Response; @@ -94,6 +109,37 @@ const LINKED_SOURCE = { updated_at: 0, manifest_state: 'absent' as const, manifest: null, + gitopsRevision: sourceRevision('application_generation_accepted', { candidateGenerationId: null }), +}; + +/** The linked source with its GitOps projection swapped for a specific state. */ +function linkedWith(revision: unknown) { + return { ...LINKED_SOURCE, gitopsRevision: revision }; +} + +const PULL_RESULT: PullResult = { + commitSha: 'sha-old', + validation: { ok: true }, + refusals: [], + warnings: [], + plan: { + blocked: false, + counts: { + add: 0, + modify: 0, + delete: 0, + rename: 0, + unchanged: 1, + localModified: 0, + localMissing: 0, + typeChanged: 0, + unmanagedCollision: 0, + invocation: 0, + }, + operations: [], + invocation: { candidateChanged: false, liveDiverged: false }, + }, + planFingerprint: 'fp-old', }; function panel() { @@ -161,6 +207,7 @@ describe('GitSourcePanel deploy-mode apply node binding', () => { return jsonRes(LINKED_SOURCE); }); render(panel()); + fireEvent.click(await screen.findByRole('button', { name: /pull now/i })); fireEvent.click(await screen.findByTestId('apply-deploy')); await waitFor(() => { @@ -177,31 +224,6 @@ describe('GitSourcePanel deploy-mode apply node binding', () => { }); describe('GitSourcePanel stale plan handling', () => { - const PULL_RESULT: PullResult = { - commitSha: 'sha-old', - validation: { ok: true }, - refusals: [], - warnings: [], - plan: { - blocked: false, - counts: { - add: 0, - modify: 0, - delete: 0, - rename: 0, - unchanged: 1, - localModified: 0, - localMissing: 0, - typeChanged: 0, - unmanagedCollision: 0, - invocation: 0, - }, - operations: [], - invocation: { candidateChanged: false, liveDiverged: false }, - }, - planFingerprint: 'fp-old', - }; - beforeEach(() => { vi.mocked(apiFetch).mockImplementation(async (url: string) => { if (String(url).includes('/git-source/pull')) { @@ -235,6 +257,37 @@ describe('GitSourcePanel stale plan handling', () => { }); }); +describe('GitSourcePanel dismiss handling', () => { + beforeEach(() => { + vi.mocked(apiFetch).mockImplementation(async (url: string) => { + if (String(url).includes('/git-source/pull')) { + return jsonRes(PULL_RESULT); + } + if (String(url).includes('/git-source/dismiss-pending')) { + return jsonRes({ + error: 'Cannot dismiss the pending update for web: cannot dismiss while an operation is in flight', + code: 'OPERATION_IN_FLIGHT', + }, false, 409); + } + return jsonRes(LINKED_SOURCE); + }); + }); + + it('surfaces an error toast and keeps the diff open when dismiss is refused as in-flight', async () => { + render(panel()); + fireEvent.click(await screen.findByRole('button', { name: /pull now/i })); + await screen.findByTestId('plan-fingerprint'); + + fireEvent.click(screen.getByTestId('dismiss')); + + await waitFor(() => { + expect(toast.error).toHaveBeenCalledWith(expect.stringMatching(/operation is in flight/i)); + }); + expect(toast.success).not.toHaveBeenCalled(); + expect(screen.getByTestId('plan-fingerprint')).toHaveTextContent('fp-old'); + }); +}); + describe('GitSourcePanel manifest summary', () => { it('renders the managed-project section when the source carries a manifest', async () => { const summary = { @@ -285,3 +338,209 @@ describe('GitSourcePanel manifest summary', () => { expect(screen.getByText('Not materialized')).toBeTruthy(); }); }); + +describe('GitSourcePanel GitOps state', () => { + it('names the waiting state rather than a generic pending update', async () => { + vi.mocked(apiFetch).mockResolvedValue( + jsonRes(linkedWith(sourceRevision('source_conflict_blocker'))), + ); + + render(panel()); + + const banner = await screen.findByTestId('git-pending'); + expect(banner).toHaveAttribute('data-state', 'source_conflict_blocker'); + expect(within(banner).getByText(SOURCE_STATE.source_conflict_blocker.line)).toBeInTheDocument(); + // The short sha stays, so the operator can still see which commit it is. + expect(within(banner).getByText('a1b2c3d')).toBeInTheDocument(); + }); + + it('offers apply wording for a candidate that needs no review', async () => { + vi.mocked(apiFetch).mockResolvedValue( + jsonRes(linkedWith(sourceRevision('candidate_ready'))), + ); + + render(panel()); + + const banner = await screen.findByTestId('git-pending'); + expect(within(banner).getByText(SOURCE_STATE.candidate_ready.line)).toBeInTheDocument(); + }); + + it('shows no banner when the accepted generation has no candidate behind it', async () => { + vi.mocked(apiFetch).mockResolvedValue(jsonRes(LINKED_SOURCE)); + + render(panel()); + + await screen.findByRole('button', { name: /pull now/i }); + expect(screen.queryByTestId('git-pending')).not.toBeInTheDocument(); + expect(screen.getByTestId('git-source-state')).toHaveTextContent( + SOURCE_STATE.application_generation_accepted.label, + ); + }); + + it('reports an application the projection could not reach', async () => { + vi.mocked(apiFetch).mockResolvedValue( + jsonRes(linkedWith(absentRevision([missingApplicationLimitation]))), + ); + + render(panel()); + + const fault = await screen.findByTestId('gitops-fault'); + expect(fault).toHaveTextContent(missingApplicationLimitation.message); + }); + + it('stays silent for a stack the model was never asked about', async () => { + // Empty limitations is the ordinary case and must not read as a failure. + vi.mocked(apiFetch).mockResolvedValue(jsonRes({ linked: false, gitopsRevision: absentRevision() })); + + render(panel()); + + await screen.findByRole('button', { name: /^save$/i }); + expect(screen.queryByTestId('gitops-fault')).not.toBeInTheDocument(); + expect(screen.queryByTestId('git-pending')).not.toBeInTheDocument(); + expect(screen.queryByTestId('git-source-state')).not.toBeInTheDocument(); + }); + + it('drops the pending card once the source is detached', async () => { + // The card is derived from the revision alone, so a detach that only + // cleared the source would keep advertising a commit for a stack Git no + // longer manages, behind a Review button that does nothing. + vi.mocked(apiFetch).mockImplementation(async (url: string) => { + if (String(url).endsWith('/git-source') && !String(url).includes('?')) { + return jsonRes(linkedWith(sourceRevision('candidate_ready'))); + } + return jsonRes({ ok: true }); + }); + render(panel()); + await screen.findByTestId('git-pending'); + + fireEvent.click(screen.getByRole('button', { name: 'Remove' })); + fireEvent.click(await screen.findByRole('button', { name: /^detach/i })); + + await waitFor(() => expect(screen.queryByTestId('git-pending')).not.toBeInTheDocument()); + }); + + it('drops the revision when a later read fails, so one stack cannot report another stack state', async () => { + // The panel is reused across stacks. A read that throws after a successful + // one has to clear the projection, or stack A's pending commit renders + // under stack B's header. + vi.mocked(apiFetch).mockResolvedValue( + jsonRes(linkedWith(sourceRevision('candidate_ready'))), + ); + const { rerender } = render( + , + ); + await screen.findByTestId('git-pending'); + + vi.mocked(apiFetch).mockRejectedValue(new Error('offline')); + rerender(); + + // Wait for the load to settle before asserting: the body is skeletons while + // it is in flight, so an assertion there would pass without the fix. + await waitFor(() => expect(toast.error).toHaveBeenCalled()); + await screen.findByLabelText(/repository url/i); + expect(screen.queryByTestId('git-pending')).not.toBeInTheDocument(); + }); + + it('re-reads after a save and shows the state the server reports', async () => { + // The save answers with a bare source row and no revision, so the panel + // cannot learn the new state from it. Keeping the old one would report a + // candidate the save has just invalidated; showing nothing would report + // "no GitOps here" for a stack that has it. + vi.mocked(apiFetch).mockResolvedValue( + jsonRes(linkedWith(sourceRevision('candidate_ready'))), + ); + render(panel()); + await screen.findByTestId('git-pending'); + + vi.mocked(apiFetch) + // The PUT. + .mockResolvedValueOnce(jsonRes({ ...LINKED_SOURCE, gitopsRevision: undefined })) + // The re-read, which is where the state actually comes from. + .mockResolvedValueOnce(jsonRes(linkedWith( + sourceRevision('source_reconcile_required', { candidateGenerationId: null }), + ))); + fireEvent.click(screen.getByRole('button', { name: /update/i })); + + await waitFor(() => expect(screen.getByTestId('git-source-state')) + .toHaveTextContent(SOURCE_STATE.source_reconcile_required.label)); + // The staged candidate is gone, so nothing is offered to review. + expect(screen.queryByTestId('git-pending')).not.toBeInTheDocument(); + }); + + it('does not go blank after a save', async () => { + // The save response carries no revision. Before the re-read, the panel + // dropped its copy and rendered nothing until the next open, which reads + // as a stack the model knows nothing about. + vi.mocked(apiFetch).mockResolvedValue( + jsonRes(linkedWith(sourceRevision('application_generation_accepted', { candidateGenerationId: null }))), + ); + render(panel()); + await screen.findByTestId('git-source-state'); + + vi.mocked(apiFetch) + .mockResolvedValueOnce(jsonRes({ ...LINKED_SOURCE, gitopsRevision: undefined })) + .mockResolvedValueOnce(jsonRes(linkedWith( + sourceRevision('application_generation_accepted', { candidateGenerationId: null }), + ))); + fireEvent.click(screen.getByRole('button', { name: /update/i })); + + await waitFor(() => expect(toast.success).toHaveBeenCalled()); + expect(screen.getByTestId('git-source-state')).toBeInTheDocument(); + }); + + it('shows no source card for an application that has no Git source', async () => { + // Guards the panel against a projection whose source facet is not + // applicable: without it the card renders "no git source" with a live + // Review button. + vi.mocked(apiFetch).mockResolvedValue(jsonRes(linkedWith(liveRevision({ + targetMode: 'inline_blueprint', + facets: facets({ + source: { status: 'not_applicable' }, + placement: { status: 'blueprint_bound', completion: 'unknown' }, + }), + })))); + render(panel()); + + await screen.findByRole('button', { name: /pull now/i }); + expect(screen.queryByTestId('git-pending')).not.toBeInTheDocument(); + expect(screen.queryByTestId('git-source-state')).not.toBeInTheDocument(); + }); + + it('still reports a waiting commit when no projection answered', async () => { + // A swallowed GitOps write leaves the flat pointer as the only evidence. + // The sidebar keeps showing it, so the panel has to agree. + vi.mocked(apiFetch).mockResolvedValue(jsonRes({ + ...LINKED_SOURCE, + pending_commit_sha: 'f00ba12345', + gitopsRevision: absentRevision(), + })); + render(panel()); + + const banner = await screen.findByTestId('git-pending'); + expect(within(banner).getByText('f00ba12')).toBeInTheDocument(); + }); + + it('does not treat a live application caveat as a fault', async () => { + vi.mocked(apiFetch).mockResolvedValue(jsonRes(linkedWith(liveRevision({ + limitations: [{ code: 'repo_identity_invalid', message: 'Repository identity could not be read.', evidence: null }], + })))); + render(panel()); + + await screen.findByTestId('git-source-state'); + expect(screen.queryByTestId('gitops-fault')).not.toBeInTheDocument(); + }); + + it('routes the pending card Review button to the pull endpoint', async () => { + vi.mocked(apiFetch).mockResolvedValue( + jsonRes(linkedWith(sourceRevision('candidate_ready'))), + ); + render(panel()); + const banner = await screen.findByTestId('git-pending'); + + fireEvent.click(within(banner).getByRole('button', { name: 'Review' })); + + await waitFor(() => expect( + vi.mocked(apiFetch).mock.calls.some(c => String(c[0]).includes('/git-source/pull')), + ).toBe(true)); + }); +}); diff --git a/frontend/src/components/stack/GitSourcePanel.tsx b/frontend/src/components/stack/GitSourcePanel.tsx index 0a2c7a9d..90cc422d 100644 --- a/frontend/src/components/stack/GitSourcePanel.tsx +++ b/frontend/src/components/stack/GitSourcePanel.tsx @@ -1,5 +1,5 @@ import { useState, useEffect, useCallback } from 'react'; -import { GitBranch, Loader2, Trash2, RefreshCw, Save, AlertCircle } from 'lucide-react'; +import { GitBranch, Loader2, Trash2, RefreshCw, Save } from 'lucide-react'; import { Modal, ModalHeader, ConfirmModal } from '@/components/ui/modal'; import { Button } from '@/components/ui/button'; import { Skeleton } from '@/components/ui/skeleton'; @@ -12,6 +12,10 @@ import { GitSourceDiffDialog, type PullResult, type PublicPendingPlan } from './ import { GitSourceFields, type ApplyMode } from './GitSourceFields'; import { GitManifestSummary, type ManifestSummary } from './GitManifestSummary'; import type { GitBrowseResult } from './GitComposeFilePicker'; +import GitOpsStateCard, { GitOpsFaultCard } from '@/components/gitops/GitOpsStateCard'; +import GitOpsCaveats from '@/components/gitops/GitOpsCaveats'; +import { SOURCE_STATE_LOOKUP, absentFault, liveSourceFacet, type LiveSourceFacet } from '@/lib/gitopsState'; +import type { GitOpsRevisionCarrier, GitOpsRevisionProjection, GitOpsSourceStatus } from '@/types/gitops'; export interface GitSource { id: number; @@ -39,6 +43,11 @@ export interface GitSource { manifest: ManifestSummary | null; } +// The GET carries the revision on both of its 200 shapes. The PUT does not, +// which is why GitSource itself stays free of it. +type GitSourceRead = GitSource & GitOpsRevisionCarrier; +type GitSourceUnlinked = { linked: false } & GitOpsRevisionCarrier; + interface GitSourcePanelProps { open: boolean; onOpenChange: (open: boolean) => void; @@ -56,6 +65,36 @@ function deriveApplyMode(source: GitSource | null, pendingMode: ApplyMode | null return source.auto_deploy_on_apply ? 'auto-deploy' : 'auto-write'; } +/** The commit the pending banner announces, or null when there is nothing to announce. */ +interface PendingCommit { + status: GitOpsSourceStatus; + /** Short-sha detail line; null when the state is known but the commit is not. */ + sha: string | null; +} + +/** + * What the pending banner shows, from the projection when one answered and from + * the flat pointer when none did. + * + * The fallback is reachable when a GitOps write failed and was swallowed while + * the pending commit still committed, and it is what this banner read before the + * projection existed. A fault suppresses it: that means an application was + * expected and could not be read, so the pointer is not evidence a candidate is + * ready. The sidebar applies the same rule, so the two surfaces cannot disagree. + */ +function derivePendingCommit( + facet: LiveSourceFacet | null, + faultCount: number, + flatPendingSha: string | null, +): PendingCommit | null { + if (facet) { + if (facet.candidateGenerationId === null) return null; + return { status: facet.status, sha: facet.fetchedCommitSha }; + } + if (faultCount > 0 || !flatPendingSha) return null; + return { status: 'candidate_ready', sha: flatPendingSha }; +} + export function GitSourcePanel({ open, onOpenChange, @@ -69,6 +108,10 @@ export function GitSourcePanel({ const [pulling, setPulling] = useState(false); const [applying, setApplying] = useState(false); const [source, setSource] = useState(null); + // Kept out of `source` on purpose: the PUT that saves this panel answers with + // a bare Git source and no revision, so carrying it on that type would make + // the save path a lie. + const [revision, setRevision] = useState(null); const [repoUrl, setRepoUrl] = useState(''); const [branch, setBranch] = useState('main'); @@ -87,6 +130,10 @@ export function GitSourcePanel({ const { activeNode } = useNodes(); const applyMode = deriveApplyMode(source, applyModeOverride); + const sourceFacet = liveSourceFacet(revision); + const faults = revision ? absentFault(revision) : []; + const pending = derivePendingCommit(sourceFacet, faults.length, source?.pending_commit_sha ?? null); + const resetToUnlinked = useCallback(() => { setSource(null); setRepoUrl(''); @@ -104,7 +151,8 @@ export function GitSourcePanel({ try { const res = await apiFetch(`/stacks/${encodeURIComponent(stackName)}/git-source`); if (res.ok) { - const data: GitSource | { linked: false } = await res.json(); + const data: GitSourceRead | GitSourceUnlinked = await res.json(); + setRevision(data.gitopsRevision); // An existing stack with no Git source attached answers 200 { linked: false }. if ('linked' in data) { resetToUnlinked(); @@ -121,14 +169,21 @@ export function GitSourcePanel({ } } else if (res.status === 404) { resetToUnlinked(); + setRevision(null); } else if (res.status === 403) { setSource(null); + setRevision(null); toast.error('You do not have permission to view this stack\'s Git source.'); } else { + setRevision(null); const err = await res.json().catch(() => ({})); toast.error(err?.error || 'Failed to load Git source.'); } } catch (e) { + // Clear alongside the other failure branches: the panel is reused across + // stacks, so a revision left behind would render one stack's state under + // another stack's header. + setRevision(null); toast.error((e as Error)?.message || 'Network error.'); } finally { setLoading(false); @@ -173,12 +228,17 @@ export function GitSourcePanel({ body: JSON.stringify(body), }); if (res.ok) { - const data: GitSource = await res.json(); - setSource(data); setToken(''); setApplyModeOverride(null); toast.success('Git source saved.'); onSourceChanged?.(); + // Re-read rather than trust the save response, which carries the source + // row without a revision. A material configuration change clears the + // staged candidate server side, so the state held here has genuinely + // moved; dropping it left the panel blank until the next open, which + // reads as "this stack has no GitOps state" rather than as a state that + // was just invalidated. + await load(); } else { const err = await res.json().catch(() => ({})); toast.error(err?.error || 'Failed to save Git source.'); @@ -231,6 +291,11 @@ export function GitSourcePanel({ if (res.ok) { toast.success('Git source removed.'); setSource(null); + // Detaching is a stronger invalidation than a save: the projection now + // describes a source that is gone, and the pending card is derived from + // the revision alone, so leaving it would advertise a waiting commit on + // a stack Git no longer manages. + setRevision(null); onSourceChanged?.(); } else { const err = await res.json().catch(() => ({})); @@ -338,11 +403,14 @@ export function GitSourcePanel({ method: 'POST', }); if (res.ok) { + toast.success('Pending update dismissed.'); setDiffOpen(false); setPull(null); await load(); onSourceChanged?.(); - toast.success('Pending update dismissed.'); + } else { + const err = await res.json().catch(() => ({})); + toast.error(err?.error || 'Failed to dismiss the pending update.'); } } catch (e) { toast.error((e as Error)?.message || 'Network error.'); @@ -373,36 +441,35 @@ export function GitSourcePanel({
    ) : ( <> - {source?.pending_commit_sha && ( -
    - -
    -

    - {source.pending_plan?.blocked ? 'Pending update blocked' : 'Pending update'} -

    -

    - Commit {source.pending_commit_sha.slice(0, 7)} - {source.pending_plan?.blocked - ? ' has local conflicts. Review the plan; apply stays disabled until they are resolved.' - : ' is ready to review.'} -

    -
    - -
    + {faults.length > 0 && } + + {pending && ( + pullNow()} + disabled={pulling} + > + Review + + )} + > + {pending.sha && ( +
    + Commit {pending.sha.slice(0, 7)} +
    + )} +     )} + +
    + {sourceFacet && ( +
    + Source state + {SOURCE_STATE_LOOKUP[sourceFacet.status]?.label ?? sourceFacet.status} +
    + )}
    Updated diff --git a/frontend/src/lib/blueprintsApi.ts b/frontend/src/lib/blueprintsApi.ts index e5bd1f2d..72f4b888 100644 --- a/frontend/src/lib/blueprintsApi.ts +++ b/frontend/src/lib/blueprintsApi.ts @@ -1,4 +1,5 @@ import { apiFetch } from './api'; +import type { GitOpsRevisionCarrier, GitOpsRevisionsCarrier } from '@/types/gitops'; export type DriftMode = 'observe' | 'suggest' | 'enforce'; export type BlueprintClassification = 'stateless' | 'stateful' | 'unknown'; @@ -41,7 +42,7 @@ export interface Blueprint { export type EffectiveApproval = 'pending' | 'approved' | 'reapproval_required'; -export interface BlueprintListItem extends Blueprint { +export interface BlueprintListItem extends Blueprint, GitOpsRevisionCarrier { deploymentCounts: Partial>; deploymentTotal: number; effectiveApproval?: EffectiveApproval; @@ -60,13 +61,24 @@ export interface BlueprintDeployment { last_error: string | null; } -export interface BlueprintSummary { +/** Note the GitOps revision sits on the envelope, a sibling of `blueprint`, not inside it. */ +export interface BlueprintSummary extends GitOpsRevisionCarrier { blueprint: Blueprint; deployments: BlueprintDeployment[]; statusCounts: Partial>; effectiveApproval?: EffectiveApproval; } +/** + * Create, update and pin all answer with the Blueprint plus its revision. + * + * Declared and deliberately unread: every caller re-reads the catalog or the + * detail immediately afterwards and that read carries the same projection. The + * field is named here so the next reader does not take these for a bare + * Blueprint and quietly drop it. + */ +export type BlueprintMutationResult = Blueprint & GitOpsRevisionCarrier; + export interface AnalyzerResult { classification: BlueprintClassification; reasons: string[]; @@ -200,13 +212,13 @@ export interface CreateBlueprintInput { enabled?: boolean; } -export async function createBlueprint(input: CreateBlueprintInput): Promise { +export async function createBlueprint(input: CreateBlueprintInput): Promise { const res = await apiFetch('/blueprints', { method: 'POST', body: JSON.stringify(input), localOnly: true, }); - return expectJson(res, 'Failed to create blueprint'); + return expectJson(res, 'Failed to create blueprint'); } export interface UpdateBlueprintInput { @@ -218,13 +230,13 @@ export interface UpdateBlueprintInput { enabled?: boolean; } -export async function updateBlueprint(id: number, input: UpdateBlueprintInput): Promise { +export async function updateBlueprint(id: number, input: UpdateBlueprintInput): Promise { const res = await apiFetch(`/blueprints/${id}`, { method: 'PUT', body: JSON.stringify(input), localOnly: true, }); - return expectJson(res, 'Failed to update blueprint'); + return expectJson(res, 'Failed to update blueprint'); } export async function deleteBlueprint(id: number): Promise { @@ -268,13 +280,13 @@ export async function applyBlueprint( return expectJson(res, 'Failed to apply blueprint'); } -export async function pinBlueprint(id: number, nodeId: number | null): Promise { +export async function pinBlueprint(id: number, nodeId: number | null): Promise { const res = await apiFetch(`/blueprints/${id}/pin`, { method: 'PUT', body: JSON.stringify({ nodeId }), localOnly: true, }); - return expectJson(res, 'Failed to update blueprint pin'); + return expectJson(res, 'Failed to update blueprint pin'); } export async function withdrawDeployment( @@ -336,7 +348,10 @@ export async function getLabelsForNode(nodeId: number): Promise { return data.labels; } -export async function addNodeLabel(nodeId: number, label: string): Promise<{ nodeId: number; label: string }> { +export async function addNodeLabel( + nodeId: number, + label: string, +): Promise<{ nodeId: number; label: string } & GitOpsRevisionsCarrier> { const res = await apiFetch(`/node-labels/${nodeId}`, { method: 'POST', body: JSON.stringify({ label }), diff --git a/frontend/src/lib/gitopsLimitations.test.ts b/frontend/src/lib/gitopsLimitations.test.ts new file mode 100644 index 00000000..a1d0ad0d --- /dev/null +++ b/frontend/src/lib/gitopsLimitations.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from 'vitest'; +import { GITOPS_LIMITATION_COPY, limitationCaveat, limitationCaveats } from './gitopsLimitations'; +import type { GitOpsLimitation } from '@/types/gitops'; + +const limitation = (code: string, evidence: unknown = null): GitOpsLimitation => ({ + code, + message: 'backend wording, not for display', + evidence, +}); + +/** + * Every code the backend can put on the live arm. + * + * Kept as a literal list rather than derived, because it is the assertion: the + * point is to notice when the backend gains a code this map has not been told + * about. Sources are `derive.ts` (projection time) and the write-time evidence + * channel that `mergePersistedLimitations` folds in. The two absent-arm codes + * (`application_row_missing`, `blueprint_application_missing`) and the + * history-row code (`history_json_invalid`) are deliberately excluded: those + * are faults and audit-row defects, which other affordances own. + */ +const LIVE_ARM_CODES = [ + 'repo_identity_invalid', + 'candidate_generation_invalid', + 'accepted_generation_invalid', + 'artifact_pointer_missing', + 'artifact_evidence_json_invalid', + 'connectivity_invalid', + 'lkg_generation_missing', + 'lkg_artifact_invalid', + 'evidence_limitations_invalid', + 'artifact_observation_invalid', + 'artifact_observation_decode_failed', + 'recovery_unproven', + 'lkg_artifact_unprovable', + 'source_acceptance_unprovable', + 'artifact_expectation_unprovable', + 'manifest_absent', + 'manifest_corrupt', + 'manifest_identity_invalid', + 'manifest_commit_unresolved', + 'manifest_commit_mismatch', + 'legacy_pending', + 'blueprint_reapproval_required', +] as const; + +describe('gitops limitation copy', () => { + it('covers every code the live arm can carry', () => { + const missing = LIVE_ARM_CODES.filter((code) => !(code in GITOPS_LIMITATION_COPY)); + expect(missing).toEqual([]); + }); + + it('carries no copy for a code that is not a live-arm caveat', () => { + // A fault replaces the state rather than qualifying it, so giving it caveat + // copy here would invite a surface to render it as the milder thing. + expect(GITOPS_LIMITATION_COPY).not.toHaveProperty('application_row_missing'); + expect(GITOPS_LIMITATION_COPY).not.toHaveProperty('blueprint_application_missing'); + expect(GITOPS_LIMITATION_COPY).not.toHaveProperty('history_json_invalid'); + }); + + it('never shows the backend message', () => { + // The stored messages are written for a log reader ("repo identity json is + // invalid") and several are raw decoder errors. + for (const code of LIVE_ARM_CODES) { + expect(limitationCaveat(limitation(code))).not.toBe('backend wording, not for display'); + } + }); + + it('names an unrecognised code rather than dropping it', () => { + // A newer node can send a code this build has never heard of. Saying + // nothing would report full confidence in a state the backend flagged. + expect(limitationCaveat(limitation('something_new_entirely'))) + .toBe('Part of this state could not be proven (something_new_entirely).'); + }); + + it('states a condition in each sentence, ending it properly', () => { + for (const code of LIVE_ARM_CODES) { + const copy = GITOPS_LIMITATION_COPY[code]; + // Narrowed rather than asserted: the coverage case above is what proves + // every code has copy, so a miss here would otherwise be reported as a + // property access on undefined instead of as the missing entry it is. + expect(copy, `no operator copy for ${code}`).toBeDefined(); + if (!copy) continue; + expect(copy.length).toBeGreaterThan(40); + expect(copy.endsWith('.')).toBe(true); + expect(copy).not.toMatch(/—/); + } + }); + + it('carries no copy for a code the backend cannot emit', () => { + // The reverse of the coverage case above: a retired code leaving stale + // wording behind is invisible without this, because the fallback only + // fires for codes that are missing rather than for ones that linger. + const documented = Object.keys(GITOPS_LIMITATION_COPY).sort(); + expect(documented).toEqual([...LIVE_ARM_CODES].sort()); + }); + + it('collapses the same caveat arriving from several places', () => { + // One condition can be recorded per target and again per application. + const caveats = limitationCaveats([ + limitation('lkg_generation_missing', 'gen-a'), + limitation('lkg_generation_missing', 'gen-b'), + limitation('legacy_pending'), + ]); + + expect(caveats).toHaveLength(2); + expect(caveats[0]).toBe(GITOPS_LIMITATION_COPY.lkg_generation_missing); + expect(caveats[1]).toBe(GITOPS_LIMITATION_COPY.legacy_pending); + }); + + it('returns nothing for a projection with no caveats', () => { + expect(limitationCaveats([])).toEqual([]); + }); +}); diff --git a/frontend/src/lib/gitopsLimitations.ts b/frontend/src/lib/gitopsLimitations.ts new file mode 100644 index 00000000..7d70fd1d --- /dev/null +++ b/frontend/src/lib/gitopsLimitations.ts @@ -0,0 +1,110 @@ +// Operator wording for the caveats a live projection can carry. +// +// A limitation on the live arm is not a failure. It says the state being shown +// is real but one piece of evidence behind it could not be proven, so a reader +// knows which part to distrust. The absent arm is different: there, a +// limitation means an application we expected could not be reached at all, and +// `absentFault` in gitopsState.ts handles that as a fault. +// +// The backend messages are written for whoever is reading a log ("repo identity +// json is invalid"), so they are deliberately not surfaced. These sentences say +// what is uncertain and what follows from it. +// +// Each entry was checked against the site that emits it rather than against the +// code's name. Several names suggest something narrower or wider than the +// condition actually tested. +// +// A backend code with no entry here degrades to the fallback below rather than +// vanishing, so drift is safe rather than silent. The accompanying test lists +// every code the live arm can carry, which is narrower than every code the +// backend emits: the absent-arm faults and the history-row defect are handled +// elsewhere and deliberately have no copy here. + +import type { GitOpsLimitation } from '@/types/gitops'; + +/** + * Keyed on an open string, not a union. + * + * The wire type is `code: string` and a limitation can be minted at write time + * and merged into the projection later, so a node running a newer build can + * legitimately send a code this build has never heard of. Closing the type here + * would only move that surprise to a type assertion. + * + * The value is optional so a lookup miss is a fact the compiler produces + * rather than one a comment asserts. This project does not set + * `noUncheckedIndexedAccess`, so a plain `Record` would type + * every miss as a `string` and make the fallback below look like dead code to + * anything that trusts the types. + */ +export const GITOPS_LIMITATION_COPY: Record = { + // --- derived while projecting ------------------------------------------- + repo_identity_invalid: + 'The stored repository identity could not be read, so this state cannot be tied back to a specific repository.', + candidate_generation_invalid: + 'The pending change points at a generation that is missing or belongs to another application, so it must be fetched again before it can be applied.', + accepted_generation_invalid: + 'The recorded accepted generation is missing or belongs to another application, so this state cannot be trusted until the source has been fetched and applied again.', + artifact_pointer_missing: + 'An artifact record this state refers to is no longer present, so what was built for this generation cannot be described.', + artifact_evidence_json_invalid: + 'An artifact record could not be read, so what is running cannot be compared against what was expected.', + connectivity_invalid: + 'The stored reachability of this node is not a value Sencho recognises, so it is being treated as unknown.', + lkg_generation_missing: + 'The generation recorded as last known good is gone, so there is nothing to fall back to.', + lkg_artifact_invalid: + 'The artifact captured with the last known good is missing or does not belong to it, so that fallback is no longer fully qualified.', + evidence_limitations_invalid: + 'The record of what could not be proven is itself unreadable, so there may be further caveats that cannot be shown.', + artifact_observation_invalid: + 'What is running on this node could not be read, so it is reported as unidentified rather than as matching.', + artifact_observation_decode_failed: + 'What is running on this node could not be read, so it is reported as unidentified rather than as matching.', + + // --- recorded at write time, merged in later ------------------------------ + recovery_unproven: + 'A recovery ran but could not be tied to a specific generation, so the pointers were left where they were rather than moved on an unproven claim.', + lkg_artifact_unprovable: + 'The artifact captured with the last known good could not be proven during recovery, so the fallback is available but no longer qualified.', + source_acceptance_unprovable: + 'The approval that authorized the generation now in place could not be restored, so this node is running an accepted generation with no approval attached to it.', + artifact_expectation_unprovable: + 'The expected artifact could not be restored during recovery, so drift between what is running and what was intended is not being checked on this node.', + manifest_absent: + 'No managed manifest was found for this stack, so the commit recorded before Sencho tracked it is kept only as evidence and not treated as current.', + manifest_corrupt: + 'The managed manifest could not be read, so the commit recorded before Sencho tracked it is kept only as evidence and not treated as current.', + manifest_identity_invalid: + 'The managed manifest does not identify this stack on this node from the repository configured now, so its commit is kept only as evidence.', + manifest_commit_unresolved: + 'The managed manifest records no commit, so the commit recorded before Sencho tracked this stack cannot be confirmed against what is on disk. Fetch to resolve one.', + manifest_commit_mismatch: + 'The managed manifest names a different commit than the one recorded as applied, so neither is treated as current. Fetch again to settle which one is on disk.', + legacy_pending: + 'A pending commit was recorded before Sencho tracked this stack and carries no proof of which repository or branch it came from. Fetch again to rebuild it.', + blueprint_reapproval_required: + 'The stored approval does not cover what this Blueprint currently asks for, so it needs approving again before it can roll out.', +}; + +/** + * Operator wording for one limitation, or a safe fallback. + * + * An unrecognised code names itself rather than being dropped. Saying nothing + * would report full confidence in a state the backend flagged, which is the one + * outcome this affordance exists to prevent. + */ +export function limitationCaveat(limitation: GitOpsLimitation): string { + return GITOPS_LIMITATION_COPY[limitation.code] + ?? `Part of this state could not be proven (${limitation.code}).`; +} + +/** + * The caveats worth showing for a projection's limitations, de-duplicated. + * + * One condition can be recorded per target and again per application, so the + * same sentence can arrive several times over. Repeating it would read as + * several separate problems. + */ +export function limitationCaveats(limitations: readonly GitOpsLimitation[]): string[] { + return [...new Set(limitations.map(limitationCaveat))]; +} diff --git a/frontend/src/lib/gitopsState.test.ts b/frontend/src/lib/gitopsState.test.ts new file mode 100644 index 00000000..d8fb1d29 --- /dev/null +++ b/frontend/src/lib/gitopsState.test.ts @@ -0,0 +1,260 @@ +import { describe, expect, it } from 'vitest'; + +import { + absentRevision, + facets, + liveRevision, + missingApplicationLimitation, + plainSource, + sourceIdentity, +} from '@/__tests__/gitopsFixtures'; +import { + GITOPS_TONE_CLASS, + RUNTIME_STATE, + SOURCE_STATE, + absentFault, + identityRefLabel, + liveSourceFacet, + pendingSourceStatus, + type GitOpsTone, +} from '@/lib/gitopsState'; +import type { GitOpsIdentityRef, GitOpsRuntimeStatus, GitOpsSourceStatus } from '@/types/gitops'; + +// Listed rather than derived from the map: this is the copy of the contract the +// test owns, so a status silently dropped from SOURCE_STATE fails here instead +// of the assertion quietly iterating one fewer key. +const SOURCE_STATUSES: GitOpsSourceStatus[] = [ + 'not_applicable', + 'never_reconciled', + 'checking_fetching', + 'application_generation_accepted', + 'candidate_ready', + 'source_review_pending', + 'source_conflict_blocker', + 'source_reconcile_required', + 'source_superseded', + 'applying', + 'source_retry_scheduled', + 'source_suspended', + 'source_failed', + 'source_unknown', + 'recovery_required', + 'recovery_failed', + 'not_live', +]; + +const RUNTIME_STATUSES: GitOpsRuntimeStatus[] = [ + 'tombstoned', + 'recovery_required', + 'deploying', + 'withdrawing', + 'failed_previous_workload_intact', + 'failed_after_mutation', + 'disk_invocation_drift', + 'rollout_artifact_drift', + 'runtime_artifact_drift', + 'artifact_verification_pending', + 'never_applied', + 'applied_not_deployed', + 'acknowledged_completion_unknown', + 'stale_acknowledgement', + 'pending_state_review', + 'evict_blocked', + 'drifted', + 'correcting', + 'fully_deployed_health_pending', + 'health_checking', + 'synced_and_healthy', + 'health_drift', + 'partially_rolled_out', + 'retry_scheduled', + 'paused', + 'recovery_failed', + 'completion_unknown', +]; + +const TONES: GitOpsTone[] = ['brand', 'success', 'warning', 'destructive', 'neutral']; + +describe('the state vocabulary', () => { + it('names every source status', () => { + expect(Object.keys(SOURCE_STATE).sort()).toEqual([...SOURCE_STATUSES].sort()); + }); + + it('names every runtime status', () => { + expect(Object.keys(RUNTIME_STATE).sort()).toEqual([...RUNTIME_STATUSES].sort()); + }); + + it('gives every state a tone from the five semantic slots and copy that stands alone', () => { + for (const meta of [...Object.values(SOURCE_STATE), ...Object.values(RUNTIME_STATE)]) { + expect(TONES).toContain(meta.tone); + expect(meta.label.trim().length).toBeGreaterThan(0); + // The line doubles as the sidebar tooltip, so it has to be a sentence. + expect(meta.line.trim()).toMatch(/\.$/); + expect(meta.line).not.toContain('—'); + expect(meta.label).not.toContain('—'); + } + }); + + it('has a card class for every tone', () => { + for (const tone of TONES) expect(GITOPS_TONE_CLASS[tone]).toBeTruthy(); + }); +}); + +describe('pendingSourceStatus', () => { + it('is null when there is no application to ask', () => { + expect(pendingSourceStatus(absentRevision())).toBeNull(); + }); + + it('is null for an application with no Git source', () => { + const revision = liveRevision({ + targetMode: 'inline_blueprint', + facets: facets({ + source: { status: 'not_applicable' }, + placement: { status: 'blueprint_bound', completion: 'unknown' }, + }), + }); + expect(pendingSourceStatus(revision)).toBeNull(); + }); + + it('is null when no candidate is waiting, even for a status that can also mean one is', () => { + // source_reconcile_required is reachable from the accepted generation with + // no candidate at all. Flagging that stack would light an indicator that is + // blank today, on a stack with nothing to review. + const revision = liveRevision({ + facets: facets({ source: plainSource('source_reconcile_required', { candidateGenerationId: null }) }), + }); + expect(pendingSourceStatus(revision)).toBeNull(); + }); + + it('is null for a retired application that still carries a candidate pointer', () => { + // Tombstoning keeps the candidate pointer as a frozen fact, so a stack + // detached while a commit was staged still has one. Reporting it would + // advertise an update on a stack Git no longer manages. + const revision = liveRevision({ + lifecycleStatus: 'detached', + facets: facets({ source: { ...sourceIdentity(), status: 'not_live', lifecycleStatus: 'detached' } }), + }); + expect(revision.facets.source).toHaveProperty('candidateGenerationId', 'gen-candidate'); + expect(pendingSourceStatus(revision)).toBeNull(); + }); + + it('reports the exact status of a waiting candidate', () => { + const statuses = [ + 'candidate_ready', + 'source_conflict_blocker', + 'source_review_pending', + 'source_reconcile_required', + ] as const; + for (const status of statuses) { + const revision = liveRevision({ facets: facets({ source: plainSource(status) }) }); + expect(pendingSourceStatus(revision)).toBe(status); + } + }); + + it('reports a candidate held behind an in-flight apply', () => { + const revision = liveRevision({ + facets: facets({ + source: { ...sourceIdentity(), status: 'applying', activeOperationId: 'op-1', activeGenerationId: 'gen-1' }, + }), + }); + expect(pendingSourceStatus(revision)).toBe('applying'); + }); +}); + +describe('liveSourceFacet', () => { + it('is null when there is no revision to read', () => { + expect(liveSourceFacet(null)).toBeNull(); + }); + + it('is null when there is no application to ask', () => { + expect(liveSourceFacet(absentRevision())).toBeNull(); + }); + + it('is null for a Blueprint-owned application, which has no Git source', () => { + const revision = liveRevision({ + targetMode: 'inline_blueprint', + facets: facets({ + source: { status: 'not_applicable' }, + placement: { status: 'blueprint_bound', completion: 'unknown' }, + }), + }); + expect(liveSourceFacet(revision)).toBeNull(); + }); + + it('returns the facet, identity fields and all, for a live Git source', () => { + const source = plainSource('source_review_pending'); + expect(liveSourceFacet(liveRevision({ facets: facets({ source }) }))).toEqual(source); + }); + + it('returns a retired source facet, which is a state to name rather than hide', () => { + // not_live is in SOURCE_STATE and reads as "the identity shown is what it + // was". Only pendingSourceStatus excludes it, because it is not an update + // waiting to be applied. + const source = { ...sourceIdentity(), status: 'not_live' as const, lifecycleStatus: 'detached' as const }; + expect(liveSourceFacet(liveRevision({ facets: facets({ source }) }))).toEqual(source); + }); +}); + +describe('absentFault', () => { + it('is empty for a stack the model was never asked about', () => { + expect(absentFault(absentRevision())).toEqual([]); + }); + + it('reports an application that was expected and could not be reached', () => { + expect(absentFault(absentRevision([missingApplicationLimitation]))).toEqual([ + missingApplicationLimitation, + ]); + }); + + it('is empty for a live application, even one carrying limitations', () => { + // A live arm's limitations are caveats on state that is being reported, not + // faults. Merging the two would recreate the conflation in reverse. + const revision = liveRevision({ + limitations: [{ code: 'repo_identity_invalid', message: 'Repository identity could not be read.', evidence: null }], + }); + expect(absentFault(revision)).toEqual([]); + }); +}); + +describe('identityRefLabel', () => { + const cases: Array<[GitOpsIdentityRef, string]> = [ + [{ kind: 'none' }, 'none'], + [{ kind: 'unknown' }, 'unknown'], + [{ kind: 'commit', sha: 'a1b2c3d4e5f6', repoUrl: 'https://example.test/a.git', ref: 'main' }, 'commit a1b2c3d'], + [{ kind: 'generation', id: 'gen-12345678-x' }, 'generation gen-1234'], + [ + { kind: 'artifact_set', id: 'art-12345678-x', qualification: 'exact', evidenceVersion: 3 }, + 'artifact art-1234 · exact', + ], + [{ kind: 'runtime_artifact', identity: 'nginx@sha256:abc', observedAt: 1 }, 'nginx@sha256:abc'], + [{ kind: 'intent', id: 'int-12345678-x', composeContentSha256: 'deadbeef' }, 'intent int-1234'], + [{ kind: 'rollout_candidate', id: 'rc-123456789' }, 'candidate rc-12345'], + [{ kind: 'rollout_generation', id: 'rg-123456789' }, 'rollout rg-12345'], + [ + { + kind: 'invocation', + authored: { + composeFileOrder: ['compose.yaml', 'compose.override.yaml'], + projectName: null, + projectDirectory: null, + envFileOrder: [], + }, + }, + 'compose.yaml, compose.override.yaml', + ], + [{ kind: 'health_run', runId: 'run-12345678-x', deployedGenerationId: null }, 'health run run-1234'], + ]; + + it.each(cases)('labels %o', (ref, expected) => { + expect(identityRefLabel(ref)).toBe(expected); + }); + + it('names an invocation that authored no compose files', () => { + expect( + identityRefLabel({ + kind: 'invocation', + authored: { composeFileOrder: [], projectName: null, projectDirectory: null, envFileOrder: [] }, + }), + ).toBe('no compose files'); + }); +}); diff --git a/frontend/src/lib/gitopsState.ts b/frontend/src/lib/gitopsState.ts new file mode 100644 index 00000000..0534b067 --- /dev/null +++ b/frontend/src/lib/gitopsState.ts @@ -0,0 +1,466 @@ +// The one place a GitOps facet status becomes words and a colour. +// +// Every surface that shows GitOps state reads these maps, so a status is named +// the same way in a sidebar tooltip, the Git source panel and the Drift tab. +// +// Both maps are Records keyed on the closed status unions in types/gitops.ts. +// That does not react to a backend change on its own, since the mirror is +// hand-written: it means the build fails here the moment someone widens the +// mirror, which is the step that would otherwise leave a status rendering blank. +// +// Each `line` states the condition the backend actually derives, not the one +// the status name suggests. Several of them differ. + +import { + Activity, + ArchiveX, + Ban, + Check, + CircleAlert, + CircleDashed, + CircleHelp, + CirclePause, + CircleSlash, + CircleX, + Clock, + Download, + Hourglass, + RefreshCw, + Rocket, + ShieldAlert, + TriangleAlert, + Undo2, + Upload, + type LucideIcon, +} from 'lucide-react'; + +import type { + GitOpsIdentityRef, + GitOpsLimitation, + GitOpsRevisionProjection, + GitOpsRuntimeStatus, + GitOpsSourceStatus, + SourceFacet, +} from '@/types/gitops'; + +/** The five semantic slots the design system defines. Fuchsia is reserved for image updates. */ +export type GitOpsTone = 'brand' | 'success' | 'warning' | 'destructive' | 'neutral'; + +export interface GitOpsStateMeta { + /** Short name of the state, rendered in mono uppercase. */ + label: string; + tone: GitOpsTone; + /** A complete sentence. Doubles as the sidebar tooltip, so it has to stand alone. */ + line: string; + icon: LucideIcon; +} + +/** Card classes per tone. Identical to the drift status cards so the families read as one. */ +export const GITOPS_TONE_CLASS: Record = { + brand: 'border-brand/40 bg-brand/[0.06] text-brand', + success: 'border-success/40 bg-success/[0.06] text-success', + warning: 'border-warning/40 bg-warning/[0.06] text-warning', + destructive: 'border-destructive/40 bg-destructive/[0.06] text-destructive', + neutral: 'border-muted bg-card/40 text-stat-subtitle', +}; + +export const SOURCE_STATE: Record = { + not_applicable: { + label: 'no git source', + tone: 'neutral', + line: 'This application is not backed by a Git source.', + icon: CircleSlash, + }, + never_reconciled: { + label: 'never reconciled', + tone: 'neutral', + // Says accepted, not fetched: a fetch that produced no materialization + // records its commit and still leaves no generation behind. + line: 'No commit from this repository has been accepted yet.', + icon: CircleDashed, + }, + checking_fetching: { + label: 'fetching', + tone: 'brand', + line: 'Sencho is fetching from the repository.', + icon: Download, + }, + application_generation_accepted: { + label: 'accepted', + tone: 'success', + line: 'The fetched commit has been accepted as the current generation.', + icon: Check, + }, + candidate_ready: { + label: 'pending update', + tone: 'brand', + // This status is reached only when review is not required, and it is the + // one status that offers apply. Saying "ready to review" would describe + // source_review_pending, which is the opposite state. + line: 'A fetched commit is ready to apply.', + icon: CircleAlert, + }, + source_review_pending: { + label: 'review required', + tone: 'warning', + line: 'A fetched commit is waiting for review before it can apply.', + icon: Hourglass, + }, + source_conflict_blocker: { + label: 'pending update blocked', + tone: 'warning', + line: 'The change plan has local conflicts. Apply stays disabled until they are resolved.', + icon: TriangleAlert, + }, + source_reconcile_required: { + label: 'reconcile required', + tone: 'warning', + // Reachable both from a stale candidate and from an accepted generation + // with no candidate at all, so the line cannot name a fetched commit. + line: 'What was reconciled no longer matches the configuration in force. Fetch again to rebuild the candidate.', + icon: RefreshCw, + }, + source_superseded: { + label: 'superseded', + tone: 'neutral', + line: 'A newer generation replaced the one this state describes.', + icon: ArchiveX, + }, + applying: { + label: 'applying', + tone: 'brand', + line: 'A commit is being applied to the stack directory.', + icon: Upload, + }, + source_retry_scheduled: { + label: 'retry scheduled', + tone: 'warning', + line: 'The last attempt failed and a retry is scheduled.', + icon: Clock, + }, + source_suspended: { + label: 'suspended', + tone: 'neutral', + line: 'Reconciliation is suspended for this source.', + icon: CirclePause, + }, + source_failed: { + label: 'source failed', + tone: 'destructive', + line: 'The last operation on this source failed.', + icon: CircleX, + }, + source_unknown: { + label: 'outcome unknown', + tone: 'warning', + line: 'An operation was interrupted, so its outcome could not be confirmed.', + icon: CircleHelp, + }, + recovery_required: { + label: 'recovering', + // In flight, not pending: the only derivation is a recovery phase of + // restoring or compensating, and no action is offered while it runs. + tone: 'brand', + line: 'Recovery is running on this stack.', + icon: Undo2, + }, + recovery_failed: { + label: 'recovery failed', + tone: 'destructive', + line: 'Recovery of this stack did not complete.', + icon: ShieldAlert, + }, + not_live: { + label: 'not live', + tone: 'neutral', + line: 'This application is no longer live. The identity shown is what it was.', + icon: CircleSlash, + }, +}; + +export const RUNTIME_STATE: Record = { + tombstoned: { + label: 'tombstoned', + tone: 'neutral', + // Still projected, so "no longer tracked" would be wrong. It is retired. + line: 'This target is retired and is no longer reconciled.', + icon: ArchiveX, + }, + never_applied: { + label: 'never applied', + tone: 'neutral', + line: 'No generation has been applied on this node yet.', + icon: CircleDashed, + }, + deploying: { + label: 'deploying', + tone: 'brand', + line: 'A deploy is in progress on this node.', + icon: Rocket, + }, + withdrawing: { + label: 'withdrawing', + tone: 'brand', + line: 'The stack is being withdrawn from this node.', + icon: Undo2, + }, + correcting: { + label: 'correcting', + tone: 'brand', + line: 'Sencho is correcting this node back to the intended state.', + icon: RefreshCw, + }, + health_checking: { + label: 'health checking', + tone: 'brand', + line: 'A health run is watching the current deploy.', + icon: Activity, + }, + fully_deployed_health_pending: { + label: 'health pending', + tone: 'brand', + line: 'The generation is deployed and its health verdict is still pending.', + icon: Hourglass, + }, + artifact_verification_pending: { + label: 'artifact unverified', + tone: 'brand', + line: 'What is running could not be identified precisely enough to compare.', + icon: CircleHelp, + }, + synced_and_healthy: { + label: 'synced and healthy', + tone: 'success', + // Claims neither acceptance nor a health run. The deriver never compares + // the deployed generation with the accepted one, and it reaches this state + // with no health run at all when the health gate is off. + line: 'This node is running its deployed generation with nothing outstanding.', + icon: Check, + }, + applied_not_deployed: { + label: 'applied not deployed', + tone: 'warning', + // The target's own applied pointer, which lags the application's accepted + // generation whenever this node is behind. + line: 'The applied generation is on disk but has not been deployed.', + icon: Upload, + }, + drifted: { + label: 'drifted', + tone: 'warning', + line: 'What is running no longer matches the intended generation.', + icon: TriangleAlert, + }, + health_drift: { + label: 'health drift', + tone: 'warning', + line: 'The deployed generation is current but its health check is failing.', + icon: Activity, + }, + disk_invocation_drift: { + label: 'invocation drift', + tone: 'warning', + line: 'The files on disk no longer match the invocation that deployed them.', + icon: TriangleAlert, + }, + rollout_artifact_drift: { + label: 'rollout artifact drift', + tone: 'warning', + line: 'The planned rollout artifact differs from the one this node holds.', + icon: TriangleAlert, + }, + runtime_artifact_drift: { + label: 'runtime artifact drift', + tone: 'warning', + line: 'The running image differs from the expected artifact for this node.', + icon: TriangleAlert, + }, + stale_acknowledgement: { + label: 'stale acknowledgement', + tone: 'warning', + line: 'This node acknowledged a generation that is no longer current.', + icon: Clock, + }, + acknowledged_completion_unknown: { + label: 'completion unknown', + tone: 'warning', + line: 'This node acknowledged the work but its outcome was never confirmed.', + icon: CircleHelp, + }, + pending_state_review: { + label: 'state review pending', + tone: 'warning', + line: 'This node is holding stateful changes for review before it proceeds.', + icon: Hourglass, + }, + evict_blocked: { + label: 'evict blocked', + tone: 'warning', + line: 'The stack cannot be removed from this node yet.', + icon: Ban, + }, + retry_scheduled: { + label: 'retry scheduled', + tone: 'warning', + line: 'The last attempt on this node failed and a retry is scheduled.', + icon: Clock, + }, + recovery_required: { + label: 'recovering', + // Same derivation as the source facet: a recovery phase in flight. + tone: 'brand', + line: 'Recovery is running on this node.', + icon: Undo2, + }, + partially_rolled_out: { + label: 'partially rolled out', + tone: 'warning', + // Derived from this target's own partial result. The fleet-wide reading + // belongs to the rollout facet, which this map does not cover. + line: 'This node reports a partial result for its last rollout.', + icon: CircleDashed, + }, + paused: { + label: 'paused', + tone: 'warning', + line: 'Work on this node is paused.', + icon: CirclePause, + }, + completion_unknown: { + label: 'completion unknown', + tone: 'warning', + line: 'An operation was interrupted, so its outcome could not be confirmed.', + icon: CircleHelp, + }, + failed_previous_workload_intact: { + label: 'failed, workload intact', + tone: 'destructive', + line: 'The deploy failed before it changed anything. The previous workload is still running.', + icon: CircleX, + }, + failed_after_mutation: { + label: 'failed after change', + tone: 'destructive', + line: 'The deploy failed after it had started changing the workload.', + icon: CircleX, + }, + recovery_failed: { + label: 'recovery failed', + tone: 'destructive', + line: 'Recovery on this node did not complete.', + icon: ShieldAlert, + }, +}; + +/** + * Read views over the two maps for a status that crossed the wire. + * + * The maps above are total over the closed unions, so indexing them yields a + * non-optional value and a miss is invisible to the compiler. That is right + * for a status this build derived and wrong for one a proxied node sent, which + * may belong to a vocabulary this build has never seen. Reading through these + * makes the miss a fact TypeScript produces, so the guard on it cannot be + * mistaken for dead code and deleted. + * + * Same objects, no copy, no cast: a total record over string-literal keys is + * assignable to a partial record over `string`. + */ +export const SOURCE_STATE_LOOKUP: Partial> = SOURCE_STATE; +export const RUNTIME_STATE_LOOKUP: Partial> = RUNTIME_STATE; + +/** + * Frontend view state: stack name to the source status of its waiting candidate. + * A key being present is what "this stack has a Git update waiting" means, so + * the value is optional: a miss is a stack with nothing waiting, not a status. + */ +export type GitSourcePendingMap = Record; + +/** A source facet that is actually describing a Git source, so it carries the identity fields. */ +export type LiveSourceFacet = Exclude; + +/** + * The Git source facet of a live application, or null when there is none to show. + * + * Two exclusions, and both mean "this surface has nothing to say", not "an + * error": the absent arm carries no facets at all, and a live application whose + * source facet is `not_applicable` is Blueprint-owned, where naming a source + * state would be a claim the model never made. + */ +export function liveSourceFacet(revision: GitOpsRevisionProjection | null): LiveSourceFacet | null { + if (!revision || revision.targetMode === 'not_applicable') return null; + const source = revision.facets.source; + return source.status === 'not_applicable' ? null : source; +} + +/** + * The source status when a fetched candidate is waiting, else null. + * + * Presence is keyed on the candidate pointer rather than on the status name. + * `source_reconcile_required` is reachable two ways: from a candidate whose + * fingerprint went stale, and from an accepted generation with no candidate at + * all. Only the first is a waiting update, so keying on the status would start + * flagging stacks that have nothing to review. + * + * A retired application is excluded before the pointer is read. Tombstoning + * keeps the candidate pointer as a frozen fact, so a stack detached while a + * commit was staged still carries one, and reporting it would advertise an + * update on a stack Git no longer manages. + */ +export function pendingSourceStatus(revision: GitOpsRevisionProjection): GitOpsSourceStatus | null { + const source = liveSourceFacet(revision); + if (!source || source.status === 'not_live') return null; + return source.candidateGenerationId === null ? null : source.status; +} + +/** + * Limitations that mean "an application we had reason to expect was unreachable". + * + * Only the absent arm can carry these. On the live arm, limitations are caveats + * on state that is being reported (an unparseable repo URL, a missing artifact + * pointer), not faults, and surfacing them as failures would recreate the same + * conflation in the opposite direction. + */ +export function absentFault(revision: GitOpsRevisionProjection): readonly GitOpsLimitation[] { + return revision.targetMode === 'not_applicable' ? revision.limitations : []; +} + +/** + * Caveats on state that is being reported, the exact complement of `absentFault`. + * + * A live limitation qualifies the answer rather than replacing it: the state + * shown is real, and one piece of evidence behind it could not be proven. The + * two arms are read through separate functions on purpose, because rendering a + * caveat as a fault would claim the state is unavailable when it is not, and + * rendering a fault as a caveat would claim a state nobody derived. + */ +export function liveCaveats(revision: GitOpsRevisionProjection): readonly GitOpsLimitation[] { + return revision.targetMode === 'not_applicable' ? [] : revision.limitations; +} + +/** One short line naming what an identity reference points at, for a drift comparison row. */ +export function identityRefLabel(ref: GitOpsIdentityRef): string { + switch (ref.kind) { + case 'none': + return 'none'; + case 'unknown': + return 'unknown'; + case 'commit': + return `commit ${ref.sha.slice(0, 7)}`; + case 'generation': + return `generation ${ref.id.slice(0, 8)}`; + case 'artifact_set': + return `artifact ${ref.id.slice(0, 8)} · ${ref.qualification}`; + case 'runtime_artifact': + return ref.identity; + case 'intent': + return `intent ${ref.id.slice(0, 8)}`; + case 'rollout_candidate': + return `candidate ${ref.id.slice(0, 8)}`; + case 'rollout_generation': + return `rollout ${ref.id.slice(0, 8)}`; + case 'invocation': + return ref.authored.composeFileOrder.join(', ') || 'no compose files'; + case 'health_run': + return `health run ${ref.runId.slice(0, 8)}`; + } +} diff --git a/frontend/src/lib/nodesApi.ts b/frontend/src/lib/nodesApi.ts index 5f443c37..ab0526c2 100644 --- a/frontend/src/lib/nodesApi.ts +++ b/frontend/src/lib/nodesApi.ts @@ -1,4 +1,5 @@ import { apiFetch } from './api'; +import type { GitOpsRevisionsCarrier } from '@/types/gitops'; export interface NodeRecord { id: number; @@ -14,6 +15,15 @@ export interface NodeRecord { cordoned_reason: string | null; } +/** + * Cordon and uncordon answer with the node plus a GitOps revision list that is + * always empty, by design: a cordon governs whether new placements may be made, + * not what a Blueprint asks for, and the reconciler leaves existing deployments + * where they are. The field is declared for shape parity with node delete and + * label add. Do not build a consumer that expects rows in it. + */ +export type NodeCordonResult = NodeRecord & GitOpsRevisionsCarrier; + async function expectJson(res: Response, fallback: string): Promise { if (!res.ok) { let detail = fallback; @@ -35,19 +45,19 @@ export async function listNodes(): Promise { return expectJson(res, 'Failed to load nodes'); } -export async function cordonNode(id: number, reason: string | null): Promise { +export async function cordonNode(id: number, reason: string | null): Promise { const res = await apiFetch(`/nodes/${id}/cordon`, { method: 'POST', body: JSON.stringify(reason ? { reason } : {}), localOnly: true, }); - return expectJson(res, 'Failed to cordon node'); + return expectJson(res, 'Failed to cordon node'); } -export async function uncordonNode(id: number): Promise { +export async function uncordonNode(id: number): Promise { const res = await apiFetch(`/nodes/${id}/uncordon`, { method: 'POST', localOnly: true, }); - return expectJson(res, 'Failed to uncordon node'); + return expectJson(res, 'Failed to uncordon node'); } diff --git a/frontend/src/types/gitops.ts b/frontend/src/types/gitops.ts new file mode 100644 index 00000000..53f1f02f --- /dev/null +++ b/frontend/src/types/gitops.ts @@ -0,0 +1,463 @@ +// Mirrors the backend GitOps read contract (the frontend never imports backend). +// Source of truth: backend/src/services/gitops/types.ts. +// +// Two conventions run through the whole contract and are worth knowing before +// reading further. Nothing is optional: absence on the wire is `null`, and a +// shape that cannot carry a field omits the key entirely rather than sending +// undefined. And every status union is closed, so a `Record` built +// over one (see lib/gitopsState.ts) stops compiling as soon as a member is +// added here. Nothing detects a backend change on its own: this file is +// hand-written, so widening it is the step that surfaces the missing cases. + +// --- scalars and closed enums --------------------------------------------- + +export type GitOpsTargetMode = 'direct' | 'inline_blueprint' | 'blueprint'; + +export type GitOpsLifecycleStatus = 'active' | 'creating' | 'detached' | 'deleted'; + +export type ArtifactQualification = + | 'unresolved' + | 'exact' + | 'qualified' + | 'stale' + | 'unavailable' + | 'local_build_unverified'; + +export type Connectivity = 'unknown' | 'reachable' | 'unreachable' | 'stale'; + +export type LkgUnavailableReason = 'generation_missing' | 'recovery_unretainable'; + +export type GitOpsAvailableAction = 'fetch' | 'apply' | 'dismiss' | 'deploy' | 'approve_legacy' | 'none'; + +// --- limitations ------------------------------------------------------------ + +/** + * A fact the projection could not establish, carried alongside the answer + * instead of dropped. + * + * `code` is an open string by contract, not a closed union: the backend adds + * codes without a schema bump, so a reader that switches exhaustively on it + * silently stops rendering the newest ones. Render `message`, which is already + * written for an operator, and use `code` only to group. + * + * Whether a limitation is a fault is decided by which arm of the projection + * carries it, not by its code: see absentFault in lib/gitopsState.ts. + */ +export interface GitOpsLimitation { + code: string; + message: string; + evidence: unknown; +} + +// --- identity --------------------------------------------------------------- + +/** Secret-free repository identity. Both fields degrade to '' when the stored identity is absent or malformed. */ +export interface RepoIdentity { + host: string; + pathname: string; +} + +export interface AuthoredInvocationIdentity { + composeFileOrder: string[]; + projectName: string | null; + projectDirectory: string | null; + envFileOrder: string[]; +} + +/** One side of a drift comparison: what a thing is identified by, whatever kind of thing it is. */ +export type GitOpsIdentityRef = + | { kind: 'none' } + | { kind: 'unknown' } + | { kind: 'commit'; sha: string; repoUrl: string; ref: string } + | { kind: 'generation'; id: string } + | { kind: 'artifact_set'; id: string; qualification: ArtifactQualification; evidenceVersion: number } + | { kind: 'runtime_artifact'; identity: string; observedAt: number | null } + | { kind: 'intent'; id: string; composeContentSha256: string } + | { kind: 'rollout_candidate'; id: string } + | { kind: 'rollout_generation'; id: string } + | { kind: 'invocation'; authored: AuthoredInvocationIdentity } + | { kind: 'health_run'; runId: string; deployedGenerationId: string | null }; + +export interface GitOpsApprovalRefs { + sourceAcceptanceRef: string | null; + placementApprovalRef: string | null; + rolloutAuthorizationRef: string | null; + legacyCombinedApprovalRef: string | null; +} + +// --- source facet ----------------------------------------------------------- + +export interface SourceIdentityFields { + configuredRepoUrl: string; + repoIdentity: RepoIdentity; + configuredRef: string; + desiredCommitSha: string | null; + fetchedCommitSha: string | null; + // Set while a fetched candidate is waiting, and kept as a frozen fact after + // the application is retired. This, not the status, is what "a Git update is + // waiting" means for a live application: source_reconcile_required is + // reachable both with a candidate and from an accepted generation without + // one. See pendingSourceStatus, which reads both this and the lifecycle. + candidateGenerationId: string | null; + acceptedGenerationId: string | null; +} + +/** + * What the Git source is doing. `not_applicable` is the Inline Blueprint case: + * a Blueprint application has no Git source, and reporting one would be a claim + * the model never made. + */ +export type SourceFacet = + | { status: 'not_applicable' } + | (SourceIdentityFields & { + status: + | 'never_reconciled' + | 'checking_fetching' + | 'application_generation_accepted' + | 'candidate_ready' + | 'source_review_pending' + | 'source_conflict_blocker' + | 'source_reconcile_required'; + }) + | (SourceIdentityFields & { status: 'source_superseded'; supersededGenerationId: string }) + | (SourceIdentityFields & { status: 'applying'; activeOperationId: string; activeGenerationId: string }) + | (SourceIdentityFields & { status: 'source_retry_scheduled'; retryAt: number; retryCount: number }) + | (SourceIdentityFields & { status: 'source_suspended'; suspendedAt: number }) + | (SourceIdentityFields & { + status: 'source_failed'; + failureStage: 'fetch' | 'validation' | 'apply' | 'create'; + failureClass: string; + failureAt: number; + retryAt: number | null; + retryCount: number; + }) + | (SourceIdentityFields & { + status: 'source_unknown'; + interruptedStage: 'fetch_started' | 'apply_started'; + interruptedAt: number; + interruptedOperationId: string | null; + interruptedGenerationId: string | null; + }) + | (SourceIdentityFields & { + status: 'recovery_required'; + recoveryRef: string | null; + recoveryGenerationId: string | null; + }) + | (SourceIdentityFields & { + status: 'recovery_failed'; + recoveryRef: string | null; + recoveryGenerationId: string | null; + failureClass: string; + failureAt: number; + }) + | (SourceIdentityFields & { status: 'not_live'; lifecycleStatus: 'detached' | 'deleted' }); + +export type GitOpsSourceStatus = SourceFacet['status']; + +// --- artifact facet --------------------------------------------------------- + +export interface ArtifactExpectedIdentity { + artifactSetId: string; + evidenceVersion: number; + qualification: ArtifactQualification; + identity: string | null; +} + +export interface ArtifactLatestEvidence { + artifactSetId: string; + evidenceVersion: number; + qualification: ArtifactQualification; + identity: string | null; +} + +/** + * `artifact_unresolved` appears in two members. Discriminate on + * `latestEvidence === null`, not on the status alone: the null-evidence member + * is the missing-pointer variant and carries no set id or freshness. + */ +export type ArtifactFacet = + | { status: 'not_applicable' } + | { + status: 'artifact_unresolved'; + generationId: string; + expected: ArtifactExpectedIdentity | null; + latestEvidence: null; + limitation: 'artifact_pointer_missing'; + } + | { + status: + | 'artifact_unresolved' + | 'artifact_resolution_pending' + | 'artifact_exact' + | 'artifact_qualified' + | 'artifact_stale' + | 'artifact_unavailable' + | 'artifact_local_build_unverified' + | 'artifact_identity_changed'; + artifactSetId: string; + generationId: string; + evidenceVersion: number; + qualification: ArtifactQualification; + freshnessAt: number; + expected: ArtifactExpectedIdentity | null; + latestEvidence: ArtifactLatestEvidence; + }; + +// --- placement and rollout facets ------------------------------------------- + +export interface FutureRolloutAuthorizationBinding { + readonly rolloutCandidateId: string; + readonly acceptedGenerationId: string; + readonly artifactSetId: string; + readonly intentRevisionId: string; + readonly requiredNodeIds: readonly number[]; + readonly sourceAcceptanceRef: string; + readonly placementApprovalRef: string; + readonly preflightFingerprint: string; +} + +export type PlacementFacet = + | { status: 'not_applicable' } + | { status: 'unbound_direct' } + | { status: 'unknown'; limitation: 'missing_intent' } + | { status: 'source_acceptance_pending'; sourceAcceptanceRef: string | null; candidateGenerationId: string } + | { status: 'placement_review_pending' } + | { status: 'rollout_authorization_pending'; rolloutAuthorizationRef: null; binding: FutureRolloutAuthorizationBinding } + | { status: 'rollout_authorization_stale'; rolloutAuthorizationRef: string; bound: FutureRolloutAuthorizationBinding } + | { status: 'stateful_confirmation_required' } + | { status: 'preflight_blocked'; reason: string; binding: FutureRolloutAuthorizationBinding } + | { status: 'blueprint_bound'; completion: 'unknown' }; + +/** `partial` arrives as the stored JSON string, not a parsed object. Nothing decodes it yet. */ +export type RolloutFacet = + | { status: 'not_applicable' } + | { status: 'rollout_not_executable'; rolloutCandidateId: string } + | { status: 'rollout_queued'; rolloutGenerationId: string } + | { status: 'canary_in_progress'; rolloutGenerationId: string } + | { status: 'batch_in_progress'; rolloutGenerationId: string } + | { status: 'rollout_paused'; pauseAt: number; pauseReason: string | null } + | { status: 'partially_rolled_out'; partial: unknown } + | { status: 'fully_deployed_health_pending'; rolloutGenerationId: string } + | { status: 'configuration_converged_artifact_qualified'; rolloutGenerationId: string } + | { status: 'exactly_converged_healthy'; rolloutGenerationId: string } + | { status: 'rollout_superseded'; rolloutGenerationId: string } + | { status: 'target_stale' } + | { status: 'target_unreachable' } + | { status: 'rollback_in_progress'; recoveryRef: string; recoveryGenerationId: string | null } + | { + status: 'rollback_partial_failed'; + recoveryRef: string; + recoveryGenerationId: string | null; + failureClass: string; + failureAt: number; + } + | { status: 'recovery_required' } + | { status: 'completion_unknown' }; + +// --- per-target facets ------------------------------------------------------ + +/** What one node is actually doing with the generation it was asked to run. */ +export type RuntimeFacet = + | { + status: + | 'tombstoned' + | 'recovery_required' + | 'deploying' + | 'withdrawing' + | 'failed_previous_workload_intact' + | 'failed_after_mutation' + | 'disk_invocation_drift' + | 'rollout_artifact_drift' + | 'runtime_artifact_drift' + | 'artifact_verification_pending' + | 'never_applied' + | 'applied_not_deployed' + | 'acknowledged_completion_unknown' + | 'stale_acknowledgement' + | 'pending_state_review' + | 'evict_blocked' + | 'drifted' + | 'correcting' + | 'fully_deployed_health_pending' + | 'health_checking' + | 'synced_and_healthy' + | 'health_drift' + | 'partially_rolled_out' + | 'retry_scheduled'; + } + | { status: 'paused'; pauseAt: number; pauseReason: string | null } + | { + status: 'recovery_failed'; + recoveryRef: string | null; + recoveryGenerationId: string | null; + failureClass: string; + failureAt: number; + } + | { + status: 'completion_unknown'; + interruptedStage: 'deploy_started' | 'blueprint_deploy_started' | 'blueprint_withdraw_started'; + interruptedAt: number; + interruptedOperationId: string | null; + interruptedGenerationId: string | null; + interruptedIntentRevisionId: string | null; + interruptedRolloutCandidateId: string | null; + }; + +export type GitOpsRuntimeStatus = RuntimeFacet['status']; + +/** `none` is "never established"; `unavailable` is "established then lost". They are not the same. */ +export type LkgFacet = + | { status: 'none' } + | { status: 'available'; generationId: string; artifactSetId: string | null } + | { status: 'unavailable' } + | { status: 'qualified'; generationId: string; artifactSetId: string }; + +export type HealthFacet = + | { status: 'not_applicable' | 'unbound' } + | { status: 'pending'; runId: string | null } + | { status: 'checking'; runId: string; deployedGenerationId: string | null } + | { status: 'passed'; runId: string; deployedGenerationId: string } + | { status: 'failed'; runId: string; deployedGenerationId: string | null } + | { status: 'unknown'; runId: string | null; limitation: 'health_unknown' }; + +/** What was observed running, and how much that observation can be trusted as proof. */ +export type ObservedArtifactIdentity = + | { kind: 'unknown' } + | { kind: 'missing' } + | { kind: 'unavailable' } + | { kind: 'exact'; identity: string; observedAt: number } + | { kind: 'qualified'; identity: string; observedAt: number } + | { kind: 'stale'; identity: string; observedAt: number } + | { kind: 'local_build_unverified'; identity: string; observedAt: number }; + +// --- application facets, targets, drift ------------------------------------- + +/** Application-level facets. Runtime, health and LKG are per-target and live on the target instead. */ +export interface GitOpsFacets { + source: SourceFacet; + artifact: ArtifactFacet; + placement: PlacementFacet; + rollout: RolloutFacet; +} + +/** One node's copy of the application. Ordered by nodeId ascending. */ +export interface GitOpsTargetProjection { + nodeId: number; + // Copied from the application, not from the target row. + stackName: string | null; + desiredGenerationId: string | null; + candidateGenerationId: string | null; + appliedGenerationId: string | null; + deployedGenerationId: string | null; + healthyGenerationId: string | null; + lkgGenerationId: string | null; + lkgArtifactSetId: string | null; + lkgUnavailableAt: number | null; + lkgUnavailableReason: LkgUnavailableReason | null; + expectedArtifactSetId: string | null; + latestArtifactSetId: string | null; + artifact: ArtifactFacet; + observedArtifactIdentity: ObservedArtifactIdentity; + intentRevisionId: string | null; + rolloutCandidateId: string | null; + rolloutGenerationId: string | null; + approvals: GitOpsApprovalRefs; + connectivity: Connectivity; + legacyAppliedRevision: number | null; + runtime: RuntimeFacet; + health: HealthFacet; + lkg: LkgFacet; + tombstoned: boolean; +} + +export type ConfiguredPolicy = + | { kind: 'git_source'; autoApplyOnWebhook: boolean; autoDeployOnApply: boolean } + | { kind: 'blueprint_drift'; driftMode: 'observe' | 'suggest' | 'enforce' } + | null; + +/** + * A classified divergence between what was intended and what was observed. + * + * The backend currently emits one class of item on its own evidence, a runtime + * artifact mismatch; every other class still needs a producer. An empty list + * therefore means "no confirmed drift", never "in sync": a state the model was + * never asked about also answers empty. + */ +export interface GitOpsDriftItem { + class: 'source' | 'managed_project' | 'invocation' | 'placement' | 'rollout' | 'runtime' | 'health'; + expected: GitOpsIdentityRef; + observed: GitOpsIdentityRef; + freshnessAt: number | null; + owner: string; + reason: string; + configuredPolicy: ConfiguredPolicy; + affectedTargets: Array<{ nodeId: number | null; stackName: string | null }>; + action: GitOpsAvailableAction; +} + +// --- the projection --------------------------------------------------------- + +/** + * Nothing to project. + * + * `limitations` carries two different facts and they must not render the same. + * Empty is the ordinary case: a stack or Blueprint the model was never asked + * about, and the right rendering is nothing at all. Non-empty is a fault, an + * application the projection had reason to believe exists and could not reach, + * and it has to be surfaced. + * + * `lifecycleStatus`, `stackName`, `blueprintId` and `rolloutGenerationId` are + * absent keys rather than nulls, so reaching for one without narrowing first is + * a compile error instead of an undefined that renders as a blank. + */ +export interface GitOpsRevisionAbsent { + schemaVersion: 1; + targetMode: 'not_applicable'; + applicationId: null; + facets: null; + targets: readonly []; + drift: readonly []; + limitations: readonly GitOpsLimitation[]; + availableActions: readonly []; + approvals: null; +} + +/** A live application. `availableActions` is never empty: "nothing to do" is ['none']. */ +export interface GitOpsRevisionLive { + schemaVersion: 1; + targetMode: GitOpsTargetMode; + applicationId: string; + lifecycleStatus: GitOpsLifecycleStatus; + stackName: string | null; + blueprintId: number | null; + rolloutGenerationId: string | null; + approvals: GitOpsApprovalRefs; + facets: GitOpsFacets; + targets: readonly GitOpsTargetProjection[]; + drift: readonly GitOpsDriftItem[]; + // Caveats on state that is being reported, not faults. The fault codes ride + // the absent arm above. + limitations: readonly GitOpsLimitation[]; + availableActions: readonly GitOpsAvailableAction[]; +} + +/** Narrow on `targetMode === 'not_applicable'`; the live modes are disjoint from it. */ +export type GitOpsRevisionProjection = GitOpsRevisionAbsent | GitOpsRevisionLive; + +// --- wire carriers ---------------------------------------------------------- + +/** Responses that describe one application: the git-source reads, drift, blueprint reads and mutations. */ +export interface GitOpsRevisionCarrier { + gitopsRevision: GitOpsRevisionProjection; +} + +/** + * Responses for a write that could move several Blueprints at once: node label + * add, cordon, uncordon, node delete. Ordered by blueprintId ascending. + * + * Empty means both "nothing moved" and "the projection faulted after the write + * committed", so it can never be reported as the first. + */ +export interface GitOpsRevisionsCarrier { + gitopsRevisions: readonly GitOpsRevisionProjection[]; +}