diff --git a/backend/src/__tests__/database-metrics.test.ts b/backend/src/__tests__/database-metrics.test.ts index 0e4b37c5..6dc9be3c 100644 --- a/backend/src/__tests__/database-metrics.test.ts +++ b/backend/src/__tests__/database-metrics.test.ts @@ -4,6 +4,7 @@ */ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import type { NotificationHistory } from '../services/DatabaseService'; let tmpDir: string; let DatabaseService: any; @@ -183,6 +184,39 @@ describe('DatabaseService - cleanupOldNotifications', () => { }); }); +describe('DatabaseService - notification history GitOps dedupe', () => { + it('inserts once and returns the existing row on a repeated dedupe_key', () => { + const first = db.addNotificationHistory(0, { + level: 'info', + message: 'source reconciled', + timestamp: Date.now(), + gitops_operation_id: 'op-1', + dedupe_key: 'gitops:app-1:op-1', + }); + const second = db.addNotificationHistory(0, { + level: 'info', + message: 'source reconciled (retry repair)', + timestamp: Date.now(), + gitops_operation_id: 'op-1', + dedupe_key: 'gitops:app-1:op-1', + }); + + expect(second.id).toBe(first.id); + const history: NotificationHistory[] = db.getNotificationHistory(0, 200); + const matches = history.filter((n) => n.dedupe_key === 'gitops:app-1:op-1'); + expect(matches).toHaveLength(1); + expect(matches[0].message).toBe('source reconciled'); + }); + + it('allows two rows with no dedupe_key, matching existing notification behavior', () => { + db.addNotificationHistory(0, { level: 'info', message: 'plain a', timestamp: Date.now() }); + db.addNotificationHistory(0, { level: 'info', message: 'plain b', timestamp: Date.now() }); + + const history: NotificationHistory[] = db.getNotificationHistory(0, 200); + expect(history.filter((n) => n.message === 'plain a' || n.message === 'plain b')).toHaveLength(2); + }); +}); + describe('DatabaseService - cleanupOldAuditLogs', () => { it('deletes audit logs older than specified days and retains recent ones', () => { const oldTimestamp = Date.now() - 120 * 24 * 60 * 60 * 1000; // 120 days ago diff --git a/backend/src/__tests__/git-source-routes.test.ts b/backend/src/__tests__/git-source-routes.test.ts index 11cafc72..52a996ad 100644 --- a/backend/src/__tests__/git-source-routes.test.ts +++ b/backend/src/__tests__/git-source-routes.test.ts @@ -30,67 +30,7 @@ import { insertHistory } from '../services/gitops/history'; import type { GitOpsApplicationRow } from '../services/gitops/types'; import { PROXY_DEPLOY_ACTOR_HEADER, PROXY_DEPLOY_SOURCE_HEADER } from '../services/license-headers'; import { withLoopbackTargetProtection } from './helpers/allowLoopbackTargets'; - -/** A minimal live Direct application row for GitOps read-path fixtures. */ -function directApplicationFixture(id: string, stackName: string): GitOpsApplicationRow { - return { - id, - lifecycle_key: `direct:${stackName}`, - lifecycle_status: 'active', - target_mode: 'direct', - stack_name: stackName, - blueprint_id: null, - configured_repo_url: 'https://github.com/example/repo.git', - repo_identity_json: '{"host":"github.com","pathname":"/example/repo.git"}', - configured_ref: 'main', - compose_paths_json: '["compose.yaml"]', - context_dir: null, - sync_env: 0, - env_path: null, - materialization_fingerprint: 'a'.repeat(64), - desired_commit_sha: null, - fetched_commit_sha: null, - fetched_resolved_ref_kind: null, - candidate_generation_id: null, - accepted_generation_id: null, - candidate_plan_blocked: 0, - review_required: 0, - artifact_set_id: null, - latest_artifact_set_id: null, - intent_revision_id: null, - rollout_candidate_id: null, - rollout_generation_id: null, - source_acceptance_ref: null, - placement_approval_ref: null, - rollout_authorization_ref: null, - legacy_combined_approval_ref: null, - preflight_fingerprint: null, - latest_operation_id: null, - active_operation_id: null, - active_operation_stage: null, - active_operation_at: null, - active_generation_id: null, - pause_at: null, - pause_reason: null, - partial_json: null, - failure_stage: null, - failure_class: null, - failure_at: null, - retry_at: null, - retry_count: 0, - suspended_at: null, - recovery_ref: null, - recovery_phase: null, - interruption_stage: null, - interruption_at: null, - interruption_operation_id: null, - interruption_generation_id: null, - evidence_fresh_at: null, - evidence_limitations_json: null, - created_at: 1, - updated_at: 1, - }; -} +import { directApplicationFixture } from './helpers/gitopsFixtures'; // ── Hoisted mocks (must come before importing the app) ───────────────── diff --git a/backend/src/__tests__/git-source-service.test.ts b/backend/src/__tests__/git-source-service.test.ts index a573d026..24f422e5 100644 --- a/backend/src/__tests__/git-source-service.test.ts +++ b/backend/src/__tests__/git-source-service.test.ts @@ -70,6 +70,11 @@ const { mockRecoveryLinkGateOrRetain: vi.fn(), })); +const { mockInvalidateNodeCaches, mockTriggerPostDeployScan } = vi.hoisted(() => ({ + mockInvalidateNodeCaches: vi.fn(), + mockTriggerPostDeployScan: vi.fn(async () => undefined), +})); + vi.mock('../services/StackUpdateRecoveryService', () => ({ StackUpdateRecoveryService: { getInstance: () => ({ @@ -86,6 +91,20 @@ vi.mock('../services/StackUpdateRecoveryService', () => ({ }, })); +vi.mock('../helpers/cacheInvalidation', async () => { + const actual = await vi.importActual( + '../helpers/cacheInvalidation', + ); + return { ...actual, invalidateNodeCaches: mockInvalidateNodeCaches }; +}); + +vi.mock('../helpers/policyGate', async () => { + const actual = await vi.importActual( + '../helpers/policyGate', + ); + return { ...actual, triggerPostDeployScan: mockTriggerPostDeployScan }; +}); + let tmpDir: string; let GitSourceService: typeof import('../services/GitSourceService').GitSourceService; @@ -1182,6 +1201,34 @@ describe('GitSourceService error mapping', () => { await expect(svc().fetchFromGit(fetchParams)).rejects.toMatchObject({ code: 'NETWORK_TIMEOUT' }); }); + it('propagates the raw transport reason onto GitSourceError.extras for retry classification', async () => { + mockFetchAtCommit.mockRejectedValueOnce(gitFailure( + 'fatal: the remote end hung up unexpectedly', + false, + )); + try { + await svc().fetchFromGit(fetchParams); + expect.fail('should have thrown'); + } catch (e) { + expect((e as InstanceType).extras?.transportReason).toBe('exit'); + } + }); + + it('propagates a non-exit transport reason (e.g. timeout) without hardcoding to exit', async () => { + mockFetchAtCommit.mockRejectedValueOnce({ + transportFailure: true as const, + reason: 'timeout', + host: 'github.com', + hasToken: false, + } satisfies TransportFailure); + try { + await svc().fetchFromGit(fetchParams); + expect.fail('should have thrown'); + } catch (e) { + expect((e as InstanceType).extras?.transportReason).toBe('timeout'); + } + }); + it('maps a TLS certificate failure to a certificate GIT_ERROR', async () => { mockFetchAtCommit.mockRejectedValueOnce(gitFailure( "fatal: unable to access 'https://github.com/example/repo.git/': SSL certificate problem: self-signed certificate", @@ -2266,6 +2313,80 @@ describe('GitSourceService.apply', () => { } }); + describe('cache invalidation and post-deploy scan', () => { + beforeEach(() => { + mockInvalidateNodeCaches.mockClear(); + mockTriggerPostDeployScan.mockClear(); + }); + + it('invalidates caches once and does not scan for an apply-only commit', async () => { + const sha = 'f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0'; + const svc = await seedPending('apply-only-cache', 'services:\n x:\n image: alpine\n', sha); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + + try { + const result = await svc.apply('apply-only-cache', sha, { deploy: false, ...skipFingerprint }); + expect(result.applied).toBe(true); + expect(result.deployed).toBe(false); + expect(mockInvalidateNodeCaches).toHaveBeenCalledTimes(1); + expect(mockTriggerPostDeployScan).not.toHaveBeenCalled(); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + } + }); + + it('invalidates caches once and scans once for a successful apply-and-deploy', async () => { + const sha = 'f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1'; + const svc = await seedPending('apply-deploy-scan', 'services:\n x:\n image: alpine\n', sha); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const { ComposeService } = await import('../services/ComposeService'); + const { HealthGateService } = await import('../services/HealthGateService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-scan'); + + try { + const result = await svc.apply('apply-deploy-scan', sha, { deploy: true, ...skipFingerprint }); + expect(result.deployed).toBe(true); + expect(mockInvalidateNodeCaches).toHaveBeenCalledTimes(1); + expect(mockTriggerPostDeployScan).toHaveBeenCalledTimes(1); + expect(mockTriggerPostDeployScan).toHaveBeenCalledWith('apply-deploy-scan', expect.any(Number)); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + deploySpy.mockRestore(); + beginSpy.mockRestore(); + } + }); + + it('invalidates caches once but does not scan when the deploy fails', async () => { + const sha = 'f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2'; + const svc = await seedPending('apply-deploy-fail-scan', 'services:\n x:\n image: alpine\n', sha); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const { ComposeService } = await import('../services/ComposeService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockRejectedValue( + new Error('compose up failed: docker unavailable'), + ); + + try { + const result = await svc.apply('apply-deploy-fail-scan', sha, { deploy: true, ...skipFingerprint }); + expect(result.deployed).toBe(false); + expect(mockInvalidateNodeCaches).toHaveBeenCalledTimes(1); + expect(mockTriggerPostDeployScan).not.toHaveBeenCalled(); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + }); + it('refuses the first complete-project apply when an unowned local file collides (audit round 9 B-1)', async () => { const sha = '9999aaaa9999aaaa9999aaaa9999aaaa9999aaaa'; mockSuccessfulClone({ diff --git a/backend/src/__tests__/gitops-approvals.test.ts b/backend/src/__tests__/gitops-approvals.test.ts index 2fc591f3..75b0d132 100644 --- a/backend/src/__tests__/gitops-approvals.test.ts +++ b/backend/src/__tests__/gitops-approvals.test.ts @@ -308,6 +308,7 @@ function directApp(id: string, stackName: string): GitOpsApplicationRow { active_generation_id: null, pause_at: null, pause_reason: null, + source_suspended_reason: null, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/__tests__/gitops-blueprint-transitions.test.ts b/backend/src/__tests__/gitops-blueprint-transitions.test.ts index 81e3f6c1..9c5b6632 100644 --- a/backend/src/__tests__/gitops-blueprint-transitions.test.ts +++ b/backend/src/__tests__/gitops-blueprint-transitions.test.ts @@ -608,6 +608,7 @@ function inlineApp(id: string, blueprintId: number): GitOpsApplicationRow { active_generation_id: null, pause_at: null, pause_reason: null, + source_suspended_reason: null, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/__tests__/gitops-create-recovery.test.ts b/backend/src/__tests__/gitops-create-recovery.test.ts index ff673f0e..85f4c614 100644 --- a/backend/src/__tests__/gitops-create-recovery.test.ts +++ b/backend/src/__tests__/gitops-create-recovery.test.ts @@ -440,6 +440,7 @@ function creatingApp(id: string, stackName: string): GitOpsApplicationRow { active_generation_id: null, pause_at: null, pause_reason: null, + source_suspended_reason: null, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/__tests__/gitops-create.test.ts b/backend/src/__tests__/gitops-create.test.ts index d7a684b6..0fdee147 100644 --- a/backend/src/__tests__/gitops-create.test.ts +++ b/backend/src/__tests__/gitops-create.test.ts @@ -706,6 +706,7 @@ function creatingApp(id: string, stackName: string): GitOpsApplicationRow { active_generation_id: null, pause_at: null, pause_reason: null, + source_suspended_reason: null, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/__tests__/gitops-deferred.test.ts b/backend/src/__tests__/gitops-deferred.test.ts index fc237647..b901e0e0 100644 --- a/backend/src/__tests__/gitops-deferred.test.ts +++ b/backend/src/__tests__/gitops-deferred.test.ts @@ -64,7 +64,11 @@ describe('gitops deferred state', () => { const app = store.getApplication('app-susp')!; expect(app.suspended_at).not.toBeNull(); expect(app.accepted_generation_id).toBe(accepted); - expect(projectOf('app-susp').facets.source.status).toBe('source_suspended'); + const sourceFacet = projectOf('app-susp').facets.source; + expect(sourceFacet.status).toBe('source_suspended'); + if (sourceFacet.status === 'source_suspended') { + expect(sourceFacet.suspendedReason).toBe('operator paused sync'); + } // A suspended source refuses new work rather than queueing it. expect(() => tx.fetchStarted('app-susp', env('op-susp-f'))).toThrow(/suspended/); @@ -89,6 +93,27 @@ describe('gitops deferred state', () => { expect(app.suspended_at).not.toBeNull(); }); + it('keeps a source-suspension reason independent of an application-wide rollout pause reason', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + seedApplied('app-susp3', 'susp3-web'); + + tx.sourceSuspended('app-susp3', 'operator paused sync', env('op-susp3')); + // A later, unrelated application-wide rollout pause must not clobber the + // suspension reason: the two events share the application row but not + // its reason field. + tx.rolloutPaused('app-susp3', null, 'awaiting approval', env('op-pause3')); + + const app = store.getApplication('app-susp3')!; + expect(app.source_suspended_reason).toBe('operator paused sync'); + expect(app.pause_reason).toBe('awaiting approval'); + + tx.sourceUnsuspended('app-susp3', env('op-unsusp3')); + expect(store.getApplication('app-susp3')?.source_suspended_reason).toBeNull(); + // Unsuspending the source must not touch the unrelated rollout pause. + expect(store.getApplication('app-susp3')?.pause_reason).toBe('awaiting approval'); + }); + it('pauses a rollout without claiming anything about health', () => { const store = GitOpsStore.getInstance(); const tx = GitOpsTransitions.getInstance(); @@ -314,6 +339,7 @@ function app(id: string, stackName: string): GitOpsApplicationRow { active_generation_id: null, pause_at: null, pause_reason: null, + source_suspended_reason: null, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/__tests__/gitops-derive.test.ts b/backend/src/__tests__/gitops-derive.test.ts index 3aa9496a..7b03c86d 100644 --- a/backend/src/__tests__/gitops-derive.test.ts +++ b/backend/src/__tests__/gitops-derive.test.ts @@ -962,6 +962,7 @@ function app(id: string, stackName: string): GitOpsApplicationRow { active_generation_id: null, pause_at: null, pause_reason: null, + source_suspended_reason: null, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/__tests__/gitops-history-read.test.ts b/backend/src/__tests__/gitops-history-read.test.ts index c6520550..8417dbcc 100644 --- a/backend/src/__tests__/gitops-history-read.test.ts +++ b/backend/src/__tests__/gitops-history-read.test.ts @@ -530,6 +530,7 @@ function application(): GitOpsApplicationRow { active_generation_id: null, pause_at: null, pause_reason: null, + source_suspended_reason: null, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/__tests__/gitops-managed-sweep.test.ts b/backend/src/__tests__/gitops-managed-sweep.test.ts index eb72f34b..87d62944 100644 --- a/backend/src/__tests__/gitops-managed-sweep.test.ts +++ b/backend/src/__tests__/gitops-managed-sweep.test.ts @@ -166,6 +166,7 @@ function creatingApp(id: string, stackName: string): GitOpsApplicationRow { active_generation_id: null, pause_at: null, pause_reason: null, + source_suspended_reason: null, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/__tests__/gitops-recovery-capture.test.ts b/backend/src/__tests__/gitops-recovery-capture.test.ts index 6f76f879..ee8b2957 100644 --- a/backend/src/__tests__/gitops-recovery-capture.test.ts +++ b/backend/src/__tests__/gitops-recovery-capture.test.ts @@ -146,6 +146,7 @@ function app(id: string, stackName: string): GitOpsApplicationRow { active_generation_id: null, pause_at: null, pause_reason: null, + source_suspended_reason: null, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/__tests__/gitops-recovery.test.ts b/backend/src/__tests__/gitops-recovery.test.ts index e0891847..eeb3a510 100644 --- a/backend/src/__tests__/gitops-recovery.test.ts +++ b/backend/src/__tests__/gitops-recovery.test.ts @@ -460,6 +460,7 @@ function app(id: string, stackName: string): GitOpsApplicationRow { active_generation_id: null, pause_at: null, pause_reason: null, + source_suspended_reason: null, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/__tests__/gitops-schema.test.ts b/backend/src/__tests__/gitops-schema.test.ts index a75132df..ca2fa446 100644 --- a/backend/src/__tests__/gitops-schema.test.ts +++ b/backend/src/__tests__/gitops-schema.test.ts @@ -293,6 +293,7 @@ function directApp(id: string, stackName: string): GitOpsApplicationRow { active_generation_id: null, pause_at: null, pause_reason: null, + source_suspended_reason: null, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/__tests__/gitops-transitions.test.ts b/backend/src/__tests__/gitops-transitions.test.ts index b7d3a6b6..0d65150d 100644 --- a/backend/src/__tests__/gitops-transitions.test.ts +++ b/backend/src/__tests__/gitops-transitions.test.ts @@ -2,7 +2,7 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; import { DatabaseService } from '../services/DatabaseService'; import { encodeArtifactEvidenceJson } from '../services/gitops/json'; -import { GitOpsStore } from '../services/gitops/store'; +import { GitOpsStore, emptyTargetRow } from '../services/gitops/store'; import { GitOpsTransitions, type EventEnvelope } from '../services/gitops/transitions'; import { projectApplication } from '../services/gitops/derive'; import type { GitOpsApplicationRow, GitOpsGenerationRow } from '../services/gitops/types'; @@ -520,6 +520,222 @@ describe('gitops transitions', () => { expect(store.getTarget('app-int', 1)?.deployed_generation_id).toBe('gen-int'); expect(store.getTarget('app-int', 1)?.failure_stage).toBeNull(); }); + + it('sourceAccepted accepts the generation without touching any target', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + const env = envelope('op-sa'); + tx.activateDirect({ application: app('app-sa', 'sa-web'), nodeId: 1, envelope: env }); + store.insertGeneration(gen('gen-sa', 'app-sa')); + tx.fetchStarted('app-sa', envelope('op-f-sa')); + tx.fetched('app-sa', 'deadbeef', envelope('op-f-sa')); + tx.candidateReady('app-sa', 'gen-sa', false, envelope('op-c-sa')); + tx.applyStarted('app-sa', 'gen-sa', envelope('op-sa')); + + const result = tx.sourceAccepted({ + applicationId: 'app-sa', + generationId: 'gen-sa', + artifactSetId: 'art-sa', + sourceAcceptanceId: 'acc-sa', + authority: 'operator', + envelope: env, + }); + + expect(result.replayed).toBe(false); + const application = store.getApplication('app-sa')!; + expect(application.accepted_generation_id).toBe('gen-sa'); + expect(application.source_acceptance_ref).toBe('acc-sa'); + expect(application.candidate_generation_id).toBeNull(); + // sourceAccepted is mode-neutral: it must not bind the Direct target. + const target = store.getTarget('app-sa', 1)!; + expect(target.applied_generation_id).toBeNull(); + expect(target.desired_generation_id).toBeNull(); + expect(target.candidate_generation_id).toBe('gen-sa'); + }); + + it('sourceAccepted refuses to accept a candidate while the source is suspended', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + const env = envelope('op-sas'); + tx.activateDirect({ application: app('app-sas', 'sas-web'), nodeId: 1, envelope: env }); + store.insertGeneration(gen('gen-sas', 'app-sas')); + tx.fetchStarted('app-sas', envelope('op-f-sas')); + tx.fetched('app-sas', 'deadbeef', envelope('op-f-sas')); + tx.candidateReady('app-sas', 'gen-sas', false, envelope('op-c-sas')); + tx.applyStarted('app-sas', 'gen-sas', envelope('op-sas')); + tx.sourceSuspended('app-sas', 'operator paused sync', envelope('op-susp-sas')); + + expect(() => tx.sourceAccepted({ + applicationId: 'app-sas', + generationId: 'gen-sas', + artifactSetId: 'art-sas', + sourceAcceptanceId: 'acc-sas', + authority: 'operator', + envelope: envelope('op-sas-2'), + })).toThrow(/suspended/); + + const application = store.getApplication('app-sas')!; + expect(application.accepted_generation_id).toBeNull(); + expect(application.candidate_generation_id).toBe('gen-sas'); + expect(application.suspended_at).not.toBeNull(); + }); + + it('targetApplied binds a Direct target only after the generation is accepted', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + const env = envelope('op-ta'); + tx.activateDirect({ application: app('app-ta', 'ta-web'), nodeId: 1, envelope: env }); + store.insertGeneration(gen('gen-ta', 'app-ta')); + tx.fetchStarted('app-ta', envelope('op-f-ta')); + tx.fetched('app-ta', 'deadbeef', envelope('op-f-ta')); + tx.candidateReady('app-ta', 'gen-ta', false, envelope('op-c-ta')); + tx.applyStarted('app-ta', 'gen-ta', envelope('op-ta')); + tx.sourceAccepted({ + applicationId: 'app-ta', + generationId: 'gen-ta', + artifactSetId: 'art-ta', + sourceAcceptanceId: 'acc-ta', + authority: 'operator', + envelope: env, + }); + + const result = tx.targetApplied(1, { + applicationId: 'app-ta', + generationId: 'gen-ta', + artifactSetId: 'art-ta', + sourceAcceptanceId: 'acc-ta', + authority: 'operator', + envelope: env, + }); + + expect(result.replayed).toBe(false); + const target = store.getTarget('app-ta', 1)!; + expect(target.applied_generation_id).toBe('gen-ta'); + expect(target.desired_generation_id).toBe('gen-ta'); + expect(target.candidate_generation_id).toBeNull(); + expect(target.expected_artifact_set_id).toBe('art-ta'); + expect(target.source_acceptance_ref).toBe('acc-ta'); + }); + + it('targetApplied refuses a delayed dispatch that would erase a newer candidate', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + const env = envelope('op-tan'); + tx.activateDirect({ application: app('app-tan', 'tan-web'), nodeId: 1, envelope: env }); + store.insertGeneration(gen('gen-tan-1', 'app-tan')); + tx.fetchStarted('app-tan', envelope('op-f-tan-1')); + tx.fetched('app-tan', 'deadbeef', envelope('op-f-tan-1')); + tx.candidateReady('app-tan', 'gen-tan-1', false, envelope('op-c-tan-1')); + tx.applyStarted('app-tan', 'gen-tan-1', envelope('op-tan')); + tx.sourceAccepted({ + applicationId: 'app-tan', + generationId: 'gen-tan-1', + artifactSetId: 'art-tan-1', + sourceAcceptanceId: 'acc-tan-1', + authority: 'operator', + envelope: env, + }); + // A newer revision arrives and supersedes generation 1 as the target's + // current candidate, before generation 1's dispatch ever binds a target. + store.insertGeneration(gen('gen-tan-2', 'app-tan')); + tx.fetchStarted('app-tan', envelope('op-f-tan-2')); + tx.fetched('app-tan', 'cafed00d', envelope('op-f-tan-2')); + tx.candidateReady('app-tan', 'gen-tan-2', false, envelope('op-c-tan-2')); + + // Generation 1's delayed dispatch must not silently erase generation 2's + // candidate, even though generation 1 is (still) the accepted generation. + expect(() => tx.targetApplied(1, { + applicationId: 'app-tan', + generationId: 'gen-tan-1', + artifactSetId: 'art-tan-1', + sourceAcceptanceId: 'acc-tan-1', + authority: 'operator', + envelope: envelope('op-ta-delayed'), + })).toThrow(/candidate/); + + const target = store.getTarget('app-tan', 1)!; + expect(target.candidate_generation_id).toBe('gen-tan-2'); + expect(target.applied_generation_id).toBeNull(); + }); + + it('targetApplied refuses a source acceptance reference that does not match what was accepted', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + const env = envelope('op-tar2'); + tx.activateDirect({ application: app('app-tar2', 'tar2-web'), nodeId: 1, envelope: env }); + store.insertGeneration(gen('gen-tar2', 'app-tar2')); + tx.fetchStarted('app-tar2', envelope('op-f-tar2')); + tx.fetched('app-tar2', 'deadbeef', envelope('op-f-tar2')); + tx.candidateReady('app-tar2', 'gen-tar2', false, envelope('op-c-tar2')); + tx.applyStarted('app-tar2', 'gen-tar2', envelope('op-tar2')); + tx.sourceAccepted({ + applicationId: 'app-tar2', + generationId: 'gen-tar2', + artifactSetId: 'art-tar2', + sourceAcceptanceId: 'acc-tar2-real', + authority: 'operator', + envelope: env, + }); + + expect(() => tx.targetApplied(1, { + applicationId: 'app-tar2', + generationId: 'gen-tar2', + artifactSetId: 'art-tar2', + sourceAcceptanceId: 'acc-tar2-forged', + authority: 'operator', + envelope: envelope('op-ta-forged'), + })).toThrow(/acceptance/); + + const target = store.getTarget('app-tar2', 1)!; + expect(target.source_acceptance_ref).not.toBe('acc-tar2-forged'); + }); + + it('targetApplied refuses to bind a target on a non-Direct application', async () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + const { blankInlineApplication } = await import('../services/gitops/blueprintProducers'); + const env = envelope('op-tam'); + + // A genuine Blueprint application: no Git identity, so the "accepted + // generation" and target row below are constructed directly rather than + // through the Direct fetch/candidate/apply path, which this mode does + // not have. + tx.activateInlineBlueprint({ application: blankInlineApplication('app-tam', 900, env.at), envelope: env }); + store.insertGeneration(gen('gen-tam', 'app-tam')); + DatabaseService.getInstance().getDb().prepare( + "UPDATE gitops_applications SET accepted_generation_id = 'gen-tam' WHERE id = 'app-tam'", + ).run(); + store.upsertTarget(emptyTargetRow('app-tam', 1, env.at)); + + expect(() => tx.targetApplied(1, { + applicationId: 'app-tam', + generationId: 'gen-tam', + artifactSetId: 'art-tam', + sourceAcceptanceId: 'acc-tam', + authority: 'operator', + envelope: env, + })).toThrow(/direct/i); + }); + + it('targetApplied refuses a generation the application has not accepted', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + const env = envelope('op-tar'); + tx.activateDirect({ application: app('app-tar', 'tar-web'), nodeId: 1, envelope: env }); + store.insertGeneration(gen('gen-tar', 'app-tar')); + tx.fetchStarted('app-tar', envelope('op-f-tar')); + tx.fetched('app-tar', 'deadbeef', envelope('op-f-tar')); + tx.candidateReady('app-tar', 'gen-tar', false, envelope('op-c-tar')); + + expect(() => tx.targetApplied(1, { + applicationId: 'app-tar', + generationId: 'gen-tar', + artifactSetId: 'art-tar', + sourceAcceptanceId: 'acc-tar', + authority: 'operator', + envelope: env, + })).toThrow(/not accepted/); + }); }); function mustProject(applicationId: string) { @@ -596,6 +812,7 @@ function app(id: string, stackName: string): GitOpsApplicationRow { active_generation_id: null, pause_at: null, pause_reason: null, + source_suspended_reason: null, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/__tests__/helpers/gitopsFixtures.ts b/backend/src/__tests__/helpers/gitopsFixtures.ts index 242f5a77..a80878ed 100644 --- a/backend/src/__tests__/helpers/gitopsFixtures.ts +++ b/backend/src/__tests__/helpers/gitopsFixtures.ts @@ -56,6 +56,7 @@ export function directApplicationFixture(id: string, stackName: string): GitOpsA active_generation_id: null, pause_at: null, pause_reason: null, + source_suspended_reason: null, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/__tests__/policy-enforcement.test.ts b/backend/src/__tests__/policy-enforcement.test.ts index c3ec63fe..6e2fb947 100644 --- a/backend/src/__tests__/policy-enforcement.test.ts +++ b/backend/src/__tests__/policy-enforcement.test.ts @@ -69,6 +69,7 @@ import { _resetTrivyMissingNotificationStateForTests, enforcePolicyForImageRefs, enforcePolicyPreDeploy, + evaluateCandidatePolicy, } from '../services/PolicyEnforcement'; function mkPolicy(overrides: Partial = {}): ScanPolicy { @@ -796,3 +797,129 @@ describe('enforcePolicyForImageRefs - risk-based inputs (KEV / fixable / optiona expect(dbStub.insertAuditLog.mock.calls[0][0].summary).toContain('policy.suppression_pass'); }); }); + +// Candidate (pre-acceptance) evaluation must not fail open the way the +// deploy-time gate deliberately does: an unresolvable scanner state must +// surface as its own `unavailable` outcome so a caller can withhold automatic +// acceptance, rather than silently reusing `ok: true`. +describe('evaluateCandidatePolicy', () => { + beforeEach(() => { + trivyStub.isTrivyAvailable.mockReset(); + trivyStub.scanImagePreflight.mockReset(); + composeStub.listStackImages.mockReset(); + dbStub.getMatchingPolicy.mockReset(); + dbStub.insertAuditLog.mockReset(); + dbStub.getGlobalSettings.mockReset().mockReturnValue({}); + dbStub.getAllVulnerabilityDetails.mockReset().mockReturnValue([]); + dbStub.getCveSuppressions.mockReset().mockReturnValue([]); + dbStub.getCveIntel.mockReset().mockReturnValue(new Map()); + notificationStub.dispatchAlert.mockReset(); + _resetTrivyMissingNotificationStateForTests(); + }); + + it('reports allowed when no matching policy exists', async () => { + dbStub.getMatchingPolicy.mockReturnValue(null); + + const result = await evaluateCandidatePolicy('web', 1, ['nginx:1.27'], { bypass: false, actor: 'u' }); + + expect(result.status).toBe('allowed'); + }); + + it('reports unavailable, not allowed, when the scanner cannot be evaluated', async () => { + dbStub.getMatchingPolicy.mockReturnValue(mkPolicy()); + trivyStub.isTrivyAvailable.mockReturnValue(false); + + const result = await evaluateCandidatePolicy('web', 1, ['nginx:1.27'], { bypass: false, actor: 'u' }); + + expect(result.status).toBe('unavailable'); + if (result.status === 'unavailable') { + expect(result.reason).toBeTruthy(); + } + }); + + it('reports blocked when a scanned image exceeds the policy severity', async () => { + dbStub.getMatchingPolicy.mockReturnValue(mkPolicy()); + trivyStub.isTrivyAvailable.mockReturnValue(true); + trivyStub.scanImagePreflight.mockResolvedValue(mkScan({ id: 7, highest_severity: 'CRITICAL', critical_count: 1 })); + + const result = await evaluateCandidatePolicy('web', 1, ['nginx:1.27'], { bypass: false, actor: 'u' }); + + expect(result.status).toBe('blocked'); + if (result.status === 'blocked') { + expect(result.violations).toHaveLength(1); + } + }); + + it('reports allowed on an authorized bypass', async () => { + dbStub.getMatchingPolicy.mockReturnValue(mkPolicy()); + trivyStub.isTrivyAvailable.mockReturnValue(true); + trivyStub.scanImagePreflight.mockResolvedValue(mkScan({ id: 8, highest_severity: 'CRITICAL', critical_count: 1 })); + + const result = await evaluateCandidatePolicy('web', 1, ['nginx:1.27'], { bypass: true, actor: 'admin' }); + + expect(result.status).toBe('allowed'); + }); + + it('never reads compose from disk; the caller supplies candidate image refs', async () => { + dbStub.getMatchingPolicy.mockReturnValue(mkPolicy()); + trivyStub.isTrivyAvailable.mockReturnValue(true); + trivyStub.scanImagePreflight.mockResolvedValue(mkScan({ id: 9, highest_severity: 'LOW' })); + + await evaluateCandidatePolicy('web', 1, ['nginx:1.27'], { bypass: false, actor: 'u' }); + + expect(composeStub.listStackImages).not.toHaveBeenCalled(); + }); + + it('reports unavailable, not allowed, for an image reference that cannot be scanned', async () => { + dbStub.getMatchingPolicy.mockReturnValue(mkPolicy()); + trivyStub.isTrivyAvailable.mockReturnValue(true); + + const result = await evaluateCandidatePolicy('web', 1, ['not a valid ref!!'], { bypass: false, actor: 'u' }); + + expect(result.status).toBe('unavailable'); + expect(trivyStub.scanImagePreflight).not.toHaveBeenCalled(); + }); + + it('reports unavailable, not blocked, when the scanner throws for every image', async () => { + dbStub.getMatchingPolicy.mockReturnValue(mkPolicy()); + trivyStub.isTrivyAvailable.mockReturnValue(true); + trivyStub.scanImagePreflight.mockRejectedValue(new Error('scan process crashed')); + + const result = await evaluateCandidatePolicy('web', 1, ['nginx:1.27'], { bypass: false, actor: 'u' }); + + expect(result.status).toBe('unavailable'); + }); + + it('still reports blocked when at least one image has a genuine scanned violation', async () => { + dbStub.getMatchingPolicy.mockReturnValue(mkPolicy()); + trivyStub.isTrivyAvailable.mockReturnValue(true); + trivyStub.scanImagePreflight + .mockRejectedValueOnce(new Error('scan process crashed')) + .mockResolvedValueOnce(mkScan({ id: 11, highest_severity: 'CRITICAL', critical_count: 1 })); + + const result = await evaluateCandidatePolicy('web', 1, ['unreachable:1', 'nginx:1.27'], { bypass: false, actor: 'u' }); + + expect(result.status).toBe('blocked'); + }); + + it('honors an explicit bypass even when the scanner is unavailable', async () => { + dbStub.getMatchingPolicy.mockReturnValue(mkPolicy()); + trivyStub.isTrivyAvailable.mockReturnValue(false); + + const result = await evaluateCandidatePolicy('web', 1, ['nginx:1.27'], { bypass: true, actor: 'admin' }); + + expect(result.status).toBe('allowed'); + }); + + it('does not attribute its audit trail to a deploy that never happened', async () => { + dbStub.getMatchingPolicy.mockReturnValue(mkPolicy()); + trivyStub.isTrivyAvailable.mockReturnValue(true); + trivyStub.scanImagePreflight.mockResolvedValue(mkScan({ id: 10, highest_severity: 'CRITICAL', critical_count: 1 })); + + await evaluateCandidatePolicy('web', 1, ['nginx:1.27'], { bypass: true, actor: 'admin' }); + + expect(dbStub.insertAuditLog).toHaveBeenCalledTimes(1); + const entry = dbStub.insertAuditLog.mock.calls[0][0]; + expect(entry.path).not.toMatch(/\/deploy$/); + }); +}); diff --git a/backend/src/__tests__/stackRouteAuth.test.ts b/backend/src/__tests__/stackRouteAuth.test.ts index c6efe976..03565c5c 100644 --- a/backend/src/__tests__/stackRouteAuth.test.ts +++ b/backend/src/__tests__/stackRouteAuth.test.ts @@ -79,6 +79,21 @@ describe('classifyStackApiPath', () => { kind: 'named-stack', stackName: 'web', action: 'stack:edit', }); }); + + // Load-bearing the same way as history/manifest above: without a rule + // here, suspend/resume/retry 403 on every remote node before the + // controller routes that use them exist. + it('maps git-source/suspend, resume, and retry to stack:edit', () => { + expect(classifyStackApiPath('POST', '/stacks/web/git-source/suspend')).toEqual({ + kind: 'named-stack', stackName: 'web', action: 'stack:edit', + }); + expect(classifyStackApiPath('POST', '/stacks/web/git-source/resume')).toEqual({ + kind: 'named-stack', stackName: 'web', action: 'stack:edit', + }); + expect(classifyStackApiPath('POST', '/stacks/web/git-source/retry')).toEqual({ + kind: 'named-stack', stackName: 'web', action: 'stack:edit', + }); + }); }); describe('static exclusions', () => { diff --git a/backend/src/helpers/stackRouteAuth.ts b/backend/src/helpers/stackRouteAuth.ts index 2a06bac4..096c0a9e 100644 --- a/backend/src/helpers/stackRouteAuth.ts +++ b/backend/src/helpers/stackRouteAuth.ts @@ -89,6 +89,9 @@ const EXACT_SUFFIX_RULES: readonly SuffixRule[] = [ { method: 'POST', suffix: '/git-source/webhook-pull', action: 'stack:edit' }, { method: 'POST', suffix: '/git-source/dismiss-pending', action: 'stack:edit' }, { method: 'POST', suffix: '/git-source/browse', action: 'stack:edit' }, + { method: 'POST', suffix: '/git-source/suspend', action: 'stack:edit' }, + { method: 'POST', suffix: '/git-source/resume', action: 'stack:edit' }, + { method: 'POST', suffix: '/git-source/retry', action: 'stack:edit' }, // Deploy { method: 'POST', suffix: '/deploy', action: 'stack:deploy' }, diff --git a/backend/src/routes/gitSources.ts b/backend/src/routes/gitSources.ts index 1155aa3c..cef891f0 100644 --- a/backend/src/routes/gitSources.ts +++ b/backend/src/routes/gitSources.ts @@ -10,7 +10,6 @@ import { classifySourceRow, satisfiesGitOpsRead } from '../services/gitops/readA import { NOT_APPLICABLE_REVISION, projectStackRevision, stackResourceSet } from '../helpers/gitopsResponse'; import { respondWithHistory } from '../helpers/gitopsHistoryPage'; import { invalidateNodeCaches } from '../helpers/cacheInvalidation'; -import { triggerPostDeployScan } from '../helpers/policyGate'; import { parseComposeSelection, defaultEnvPath } from '../helpers/gitSourceSelection'; import { isValidGitSourcePath, isValidStackName } from '../utils/validation'; import { sendGitSourceError, webhookPullStatus } from '../utils/gitSourceHttp'; @@ -563,7 +562,7 @@ stackGitSourceRouter.post('/:stackName/git-source/apply', async (req: Request, r requirePlanFingerprint: true, }, ); - invalidateNodeCaches(req.nodeId); + // Cache invalidation and the post-deploy scan now run inside GitSourceService.apply() itself. const shortSha = commitSha.trim().slice(0, 7); if (result.deployed) { console.log('[GitSource] Applied commit %s to %s (deployed)', sanitizeForLog(shortSha), sanitizeForLog(stackName)); @@ -573,11 +572,6 @@ stackGitSourceRouter.post('/:stackName/git-source/apply', async (req: Request, r console.log('[GitSource] Applied commit %s to %s', sanitizeForLog(shortSha), sanitizeForLog(stackName)); } res.json(result); - if (result.deployed) { - triggerPostDeployScan(stackName, req.nodeId).catch(err => - console.error(`[Security] Post-deploy scan failed for ${sanitizeForLog(stackName)}:`, err), - ); - } } catch (error) { sendGitSourceError(res, error); } diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index 07376a03..d999877d 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -573,6 +573,14 @@ export interface NotificationHistory { container_name?: string; actor_username?: string | null; suppression_match?: string | null; + /** The GitOps operation this notification reports on, if any. */ + gitops_operation_id?: string | null; + /** + * Unique across all notifications when set. Lets fanout repair re-run + * safely: inserting the same key again is a no-op that returns the + * existing row instead of creating a duplicate. + */ + dedupe_key?: string | null; } export interface FleetSnapshot { @@ -1164,6 +1172,7 @@ export class DatabaseService { this.migratePolicyEvaluationColumn(); this.migrateNotificationCategory(); this.migrateNotificationActor(); + this.migrateNotificationGitOpsDedupe(); this.migrateMeshTables(); this.migrateNodeLabels(); this.migrateBlueprints(); @@ -1963,6 +1972,10 @@ export class DatabaseService { // from the CREATE TABLE; older DBs need the additive column here. maybeAddCol('gitops_generations', 'resolved_ref_kind', 'TEXT NULL'); maybeAddCol('gitops_applications', 'fetched_resolved_ref_kind', 'TEXT NULL'); + // Source suspension reason, distinct from the rollout pause_reason + // existing installs already have. New installs get it from the + // CREATE TABLE; older DBs need the additive column here. + maybeAddCol('gitops_applications', 'source_suspended_reason', 'TEXT NULL'); // Distributed API model columns maybeAddCol('nodes', 'api_url', "TEXT DEFAULT ''"); @@ -2799,6 +2812,29 @@ stmt.run('gitops_schema_version', '1'); } } + /** + * A GitOps history/operation reference and a dedupe key, so notification + * fanout for a settled GitOps attempt can be repaired from durable state + * (retried at startup after a crash between commit and fanout) without + * ever inserting a duplicate notification for the same attempt. + */ + private migrateNotificationGitOpsDedupe(): void { + this.tryAddColumn('notification_history', 'gitops_operation_id', 'TEXT'); + this.tryAddColumn('notification_history', 'dedupe_key', 'TEXT'); + try { + this.db.prepare( + 'CREATE UNIQUE INDEX IF NOT EXISTS idx_notif_history_dedupe_key ON notification_history(dedupe_key) WHERE dedupe_key IS NOT NULL' + ).run(); + } catch (err) { + // Unlike a pure performance index, this one is the ON CONFLICT + // target every addNotificationHistory() insert names. If it is + // missing, every notification write in the product fails, not + // just GitOps ones, so a silent catch here would turn into an + // unexplained total outage instead of a diagnosable startup log. + console.error('[DatabaseService] Failed to create notification dedupe index:', err); + } + } + private migrateMeshTables(): void { try { if (isPilotMode()) { @@ -4738,8 +4774,13 @@ stmt.run('gitops_schema_version', '1'); } public addNotificationHistory(nodeId: number, notification: Omit): NotificationHistory { + const dedupeKey = notification.dedupe_key ?? null; const stmt = this.db.prepare( - 'INSERT INTO notification_history (node_id, level, message, timestamp, is_read, stack_name, container_name, category, actor_username) VALUES (?, ?, ?, ?, 0, ?, ?, ?, ?)' + `INSERT INTO notification_history ( + node_id, level, message, timestamp, is_read, stack_name, container_name, + category, actor_username, gitops_operation_id, dedupe_key + ) VALUES (?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?) + ON CONFLICT(dedupe_key) WHERE dedupe_key IS NOT NULL DO NOTHING` ); const result = stmt.run( nodeId, @@ -4750,8 +4791,20 @@ stmt.run('gitops_schema_version', '1'); notification.container_name ?? null, notification.category ?? null, notification.actor_username ?? null, + notification.gitops_operation_id ?? null, + dedupeKey, ); + // A repair replaying a settled GitOps attempt must not create a + // duplicate notification; the conflict is not an error, it is proof + // this exact attempt was already reported. + if (result.changes === 0 && dedupeKey !== null) { + const existing = this.db.prepare( + 'SELECT * FROM notification_history WHERE dedupe_key = ?', + ).get(dedupeKey); + return this.mapNotificationRow(existing); + } + return { id: result.lastInsertRowid as number, level: notification.level, @@ -4762,6 +4815,8 @@ stmt.run('gitops_schema_version', '1'); stack_name: notification.stack_name, container_name: notification.container_name, actor_username: notification.actor_username, + gitops_operation_id: notification.gitops_operation_id, + dedupe_key: dedupeKey, }; } diff --git a/backend/src/services/GitSourceService.ts b/backend/src/services/GitSourceService.ts index f4db48f1..345e2560 100644 --- a/backend/src/services/GitSourceService.ts +++ b/backend/src/services/GitSourceService.ts @@ -11,7 +11,8 @@ import { ComposeService } from './ComposeService'; import { StackOpLockService } from './StackOpLockService'; import { HealthGateService } from './HealthGateService'; import { NodeRegistry } from './NodeRegistry'; -import { assertPolicyGateAllows, buildSystemPolicyGateOptions } from '../helpers/policyGate'; +import { assertPolicyGateAllows, buildSystemPolicyGateOptions, triggerPostDeployScan } from '../helpers/policyGate'; +import { invalidateNodeCaches } from '../helpers/cacheInvalidation'; import { isDebugEnabled } from '../utils/debug'; import { sanitizeForLog } from '../utils/safeLog'; import { isPathWithinBase, isValidRelativeStackPath } from '../utils/validation'; @@ -27,7 +28,7 @@ import type { ComposeInputEntry, GitProjectManifest, GitSourceManifestState, Inv import type { GitChangePlan, PublicGitChangePlan, GitChangePlanCounts, PublicGitChangePlanOperation } from '../types/gitChangePlan'; import { GIT_CHANGE_PLAN_SCHEMA_VERSION } from '../types/gitChangePlan'; import type { NotificationCategory } from './NotificationService'; -import { classifyGitFailure, isTransportFailure } from './git/errors'; +import { classifyGitFailure, isTransportFailure, type TransportFailureReason } from './git/errors'; import type { RefKind, SshDeployKeyAuth } from './git/types'; import { nativeGitTransport, verifyFastForward } from './git/nativeGitTransport'; import { fingerprintFromKnownHostsLine } from './git/sshTrust'; @@ -82,7 +83,18 @@ export class GitSourceError extends Error { constructor( public code: GitSourceErrorCode, message: string, - public extras?: { plan?: PublicGitChangePlan; planFingerprint?: string }, + public extras?: { + plan?: PublicGitChangePlan; + planFingerprint?: string; + /** + * The raw structured reason from the native transport failure, + * kept alongside the sanitized `code`/`message` operators see. + * Consumed by GitOps retry/backoff classification, which needs + * more than the public error code to tell a transient network + * condition from a permanent configuration one. + */ + transportReason?: TransportFailureReason; + }, ) { super(message); this.name = 'GitSourceError'; @@ -1259,9 +1271,9 @@ export class GitSourceService { // A ref that resolved before but no longer does is a deletion // or force-push, distinct from a mis-typed ref on first link. if (classified.code === 'REF_NOT_FOUND' && hasPriorHistory) { - throw new GitSourceError('REF_DELETED', REF_DELETED_MESSAGE); + throw new GitSourceError('REF_DELETED', REF_DELETED_MESSAGE, { transportReason: e.reason }); } - throw new GitSourceError(classified.code, classified.message); + throw new GitSourceError(classified.code, classified.message, { transportReason: e.reason }); } throw e; } finally { @@ -2642,6 +2654,12 @@ export class GitSourceService { `Git apply succeeded for ${stackName} (${commitSha.slice(0, 7)}, op ${pending.operationId.slice(0, 8)}, plan ${plan.fingerprint.slice(0, 12)})`, actor, ); + // Promotion has committed and rewritten the authoritative Compose + // files, so cached stats/statuses/project-name state is stale here + // whether or not a deploy follows. This must fire exactly once per + // successful promotion, from every trigger, not only the manual + // apply route (which used to invalidate here itself). + invalidateNodeCaches(nodeId); } else { throw new GitSourceError('PLAN_UNAVAILABLE', 'Pending update cannot be reviewed; pull again.'); } @@ -2725,6 +2743,12 @@ export class GitSourceService { recoverySvc.linkGateOrRetain(recoveryId, healthGateId); } console.log(`[GitSource] Applied and deployed ${stackName} at ${commitSha.slice(0, 7)}`); + // Fire-and-forget, matching the manual apply route's prior + // placement: the scan runs only after a successful deploy and + // must never delay or fail the apply response. + triggerPostDeployScan(stackName, nodeId).catch((err) => + console.error(`[Security] Post-deploy scan failed for ${sanitizeForLog(stackName)}:`, err), + ); return { applied: true, deployed: true, recoveryId }; } catch (e) { // R1: do not auto-compensate. Keep applied files and leave the diff --git a/backend/src/services/PolicyEnforcement.ts b/backend/src/services/PolicyEnforcement.ts index 8ac9cddc..a594a0e4 100644 --- a/backend/src/services/PolicyEnforcement.ts +++ b/backend/src/services/PolicyEnforcement.ts @@ -74,6 +74,20 @@ export interface PolicyEnforcementResult { trivyMissing?: boolean; } +/** + * Candidate (pre-acceptance) policy outcome. Unlike the deploy-time gate, + * which deliberately fails open when the scanner is unavailable so an + * operator is never blocked from deploying, an unresolvable scanner state + * here is its own outcome: automatic source acceptance must not read + * `unavailable` as `allowed`, or a GitOps source could accept a candidate + * nothing actually proved safe. + */ +export type CandidatePolicyEvaluation = { policy?: ScanPolicy } & ( + | { status: 'allowed' } + | { status: 'blocked'; violations: PolicyViolation[] } + | { status: 'unavailable'; reason: string } +); + const TRIVY_MISSING_NOTIFY_COOLDOWN_MS = 60 * 60 * 1000; // Growth bounded by configured-policy fanout (only stacks with an enabled // block_on_deploy policy can land here), not by total stack churn. Cleared @@ -472,3 +486,50 @@ export async function enforcePolicyForImageRefs( ); return { ok: false, bypassed: false, policy, violations }; } + +/** + * Tri-state candidate evaluation for GitOps source acceptance, built on the + * same evaluator the deploy-time gate uses, with the candidate's own image + * refs supplied directly rather than read from disk. Has side effects: + * writes a policy.bypass/policy.suppression_pass audit row when applicable, + * and may dispatch the once-per-hour Trivy-missing operator notification. + */ +export async function evaluateCandidatePolicy( + stackName: string, + nodeId: number, + imageRefs: string[], + opts: PolicyEnforcementOptions, +): Promise { + const result = await enforcePolicyForImageRefs(stackName, nodeId, imageRefs, { + ...opts, + // Undefaulted, these attribute the audit row to a deploy path + // (enforcePolicyForImageRefs's own default), which never happened + // for a pre-acceptance candidate. + auditMethod: opts.auditMethod ?? 'POST', + auditPath: opts.auditPath ?? `/api/stacks/${stackName}/git-source/candidate`, + // The deploy gate silently skips an unscannable ref (fail-open, + // since it must never block an operator's deploy on its own + // inability to evaluate). Candidate evaluation is the opposite: an + // unscannable ref must surface as evidence, not vanish, so it can be + // told apart from a genuinely clean scan below. + }, undefined, true); + // Only trivyMissing forgoes bypass consideration below it because it is + // the one path that returns bypassed: false unconditionally; every other + // branch of the shared evaluator already honors opts.bypass itself. + if (result.trivyMissing) { + if (opts.bypass) return { status: 'allowed', policy: result.policy }; + return { status: 'unavailable', policy: result.policy, reason: 'Vulnerability scanner is unavailable' }; + } + if (!result.ok) { + // A violation with no `error` is a genuine scanned policy match; one + // with `error` set is an invalid ref, a scan failure, or an + // evaluation failure -- evidence Sencho could not prove either way, + // not a proven violation. All-unproven must not read as `blocked`. + const hasGenuineViolation = result.violations.some((v) => !v.error); + if (!hasGenuineViolation) { + return { status: 'unavailable', policy: result.policy, reason: 'Candidate could not be fully evaluated' }; + } + return { status: 'blocked', policy: result.policy, violations: result.violations }; + } + return { status: 'allowed', policy: result.policy }; +} diff --git a/backend/src/services/gitops/blueprintProducers.ts b/backend/src/services/gitops/blueprintProducers.ts index a08fd4b2..7dfc5f17 100644 --- a/backend/src/services/gitops/blueprintProducers.ts +++ b/backend/src/services/gitops/blueprintProducers.ts @@ -381,6 +381,7 @@ export function blankInlineApplication(id: string, blueprintId: number, at: numb active_generation_id: null, pause_at: null, pause_reason: null, + source_suspended_reason: null, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/services/gitops/derive.ts b/backend/src/services/gitops/derive.ts index 1ecb37b7..d8826380 100644 --- a/backend/src/services/gitops/derive.ts +++ b/backend/src/services/gitops/derive.ts @@ -234,7 +234,14 @@ function deriveSource(app: GitOpsApplicationRow, limitations: GitOpsLimitation[] interruptedGenerationId: app.interruption_generation_id, }; } - if (app.suspended_at) return { ...identity, status: 'source_suspended', suspendedAt: app.suspended_at }; + if (app.suspended_at) { + return { + ...identity, + status: 'source_suspended', + suspendedAt: app.suspended_at, + suspendedReason: app.source_suspended_reason, + }; + } if (app.failure_stage === 'fetch' || app.failure_stage === 'validation' || app.failure_stage === 'apply' || app.failure_stage === 'create') { return { ...identity, diff --git a/backend/src/services/gitops/directApplication.ts b/backend/src/services/gitops/directApplication.ts index 33135e40..c3693dd9 100644 --- a/backend/src/services/gitops/directApplication.ts +++ b/backend/src/services/gitops/directApplication.ts @@ -156,6 +156,7 @@ export function buildDirectApplicationRow(args: { active_generation_id: null, pause_at: null, pause_reason: null, + source_suspended_reason: null, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/services/gitops/history.ts b/backend/src/services/gitops/history.ts index 2695d73f..bee1642f 100644 --- a/backend/src/services/gitops/history.ts +++ b/backend/src/services/gitops/history.ts @@ -68,10 +68,12 @@ export type GitOpsHistoryStage = | 'rollout_candidate_opened' | 'rollout_paused' | 'rollout_unpaused' + | 'source_accepted' | 'source_conflict_blocker' | 'source_retry_scheduled' | 'source_suspended' | 'source_unsuspended' + | 'target_applied' | 'target_tombstoned'; export type HistoryInsert = { diff --git a/backend/src/services/gitops/schema.ts b/backend/src/services/gitops/schema.ts index b7d190a5..1c311e24 100644 --- a/backend/src/services/gitops/schema.ts +++ b/backend/src/services/gitops/schema.ts @@ -95,6 +95,11 @@ CREATE TABLE IF NOT EXISTS gitops_applications ( active_generation_id TEXT NULL, pause_at INTEGER NULL, pause_reason TEXT NULL, + -- Distinct from pause_reason: sourceSuspended/sourceUnsuspended write this + -- field, not the one rolloutPaused/rolloutUnpaused share across app and + -- target rows, so suspending a source can never clobber an unrelated + -- rollout pause reason (or the reverse). + source_suspended_reason TEXT NULL, partial_json TEXT NULL, failure_stage TEXT NULL CHECK ( failure_stage IS NULL OR failure_stage IN ( diff --git a/backend/src/services/gitops/store.ts b/backend/src/services/gitops/store.ts index e839f95b..4069d6a2 100644 --- a/backend/src/services/gitops/store.ts +++ b/backend/src/services/gitops/store.ts @@ -365,12 +365,12 @@ export class GitOpsStore { intent_revision_id, rollout_candidate_id, rollout_generation_id, source_acceptance_ref, placement_approval_ref, rollout_authorization_ref, legacy_combined_approval_ref, preflight_fingerprint, latest_operation_id, active_operation_id, active_operation_stage, - active_operation_at, active_generation_id, pause_at, pause_reason, partial_json, + active_operation_at, active_generation_id, pause_at, pause_reason, source_suspended_reason, partial_json, failure_stage, failure_class, failure_at, retry_at, retry_count, suspended_at, recovery_ref, recovery_phase, interruption_stage, interruption_at, interruption_operation_id, interruption_generation_id, evidence_fresh_at, evidence_limitations_json, created_at, updated_at - ) VALUES (${Array(55).fill('?').join(', ')})`, + ) VALUES (${Array(56).fill('?').join(', ')})`, ).run( row.id, row.lifecycle_key, row.lifecycle_status, row.target_mode, row.stack_name, row.blueprint_id, row.configured_repo_url, row.repo_identity_json, row.configured_ref, row.compose_paths_json, @@ -380,7 +380,7 @@ export class GitOpsStore { row.intent_revision_id, row.rollout_candidate_id, row.rollout_generation_id, row.source_acceptance_ref, row.placement_approval_ref, row.rollout_authorization_ref, row.legacy_combined_approval_ref, row.preflight_fingerprint, row.latest_operation_id, row.active_operation_id, row.active_operation_stage, - row.active_operation_at, row.active_generation_id, row.pause_at, row.pause_reason, row.partial_json, + row.active_operation_at, row.active_generation_id, row.pause_at, row.pause_reason, row.source_suspended_reason, row.partial_json, row.failure_stage, row.failure_class, row.failure_at, row.retry_at, row.retry_count, row.suspended_at, row.recovery_ref, row.recovery_phase, row.interruption_stage, row.interruption_at, row.interruption_operation_id, row.interruption_generation_id, row.evidence_fresh_at, diff --git a/backend/src/services/gitops/transitions.ts b/backend/src/services/gitops/transitions.ts index 973f0acb..5ebf736a 100644 --- a/backend/src/services/gitops/transitions.ts +++ b/backend/src/services/gitops/transitions.ts @@ -372,28 +372,10 @@ export class GitOpsTransitions { applied(args: AppliedArgs): TransitionResult { return this.mutateApp(args.applicationId, args.envelope, 'applied', 'committed', (app) => { const targets = this.acceptanceTargets(app, args); - this.insertAcceptanceRecords(app, args); - app.accepted_generation_id = args.generationId; - app.artifact_set_id = args.artifactSetId; - app.latest_artifact_set_id = args.artifactSetId; - app.source_acceptance_ref = args.sourceAcceptanceId; - app.candidate_generation_id = null; - app.candidate_plan_blocked = 0; - app.review_required = 0; - if (args.activateCreating && app.lifecycle_status === 'creating') { - app.lifecycle_status = 'active'; - } - this.clearActive(app); - this.clearAppFailure(app, ['apply', 'fetch', 'validation']); - this.clearInterruption(app, 'apply_started'); + this.applySourceAcceptanceMutation(app, args); for (const target of targets) { if (app.target_mode === 'direct') { - target.desired_generation_id = args.generationId; - target.applied_generation_id = args.generationId; - target.expected_artifact_set_id = args.artifactSetId; - target.latest_artifact_set_id = args.artifactSetId; - target.source_acceptance_ref = args.sourceAcceptanceId; - target.candidate_generation_id = null; + this.applyTargetAcceptanceMutation(target, args); } this.store().upsertTarget(target); } @@ -404,6 +386,63 @@ export class GitOpsTransitions { }); } + /** + * Mode-neutral half of `applied`: accept the candidate at the application + * level without binding any target. A Direct dispatch calls this before + * promotion; `targetApplied` binds the target only after promotion commits. + */ + sourceAccepted(args: AppliedArgs): TransitionResult { + return this.mutateApp(args.applicationId, args.envelope, 'source_accepted', 'committed', (app) => { + // Unlike applied() (preserved byte-identical, predates suspension), + // this new entry point is the one a suspended source must refuse: no + // new acceptance while suspended, so the check lives here rather than + // in the shared requireAcceptableCandidate guard. + if (app.suspended_at) throw new GitOpsTransitionError('source is suspended'); + this.requireAcceptableCandidate(app, args); + this.applySourceAcceptanceMutation(app, args); + }, { + generationId: args.generationId, + artifactSetId: args.artifactSetId, + sourceAcceptanceRef: args.sourceAcceptanceId, + }); + } + + /** + * Direct-only half of `applied`: bind one target to an already-accepted + * generation. Refuses a generation the application has not accepted, so a + * dispatch cannot bind a target to source content nothing authorized. + */ + targetApplied(nodeId: number, args: AppliedArgs): TransitionResult { + const app = this.requireApp(args.applicationId); + if (app.target_mode !== 'direct') { + throw new GitOpsTransitionError('target application is not direct'); + } + if (app.accepted_generation_id !== args.generationId) { + throw new GitOpsTransitionError('generation is not accepted'); + } + // The application's accepted_generation_id does not move again until a + // later sourceAccepted call, so a delayed dispatch for a superseded-but- + // still-accepted generation would otherwise pass the check above even + // after a newer candidate has already been staged for this target. Only + // an acceptance reference this application actually recorded may bind a + // target; a caller passing any other id would otherwise write + // unverifiable authorization evidence straight onto the target row. + if (app.source_acceptance_ref !== args.sourceAcceptanceId) { + throw new GitOpsTransitionError('source acceptance reference does not match the accepted generation'); + } + return this.mutateTarget(args.applicationId, nodeId, args.envelope, 'target_applied', args.generationId, (target) => { + if (target.target_status !== 'active') { + throw new GitOpsTransitionError('cannot apply to a tombstoned target'); + } + if (target.candidate_generation_id !== args.generationId) { + throw new GitOpsTransitionError('target candidate does not match applied generation'); + } + const before = { appliedGenerationId: target.applied_generation_id }; + this.applyTargetAcceptanceMutation(target, args); + return { before, after: { appliedGenerationId: args.generationId } }; + }); + } + /** * The single transaction that makes a create-from-Git durable. * @@ -841,7 +880,7 @@ export class GitOpsTransitions { this.clearActive(app); } app.suspended_at = envelope.at; - app.pause_reason = reason; + app.source_suspended_reason = reason; }); } @@ -850,7 +889,7 @@ export class GitOpsTransitions { return this.mutateApp(applicationId, envelope, 'source_unsuspended', 'committed', (app) => { if (!app.suspended_at) throw new GitOpsTransitionError('source is not suspended'); app.suspended_at = null; - app.pause_reason = null; + app.source_suspended_reason = null; }); } @@ -1893,7 +1932,8 @@ export class GitOpsTransitions { * Guard every precondition of `applied` and return the active targets the * acceptance has to bind. */ - private acceptanceTargets(app: GitOpsApplicationRow, args: AppliedArgs): GitOpsTargetCurrentRow[] { + /** Guard every application-level precondition of accepting a candidate. */ + private requireAcceptableCandidate(app: GitOpsApplicationRow, args: AppliedArgs): void { if (app.candidate_generation_id !== args.generationId) { throw new GitOpsTransitionError('applied generation is not the current candidate'); } @@ -1913,6 +1953,10 @@ export class GitOpsTransitions { throw new GitOpsTransitionError('live apply belongs to a different operation'); } } + } + + private acceptanceTargets(app: GitOpsApplicationRow, args: AppliedArgs): GitOpsTargetCurrentRow[] { + this.requireAcceptableCandidate(app, args); const targets = this.store().listTargets(app.id).filter((row) => row.target_status === 'active'); if (app.target_mode === 'direct') { for (const target of targets) { @@ -1924,6 +1968,34 @@ export class GitOpsTransitions { return targets; } + /** The mode-neutral application-row mutation `applied` and `sourceAccepted` share. */ + private applySourceAcceptanceMutation(app: GitOpsApplicationRow, args: AppliedArgs): void { + this.insertAcceptanceRecords(app, args); + app.accepted_generation_id = args.generationId; + app.artifact_set_id = args.artifactSetId; + app.latest_artifact_set_id = args.artifactSetId; + app.source_acceptance_ref = args.sourceAcceptanceId; + app.candidate_generation_id = null; + app.candidate_plan_blocked = 0; + app.review_required = 0; + if (args.activateCreating && app.lifecycle_status === 'creating') { + app.lifecycle_status = 'active'; + } + this.clearActive(app); + this.clearAppFailure(app, ['apply', 'fetch', 'validation']); + this.clearInterruption(app, 'apply_started'); + } + + /** The Direct-only target-row mutation `applied` and `targetApplied` share. */ + private applyTargetAcceptanceMutation(target: GitOpsTargetCurrentRow, args: AppliedArgs): void { + target.desired_generation_id = args.generationId; + target.applied_generation_id = args.generationId; + target.expected_artifact_set_id = args.artifactSetId; + target.latest_artifact_set_id = args.artifactSetId; + target.source_acceptance_ref = args.sourceAcceptanceId; + target.candidate_generation_id = null; + } + /** Seed the unresolved artifact row and the source acceptance this apply proves. */ private insertAcceptanceRecords(app: GitOpsApplicationRow, args: AppliedArgs): void { const artifact: GitOpsArtifactSetRow = { @@ -2247,7 +2319,7 @@ export class GitOpsTransitions { legacy_combined_approval_ref=?, preflight_fingerprint=?, latest_operation_id=?, active_operation_id=?, active_operation_stage=?, active_operation_at=?, active_generation_id=?, - pause_at=?, pause_reason=?, partial_json=?, + pause_at=?, pause_reason=?, source_suspended_reason=?, partial_json=?, failure_stage=?, failure_class=?, failure_at=?, retry_at=?, retry_count=?, suspended_at=?, recovery_ref=?, recovery_phase=?, interruption_stage=?, interruption_at=?, interruption_operation_id=?, @@ -2264,7 +2336,7 @@ export class GitOpsTransitions { app.legacy_combined_approval_ref, app.preflight_fingerprint, app.latest_operation_id, app.active_operation_id, app.active_operation_stage, app.active_operation_at, app.active_generation_id, - app.pause_at, app.pause_reason, app.partial_json, + app.pause_at, app.pause_reason, app.source_suspended_reason, app.partial_json, app.failure_stage, app.failure_class, app.failure_at, app.retry_at, app.retry_count, app.suspended_at, app.recovery_ref, app.recovery_phase, app.interruption_stage, app.interruption_at, app.interruption_operation_id, diff --git a/backend/src/services/gitops/types.ts b/backend/src/services/gitops/types.ts index 94ea799c..c9f1ffa8 100644 --- a/backend/src/services/gitops/types.ts +++ b/backend/src/services/gitops/types.ts @@ -71,6 +71,8 @@ export type GitOpsApplicationRow = { active_generation_id: string | null; pause_at: number | null; pause_reason: string | null; + /** sourceSuspended/sourceUnsuspended's own reason field; independent of pause_reason. */ + source_suspended_reason: string | null; partial_json: string | null; failure_stage: ApplicationFailureStage | null; failure_class: string | null; @@ -402,7 +404,7 @@ export type SourceFacet = | (SourceIdentityFields & { status: 'source_superseded'; supersededGenerationId: string }) | (SourceIdentityFields & { status: 'applying'; activeOperationId: string; activeGenerationId: string }) | (SourceIdentityFields & { status: 'source_retry_scheduled'; retryAt: number; retryCount: number }) - | (SourceIdentityFields & { status: 'source_suspended'; suspendedAt: number }) + | (SourceIdentityFields & { status: 'source_suspended'; suspendedAt: number; suspendedReason: string | null }) | (SourceIdentityFields & { status: 'source_failed'; failureStage: 'fetch' | 'validation' | 'apply' | 'create'; diff --git a/frontend/src/types/gitops.ts b/frontend/src/types/gitops.ts index 53f1f02f..3be624dd 100644 --- a/frontend/src/types/gitops.ts +++ b/frontend/src/types/gitops.ts @@ -122,7 +122,7 @@ export type SourceFacet = | (SourceIdentityFields & { status: 'source_superseded'; supersededGenerationId: string }) | (SourceIdentityFields & { status: 'applying'; activeOperationId: string; activeGenerationId: string }) | (SourceIdentityFields & { status: 'source_retry_scheduled'; retryAt: number; retryCount: number }) - | (SourceIdentityFields & { status: 'source_suspended'; suspendedAt: number }) + | (SourceIdentityFields & { status: 'source_suspended'; suspendedAt: number; suspendedReason: string | null }) | (SourceIdentityFields & { status: 'source_failed'; failureStage: 'fetch' | 'validation' | 'apply' | 'create';