From 1ad20da31f4f452d318fed6523b50c9f93034359 Mon Sep 17 00:00:00 2001 From: Anso Date: Mon, 7 Sep 2026 14:20:47 -0400 Subject: [PATCH 1/3] Surface runtime build identity across About, shell, and Admiral Account (#1899) * feat: surface runtime build identity across About, shell, and Admiral Account Add one canonical source for the control instance's runtime build identity so a development image is visibly identified even when its packaged semver still matches the previous stable release. SelfIdentityService now retains the running image reference and assembles a BuildInfo (version, channel, imageRef, imageId, revision) via a bounded, failure-isolated image-inspect step during initialization. getBuildInfo() is a cached read that never triggers Docker. classifyBuildChannel() labels the running reference stable, dev, preview, or unknown. A new proxy-exempt GET /api/build-info returns that identity to a signed-in human session, redacting hardened image references to non-admins with a restricted flag so the UI shows Restricted rather than Unknown. Public /api/meta gains a bounded buildChannel enum without leaking the image reference. A BuildInfoProvider context shares one fetch across the About section, the sidebar DEV/PREVIEW chip, the mobile tab bar, and the Admiral Account Channel and Current image rows, keeping the control-instance identity separate from remote nodes. * fix: move chipDetail helper out of the component file to satisfy fast refresh SidebarBrand.tsx exported both the SidebarBrand component and the chipDetail helper, which trips react-refresh/only-export-components and fails the lint gate. The helper now lives in its own chipDetail.ts module so the component file exports only the component; the direct unit test keeps its coverage by importing from the new module. * fix(build-info): resolve revision-enrichment race and harden identity rendering - Await the detached revision enrichment before serving /api/build-info so a successful response never freezes a transient null revision. - Lay the mobile DEV/PREVIEW pill in-flow as a non-overlapping flex sibling instead of an absolutely positioned overlay. - Wrap long image and revision tokens with break-all so they do not overflow the About panel. - Surface a toast when copying the image id fails instead of leaving an unhandled rejection. --- .../src/__tests__/build-info-route.test.ts | 225 ++++++++++++++++++ .../__tests__/classify-build-channel.test.ts | 42 ++++ backend/src/__tests__/monitor-service.test.ts | 35 +++ .../__tests__/self-identity-buildinfo.test.ts | 197 +++++++++++++++ backend/src/helpers/proxyExemptPaths.ts | 1 + backend/src/helpers/selfUpdateCompose.ts | 45 +++- backend/src/index.ts | 2 + backend/src/routes/buildInfo.ts | 47 ++++ backend/src/routes/meta.ts | 8 +- backend/src/services/SelfIdentityService.ts | 89 +++++++ backend/src/services/selfDevBuildDetect.ts | 8 +- docs/features/licensing.mdx | 2 +- docs/operations/verifying-images.mdx | 14 ++ docs/reference/settings.mdx | 12 +- frontend/src/App.tsx | 13 +- frontend/src/components/EditorLayout.tsx | 3 + frontend/src/components/MobileTabBar.test.tsx | 73 +++++- frontend/src/components/MobileTabBar.tsx | 19 +- .../src/components/settings/AboutSection.tsx | 102 +++++++- .../components/settings/LicenseSection.tsx | 24 +- .../settings/__tests__/AboutSection.test.tsx | 76 ++++++ .../AboutSection.whatsNewEmpty.test.tsx | 4 + .../__tests__/LicenseSection.test.tsx | 76 ++++++ .../components/sidebar/SidebarBrand.test.tsx | 72 ++++++ .../src/components/sidebar/SidebarBrand.tsx | 36 ++- .../src/components/sidebar/StackSidebar.tsx | 6 +- frontend/src/components/sidebar/chipDetail.ts | 11 + .../src/context/BuildInfoProvider.test.tsx | 135 +++++++++++ frontend/src/context/BuildInfoProvider.tsx | 103 ++++++++ frontend/src/hooks/useBuildInfo.ts | 13 + 30 files changed, 1464 insertions(+), 29 deletions(-) create mode 100644 backend/src/__tests__/build-info-route.test.ts create mode 100644 backend/src/__tests__/classify-build-channel.test.ts create mode 100644 backend/src/__tests__/self-identity-buildinfo.test.ts create mode 100644 backend/src/routes/buildInfo.ts create mode 100644 frontend/src/components/sidebar/SidebarBrand.test.tsx create mode 100644 frontend/src/components/sidebar/chipDetail.ts create mode 100644 frontend/src/context/BuildInfoProvider.test.tsx create mode 100644 frontend/src/context/BuildInfoProvider.tsx create mode 100644 frontend/src/hooks/useBuildInfo.ts diff --git a/backend/src/__tests__/build-info-route.test.ts b/backend/src/__tests__/build-info-route.test.ts new file mode 100644 index 00000000..1b6a5bbc --- /dev/null +++ b/backend/src/__tests__/build-info-route.test.ts @@ -0,0 +1,225 @@ +/** + * Route coverage for the canonical build identity: + * + * - GET /api/build-info is proxy-exempt (always served by the control + * instance), requires a signed-in human session (rejects machine / API-token + * credentials), redacts hardened image references to non-admins via + * `restricted: true`, and never mislabels a redacted field "Unknown". + * - GET /api/meta exposes only the bounded `buildChannel` enum on the public + * surface and never leaks the running image reference. + */ +import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest'; +import request from 'supertest'; +import jwt from 'jsonwebtoken'; +import bcrypt from 'bcrypt'; +import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb'; +import { withLoopbackTargetProtection } from './helpers/allowLoopbackTargets'; + +let tmpDir: string; +let app: import('express').Express; +let adminAuth: string; +let viewerAuth: string; +let machineAuth: string; +let remoteNodeId: number; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; +let SelfIdentityService: typeof import('../services/SelfIdentityService').default; + +const IMAGE_ID = 'b'.repeat(64); +const DIGEST = 'a'.repeat(64); + +function mockBuildInfo(over: Record = {}) { + const svc = SelfIdentityService.getInstance(); + vi.spyOn(svc, 'getBuildInfo').mockReturnValue({ + version: '0.97.1', + channel: 'dev', + imageRef: 'ghcr.io/studio-saelix/sencho-dev:dev-abc1234', + imageId: IMAGE_ID, + revision: 'dev-abc1234', + ...over, + }); +} + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ app } = await import('../index')); + ({ DatabaseService } = await import('../services/DatabaseService')); + SelfIdentityService = (await import('../services/SelfIdentityService')).default; + + const db = DatabaseService.getInstance(); + remoteNodeId = db.addNode({ + name: 'build-info-remote', + type: 'remote', + compose_dir: '/tmp', + is_default: false, + api_url: 'http://127.0.0.1:1', + api_token: 'build-info-remote-token', + }); + // A signed-in non-admin human session (role resolved from the DB row). + db.addUser({ + username: 'build-info-viewer', + password_hash: await bcrypt.hash('pw', 1), + role: 'viewer', + }); + + adminAuth = `Bearer ${jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' })}`; + viewerAuth = `Bearer ${jwt.sign({ username: 'build-info-viewer' }, TEST_JWT_SECRET, { expiresIn: '1m' })}`; + // node_proxy machine credential: authMiddleware maps it to role admin with + // userId 0, so requireUserSession must reject it as not a human session. + machineAuth = `Bearer ${jwt.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' })}`; +}); + +afterAll(() => cleanupTestDb(tmpDir)); + +afterEach(() => vi.restoreAllMocks()); + +describe('GET /api/build-info auth', () => { + it('requires authentication', async () => { + mockBuildInfo(); + const res = await request(app).get('/api/build-info'); + expect(res.status).toBe(401); + }); + + it('rejects node_proxy machine credentials (not a human session)', async () => { + mockBuildInfo(); + const res = await request(app).get('/api/build-info').set('Authorization', machineAuth); + expect(res.status).toBe(403); + expect(res.body.code).toBe('SESSION_REQUIRED'); + }); +}); + +describe('GET /api/build-info is proxy-exempt', () => { + it('serves locally even when x-node-id targets a remote node', async () => { + mockBuildInfo(); + // The remote node's api_url is a closed loopback port. A 502 would mean the + // proxy intercepted the request; anything else proves the local handler + // matched, exactly as the existing /api/nodes proxy-exempt test asserts. + const res = await withLoopbackTargetProtection(() => request(app) + .get('/api/build-info') + .set('Authorization', adminAuth) + .set('x-node-id', String(remoteNodeId))); + expect(res.status).not.toBe(502); + expect(res.body.channel).toBe('dev'); + }); +}); + +describe('GET /api/build-info as admin', () => { + it('returns the full dev identity for a dev image (regression: semver still previous stable)', async () => { + mockBuildInfo(); + const res = await request(app).get('/api/build-info').set('Authorization', adminAuth); + expect(res.status).toBe(200); + expect(res.body.version).toBe('0.97.1'); + expect(res.body.channel).toBe('dev'); + expect(res.body.imageChannel).toBe('community'); + expect(res.body.imageRef).toBe('ghcr.io/studio-saelix/sencho-dev:dev-abc1234'); + expect(res.body.imageId).toBe(IMAGE_ID); + expect(res.body.revision).toBe('dev-abc1234'); + expect(res.body.restricted).toBe(false); + }); + + it('returns the bounded imageChannel for a hardened image', async () => { + mockBuildInfo({ + channel: 'stable', + imageRef: 'ghcr.io/studio-saelix/sencho-hardened:0.97.1', + }); + const res = await request(app).get('/api/build-info').set('Authorization', adminAuth); + expect(res.status).toBe(200); + expect(res.body.channel).toBe('stable'); + expect(res.body.imageChannel).toBe('hardened'); + expect(res.body.imageRef).toBe('ghcr.io/studio-saelix/sencho-hardened:0.97.1'); + expect(res.body.restricted).toBe(false); + }); +}); + +describe('GET /api/build-info as a non-admin', () => { + it('redacts a hardened image reference and revision to a non-admin via restricted:true', async () => { + mockBuildInfo({ + channel: 'stable', + imageRef: 'ghcr.io/studio-saelix/sencho-hardened:0.97.1', + revision: `sha256:${DIGEST}`, + }); + const res = await request(app).get('/api/build-info').set('Authorization', viewerAuth); + expect(res.status).toBe(200); + // The build channel stays stable; the procurement channel is what gates + // redaction. Both reference fields are nulled with restricted:true so the + // UI can label them "Restricted", never "Unknown". + expect(res.body.channel).toBe('stable'); + expect(res.body.imageChannel).toBe('hardened'); + expect(res.body.imageRef).toBeNull(); + expect(res.body.revision).toBeNull(); + expect(res.body.restricted).toBe(true); + // The image ID is not a registry reference and is always returned. + expect(res.body.imageId).toBe(IMAGE_ID); + }); + + it('does not redact a community image for a non-admin', async () => { + mockBuildInfo(); + const res = await request(app).get('/api/build-info').set('Authorization', viewerAuth); + expect(res.status).toBe(200); + expect(res.body.imageRef).toBe('ghcr.io/studio-saelix/sencho-dev:dev-abc1234'); + expect(res.body.revision).toBe('dev-abc1234'); + expect(res.body.restricted).toBe(false); + }); + + it('reports unknown procurement channel on bare metal without a running reference', async () => { + mockBuildInfo({ channel: 'unknown', imageRef: null, revision: null }); + const res = await request(app).get('/api/build-info').set('Authorization', viewerAuth); + expect(res.status).toBe(200); + // No reference means no classification, and no redaction can apply. + expect(res.body.channel).toBe('unknown'); + expect(res.body.imageChannel).toBe('unknown'); + expect(res.body.imageRef).toBeNull(); + expect(res.body.revision).toBeNull(); + expect(res.body.restricted).toBe(false); + }); +}); + +describe('GET /api/build-info awaits revision enrichment', () => { + it('blocks the response until the enrichment settle promise resolves', async () => { + mockBuildInfo(); + const svc = SelfIdentityService.getInstance(); + let release!: () => void; + vi.spyOn(svc, 'whenRevisionResolved').mockImplementation( + () => new Promise((res) => { release = res; }), + ); + + let settled = false; + const pending = request(app) + .get('/api/build-info') + .set('Authorization', adminAuth) + .then((res) => { settled = true; return res; }); + + // Give the route a tick to reach the await. It must not have responded yet, + // proving a transient null is never the settled value of a success. + await new Promise((r) => setTimeout(r, 10)); + expect(settled).toBe(false); + + release(); + const res = await pending; + expect(res.status).toBe(200); + expect(res.body.revision).toBe('dev-abc1234'); + }); +}); + +describe('GET /api/meta buildChannel', () => { + it('exposes the bounded build channel on the public endpoint', async () => { + mockBuildInfo(); + const res = await request(app).get('/api/meta'); + expect(res.status).toBe(200); + expect(res.body.buildChannel).toBe('dev'); + }); + + it('never leaks the running image reference on the public endpoint', async () => { + mockBuildInfo(); + const res = await request(app).get('/api/meta'); + const body = JSON.stringify(res.body); + expect(body).not.toContain('ghcr.io/studio-saelix/sencho-dev'); + expect(body).not.toContain('dev-abc1234'); + }); + + it('omits buildChannel when the running image reference is unknown', async () => { + mockBuildInfo({ imageRef: null, channel: 'unknown', revision: null }); + const res = await request(app).get('/api/meta'); + expect(res.status).toBe(200); + expect(res.body.buildChannel).toBeUndefined(); + }); +}); \ No newline at end of file diff --git a/backend/src/__tests__/classify-build-channel.test.ts b/backend/src/__tests__/classify-build-channel.test.ts new file mode 100644 index 00000000..825fa1d4 --- /dev/null +++ b/backend/src/__tests__/classify-build-channel.test.ts @@ -0,0 +1,42 @@ +/** + * Truth table for classifyBuildChannel, the canonical build-identity classifier. + * Answers "is this a dev, preview, or stable build" from the image reference + * alone, independent of the packaged semver. + */ +import { describe, it, expect } from 'vitest'; +import { classifyBuildChannel } from '../helpers/selfUpdateCompose'; + +describe('classifyBuildChannel', () => { + it('classifies the dev repository as dev regardless of tag', () => { + expect(classifyBuildChannel('ghcr.io/studio-saelix/sencho-dev:dev')).toBe('dev'); + expect(classifyBuildChannel('ghcr.io/studio-saelix/sencho-dev:dev-abc1234')).toBe('dev'); + expect(classifyBuildChannel('ghcr.io/studio-saelix/sencho-dev:latest')).toBe('dev'); + expect(classifyBuildChannel('ghcr.io/studio-saelix/sencho-dev@sha256:abc')).toBe('dev'); + }); + + it('classifies stable-repo preview tags as preview', () => { + expect(classifyBuildChannel('saelix/sencho:pr-42')).toBe('preview'); + expect(classifyBuildChannel('saelix/sencho:preview-abc1234')).toBe('preview'); + expect(classifyBuildChannel('ghcr.io/studio-saelix/sencho:pr-7')).toBe('preview'); + }); + + it('classifies stable-repo release/floating tags as stable', () => { + expect(classifyBuildChannel('saelix/sencho:0.97.1')).toBe('stable'); + expect(classifyBuildChannel('saelix/sencho:latest')).toBe('stable'); + expect(classifyBuildChannel('ghcr.io/studio-saelix/sencho:v1.2.3')).toBe('stable'); + expect(classifyBuildChannel('ghcr.io/studio-saelix/sencho-hardened:1.2.3')).toBe('stable'); + expect(classifyBuildChannel('ghcr.io/studio-saelix/sencho-hardened:latest')).toBe('stable'); + }); + + it('treats a dev- tag as preview-matching-tolerant (still dev repo wins)', () => { + // dev repo takes precedence over the preview/stable tag classification + expect(classifyBuildChannel('ghcr.io/studio-saelix/sencho-dev:pr-42')).toBe('dev'); + }); + + it('classifies unknown repositories as unknown', () => { + expect(classifyBuildChannel('ubuntu:22.04')).toBe('unknown'); + expect(classifyBuildChannel('registry.example.com/private/app:1.0')).toBe('unknown'); + expect(classifyBuildChannel('')).toBe('unknown'); + expect(classifyBuildChannel(' ')).toBe('unknown'); + }); +}); \ No newline at end of file diff --git a/backend/src/__tests__/monitor-service.test.ts b/backend/src/__tests__/monitor-service.test.ts index a35e387b..2a1bea18 100644 --- a/backend/src/__tests__/monitor-service.test.ts +++ b/backend/src/__tests__/monitor-service.test.ts @@ -1621,6 +1621,41 @@ describe('MonitorService - Sencho dev build check', () => { expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', 'node_update_available', expect.anything()); }); + + // C1: update eligibility is compose-declared. A dev *running* identity must + // not make a stable-declared pin eligible for the dev-build detector, and an + // unknown running identity must still gate before any registry call. These + // two disagreement directions pin that the running identity surfaced by + // SelfIdentityService never changes update behavior. + it('does not treat a stable-declared pin as eligible even when the running identity is a dev build', async () => { + // beforeEach() already mints a dev running imageId; the declared pin is stable. + // Reset the version-update inputs: an earlier suppression test queues a + // once-value that a stable-pin run would otherwise drain into the version + // path, which is not what this test observes. + mockGetLatestVersionInfo.mockReset(); + mockGetPinInfo.mockResolvedValue(STABLE_PIN); + + await runEvaluate(); + + expect(mockDetectSelfDevBuildUpdate).not.toHaveBeenCalled(); + expect(devBuildCalls()).toHaveLength(0); + expect(mockSetSystemState).not.toHaveBeenCalledWith('sencho_dev_build_available_digest', expect.anything()); + }); + + it('still gates the registry comparison on a known running image id for an eligible dev-declared pin', async () => { + mockGetPinInfo.mockResolvedValue(DEV_PIN); + mockGetIdentity.mockReturnValue({ + containerId: null, containerName: null, composeProjectName: null, + imageId: null, networkNames: [], volumeNames: [], + }); + + await runEvaluate(); + + expect(mockDetectSelfDevBuildUpdate).not.toHaveBeenCalled(); + expect(devBuildCalls()).toHaveLength(0); + // Unknown running id takes the retry-sooner path, not the detector path. + expect((MonitorService.getInstance() as any).lastDevBuildCheckGateMs).toBe(5 * 60 * 1000); + }); }); // ── Per-container parallel fan-out ──────────────────────────────────── diff --git a/backend/src/__tests__/self-identity-buildinfo.test.ts b/backend/src/__tests__/self-identity-buildinfo.test.ts new file mode 100644 index 00000000..e5445e5b --- /dev/null +++ b/backend/src/__tests__/self-identity-buildinfo.test.ts @@ -0,0 +1,197 @@ +/** + * Unit tests for SelfIdentityService.getBuildInfo(): the canonical runtime + * build identity (version, channel, imageRef, imageId, revision), the detached + * bounded revision enrichment, and the failure-isolation guarantee. + */ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + +// ── Hoisted mocks ────────────────────────────────────────────────────── + +const { mockContainer, mockDocker, mockInspectImage, mockGetSenchoVersion } = vi.hoisted(() => { + const mockContainer = { inspect: vi.fn() }; + const mockDocker = { + getContainer: vi.fn(() => mockContainer), + getImage: vi.fn(), + listImages: vi.fn().mockResolvedValue([]), + listVolumes: vi.fn().mockResolvedValue({ Volumes: [] }), + listNetworks: vi.fn().mockResolvedValue([]), + listContainers: vi.fn().mockResolvedValue([]), + }; + return { + mockContainer, + mockDocker, + mockInspectImage: vi.fn(), + mockGetSenchoVersion: vi.fn(() => '0.97.1'), + }; +}); + +vi.mock('../services/NodeRegistry', () => ({ + NodeRegistry: { + getInstance: () => ({ + getDocker: () => mockDocker, + getDefaultNodeId: () => 1, + }), + }, +})); + +// Replace defaultInspectImage so enrichment is deterministic and isolated. +vi.mock('../services/selfDevBuildDetect', () => ({ + defaultInspectImage: (...args: unknown[]) => mockInspectImage(...args), +})); + +vi.mock('../services/CapabilityRegistry', () => ({ + getSenchoVersion: () => mockGetSenchoVersion(), +})); + +vi.mock('child_process', () => ({ exec: vi.fn(), execFile: vi.fn() })); +vi.mock('util', () => ({ promisify: () => vi.fn() })); + +import SelfIdentityService from '../services/SelfIdentityService'; + +const FULL_IMAGE_ID_HEX = 'b'.repeat(64); +const DIGEST = 'a'.repeat(64); + +const originalHostname = process.env.HOSTNAME; + +beforeEach(() => { + vi.clearAllMocks(); + mockContainer.inspect.mockReset(); + mockDocker.getImage.mockReset(); + mockInspectImage.mockReset(); + mockGetSenchoVersion.mockReturnValue('0.97.1'); + SelfIdentityService.getInstance().resetForTesting(); +}); + +afterEach(() => { + vi.restoreAllMocks(); + if (originalHostname === undefined) delete process.env.HOSTNAME; + else process.env.HOSTNAME = originalHostname; +}); + +async function initWith(configImage: string | undefined): Promise { + process.env.HOSTNAME = 'sencho-1'; + mockContainer.inspect.mockResolvedValue({ + Id: 'a'.repeat(64), + Name: '/sencho', + Image: 'sha256:' + FULL_IMAGE_ID_HEX, + ...(configImage !== undefined ? { Config: { Image: configImage } } : {}), + NetworkSettings: { Networks: {} }, + Mounts: [], + }); + const svc = SelfIdentityService.getInstance(); + await svc.initialize(); + return svc; +} + +/** Enrichment runs detached; poll until the condition holds so assertions are stable. */ +async function until(assert: () => void): Promise { + await vi.waitFor(assert, { timeout: 2000 }); +} + +describe('SelfIdentityService.getBuildInfo', () => { + it('identifies a dev image as DEV even when the packaged semver matches the previous stable', async () => { + // The regression case from the ticket: dev image, version still 0.97.1. + // The inspect mock must be set before initialize(): enrichment fires + // detached during initialize(), before the awaited call returns. + mockInspectImage.mockResolvedValue({ + RepoDigests: [`ghcr.io/studio-saelix/sencho-dev@sha256:${DIGEST}`], + Os: 'linux', + Architecture: 'amd64', + }); + const svc = await initWith('ghcr.io/studio-saelix/sencho-dev:dev'); + await until(() => expect(svc.getBuildInfo().revision).toBe(`sha256:${DIGEST}`)); + + const info = svc.getBuildInfo(); + expect(info.version).toBe('0.97.1'); + expect(info.channel).toBe('dev'); + expect(info.imageRef).toBe('ghcr.io/studio-saelix/sencho-dev:dev'); + expect(info.imageId).toBe(FULL_IMAGE_ID_HEX); + }); + + it('derives the revision from a pinned dev- tag without an image inspect', async () => { + const svc = await initWith('ghcr.io/studio-saelix/sencho-dev:dev-abc1234'); + await until(() => expect(svc.getBuildInfo().revision).toBe('dev-abc1234')); + + const info = svc.getBuildInfo(); + expect(info.channel).toBe('dev'); + expect(mockInspectImage).not.toHaveBeenCalled(); + }); + + it('classifies a stable image as stable', async () => { + mockInspectImage.mockResolvedValue({ + RepoDigests: [`ghcr.io/studio-saelix/sencho@sha256:${DIGEST}`], + Os: 'linux', + Architecture: 'amd64', + }); + const svc = await initWith('ghcr.io/studio-saelix/sencho:0.97.1'); + await until(() => expect(svc.getBuildInfo().revision).toBe(`sha256:${DIGEST}`)); + + expect(svc.getBuildInfo().channel).toBe('stable'); + }); + + it('reads unknown for partial metadata (no running image reference)', async () => { + const svc = await initWith(undefined); + + const info = svc.getBuildInfo(); + expect(info.imageRef).toBeNull(); + expect(info.channel).toBe('unknown'); + expect(info.revision).toBeNull(); + expect(info.imageId).toBe(FULL_IMAGE_ID_HEX); + }); + + it('keeps the dev channel but null revision when image inspection fails', async () => { + mockInspectImage.mockRejectedValue(new Error('docker daemon unreachable')); + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const svc = await initWith('ghcr.io/studio-saelix/sencho-dev:dev'); + await until(() => expect(warnSpy).toHaveBeenCalled()); + + const info = svc.getBuildInfo(); + expect(info.channel).toBe('dev'); + expect(info.revision).toBeNull(); + // C2: a failed enrichment must not corrupt the already-captured core identity. + expect(info.imageRef).toBe('ghcr.io/studio-saelix/sencho-dev:dev'); + expect(info.imageId).toBe(FULL_IMAGE_ID_HEX); + }); + + it('never inspects the image again on repeated reads', async () => { + mockInspectImage.mockResolvedValue({ + RepoDigests: [`ghcr.io/studio-saelix/sencho-dev@sha256:${DIGEST}`], + Os: 'linux', + Architecture: 'amd64', + }); + const svc = await initWith('ghcr.io/studio-saelix/sencho-dev:dev'); + await until(() => expect(mockInspectImage).toHaveBeenCalledTimes(1)); + + for (let i = 0; i < 5; i++) svc.getBuildInfo(); + expect(mockInspectImage).toHaveBeenCalledTimes(1); + }); + + it('exposes revision only after enrichment settles, and whenRevisionResolved awaits that', async () => { + let resolveInspect!: (v: { RepoDigests: string[]; Os: string; Architecture: string }) => void; + mockInspectImage.mockReturnValue(new Promise((res) => { resolveInspect = res; })); + + const svc = await initWith('ghcr.io/studio-saelix/sencho-dev:dev'); + // initialize() returned without awaiting the detached enrichment, so the + // revision is still transiently null and the settle promise is pending. + expect(svc.getBuildInfo().revision).toBeNull(); + + // A reader that awaits the settle promise (the build-info route) blocks + // until enrichment lands, then observes the resolved digest, so a single + // successful read never freezes a transient null. + const settled = svc.whenRevisionResolved(); + resolveInspect({ + RepoDigests: [`ghcr.io/studio-saelix/sencho-dev@sha256:${DIGEST}`], + Os: 'linux', + Architecture: 'amd64', + }); + await settled; + expect(svc.getBuildInfo().revision).toBe(`sha256:${DIGEST}`); + }); + + it('resolves whenRevisionResolved immediately when no enrichment ever started', async () => { + process.env.HOSTNAME = undefined; + const svc = SelfIdentityService.getInstance(); + await svc.whenRevisionResolved(); + expect(svc.getBuildInfo().revision).toBeNull(); + }); +}); diff --git a/backend/src/helpers/proxyExemptPaths.ts b/backend/src/helpers/proxyExemptPaths.ts index 1223d95f..ccad56b1 100644 --- a/backend/src/helpers/proxyExemptPaths.ts +++ b/backend/src/helpers/proxyExemptPaths.ts @@ -13,6 +13,7 @@ export const PROXY_EXEMPT_PREFIXES: readonly string[] = [ '/api/fleet/', '/api/webhooks', '/api/meta', + '/api/build-info', ]; /** Returns true when the path should bypass the remote proxy (handled locally). */ diff --git a/backend/src/helpers/selfUpdateCompose.ts b/backend/src/helpers/selfUpdateCompose.ts index 2c5f9c17..2ce0f100 100644 --- a/backend/src/helpers/selfUpdateCompose.ts +++ b/backend/src/helpers/selfUpdateCompose.ts @@ -104,13 +104,50 @@ export function isSenchoDevFloatingTag(imageRef: string): boolean { // A digest pin disqualifies the reference (e.g., `@sha256:...`) if (ref.includes('@sha256:') || ref.startsWith('sha256:')) return false; - // Extract the tag using the same logic as classifyImagePin. + const tag = extractTagFromRef(ref); + return tag === 'dev'; +} + +/** + * Build channel of a running or declared image, derived from the image + * reference alone. This is the canonical build identity classifier used by + * `/api/build-info` and the shell/About surfaces: it answers "is this a dev, + * preview, or stable build" from the ref, independent of the packaged semver + * (a dev image carries the last released version, so version alone cannot + * identify it). + * + * - dev repo (`ghcr.io/studio-saelix/sencho-dev`): always `'dev'`. An + * arbitrary `:dev-` tag still reads `dev` (a deliberate operator + * choice); the immutable revision is surfaced separately from the digest. + * - stable repos (`ghcr.io/studio-saelix/sencho-hardened`, `saelix/sencho`, + * `ghcr.io/studio-saelix/sencho`): `pr-` and `preview-` tags are + * `'preview'` (CI builds on the stable repo that are not releases); + * everything else is `'stable'`. + * - any other repository: `'unknown'`. + */ +export type BuildChannel = 'stable' | 'dev' | 'preview' | 'unknown'; + +export function classifyBuildChannel(imageRef: string): BuildChannel { + const repository = normalizeImageRepository(imageRef); + if (repository === 'ghcr.io/studio-saelix/sencho-dev') return 'dev'; + if ( + repository === 'ghcr.io/studio-saelix/sencho-hardened' || + repository === 'saelix/sencho' || + repository === 'ghcr.io/studio-saelix/sencho' + ) { + const tag = extractTagFromRef(imageRef.trim()); + if (tag && (/^pr-\d+$/.test(tag) || /^preview-[0-9a-f]{7,40}$/.test(tag))) return 'preview'; + return 'stable'; + } + return 'unknown'; +} + +/** Extract the tag portion of an image ref (`.../repo:tag`), or '' when absent. */ +function extractTagFromRef(ref: string): string { const lastSlash = ref.lastIndexOf('/'); const lastColon = ref.lastIndexOf(':'); // A colon after the last slash is a tag separator; before it is a registry port. - const tag = lastColon > lastSlash ? ref.slice(lastColon + 1) : ''; - - return tag === 'dev'; + return lastColon > lastSlash ? ref.slice(lastColon + 1) : ''; } /** diff --git a/backend/src/index.ts b/backend/src/index.ts index 2fe9e10d..587e5240 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -19,6 +19,7 @@ import { mfaRouter } from './routes/mfa'; import { ssoRouter } from './routes/sso'; import { licenseRouter, systemUpdateRouter } from './routes/license'; import { imageChannelRouter } from './routes/imageChannel'; +import { buildInfoRouter } from './routes/buildInfo'; import { webhooksRouter } from './routes/webhooks'; import { usersRouter } from './routes/users'; import { gitSourcesRouter, stackGitSourceRouter } from './routes/gitSources'; @@ -115,6 +116,7 @@ app.use('/api/', createRemoteProxyMiddleware()); app.use('/api/license', licenseRouter); app.use('/api/license/image-channel', imageChannelRouter); +app.use('/api/build-info', buildInfoRouter); app.use('/api/system', systemUpdateRouter); app.use('/api/permissions', permissionsRouter); app.use('/api/convert', convertRouter); diff --git a/backend/src/routes/buildInfo.ts b/backend/src/routes/buildInfo.ts new file mode 100644 index 00000000..7d999166 --- /dev/null +++ b/backend/src/routes/buildInfo.ts @@ -0,0 +1,47 @@ +import { Router, type Request, type Response } from 'express'; +import { requireUserSession } from '../middleware/tierGates'; +import { classifyImageChannel, type ImageChannel } from '../helpers/imageChannel'; +import type { BuildChannel } from '../helpers/selfUpdateCompose'; +import SelfIdentityService from '../services/SelfIdentityService'; + +export const buildInfoRouter = Router(); + +/** Wire shape of GET /api/build-info. `restricted: true` implies `imageRef` and + * `revision` are nulled for a hardened image viewed by a non-admin. */ +interface BuildInfoResponse { + version: string | null; + channel: BuildChannel; + imageChannel: ImageChannel; + imageRef: string | null; + imageId: string | null; + revision: string | null; + restricted: boolean; +} + +// Canonical runtime build identity of the control instance. Proxy-exempt (see +// helpers/proxyExemptPaths.ts) so it is always served by the local hub, never +// forwarded to a remote node. The running image reference can carry a private +// registry/repository name, so the endpoint requires a human session and +// redacts hardened-image references to non-admins via `restricted: true` (the +// UI shows "Restricted", never "Unknown", when set). +buildInfoRouter.get('/', async (req: Request, res: Response): Promise => { + if (!requireUserSession(req, res)) return; + const service = SelfIdentityService.getInstance(); + // Await the detached revision enrichment so a transient null is never the + // settled value of a successful response. Bounded by the inspect timeout and + // never rejects, so this adds at most a short wait on the first read. + await service.whenRevisionResolved(); + const identity = service.getBuildInfo(); + const isAdmin = req.user?.role === 'admin'; + const imageChannel = identity.imageRef ? classifyImageChannel(identity.imageRef) : 'unknown'; + const restricted = !isAdmin && imageChannel === 'hardened'; + res.json({ + version: identity.version, + channel: identity.channel, + imageChannel, + imageRef: restricted ? null : identity.imageRef, + imageId: identity.imageId, + revision: restricted ? null : identity.revision, + restricted, + } satisfies BuildInfoResponse); +}); \ No newline at end of file diff --git a/backend/src/routes/meta.ts b/backend/src/routes/meta.ts index 93dcba01..3e6d8402 100644 --- a/backend/src/routes/meta.ts +++ b/backend/src/routes/meta.ts @@ -1,9 +1,10 @@ import { Router, type Request, type Response } from 'express'; import { getActiveCapabilities, getSenchoVersion } from '../services/CapabilityRegistry'; import { classifyImageChannel } from '../helpers/imageChannel'; -import { isRepinBlocked } from '../helpers/selfUpdateCompose'; +import { classifyBuildChannel, isRepinBlocked } from '../helpers/selfUpdateCompose'; import { MeshService } from '../services/MeshService'; import SelfUpdateService from '../services/SelfUpdateService'; +import SelfIdentityService from '../services/SelfIdentityService'; // Captured at boot. Exposed via /api/health and /api/meta so the Fleet update // overlay can distinguish a brand-new process from the old one still mid-pull. @@ -41,6 +42,7 @@ metaRouter.get('/meta', async (_req: Request, res: Response): Promise => { const updateError = selfUpdate.getLastError(); const pin = await selfUpdate.getPinInfo({ cacheOnly: true }); const updateBlocked = pin ? isRepinBlocked(pin.pinKind) : false; + const runningRef = SelfIdentityService.getInstance().getBuildInfo().imageRef; res.json({ version: getSenchoVersion(), capabilities: getActiveCapabilities(), @@ -50,6 +52,10 @@ metaRouter.get('/meta', async (_req: Request, res: Response): Promise => { imagePinKind: pin.pinKind, imageChannel: classifyImageChannel(pin.composeImageRef), } : {}), + // Bounded build channel of the RUNNING image (stable|dev|preview|unknown). + // Like imagePinKind, this is a non-sensitive enum; no image reference is + // ever exposed on this public endpoint. + ...(runningRef ? { buildChannel: classifyBuildChannel(runningRef) } : {}), updateBlocked, ...(updateError ? { updateError: 'update_failed' } : {}), }); diff --git a/backend/src/services/SelfIdentityService.ts b/backend/src/services/SelfIdentityService.ts index e134bb01..29454db5 100644 --- a/backend/src/services/SelfIdentityService.ts +++ b/backend/src/services/SelfIdentityService.ts @@ -1,5 +1,10 @@ import fs from 'fs/promises'; import DockerController from './DockerController'; +import { classifyBuildChannel, isSenchoDevRepository, type BuildChannel } from '../helpers/selfUpdateCompose'; +import { defaultInspectImage } from './selfDevBuildDetect'; +import { parseImageRef, selectLocalRepoDigest } from './registry-api'; +import { getSenchoVersion } from './CapabilityRegistry'; +import { withTimeout } from '../utils/withTimeout'; /** * Identifies the Docker resources that belong to the running Sencho container @@ -20,17 +25,32 @@ import DockerController from './DockerController'; * stays in its empty state, every `isOwn*()` returns false, and today's * behavior is preserved. */ +/** Canonical runtime build identity of the running Sencho container. */ +export interface BuildInfo { + version: string | null; + channel: BuildChannel; + /** The image reference the running container was started with, null when unknown. */ + imageRef: string | null; + /** Running image sha256 hex (no prefix), null when unknown. */ + imageId: string | null; + /** Validated registry digest or pinned `dev-` tag, null when unknown. */ + revision: string | null; +} + class SelfIdentityService { private static instance: SelfIdentityService; private containerId: string | null = null; private containerName: string | null = null; private composeProjectName: string | null = null; private imageIdHex: string | null = null; + private imageRef: string | null = null; + private revision: string | null = null; private networkIds = new Set(); private networkNames = new Set(); private volumeNames = new Set(); private initialized = false; private initializePromise: Promise | null = null; + private enrichmentPromise: Promise | null = null; public static getInstance(): SelfIdentityService { if (!SelfIdentityService.instance) { @@ -58,6 +78,14 @@ class SelfIdentityService { this.containerName = (info.Name || '').replace(/^\//, '') || null; this.composeProjectName = info.Config?.Labels?.['com.docker.compose.project'] ?? null; this.imageIdHex = SelfIdentityService.stripSha(info.Image ?? '') || null; + this.imageRef = info.Config?.Image ?? null; + // Bounded revision enrichment runs detached so it never blocks the callers + // awaiting initialize() (Docker event monitoring, resources discovery). Core + // identity above is already captured; enrichment only adds the registry + // digest / pinned dev- and is failure-isolated. The promise is retained + // so a reader that needs the settled revision can await it (see + // whenRevisionResolved) instead of observing a transient null. + this.enrichmentPromise = this.enrichRevision(this.imageRef, this.imageIdHex); const nets = info.NetworkSettings?.Networks ?? {}; for (const [name, net] of Object.entries(nets)) { @@ -118,6 +146,64 @@ class SelfIdentityService { } } + /** + * Canonical runtime build identity. All fields are captured fields or derived + * synchronously from them; this read never triggers a Docker call. `revision` + * is populated by the detached enrichment step fired during initialize() and + * reads null until that resolves (or if it fails). + */ + getBuildInfo(): BuildInfo { + return { + version: getSenchoVersion(), + channel: this.imageRef ? classifyBuildChannel(this.imageRef) : 'unknown', + imageRef: this.imageRef, + imageId: this.imageIdHex, + revision: this.revision, + }; + } + + /** + * Resolves once the detached revision enrichment has settled (success or + * failure), or immediately when none was started. Awaiting cannot hang or + * throw (enrichment is bounded and failure-isolated). A reader that needs + * the final `revision` awaits this before getBuildInfo() so a successful + * response never freezes a transient null. + */ + async whenRevisionResolved(): Promise { + if (this.enrichmentPromise) await this.enrichmentPromise; + } + + /** + * Resolve the immutable revision from the running image. For a dev-repo image + * carrying a pinned `dev-` tag, the tag itself is the revision. Otherwise + * the running image's `RepoDigests` are inspected for a digest matching the + * running reference. Any failure (inspect rejection, timeout, no matching + * digest) leaves `revision` null; enrichment never throws to the caller. + */ + private async enrichRevision(imageRef: string | null, imageIdHex: string | null): Promise { + try { + let revision: string | null = null; + if (imageRef && isSenchoDevRepository(imageRef)) { + const tag = parseImageRef(imageRef)?.tag; + if (tag && /^dev-[0-9a-f]{7,40}$/.test(tag)) revision = tag; + } + if (!revision && imageRef && imageIdHex) { + revision = await this.resolveDigestRevision(imageRef, imageIdHex); + } + this.revision = revision; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + console.warn('[SelfIdentity] build revision enrichment failed:', message); + } + } + + private async resolveDigestRevision(imageRef: string, imageIdHex: string): Promise { + const parsed = parseImageRef(imageRef); + if (!parsed) return null; + const inspected = await withTimeout(defaultInspectImage(imageIdHex), 2000, 'build revision inspect'); + return selectLocalRepoDigest(inspected.RepoDigests ?? [], parsed); + } + /** True when the given container ID or name matches the running Sencho container. Accepts short or full IDs. */ isOwnContainer(idOrName: string): boolean { if (!idOrName) return false; @@ -205,11 +291,14 @@ class SelfIdentityService { this.containerName = null; this.composeProjectName = null; this.imageIdHex = null; + this.imageRef = null; + this.revision = null; this.networkIds.clear(); this.networkNames.clear(); this.volumeNames.clear(); this.initialized = false; this.initializePromise = null; + this.enrichmentPromise = null; } private static stripSha(s: string): string { diff --git a/backend/src/services/selfDevBuildDetect.ts b/backend/src/services/selfDevBuildDetect.ts index 65cf4cfa..2ee64ef6 100644 --- a/backend/src/services/selfDevBuildDetect.ts +++ b/backend/src/services/selfDevBuildDetect.ts @@ -27,13 +27,17 @@ export type SelfDevBuildDetectResult = | { kind: 'inconclusive'; reason: string }; /** The subset of `docker image inspect` output the detector reads. */ -interface InspectedImage { +export interface InspectedImage { RepoDigests: string[]; Os: string; Architecture: string; } -async function defaultInspectImage(imageId: string): Promise { +/** Bounded read of `docker image inspect` (RepoDigests, OS, architecture) for a + * resolved image. Reused by `SelfIdentityService` for revision enrichment; + * callers wrap it so any rejection stays isolated from the fields already + * captured. */ +export async function defaultInspectImage(imageId: string): Promise { const inspect = await DockerController.getInstance().getDocker().getImage(`sha256:${imageId}`).inspect(); return { RepoDigests: inspect.RepoDigests ?? [], Os: inspect.Os, Architecture: inspect.Architecture }; } diff --git a/docs/features/licensing.mdx b/docs/features/licensing.mdx index fc0f82dc..67574fc3 100644 --- a/docs/features/licensing.mdx +++ b/docs/features/licensing.mdx @@ -108,7 +108,7 @@ The **Plan** card lists: - **License** (Community only): a link to view the AGPLv3 source on GitHub. - **Recovery Vault** (Admiral only): confirms the entitlement is included in your subscription. - **Hardened Build** (Admiral only): a **Switch to Hardened** button; see [Switching to Hardened Build](#switching-to-hardened-build) below. -- **Current image** and **Channel**: the image reference and channel (`Community` or `Hardened`) this control plane is currently running. +- **Current image** and **Channel**: the image reference and channel (`Community` or `Hardened`) this control plane is currently running. These rows reflect the running build, not the configured target, so a compose edit does not change them until the container is recreated. A hardened image is shown as `Restricted` to non-administrators. - **Customer**, **Product**, and **License key** (active licenses only): purchase metadata and your key masked to its last four characters (`****-****-****-XXXX`). The full key is never re-displayed after activation. ## Switching to Hardened Build diff --git a/docs/operations/verifying-images.mdx b/docs/operations/verifying-images.mdx index 07d25f12..27376e32 100644 --- a/docs/operations/verifying-images.mdx +++ b/docs/operations/verifying-images.mdx @@ -154,3 +154,17 @@ Maintainers publish these from open PRs for external validation. They are unsign |---|---|---| | `pr-` | `saelix/sencho:pr-1526` | Each re-run of the preview workflow for that PR | | `preview-` | `saelix/sencho:preview-abc1234` | Never (immutable per build) | + +## Reading your running build + +Sencho surfaces the build it is actually running in **Settings → About → Build** (and in the **Channel** and **Current image** rows of **Settings → Admiral Account**). These fields come from the running container's identity, not from the compose file, so a compose edit does not change them until the container is recreated. + +| Field | Meaning | +|---|---| +| **Version** | The packaged semantic version of the build. A dev build keeps the last stable version here, which is why the Channel row matters. | +| **Channel** | The build track of the running image: `Dev`, `Preview`, `Stable`, or `Unknown`. A `dev` or `dev-` image reads `Dev`; a `pr-` or `preview-` image reads `Preview`; a release image reads `Stable`. | +| **Current image** | The image reference the container was started with, for example `ghcr.io/studio-saelix/sencho-dev:dev-a1b2c3d`. | +| **Revision** | The immutable digest this build resolves to, or the pinned `dev-` tag when the running image is on the integration track. | +| **Image ID** | The first twelve characters of the running image's sha256 identifier; click to copy the full id. | + +These fields describe the **control plane** instance you are logged in to, not remote nodes. When identity metadata cannot be determined, the reference fields read `Unknown` rather than guessing. A hardened image viewed by a non-administrator shows `Restricted` for the reference fields instead of `Unknown`. diff --git a/docs/reference/settings.mdx b/docs/reference/settings.mdx index 44bc7625..49dc6c4e 100644 --- a/docs/reference/settings.mdx +++ b/docs/reference/settings.mdx @@ -167,8 +167,8 @@ Activate, view, or deactivate the license for this Sencho control plane, and see | **Sencho Admiral** | The active license on this control plane, with a tier badge. Community instances see an upgrade prompt here instead. | | **Recovery Vault** | Whether the current subscription includes Recovery Vault entitlement. | | **Hardened Build** | Switches this control plane between the Community image channel and the Admiral Hardened Build channel. Review entitlement and registry access before switching; see [Plans](/features/licensing#feature-breakdown) for what Hardened Build changes. | -| **Current image** | The image reference this control plane is currently running, so you can confirm which channel took effect after a switch. | -| **Channel** | The active image channel (Community or Hardened). | +| **Current image** | The image reference this control plane is currently running, so you can confirm which channel took effect after a switch. A hardened image is shown as `Restricted` to non-administrators. | +| **Channel** | The running image channel this control plane is on: Community, Hardened, or Unknown. Reflects the running build, not the configured target. | | **Customer** | The customer name on file with Lemon Squeezy (paid plans only). | | **Product** | The product (paid plans only). | | **License key** | The active key, masked to the last four characters. | @@ -751,9 +751,15 @@ Displays instance information at a glance. | Field | Description | |-------|-------------| -| **Version** | Current Sencho version. | +| **Version** | The semantic version of this control plane instance. | +| **Channel** | The build track this control plane is running: `Dev`, `Preview`, `Stable`, or `Unknown`. A dev build is identified here even when its packaged version still matches the previous stable release. | +| **Current image** | The image reference this control plane was started with. A compose edit changes the configured target until the container is recreated; this row always shows the running image, never the configured one. | +| **Revision** | The immutable digest or pinned `dev-` tag this build resolves to, or `Unknown` when it cannot be determined. | +| **Image ID** | The first twelve characters of the running image's sha256 identifier. Click to copy the full id. Hidden when no image identity is available. | | **Tier** | Community or Admiral badge. | | **Plan Status** | active, trial, expired, or community (Admiral entitlement state, not the AGPL software license). | | **Instance ID** | First eight characters of the unique identifier for this Sencho control plane (used by the license server to identify it). | +The Build rows describe the **control plane** instance (the Sencho you are logged in to), not remote nodes. When identity metadata is unavailable, the reference rows read `Unknown` rather than inferring a value. For a hardened image seen by a non-administrator, the reference fields read `Restricted` instead. See [Verifying images](/operations/verifying-images) for how the build tracks and immutable tags relate. + The **Links** section contains Source code, AGPLv3 License, Licensing documentation, and Changelog links. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 461b5779..41521525 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -4,6 +4,7 @@ import { AuthProvider, useAuth } from './context/AuthContext'; import { useReducedMotion } from './hooks/use-theme'; import { NodeProvider } from './context/NodeContext'; import { LicenseProvider } from './context/LicenseContext'; +import { BuildInfoProvider } from './context/BuildInfoProvider'; import { Login } from './components/Login'; import { Setup } from './components/Setup'; import EditorLayout from './components/EditorLayout'; @@ -67,11 +68,13 @@ function AppContent() { )} - - {/* Portal lives inside LicenseProvider so the editor surface and its - portalled overlays can read license state via useLicense(). - Outer DeployFeedbackProvider is still an ancestor through App. */} - + + + {/* Portal lives inside LicenseProvider so the editor surface and its + portalled overlays can read license state via useLicense(). + Outer DeployFeedbackProvider is still an ancestor through App. */} + + diff --git a/frontend/src/components/EditorLayout.tsx b/frontend/src/components/EditorLayout.tsx index bd982186..0284d3fb 100644 --- a/frontend/src/components/EditorLayout.tsx +++ b/frontend/src/components/EditorLayout.tsx @@ -22,6 +22,7 @@ import { useOverlayState } from './EditorLayout/hooks/useOverlayState'; import { useStackActions, NODE_SWITCH_PENDING_TOKEN } from './EditorLayout/hooks/useStackActions'; import { useSelectedStackLiveRefresh } from './EditorLayout/hooks/useSelectedStackLiveRefresh'; import { useTheme } from '@/hooks/use-theme'; +import { useBuildInfo } from '@/hooks/useBuildInfo'; import { ThemeQuickSwitch } from './theme/ThemeQuickSwitch'; import { useNotifications } from './EditorLayout/hooks/useNotifications'; import { useContainerStats } from './EditorLayout/hooks/useContainerStats'; @@ -453,6 +454,7 @@ export default function EditorLayout() { const stackMuteActions = useStackMuteActions(stackDisplayName, openMuteRulesWithPrefill); const { isDarkMode } = useTheme(); + const { buildInfo } = useBuildInfo(); // ---- Mobile shell (below md) --------------------------------------------- // Desktop renders the persistent sidebar + workspace untouched. On a phone we @@ -974,6 +976,7 @@ export default function EditorLayout() { const sidebarEl = ( openSettings('nodes')} diff --git a/frontend/src/components/MobileTabBar.test.tsx b/frontend/src/components/MobileTabBar.test.tsx index ad917026..162a9914 100644 --- a/frontend/src/components/MobileTabBar.test.tsx +++ b/frontend/src/components/MobileTabBar.test.tsx @@ -1,15 +1,37 @@ -import { describe, it, expect, vi } from 'vitest'; +import { describe, it, expect, vi, beforeEach } from 'vitest'; import { render, screen, fireEvent } from '@testing-library/react'; import { Home, Radar, Clock } from 'lucide-react'; import { MobileTabBar } from './MobileTabBar'; +import { useBuildInfo } from '@/hooks/useBuildInfo'; +import type { BuildInfo } from '@/context/BuildInfoProvider'; import type { NavItem } from './EditorLayout/hooks/useViewNavigationState'; +vi.mock('@/hooks/useBuildInfo', () => ({ + useBuildInfo: vi.fn(() => ({ buildInfo: null, status: 'ready', retry: vi.fn() })), +})); + +const mockUseBuildInfo = vi.mocked(useBuildInfo); + const allItems: NavItem[] = [ { value: 'dashboard', label: 'Home', icon: Home }, { value: 'fleet', label: 'Fleet', icon: Radar }, { value: 'scheduled-ops', label: 'Schedules', icon: Clock }, ]; +function buildInfo(channel: BuildInfo['channel']): BuildInfo { + return { + version: '0.97.1', + channel, + imageChannel: 'community', + imageRef: channel === 'dev' ? 'ghcr.io/studio-saelix/sencho-dev:dev' : 'ghcr.io/studio-saelix/sencho:0.97.1', + imageId: 'a'.repeat(64), + revision: null, + restricted: false, + }; +} + +const noPill = { buildInfo: null, status: 'ready' as const, retry: vi.fn() }; + function renderBar(over: Partial> = {}) { const props: React.ComponentProps = { navItems: allItems, @@ -76,3 +98,52 @@ describe('MobileTabBar', () => { expect(screen.getByRole('button', { name: 'Stacks' })).not.toHaveAttribute('aria-current'); }); }); + +describe('MobileTabBar build-identity pill', () => { + beforeEach(() => { + mockUseBuildInfo.mockReturnValue(noPill); + }); + + it('shows a text DEV pill for a dev build, not an interactive control', () => { + mockUseBuildInfo.mockReturnValue({ buildInfo: buildInfo('dev'), status: 'ready', retry: vi.fn() }); + renderBar(); + expect(screen.getByText('DEV')).toBeInTheDocument(); + // The pill is a plain span: it adds no competing tap target in the tab row. + expect(screen.queryByRole('button', { name: 'DEV' })).not.toBeInTheDocument(); + }); + + it('shows a text PREVIEW pill for a preview build', () => { + mockUseBuildInfo.mockReturnValue({ buildInfo: buildInfo('preview'), status: 'ready', retry: vi.fn() }); + renderBar(); + expect(screen.getByText('PREVIEW')).toBeInTheDocument(); + expect(screen.queryByRole('button', { name: 'PREVIEW' })).not.toBeInTheDocument(); + }); + + it('renders no pill for a stable build', () => { + mockUseBuildInfo.mockReturnValue({ buildInfo: buildInfo('stable'), status: 'ready', retry: vi.fn() }); + renderBar(); + expect(screen.queryByText('DEV')).not.toBeInTheDocument(); + expect(screen.queryByText('PREVIEW')).not.toBeInTheDocument(); + }); + + it('keeps the tab touch targets present alongside the pill', () => { + mockUseBuildInfo.mockReturnValue({ buildInfo: buildInfo('dev'), status: 'ready', retry: vi.fn() }); + renderBar(); + expect(screen.getByRole('button', { name: 'Home' })).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'Stacks' })).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'Settings' })).toBeInTheDocument(); + }); + + it('lays the pill in-flow as a non-overlapping sibling of the tabs', () => { + mockUseBuildInfo.mockReturnValue({ buildInfo: buildInfo('dev'), status: 'ready', retry: vi.fn() }); + renderBar(); + const pill = screen.getByText('DEV'); + // In-flow (flex sibling), not absolutely positioned over the Settings tab. + expect(pill).not.toHaveClass('absolute'); + expect(pill).toHaveClass('self-center', 'shrink-0'); + // Sibling of the tab buttons inside the nav, so flexbox reserves its own + // region rather than letting it overlap the rightmost tab. + const settingsTab = screen.getByRole('button', { name: 'Settings' }); + expect(pill.parentElement).toBe(settingsTab.parentElement); + }); +}); diff --git a/frontend/src/components/MobileTabBar.tsx b/frontend/src/components/MobileTabBar.tsx index 8edb096b..b9952a7a 100644 --- a/frontend/src/components/MobileTabBar.tsx +++ b/frontend/src/components/MobileTabBar.tsx @@ -1,6 +1,7 @@ import { Home, Layers, Radar, Clock, Settings as SettingsIcon } from 'lucide-react'; import type { LucideIcon } from 'lucide-react'; import { cn } from '@/lib/utils'; +import { useBuildInfo } from '@/hooks/useBuildInfo'; import type { NavItem, ActiveView } from './EditorLayout/hooks/useViewNavigationState'; import type { MobileView } from './EditorLayout/mobile-surface'; @@ -45,7 +46,12 @@ export function MobileTabBar({ onNavigate, onSettings, }: MobileTabBarProps) { + const { buildInfo } = useBuildInfo(); + const has = (value: ActiveView) => navItems.some(i => i.value === value); + const channel = buildInfo?.channel; + const showPill = channel === 'dev' || channel === 'preview'; + const pillLabel = channel === 'dev' ? 'DEV' : 'PREVIEW'; const tabs: Tab[] = [ { id: 'home', label: 'Home', icon: Home }, @@ -79,7 +85,7 @@ export function MobileTabBar({ aria-label="Primary mobile" data-sn-glass="mobile-tabbar" className={cn( - 'md:hidden flex shrink-0 items-stretch', + 'md:hidden relative flex shrink-0 items-stretch', 'border-t border-hairline', 'bg-[color-mix(in_oklch,var(--card)_70%,transparent)] backdrop-blur-md backdrop-saturate-150', 'pb-[max(8px,env(safe-area-inset-bottom))]', @@ -108,6 +114,17 @@ export function MobileTabBar({ ); })} + {showPill ? ( + + {pillLabel} + + ) : null} ); } diff --git a/frontend/src/components/settings/AboutSection.tsx b/frontend/src/components/settings/AboutSection.tsx index 83c8016e..6a1d6bd0 100644 --- a/frontend/src/components/settings/AboutSection.tsx +++ b/frontend/src/components/settings/AboutSection.tsx @@ -1,5 +1,11 @@ +import { useState } from 'react'; import { useLicense } from '@/context/LicenseContext'; +import { useBuildInfo } from '@/hooks/useBuildInfo'; import { TierBadge } from '@/components/TierBadge'; +import { Badge } from '@/components/ui/badge'; +import { FlaskConical } from 'lucide-react'; +import { copyToClipboard } from '@/lib/clipboard'; +import { toast } from '@/components/ui/toast-store'; import { TogglePill } from '@/components/ui/toggle-pill'; import { useWhatsNewPreference } from '@/hooks/useWhatsNewPreference'; import { whatsNewEntries } from '@/whats-new/entries'; @@ -15,16 +21,110 @@ import { const linkClassName = 'font-mono text-[10px] leading-3 uppercase tracking-[0.18em] text-brand hover:text-brand/80 transition-colors'; +const mono = 'font-mono text-sm text-stat-value'; + +function BuildChannelChip({ label }: { label: string }) { + if (label === 'Dev') { + return ( + + Dev + + ); + } + if (label === 'Preview') { + return ( + + Preview + + ); + } + return {label}; +} + export function AboutSection() { const { license } = useLicense(); + const { buildInfo, status } = useBuildInfo(); const { enabled: whatsNewEnabled, setEnabled: setWhatsNewEnabled } = useWhatsNewPreference(); + const [copied, setCopied] = useState(false); + + // Loading surfaces a placeholder and error surfaces "Unknown" (truthful); + // the reference and revision fields below read "Restricted" for a redacted + // hardened image, before their null-check. + const channelLabel = (() => { + if (status === 'loading') return '…'; + if (status === 'error' || !buildInfo) return 'Unknown'; + switch (buildInfo.channel) { + case 'dev': return 'Dev'; + case 'preview': return 'Preview'; + case 'stable': return 'Stable'; + default: return 'Unknown'; + } + })(); + const resolveLabel = (value: string | null | undefined) => + buildInfo?.restricted + ? 'Restricted' + : status === 'loading' + ? '…' + : status === 'error' + ? 'Unknown' + : value ?? 'Unknown'; + + const imageRefLabel = resolveLabel(buildInfo?.imageRef); + const revisionLabel = resolveLabel(buildInfo?.revision); + const imageIdLabel = + status === 'loading' ? '…' + : status === 'error' || !buildInfo?.imageId ? 'Unknown' + : `sha256:${buildInfo.imageId.slice(0, 12)}`; + + const copyImageId = async () => { + if (!buildInfo?.imageId) return; + try { + await copyToClipboard(`sha256:${buildInfo.imageId}`); + setCopied(true); + window.setTimeout(() => setCopied(false), 1500); + } catch { + toast.error('Could not copy the image id.'); + } + }; return (
- v{__APP_VERSION__} + v{buildInfo?.version ?? __APP_VERSION__} + + + + + {imageRefLabel} + + + {revisionLabel} + + {imageIdLabel !== 'Unknown' && imageIdLabel !== '…' ? ( + + + + ) : null} diff --git a/frontend/src/components/settings/LicenseSection.tsx b/frontend/src/components/settings/LicenseSection.tsx index d1981dc8..7a2eb023 100644 --- a/frontend/src/components/settings/LicenseSection.tsx +++ b/frontend/src/components/settings/LicenseSection.tsx @@ -3,6 +3,7 @@ import { Button } from '@/components/ui/button'; import { Input } from '@/components/ui/input'; import { toast } from '@/components/ui/toast-store'; import { useLicense } from '@/context/LicenseContext'; +import { useBuildInfo } from '@/hooks/useBuildInfo'; import { TierBadge } from '@/components/TierBadge'; import { Crown, CheckCircle, XCircle, Clock, ExternalLink, @@ -47,8 +48,8 @@ function formatChannel(channel: ImageChannel): string { return 'Community'; case 'hardened': return 'Hardened'; - default: - return 'Custom'; + case 'unknown': + return 'Unknown'; } } @@ -64,6 +65,7 @@ function getTierMastheadValue(tier?: string): string { export function LicenseSection() { const { license, isPaid, activate, deactivate } = useLicense(); + const { buildInfo, status: buildInfoStatus } = useBuildInfo(); const [licenseKeyInput, setLicenseKeyInput] = useState(''); const [isActivating, setIsActivating] = useState(false); const [isDeactivating, setIsDeactivating] = useState(false); @@ -253,16 +255,28 @@ export function LicenseSection() { ) : null} - {channelStatus?.composeImageRef ?? formatChannel(channelStatus?.channel ?? 'unknown')} + {buildInfo?.restricted + ? 'Restricted' + : buildInfoStatus === 'loading' + ? '…' + : buildInfoStatus === 'error' + ? 'Unknown' + : buildInfo?.imageRef ?? 'Unknown'} - {formatChannel(channelStatus?.channel ?? 'unknown')} + + {buildInfoStatus === 'loading' + ? '…' + : buildInfoStatus === 'error' || !buildInfo + ? 'Unknown' + : formatChannel(buildInfo.imageChannel)} + {channelStatus?.operation?.state === 'failed' ? ( ({ useWhatsNewPreference: () => ({ enabled: true, setEnabled: mockSetEnabled, hasUnseen: false, markSeen: vi.fn() }), })); +vi.mock('@/hooks/useBuildInfo', () => ({ + useBuildInfo: vi.fn(() => ({ buildInfo: null, status: 'ready', retry: vi.fn() })), +})); +import { useBuildInfo } from '@/hooks/useBuildInfo'; +import type { BuildInfo } from '@/context/BuildInfoProvider'; + +const { mockCopyToClipboard, mockToastError } = vi.hoisted(() => ({ + mockCopyToClipboard: vi.fn(), + mockToastError: vi.fn(), +})); +vi.mock('@/lib/clipboard', () => ({ copyToClipboard: mockCopyToClipboard })); +vi.mock('@/components/ui/toast-store', () => ({ + toast: { error: mockToastError, success: vi.fn(), info: vi.fn(), loading: vi.fn(), dismiss: vi.fn() }, +})); + +const mockUseBuildInfo = vi.mocked(useBuildInfo); + +function buildInfo(over: Partial = {}): BuildInfo { + return { + version: '0.97.1', + channel: 'dev', + imageChannel: 'community', + imageRef: 'ghcr.io/studio-saelix/sencho-dev:dev', + imageId: 'a'.repeat(64), + revision: 'dev-abc1234', + restricted: false, + ...over, + }; +} + // The shipped entries.json is empty, so populate it here; the empty state has its own file. vi.mock('@/whats-new/entries', () => ({ whatsNewEntries: [{ id: 'entry-a', title: 'A feature', blurb: 'Does a thing.' }], @@ -91,3 +121,49 @@ describe('AboutSection Preferences', () => { expect(mockSetEnabled).toHaveBeenCalledWith(false); }); }); + +describe('AboutSection Build identity', () => { + it('shows the runtime channel, current image, revision and version', () => { + mockUseBuildInfo.mockReturnValue({ buildInfo: buildInfo(), status: 'ready', retry: vi.fn() }); + render(); + expect(screen.getByText('Dev')).toBeTruthy(); + expect(screen.getByText('ghcr.io/studio-saelix/sencho-dev:dev')).toBeTruthy(); + expect(screen.getByText('dev-abc1234')).toBeTruthy(); + expect(screen.getByText('v0.97.1')).toBeTruthy(); + }); + + it('labels redacted hardened reference fields Restricted, not Unknown', () => { + mockUseBuildInfo.mockReturnValue({ + buildInfo: buildInfo({ channel: 'stable', restricted: true, imageRef: null, revision: null }), + status: 'ready', + retry: vi.fn(), + }); + render(); + expect(screen.getByText('Stable')).toBeTruthy(); + expect(screen.getAllByText('Restricted').length).toBeGreaterThanOrEqual(2); + expect(screen.queryByText('Unknown')).toBeNull(); + }); + + it('shows Unknown for reference fields when build info is unavailable', () => { + mockUseBuildInfo.mockReturnValue({ buildInfo: null, status: 'error', retry: vi.fn() }); + render(); + expect(screen.getAllByText('Unknown').length).toBeGreaterThan(0); + }); + + it('wraps long image and revision tokens so they do not overflow', () => { + mockUseBuildInfo.mockReturnValue({ buildInfo: buildInfo(), status: 'ready', retry: vi.fn() }); + render(); + expect(screen.getByText('ghcr.io/studio-saelix/sencho-dev:dev')).toHaveClass('break-all'); + expect(screen.getByText('dev-abc1234')).toHaveClass('break-all'); + }); + + it('surfaces an error toast when copying the image id fails', async () => { + mockUseBuildInfo.mockReturnValue({ buildInfo: buildInfo(), status: 'ready', retry: vi.fn() }); + mockCopyToClipboard.mockRejectedValue(new Error('clipboard blocked')); + render(); + + await userEvent.click(screen.getByRole('button', { name: /sha256:/ })); + expect(mockCopyToClipboard).toHaveBeenCalledWith(`sha256:${'a'.repeat(64)}`); + expect(mockToastError).toHaveBeenCalledWith('Could not copy the image id.'); + }); +}); diff --git a/frontend/src/components/settings/__tests__/AboutSection.whatsNewEmpty.test.tsx b/frontend/src/components/settings/__tests__/AboutSection.whatsNewEmpty.test.tsx index 181a3c7d..8c0b92fa 100644 --- a/frontend/src/components/settings/__tests__/AboutSection.whatsNewEmpty.test.tsx +++ b/frontend/src/components/settings/__tests__/AboutSection.whatsNewEmpty.test.tsx @@ -43,6 +43,10 @@ vi.mock('@/hooks/useWhatsNewPreference', () => ({ useWhatsNewPreference: () => ({ enabled: true, setEnabled: vi.fn(), hasUnseen: false, markSeen: vi.fn() }), })); +vi.mock('@/hooks/useBuildInfo', () => ({ + useBuildInfo: () => ({ buildInfo: null, status: 'ready', retry: vi.fn() }), +})); + describe("AboutSection with no What's New entries authored", () => { it('hides the Preferences section entirely, so no toggle describes an absent icon', () => { render(); diff --git a/frontend/src/components/settings/__tests__/LicenseSection.test.tsx b/frontend/src/components/settings/__tests__/LicenseSection.test.tsx index 257bc0fe..d8d99605 100644 --- a/frontend/src/components/settings/__tests__/LicenseSection.test.tsx +++ b/frontend/src/components/settings/__tests__/LicenseSection.test.tsx @@ -1,6 +1,7 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import { render, screen } from '@testing-library/react'; import type { LicenseInfo } from '@/context/LicenseContext'; +import type { BuildInfoContextType } from '@/context/BuildInfoProvider'; const useLicenseMock = vi.fn(); @@ -12,6 +13,12 @@ vi.mock('../MastheadStatsContext', () => ({ useMastheadStats: () => {}, })); +const useBuildInfoMock = vi.fn<() => BuildInfoContextType>(() => ({ buildInfo: null, status: 'ready', retry: vi.fn() })); + +vi.mock('@/hooks/useBuildInfo', () => ({ + useBuildInfo: () => useBuildInfoMock(), +})); + vi.mock('@/components/TierBadge', () => ({ TierBadge: () => tier, })); @@ -135,3 +142,72 @@ describe('LicenseSection pricing link', () => { expect(screen.getByText('See pricing')).toBeTruthy(); }); }); + +describe('LicenseSection build rows (running identity consistency)', () => { + beforeEach(() => { + useLicenseMock.mockReset(); + useBuildInfoMock.mockReset(); + useBuildInfoMock.mockReturnValue({ buildInfo: null, status: 'ready', retry: vi.fn() }); + mockLicense(baseLicense()); + }); + + it('renders the running Community channel and image ref, regardless of configured target', () => { + useBuildInfoMock.mockReturnValue({ + buildInfo: { + version: '0.97.1', + channel: 'stable', + imageChannel: 'community', + imageRef: 'ghcr.io/studio-saelix/sencho:0.97.1', + imageId: 'a'.repeat(64), + revision: null, + restricted: false, + }, + status: 'ready', + retry: vi.fn(), + }); + render(); + // The configured/compose target is hardened, but the running build is + // Community: the row must show the running image, never the target. + expect(screen.getByText('ghcr.io/studio-saelix/sencho:0.97.1')).toBeTruthy(); + expect(screen.queryByText('Hardened')).toBeNull(); + }); + + it('renders a hardened running image as Hardened channel and Restricted image, not Unknown', () => { + useBuildInfoMock.mockReturnValue({ + buildInfo: { + version: '0.97.1', + channel: 'stable', + imageChannel: 'hardened', + imageRef: null, + imageId: 'b'.repeat(64), + revision: null, + restricted: true, + }, + status: 'ready', + retry: vi.fn(), + }); + render(); + expect(screen.getByText('Hardened')).toBeTruthy(); + expect(screen.getByText('Restricted')).toBeTruthy(); + expect(screen.queryByText('Unknown')).toBeNull(); + }); + + it('labels an unclassifiable running image Channel Unknown, never Custom', () => { + useBuildInfoMock.mockReturnValue({ + buildInfo: { + version: '0.97.1', + channel: 'unknown', + imageChannel: 'unknown', + imageRef: null, + imageId: null, + revision: null, + restricted: false, + }, + status: 'ready', + retry: vi.fn(), + }); + render(); + expect(screen.getAllByText('Unknown').length).toBeGreaterThan(0); + expect(screen.queryByText('Custom')).toBeNull(); + }); +}); diff --git a/frontend/src/components/sidebar/SidebarBrand.test.tsx b/frontend/src/components/sidebar/SidebarBrand.test.tsx new file mode 100644 index 00000000..19b6f59a --- /dev/null +++ b/frontend/src/components/sidebar/SidebarBrand.test.tsx @@ -0,0 +1,72 @@ +import { describe, it, expect } from 'vitest'; +import { render, screen } from '@testing-library/react'; +import { SidebarBrand } from './SidebarBrand'; +import { chipDetail } from './chipDetail'; +import type { BuildInfo } from '@/context/BuildInfoProvider'; + +function info(channel: BuildInfo['channel']): BuildInfo { + return { + version: '0.97.1', + channel, + imageChannel: 'community', + imageRef: channel === 'dev' ? 'ghcr.io/studio-saelix/sencho-dev:dev' : 'ghcr.io/studio-saelix/sencho:0.97.1', + imageId: 'a'.repeat(64), + revision: null, + restricted: false, + }; +} + +describe('SidebarBrand build-identity chip', () => { + it('shows a DEV text chip for a dev build', () => { + render(); + const chip = screen.getByText('DEV'); + expect(chip).toBeInTheDocument(); + // Text is the cue, not color alone. + expect(chip.tagName).toBe('SPAN'); + expect(chip.textContent).toContain('DEV'); + }); + + it('shows a PREVIEW text chip for a preview build', () => { + render(); + expect(screen.getByText('PREVIEW')).toBeInTheDocument(); + }); + + it('renders no chip for a stable build', () => { + render(); + expect(screen.queryByText('DEV')).not.toBeInTheDocument(); + expect(screen.queryByText('PREVIEW')).not.toBeInTheDocument(); + }); + + it('renders no chip when build info is unavailable', () => { + render(); + expect(screen.queryByText('DEV')).not.toBeInTheDocument(); + expect(screen.queryByText('PREVIEW')).not.toBeInTheDocument(); + }); + + it('prefers the runtime version when available', () => { + render(); + expect(screen.getByText('v0.97.1')).toBeInTheDocument(); + }); +}); + +describe('chipDetail', () => { + it('reads Restricted for a redacted hardened reference', () => { + const b: BuildInfo = { ...info('stable'), restricted: true, imageRef: null, revision: null }; + expect(chipDetail(b)).toBe('Restricted'); + }); + + it('combines the reference and revision when both are present', () => { + const b: BuildInfo = { ...info('dev'), revision: 'dev-abc1234' }; + expect(chipDetail(b)).toBe('ghcr.io/studio-saelix/sencho-dev:dev · dev-abc1234'); + }); + + it('reads the reference alone when the revision is unknown', () => { + const b: BuildInfo = { ...info('dev'), revision: null }; + expect(chipDetail(b)).toBe('ghcr.io/studio-saelix/sencho-dev:dev'); + }); + + it('reads Unknown when the reference is absent and not restricted', () => { + const b: BuildInfo = { ...info('dev'), imageRef: null }; + expect(chipDetail(b)).toBe('Unknown'); + }); +}); \ No newline at end of file diff --git a/frontend/src/components/sidebar/SidebarBrand.tsx b/frontend/src/components/sidebar/SidebarBrand.tsx index 8e5f0a78..08d867e5 100644 --- a/frontend/src/components/sidebar/SidebarBrand.tsx +++ b/frontend/src/components/sidebar/SidebarBrand.tsx @@ -1,8 +1,17 @@ +import { FlaskConical } from 'lucide-react'; +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/components/ui/tooltip'; +import type { BuildInfo } from '@/context/BuildInfoProvider'; +import { chipDetail } from './chipDetail'; + interface SidebarBrandProps { isDarkMode: boolean; + buildInfo?: BuildInfo | null; } -export function SidebarBrand({ isDarkMode }: SidebarBrandProps) { +export function SidebarBrand({ isDarkMode, buildInfo }: SidebarBrandProps) { + const channel = buildInfo?.channel; + const showChip = channel === 'dev' || channel === 'preview'; + return (
-
+
Sencho - v{__APP_VERSION__} + v{buildInfo?.version ?? __APP_VERSION__} + {showChip ? ( + + + + + {channel === 'dev' ? : null} + {channel === 'dev' ? 'DEV' : 'PREVIEW'} + + + + {chipDetail(buildInfo)} + + + + ) : null}
); diff --git a/frontend/src/components/sidebar/StackSidebar.tsx b/frontend/src/components/sidebar/StackSidebar.tsx index 1f1ecb25..4d78caf0 100644 --- a/frontend/src/components/sidebar/StackSidebar.tsx +++ b/frontend/src/components/sidebar/StackSidebar.tsx @@ -11,10 +11,12 @@ import { StackList, type StackListProps } from './StackList'; import type { FilterChip } from './sidebar-types'; import type { BulkAction } from '@/hooks/useBulkStackActions'; import type { SidebarActivitySummary } from './useSidebarActivitySummary'; +import type { BuildInfo } from '@/context/BuildInfoProvider'; import { isStacksListSettled } from './stacksLoadUi'; export interface StackSidebarProps { isDarkMode: boolean; + buildInfo?: BuildInfo | null; nodeSwitcherSlot: ReactNode; createStackSlot: ReactNode | null; onScan: () => void; @@ -43,7 +45,7 @@ export interface StackSidebarProps { export function StackSidebar(props: StackSidebarProps) { const { - isDarkMode, nodeSwitcherSlot, createStackSlot, onScan, isScanning, canCreate, + isDarkMode, buildInfo, nodeSwitcherSlot, createStackSlot, onScan, isScanning, canCreate, searchQuery, onSearchChange, filterChip, filterCounts, onFilterChipChange, list, activitySummary, onActivityAction, bulkMode, selectedFiles, onToggleBulkMode, onToggleSelect, onClearSelection, onBulkAction, @@ -76,7 +78,7 @@ export function StackSidebar(props: StackSidebarProps) { its kicker chip), so the in-sidebar brand and node rows are redundant there and hidden to save vertical space. */}
- +
{nodeSwitcherSlot}
{canCreate && createStackSlot !== null && ( diff --git a/frontend/src/components/sidebar/chipDetail.ts b/frontend/src/components/sidebar/chipDetail.ts new file mode 100644 index 00000000..e98ed8ae --- /dev/null +++ b/frontend/src/components/sidebar/chipDetail.ts @@ -0,0 +1,11 @@ +import type { BuildInfo } from '@/context/BuildInfoProvider'; + +/** Detail shown under the DEV/PREVIEW chip, or the truthful Unknown / Restricted + * states when the running reference is unavailable or redacted for this user. */ +export function chipDetail(buildInfo: BuildInfo | null | undefined): string { + if (buildInfo?.restricted) return 'Restricted'; + if (buildInfo?.imageRef) { + return buildInfo.revision ? `${buildInfo.imageRef} · ${buildInfo.revision}` : buildInfo.imageRef; + } + return 'Unknown'; +} \ No newline at end of file diff --git a/frontend/src/context/BuildInfoProvider.test.tsx b/frontend/src/context/BuildInfoProvider.test.tsx new file mode 100644 index 00000000..f9b498ee --- /dev/null +++ b/frontend/src/context/BuildInfoProvider.test.tsx @@ -0,0 +1,135 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { render, screen, act } from '@testing-library/react'; +import type { ReactNode } from 'react'; +import { BuildInfoProvider } from './BuildInfoProvider'; +import { useBuildInfo } from '@/hooks/useBuildInfo'; + +const { currentUserRef } = vi.hoisted(() => ({ + currentUserRef: { value: null as { username: string; role: string } | null }, +})); + +vi.mock('./AuthContext', () => ({ + useAuth: () => ({ user: currentUserRef.value }), +})); + +vi.mock('@/lib/api', () => ({ + apiFetch: vi.fn(), +})); +import { apiFetch } from '@/lib/api'; + +const restrictedViewer = { + version: '0.97.1', + channel: 'stable', + imageChannel: 'hardened', + imageRef: null, + imageId: 'b'.repeat(64), + revision: null, + restricted: true, +}; + +const adminCommunity = { + version: '0.97.1', + channel: 'dev', + imageChannel: 'community', + imageRef: 'ghcr.io/studio-saelix/sencho-dev:dev', + imageId: 'a'.repeat(64), + revision: null, + restricted: false, +}; + +function json(data: unknown, status = 200): Response { + return new Response(JSON.stringify(data), { status }); +} + +function Harness() { + const { buildInfo, status } = useBuildInfo(); + return
{status}:{buildInfo?.imageRef ?? 'none'}
; +} + +function wrapper({ children }: { children: ReactNode }) { + return {children}; +} + +async function flush() { + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); +} + +describe('BuildInfoProvider', () => { + beforeEach(() => { + currentUserRef.value = null; + vi.mocked(apiFetch).mockReset(); + }); + + it('clears permission-filtered state and refetches when the auth identity changes', async () => { + currentUserRef.value = { username: 'admin', role: 'admin' }; + vi.mocked(apiFetch).mockResolvedValue(json(adminCommunity)); + const { rerender } = render(, { wrapper }); + await flush(); + expect(screen.getByTestId('s').textContent).toContain('ready'); + expect(screen.getByTestId('s').textContent).toContain('ghcr.io/studio-saelix/sencho-dev:dev'); + + // A viewer session must never see the admin-cached ref: the provider + // clears immediately and rejects the stale in-flight completion. + currentUserRef.value = { username: 'viewer', role: 'viewer' }; + vi.mocked(apiFetch).mockResolvedValue(json(restrictedViewer)); + rerender(); + // Cleared synchronously on the identity change, before the refetch lands. + expect(screen.getByTestId('s').textContent).toBe('loading:none'); + await flush(); + expect(screen.getByTestId('s').textContent).toBe('ready:none'); + }); + + it('rejects a stale in-flight completion from a prior identity', async () => { + currentUserRef.value = { username: 'admin', role: 'admin' }; + let resolveAdmin!: (r: Response) => void; + vi.mocked(apiFetch).mockReturnValue(new Promise((res) => { resolveAdmin = res; })); + const { rerender } = render(, { wrapper }); + + currentUserRef.value = { username: 'viewer', role: 'viewer' }; + vi.mocked(apiFetch).mockResolvedValue(json(restrictedViewer)); + rerender(); + await flush(); + expect(screen.getByTestId('s').textContent).toBe('ready:none'); + + // The old admin request resolving later must not surface its ref. + await act(async () => { resolveAdmin(json(adminCommunity)); }); + expect(screen.getByTestId('s').textContent).toBe('ready:none'); + }); + + it('shares one fetch across two consumers', async () => { + currentUserRef.value = { username: 'admin', role: 'admin' }; + vi.mocked(apiFetch).mockResolvedValue(json(adminCommunity)); + render( + + + + , + ); + await flush(); + expect(apiFetch).toHaveBeenCalledTimes(1); + expect(apiFetch).toHaveBeenCalledWith('/build-info', expect.objectContaining({ localOnly: true })); + expect(screen.getAllByTestId('s')).toHaveLength(2); + for (const el of screen.getAllByTestId('s')) { + expect(el.textContent).toContain('ghcr.io/studio-saelix/sencho-dev:dev'); + } + }); + + it('surfaces error as Unknown and retries on focus', async () => { + currentUserRef.value = { username: 'admin', role: 'admin' }; + vi.mocked(apiFetch).mockRejectedValue(new Error('down')); + render(, { wrapper }); + await flush(); + expect(screen.getByTestId('s').textContent).toBe('error:none'); + + vi.mocked(apiFetch).mockResolvedValue(json(adminCommunity)); + await act(async () => { + window.dispatchEvent(new Event('focus')); + await Promise.resolve(); + await Promise.resolve(); + }); + expect(screen.getByTestId('s').textContent).toBe('ready:ghcr.io/studio-saelix/sencho-dev:dev'); + }); +}); \ No newline at end of file diff --git a/frontend/src/context/BuildInfoProvider.tsx b/frontend/src/context/BuildInfoProvider.tsx new file mode 100644 index 00000000..614a124b --- /dev/null +++ b/frontend/src/context/BuildInfoProvider.tsx @@ -0,0 +1,103 @@ +import { createContext, useCallback, useEffect, useRef, useState, type ReactNode } from 'react'; +import { apiFetch } from '@/lib/api'; +import { useAuth } from './AuthContext'; + +export type BuildChannel = 'stable' | 'dev' | 'preview' | 'unknown'; +export type ImageChannel = 'community' | 'hardened' | 'unknown'; +export type BuildInfoStatus = 'loading' | 'ready' | 'error'; + +/** Canonical runtime build identity of the control instance. Mirrors the + * /api/build-info response: imageRef and revision are nulled for hardened + * images when the viewer is not an admin, with `restricted` marking that + * redaction (the UI shows "Restricted", never "Unknown", when set). + * `restricted === true` implies `imageRef === null && revision === null`. */ +export interface BuildInfo { + version: string | null; + channel: BuildChannel; + imageChannel: ImageChannel; + imageRef: string | null; + imageId: string | null; + revision: string | null; + restricted: boolean; +} + +export interface BuildInfoContextType { + buildInfo: BuildInfo | null; + status: BuildInfoStatus; + retry: () => void; +} + +// eslint-disable-next-line react-refresh/only-export-components +export const BuildInfoContext = createContext(undefined); + +/** Single owner of the control-instance build identity. Mounted inside the + * authenticated subtree. One shared fetch (localOnly, so it always targets the + * local hub); permission-filtered state is cleared the moment the auth identity + * or role changes, and stale in-flight completions from a prior session are + * rejected by a generation counter. Focus retries an error so it never sticks + * failed; consumers show a placeholder while loading and "Unknown" on error. */ +export function BuildInfoProvider({ children }: { children: ReactNode }) { + const { user } = useAuth(); + + const [buildInfo, setBuildInfo] = useState(null); + const [status, setStatus] = useState('loading'); + + const generationRef = useRef(0); + const statusRef = useRef('loading'); + const identityRef = useRef(null); + + const load = useCallback(async () => { + const gen = ++generationRef.current; + setStatus('loading'); + try { + const res = await apiFetch('/build-info', { localOnly: true }); + if (gen !== generationRef.current) return; + if (res.ok) { + const data = (await res.json()) as BuildInfo; + if (gen !== generationRef.current) return; + setBuildInfo(data); + setStatus('ready'); + } else { + console.error(`[BuildInfo] fetch returned ${res.status}`); + setStatus('error'); + } + } catch (err) { + if (gen !== generationRef.current) return; + console.error('[BuildInfo] fetch failed', err); + setStatus('error'); + } + }, []); + + // Reload whenever the signed-in identity or role changes. Clearing state + // synchronously (before the replacement fetch resolves) guarantees a + // hardened admin's reference never surfaces in a viewer session. + useEffect(() => { + const identity = user ? `${user.username}:${user.role}` : null; + if (identity !== identityRef.current) { + identityRef.current = identity; + setBuildInfo(null); + setStatus('loading'); + void load(); + } + }, [user, load]); + + useEffect(() => { + statusRef.current = status; + }, [status]); + + // A transient failure (auth expiry, hub restart) should not stick: retry the + // moment the tab regains focus. + useEffect(() => { + const onFocus = () => { + if (statusRef.current === 'error') void load(); + }; + window.addEventListener('focus', onFocus); + return () => window.removeEventListener('focus', onFocus); + }, [load]); + + return ( + void load() }}> + {children} + + ); +} \ No newline at end of file diff --git a/frontend/src/hooks/useBuildInfo.ts b/frontend/src/hooks/useBuildInfo.ts new file mode 100644 index 00000000..7b13f06a --- /dev/null +++ b/frontend/src/hooks/useBuildInfo.ts @@ -0,0 +1,13 @@ +import { useContext } from 'react'; +import { BuildInfoContext, type BuildInfoContextType } from '@/context/BuildInfoProvider'; + +/** Consumer of the single shared control-instance build identity. The shell, + * About, and Admiral Account all read the same fetched BuildInfo by reference + * instead of issuing their own fetches. */ +export function useBuildInfo(): BuildInfoContextType { + const context = useContext(BuildInfoContext); + if (context === undefined) { + throw new Error('useBuildInfo must be used within a BuildInfoProvider'); + } + return context; +} \ No newline at end of file From 69e76090ccc6c7ef9a2b358de4bf4c6bbaa746fc Mon Sep 17 00:00:00 2001 From: Anso Date: Tue, 8 Sep 2026 12:41:30 -0400 Subject: [PATCH 2/3] feat(gitops): durable reconcile-attempt reservation, coalescing, and recovery (#1893) * feat(gitops): add reconcile trigger normalization and coalescing keys The source controller needs a single normalized shape for every trigger that can request evaluation (manual, API, webhook, poll, retry, config change, startup, resume, and future provider/schedule/binding-change producers), and a way to decide whether two concurrent submissions describe the same work. Add ReconcileTrigger, the discriminated ReconcileRequest (fetch vs apply), and coalesceKey/deliveryKey. A fetch coalesces by application alone, since it has one live outcome regardless of trigger; an apply coalesces only when the commit, plan fingerprint, and deploy choice all match, so two applies that differ in any of those can never be joined and have one silently receive the other's result. * feat(gitops): add exhaustive failure classification and retry backoff The controller needs to tell a transient network condition from a permanent configuration one from a broad range of Git-source, policy, and target failures, since several distinct causes collapse onto the same public error code and a wrong call either retries forever on a bad URL or gives up on a DNS blip. Add classifyFailure, built on two total lookup records (one keyed by TransportFailureReason, one by GitSourceErrorCode) so adding a new value to either source union fails the build until this classifier accounts for it, rather than silently defaulting. A tip-changed race classifies as supersession, not backoff; an unrecognized exit-coded git error gets a lower retry ceiling than a plain network timeout; a deploy or health failure after a successful apply is its own class that must never refetch or reapply. Add nextRetryAt: bounded exponential backoff (60s doubling, capped at one hour) with +-10% jitter, honoring a provider-supplied retry floor when it is larger than the computed delay. * feat(gitops): derive normalized reconcile outcomes from the source facet Silence is not an acceptable GitOps result: every reconcile attempt needs to settle into one named outcome an operator can act on, not a bare success/failure. Deriving that outcome from the existing source-facet projection (rather than a second, parallel status source) is what keeps "no source change" from silently collapsing into "converged" the way a bare commit-SHA comparison would. Add outcomeFromSourceFacet, exhaustive over all 17 SourceFacet statuses, each mapped to a ReconcileOutcome, a human reason, and a next action (review, resume, retry, resolve_conflict, configure_credentials, view_target_results, or none). converged is deliberately never produced here: it requires target and health evidence this source-only projection does not carry, so a later composition over source + target + health is the only thing allowed to report it. * feat(gitops): add the portable accepted-generation contract Direct and Blueprint dispatch need to consume the exact same description of what an accepted generation contains, without either side inventing a node id, local candidate path, target project name, or placement field into it, since that is exactly the kind of drift that would let a stale acceptance authorize a routing decision made after it. Add six additive, nullable columns to gitops_generations (portable manifest, Compose inputs, source/security policy evidence, support and compatibility requirements), all decoded honestly: a legacy row missing one records an explicit limitation rather than inventing evidence, and a legacy pending candidate lacking the new contract must be re-evaluated before it can be accepted or dispatched. Add gitops/handoff.ts: the AcceptedGeneration type built from a generation row via buildAcceptedGeneration, a compile-time assertion that the contract cannot carry a target-mode-specific field, and the TargetAdapter boundary with BlueprintTargetAdapter failing closed until Blueprint rollout orchestration exists. Current target mode and binding travel separately in DispatchContext, re-read under the dispatch lock rather than carried on the generation itself. * feat(gitops): add controller-owned bookkeeping columns to gitops_applications The source controller needs somewhere to persist source policy, poll cadence, and a durable attempt sequence per application, and it needs to work identically for Direct and Blueprint applications. stack_git_sources (the existing home for auto_apply_on_webhook/auto_deploy_on_apply) is Direct-only and keyed by stack name, so it cannot represent a Blueprint application at all. Add source_policy (manual|review|automatic, default manual), poll_interval_secs (NULL inherits the global default, 0 disables), next_poll_at (the durable scheduling cursor), and attempt_seq to gitops_applications instead. All default to values that start no unattended work: an upgraded installation begins polling nothing and stays on manual policy until explicitly migrated or configured. * feat(gitops): add durable attempt reservation and poll/retry queries History dedupe alone is not execution idempotency: mutateApp writes the application row and only then inserts history, so a dedupe conflict still commits the row write. The controller needs a reservation that runs before any side effect and touches nothing else, so a duplicate or restarted submission can be told apart from new work without repeating it. Add reserveReconcileAttempt/settleReconcileAttempt: a bare history insert in its own transaction, deliberately not through mutateApp, using the existing history dedupe index as the reservation/idempotency check itself. A repeated reservation or settlement for the same operation is a no-op, never overwriting the first settled result. Add the store queries a controller needs to drive this: getSettledAttempt and latestSettledAttempt (exact vs. most-recent lookup, the latter tie-broken by rowid since the id column is a random UUID unrelated to recency), listUnsettledReconcileAttempts (a reservation with no matching settled row, for startup recovery), and listSourcesDueForPoll / listApplicationsDueForRetry (excluding suspended, in-flight, and Blueprint-mode applications). * feat(gitops): add controller-facing reconcile entry point to GitSourceService Adds GitSourceService.reconcile(), a single normalized entry point that takes a fetch or apply request and returns a normalized outcome, next action, and reason instead of a thrown error or a raw boolean. It owns the git mutex for the whole evaluation and calls the same private fetch/apply bodies the existing pull()/apply() routes use, so it never nests locks. Resolves the live application for the stack first and fails closed, without attempting any work, when the request's application id no longer matches it or when no application exists at all. A fetch or apply failure that the underlying transition never persisted (missing config, a stale commit, lock contention) is classified through the existing retry disposition table so an unretryable failure is never reported as retryable, and a stale-looking success is never reported in place of a real failure. * feat(gitops): dispatch accepted generations to their target Adds GitSourceService.dispatchAcceptedGeneration(), which routes a portable accepted-generation contract to its target: Blueprint mode delegates to the existing BlueprintTargetAdapter (rollout orchestration is not built yet, so it always blocks), and Direct mode dispatches by driving reconcile() with the generation's commit, since there is no separate generation-based promotion pipeline yet. The deploy flag is read from the stack's own auto_deploy_on_apply setting, matching how every other producer decides it. Also fixes a case reconcile() got wrong: when a promotion succeeds but the following deploy fails, the source itself has genuinely changed, so reporting it as an unchanged failure was as untruthful as reporting it a plain success would have been. It now reports a result that neither claims, pointing at the target instead of asking for a source retry that could never succeed. * feat(gitops): add background poll and retry driver for source reconciliation Adds SourceController, a self-rescheduling background timer that finds sources whose poll interval or retry time has arrived and evaluates each through the existing reconcile() entry point. A per-application in-flight set is what keeps one slow evaluation from blocking the rest of the fleet; the tick never waits on any evaluation before scheduling the next one, and a scan that throws still reschedules rather than stopping the driver permanently. This delivery only covers detection: a tick issues a fetch, the same step a manual pull performs, so a new candidate is staged for review but not automatically accepted or dispatched, and a retry re-issues a plain fetch rather than resuming whatever stage previously failed. Both are documented as open follow-on work rather than silent gaps. Not yet wired into the process lifecycle; that follows separately. * feat(gitops): start and stop the source reconciliation driver with the process Wires SourceController into the background-service lifecycle: it starts after the existing GitOps recovery and orphan-sweep steps, so it never scans an application still carrying a stale in-progress marker from a killed process, and stops alongside every other background timer on shutdown. * feat(gitops): add suspend, resume, and explicit retry for a source Adds GitSourceService.suspend()/resume()/retry(), the service-layer methods behind an upcoming suspend/resume/retry control surface. Suspending a source now genuinely stops it: pullLocked and applyLockedBody both check suspension before doing any work, not just inside the transition bookkeeping, which used to reject but then let the fetch or apply proceed anyway. A refused suspend surfaces as a real error rather than a silent no-op, since an operator believing a source is suspended when it isn't is the exact failure this exists to prevent; a refused resume stays silent, since the row read back after the attempt already reports the true state either way. A webhook delivery to a suspended source is reported as skipped rather than a failed pull, so a long suspension does not read to the Git host as a broken webhook. * feat(gitops): add suspend, resume, and retry routes for a git source Adds POST endpoints for suspending a source (with an optional, length- capped reason), resuming it, and explicitly retrying it, wired to the existing service-layer methods behind the same stack:edit permission as pull and apply. Neither route can trigger a deploy today, so unlike apply and webhook-pull they need no conditional stack:deploy check. * fix(gitops): close path-injection gaps at two candidate-file sinks Adds the inline resolve-and-prefix-check barrier this codebase already uses at other filesystem sinks derived from a stack name or a stored candidate path, closing two sinks that lacked it: the manifest write in GitProjectManifestService, and the pending-candidate access check before promoting an apply. The apply-side check now shares the same strict candidate-path validator the registry-delivery path already uses on the identical stored field, rather than a looser check, so both call sites treat a tampered candidate reference the same way. Fixed two test fixtures that had never matched the shape a real candidate path takes, which the stricter check would otherwise have rejected. * fix(gitops): confine the manifest write directory to the managed area The prior fix checked the manifest write/rename targets against the resolved managed directory, but left the mkdir call on that directory itself unconfined and checked it against the data root rather than the narrower managed area every sibling barrier in this file uses. Aligns it with the established convention and the actual boundary that matters. * feat(gitops): wire durable attempt reservation and coalescing into reconcile() reconcile() now reserves a durable attempt before doing any fetch or apply and settles it with the normalized result once execution finishes, using the reservation and settlement primitives that already existed but had no caller. A request carrying a stable external delivery id reserves under a producer- and intent-namespaced key, so a redelivery reuses the same attempt instead of minting a second one; a request with no such identity gets a freshly allocated attemptSeq- based id, bumped atomically with its reservation. Concurrent submissions that would do the same work now coalesce: the first becomes the leader and actually runs, and any submission that joins while it is still in flight awaits the leader's real result instead of running a duplicate fetch or apply. Each still gets its own durable attempt and its own settled row, including a concurrent redelivery of the same external event, which joins the in-flight leader rather than falling back to a snapshot of the row from before the leader's work landed. Startup gains an attempt-recovery phase, run before the managed-area sweep and before SourceController starts: every attempt reserved but never settled, most likely from a crash between the two, is resolved from durable state without re-executing anything. One row failing to recover no longer blocks the rest; it is skipped and logged, and recovery keeps paging until nothing unsettled remains. A settlement failure is caught and logged rather than turning an already-successful fetch or apply into a thrown error for the caller, and a settled attempt's stored result is now decoded through a validated outcome/next-action check instead of a blind cast, logging rather than silently reporting unknown when a stored row is corrupt or unreadable. * fix(gitops): remove unused store variable from coalescing test * fix(gitops): make reconcile-attempt recovery leader-aware and cursor-paginated A fresh audit found real gaps in the reservation/coalescing wiring from the previous commit: recovery paged by "still unsettled" status rather than a cursor, so once a permanently unrecoverable row occupied every slot in a page, every genuinely recoverable row beyond it was silently never reached; a follower's outcome was reconstructed independently from row state rather than from its leader's actual stored result, so a leader and its follower could durably disagree; and an attempt resolved on the live path was returned but never actually settled, leaving it open indefinitely until the next restart. listUnsettledReconcileAttempts now takes an optional (created_at, id) cursor, matching the pagination shape queryHistoryRows already uses, so paging always advances regardless of which rows settle. Recovery is now two-pass: independent attempts settle first from row state, followers are deferred, then each deferred follower settles from its leader's now-settled result. A follower whose leader is a real but still-unresolved reservation is left unsettled rather than guessed at independently, since the leader could still settle to something else later, including when the leader simply failed to settle in this same pass rather than "never will". The same leader-aware resolution now backs the live reconcile() path too, via a shared helper, so an already-reserved attempt is durably settled instead of merely returning a value. Also fixes a narrower race in reconcile() itself: two submissions can share an operation id (a stable external delivery id) while running under different coalesce keys, since an apply's coalesce key includes its commit sha, plan fingerprint, and deploy flag, which the delivery id does not carry. reconcile() now checks in-process executions by operation id directly before falling back to a durable-state resolution, so such a submission joins the real in-flight leader instead of settling a stale pre-execution snapshot ahead of it. * feat(gitops): route pull, apply, and webhook producers through reservation A fresh audit found the same "primitive built, real caller does not use it" pattern one layer deeper than the previous commit fixed: reconcile(), the controller-facing entry point, reserved and coalesced durable attempts correctly, but the actual production producers, the manual pull button, the manual apply button, and the webhook route, all called pullLocked/applyWithSharedLock directly, bypassing reservation entirely. pullLocked also minted its own independent operation id rather than using a reserved attempt's, breaking the "one operation id spans an attempt and its stage evidence" invariant. pullLocked, applyWithSharedLock, applyLocked, and applyLockedBody now accept an optional operation id, using it in place of their own default when a caller supplies one. pull() and apply() reserve a durable attempt and coalesce with a concurrent call to themselves whenever a real GitOps application exists for the stack, the same definition pullLocked itself already used to decide whether it has any GitOps bookkeeping to do at all. handleWebhookPull()'s fetch step and its conditional auto-apply step reserve too, without a coalescing map: the route's own debounce window plus its single per-stack lock acquisition already prevent a concurrent duplicate from reaching that point, so there is nothing to coalesce there. Fixed along the way: apply()'s coalesce key computed its deploy flag differently than the code that actually executed the apply, so two concurrent applies that genuinely differed in deploy behavior could share a key and one could silently receive the other's result; deploy is now resolved once, the same way applyLockedBody itself resolves it, before it drives either the key or the execution. A policy-bypassing apply is now routed through a fresh, never-shared coalescing map, since bypassPolicy changes behavior but was not part of the key. A coalesced follower's own reservation is now settled in a finally rather than only after a successful await, so a rejecting leader (the ordinary path for a producer that preserves its own throw contract, unlike reconcile()'s internal error handling) no longer leaves the follower's attempt open until the next restart. A reservation bookkeeping failure no longer turns a manual pull or apply that would otherwise have succeeded into a hard failure; it logs and falls through to unreserved execution instead. * fix(gitops): make the boot sweep consult claimant pointers before reaping a candidate The boot-time managed-area sweep decided whether to delete a staged candidate directory purely from file age and a completeness marker, with no awareness of the database. A fresh audit gave the concrete failure: a reconcile stages a candidate, the process crashes before settlement, the installation stays down more than a day, startup settles the attempt from a snapshot rather than real stage evidence, and the sweep then deletes the still-needed candidate out from under it. sweepManagedArea now takes the set of candidate directory basenames still referenced by the stack, and never reaps one of them regardless of age or completeness. GitSourceService computes that set from three independent sources: the live application's current candidate generation, its accepted-but-not-yet-promoted generation (the sourceAccepted-committed, targetApplied-not-yet-committed window; that path has no production caller yet, so this is forward-looking coverage for it), and the pending fetch record's own candidate reference, which is written outside the transaction that mints a generation and can therefore be the only claimant for a candidate that failed validation or was staged while no live application existed to read a pointer from. * fix(gitops): fail closed on reservation failure and a torn-down application A fresh audit found that a reservation-bookkeeping failure (a transient DB error, an application torn down in the window between resolving it and reserving against it) fell through to unreserved execution. That directly defeated this delivery's own purpose: a manual apply could still promote Compose files and deploy with zero durable record of it happening. Reservation failure now fails closed for pull, apply, and the webhook route: the operation is refused, logged, and recorded to the stack's own activity history, distinguishing a torn-down application (never retryable) from a transient failure (worth retrying). Closing that hole surfaced a second, related gap: pull, apply, and the webhook route only checked for a live (active) application before deciding whether to reserve at all, so a stack whose GitOps tracking was explicitly torn down while its Git source configuration survived could still run fully untracked, the same failure mode reached a different way. Refusing this case took two attempts to get right, both caught by review before landing: the first version refused on any tombstoned state, which would have permanently and unrecoverably blocked pull and apply for an application deliberately tombstoned as deleted while its config survives for a future rebuild, a state two existing production paths produce on purpose; narrowing to detached only still misfired on a routine, fully completed detach, since that same operation deletes the source row in the same transaction, so the refusal must also confirm the source row actually survived before firing. Both are covered by regression tests now, alongside the original reservation-failure fix. * fix(gitops): unify fetch-intent coalescing across pull and reconcile A manual pull and a concurrently poll-triggered reconcile for the same application previously ran their own separate in-flight maps and could each start a clone for the same fetch. They now share one coalescing map, with reconcile's fetch path classifying its own outcome instead of relying on generic row-state derivation, so a pre-transition failure the row does not yet reflect is never durably recorded as a plain success. Apply-intent coalescing stays producer-local for now; unifying it needs deploy-failure awareness threaded into the shared settlement path first, which is a separately scoped follow-up. * feat(gitops): recognize a webhook delivery id for traceability The real webhook trigger endpoint is generic and HMAC-signed, with no delivery identity of its own. It now extracts one from a recognized provider header (GitHub, GitLab, Bitbucket, or a generic fallback) when the caller sends one, and threads it through to the git-pull execution path as a plain traceability breadcrumb on failure logs. Redelivery dedup is deliberately not implemented here: an earlier attempt routed the delivery id through the durable attempt reservation itself, which silently dropped a redelivery's history through the existing dedupe index instead of recording it. Building real dedup needs settlement to reflect classified outcomes rather than generic row-state derivation for both a settled and a crashed-mid-flight prior attempt, which is a wider, separately scoped change shared with the same gap already deferred for apply-intent reconcile. * fix(gitops): thread the reserved attempt's operation id into the pending fetch record The pending fetch record stamped its own independent random operation id instead of the reserved attempt's real one, so an apply falling back to it (when it holds no reservation of its own) inherited an identity unrelated to the fetch that actually produced the candidate. One id now spans reservation, fetch, generation, and the pending record. Also fixes the short "op" token rendered in activity log lines: a fixed prefix stopped discriminating between attempts once operation ids became structured (:attempt:), since the prefix is now the same applicationId every time. A shared helper renders the actual attempt-discriminating suffix instead, applied consistently across pull, apply, and create so each event's logged identity matches what its own durable history actually recorded. * test(gitops): cover the reconcile-recovery-then-sweep startup ordering Reconcile-attempt recovery and the managed-area sweep must run in that fixed order before the source controller's own poll loop starts, but the guarantee lived only in a comment inside startServer, a function with roughly two dozen unrelated service initializations that makes it impractical to exercise end to end in a test. Extracted the two steps into their own function so they're directly testable in isolation, without moving the source controller's own start call: that stays exactly where it was, since pulling it earlier would have crossed a separate, already-documented ordering requirement for registry delivery recovery. A structural test guards the one property that can't be covered by driving the function directly: that the real startServer body still calls the extracted function before starting the controller. * fix(gitops): complete durable reconciliation execution * fix(gitops): resolve static analysis findings --- .../bootstrap-startup-gitops-order.test.ts | 99 + .../__tests__/git-project-manifest.test.ts | 78 +- .../git-source-apply-recovery.test.ts | 40 +- .../src/__tests__/git-source-routes.test.ts | 258 ++ .../src/__tests__/git-source-service.test.ts | 2962 ++++++++++++++++- .../src/__tests__/gitops-approvals.test.ts | 10 + backend/src/__tests__/gitops-backoff.test.ts | 157 + .../gitops-blueprint-transitions.test.ts | 4 + .../__tests__/gitops-create-recovery.test.ts | 10 + backend/src/__tests__/gitops-create.test.ts | 10 + backend/src/__tests__/gitops-deferred.test.ts | 10 + backend/src/__tests__/gitops-derive.test.ts | 10 + .../__tests__/gitops-direct-producers.test.ts | 9 +- backend/src/__tests__/gitops-handoff.test.ts | 115 + .../src/__tests__/gitops-history-read.test.ts | 4 + .../__tests__/gitops-managed-sweep.test.ts | 4 + backend/src/__tests__/gitops-outcomes.test.ts | 152 + .../gitops-reconcile-attempts.test.ts | 387 +++ .../__tests__/gitops-recovery-capture.test.ts | 10 + backend/src/__tests__/gitops-recovery.test.ts | 10 + backend/src/__tests__/gitops-schema.test.ts | 65 + .../src/__tests__/gitops-transitions.test.ts | 10 + backend/src/__tests__/gitops-triggers.test.ts | 85 + .../src/__tests__/helpers/gitopsFixtures.ts | 4 + .../src/__tests__/source-controller.test.ts | 207 ++ .../src/__tests__/webhooks-git-source.test.ts | 139 + .../src/__tests__/webhooks-trigger.test.ts | 49 + backend/src/bootstrap/shutdown.ts | 4 + backend/src/bootstrap/startup.ts | 42 +- backend/src/routes/gitSources.ts | 82 +- backend/src/routes/webhooks.ts | 31 +- backend/src/services/DatabaseService.ts | 18 + .../src/services/GitProjectManifestService.ts | 123 +- backend/src/services/GitSourceService.ts | 1429 +++++++- backend/src/services/WebhookService.ts | 40 +- .../src/services/gitops/SourceController.ts | 158 + backend/src/services/gitops/backoff.ts | 140 + .../src/services/gitops/blueprintProducers.ts | 6 +- .../src/services/gitops/directApplication.ts | 10 + backend/src/services/gitops/handoff.ts | 149 + backend/src/services/gitops/history.ts | 2 + backend/src/services/gitops/migrate.ts | 6 + backend/src/services/gitops/outcomes.ts | 211 ++ backend/src/services/gitops/schema.ts | 25 + backend/src/services/gitops/store.ts | 181 +- backend/src/services/gitops/transitions.ts | 106 +- backend/src/services/gitops/triggers.ts | 77 + backend/src/services/gitops/types.ts | 12 + 48 files changed, 7501 insertions(+), 249 deletions(-) create mode 100644 backend/src/__tests__/bootstrap-startup-gitops-order.test.ts create mode 100644 backend/src/__tests__/gitops-backoff.test.ts create mode 100644 backend/src/__tests__/gitops-handoff.test.ts create mode 100644 backend/src/__tests__/gitops-outcomes.test.ts create mode 100644 backend/src/__tests__/gitops-reconcile-attempts.test.ts create mode 100644 backend/src/__tests__/gitops-triggers.test.ts create mode 100644 backend/src/__tests__/source-controller.test.ts create mode 100644 backend/src/services/gitops/SourceController.ts create mode 100644 backend/src/services/gitops/backoff.ts create mode 100644 backend/src/services/gitops/handoff.ts create mode 100644 backend/src/services/gitops/outcomes.ts create mode 100644 backend/src/services/gitops/triggers.ts diff --git a/backend/src/__tests__/bootstrap-startup-gitops-order.test.ts b/backend/src/__tests__/bootstrap-startup-gitops-order.test.ts new file mode 100644 index 00000000..6ccf2e57 --- /dev/null +++ b/backend/src/__tests__/bootstrap-startup-gitops-order.test.ts @@ -0,0 +1,99 @@ +import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest'; +import fs from 'fs'; +import path from 'path'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; + +let tmpDir: string; +let runGitOpsSourceRecovery: typeof import('../bootstrap/startup').runGitOpsSourceRecovery; +let GitSourceService: typeof import('../services/GitSourceService').GitSourceService; +let gitSourceServiceModule: typeof import('../services/GitSourceService'); + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ runGitOpsSourceRecovery } = await import('../bootstrap/startup')); + gitSourceServiceModule = await import('../services/GitSourceService'); + ({ GitSourceService } = gitSourceServiceModule); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +afterAll(() => { + cleanupTestDb(tmpDir); +}); + +describe('runGitOpsSourceRecovery', () => { + it('recovers unsettled reconcile attempts before sweeping the managed area', async () => { + const order: string[] = []; + // Recovery yields before recording itself: if the sweep were ever + // started concurrently instead of strictly after recovery resolves, + // the sweep's own synchronous push would land first and this would + // catch it, rather than merely proving call order at invocation + // time. + vi.spyOn(GitSourceService.getInstance(), 'recoverUnsettledReconcileAttempts') + .mockImplementation(async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + order.push('recover'); + }); + vi.spyOn(gitSourceServiceModule, 'sweepGitManifestOrphans') + .mockImplementation(async () => { order.push('sweep'); }); + + await runGitOpsSourceRecovery(); + + expect(order).toEqual(['recover', 'sweep']); + }); + + it('still runs the sweep when recovery itself throws, tolerating the failure', async () => { + const order: string[] = []; + vi.spyOn(GitSourceService.getInstance(), 'recoverUnsettledReconcileAttempts') + .mockImplementation(async () => { throw new Error('simulated recovery failure'); }); + vi.spyOn(gitSourceServiceModule, 'sweepGitManifestOrphans') + .mockImplementation(async () => { order.push('sweep'); }); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + + await expect(runGitOpsSourceRecovery()).resolves.toBeUndefined(); + + expect(order).toEqual(['sweep']); + expect(errorSpy).toHaveBeenCalled(); + }); + + it('still resolves when the sweep itself throws, tolerating the failure rather than aborting startup', async () => { + const recoverSpy = vi.spyOn(GitSourceService.getInstance(), 'recoverUnsettledReconcileAttempts') + .mockImplementation(async () => {}); + vi.spyOn(gitSourceServiceModule, 'sweepGitManifestOrphans') + .mockImplementation(async () => { throw new Error('simulated sweep failure'); }); + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + + await expect(runGitOpsSourceRecovery()).resolves.toBeUndefined(); + + expect(recoverSpy).toHaveBeenCalled(); + expect(warnSpy).toHaveBeenCalled(); + }); +}); + +describe('startServer source ordering', () => { + it('calls runGitOpsSourceRecovery before starting SourceController, so recovery and the sweep always precede the controller poll loop', () => { + // A structural check, not a behavioral one: startServer drives ~25 + // unrelated services and isn't practical to run end to end in a + // test. What matters here is the one property a future edit could + // silently break: SourceController must never start before + // recovery and the sweep have both awaited to completion. Comments + // are stripped first so a mention of either symbol in prose can't + // satisfy the match, and matching is whitespace/chaining-tolerant + // so a harmless reformat (line-wrapped method chain, a `const` + // extracted for the controller instance) doesn't false-fail this. + const source = fs.readFileSync(path.join(__dirname, '../bootstrap/startup.ts'), 'utf-8'); + const startServerStart = source.indexOf('export async function startServer'); + // startServer is the last top-level declaration in this file today; + // if that ever changes, bound this slice to its closing brace + // instead of running to end of file. + const startServerBody = source.slice(startServerStart); + const code = startServerBody.replace(/\/\*[\s\S]*?\*\//g, '').replace(/\/\/[^\n]*/g, ''); + const reconcileCallIndex = code.search(/await\s+runGitOpsSourceRecovery\s*\(/); + const controllerStartIndex = code.search(/SourceController\s*\.\s*getInstance\s*\(\s*\)\s*\.\s*start\s*\(/); + expect(reconcileCallIndex).toBeGreaterThan(-1); + expect(controllerStartIndex).toBeGreaterThan(-1); + expect(reconcileCallIndex).toBeLessThan(controllerStartIndex); + }); +}); diff --git a/backend/src/__tests__/git-project-manifest.test.ts b/backend/src/__tests__/git-project-manifest.test.ts index e8790b4d..464e3012 100644 --- a/backend/src/__tests__/git-project-manifest.test.ts +++ b/backend/src/__tests__/git-project-manifest.test.ts @@ -92,6 +92,7 @@ function makeClone(files: Record): string { } const REPO = { repo_url: 'https://github.com/example/repo.git', branch: 'main' }; +const NO_CANDIDATE_CLAIMS = { complete: true as const, dirs: new Set() }; function seedGitSource(stackName: string): void { DatabaseService.getInstance().upsertGitSource({ @@ -731,6 +732,59 @@ describe('promoteGeneration', () => { }); describe('sweepManagedArea (crash recovery)', () => { + it('does not delete a candidate when its completion marker cannot be inspected', async () => { + const svc = GitProjectManifestService.getInstance(); + const stackName = 'sweep-candidate-marker-io'; + const candidateAbs = path.join(tmpDir, 'git-managed', '1', stackName, 'generations', 'candidate-marker-io'); + const markerPath = path.join(candidateAbs, CANDIDATE_COMPLETE_MARKER); + fs.mkdirSync(candidateAbs, { recursive: true }); + fs.writeFileSync(markerPath, 'complete'); + const originalAccess = fs.promises.access.bind(fs.promises); + const accessSpy = vi.spyOn(fs.promises, 'access').mockImplementation(async (...args: Parameters) => { + if (String(args[0]) === markerPath) { + throw Object.assign(new Error('candidate marker permission denied'), { code: 'EACCES' }); + } + return originalAccess(...args); + }); + + try { + await expect(svc.sweepManagedArea(stackName, { + repoUrl: REPO.repo_url, + branch: REPO.branch, + stackExists: true, + candidateClaims: NO_CANDIDATE_CLAIMS, + })).rejects.toThrow(/candidate marker permission denied/); + expect(fs.existsSync(candidateAbs)).toBe(true); + } finally { + accessSpy.mockRestore(); + } + }); + + it('surfaces a generations-directory read failure', async () => { + const svc = GitProjectManifestService.getInstance(); + const stackName = 'sweep-generations-read-io'; + const generationsDir = path.join(tmpDir, 'git-managed', '1', stackName, 'generations'); + fs.mkdirSync(generationsDir, { recursive: true }); + const originalReaddir = fs.promises.readdir.bind(fs.promises); + const readdirSpy = vi.spyOn(fs.promises, 'readdir').mockImplementation(async (...args: Parameters) => { + if (String(args[0]) === generationsDir) { + throw Object.assign(new Error('generations directory unavailable'), { code: 'EIO' }); + } + return originalReaddir(...args); + }); + + try { + await expect(svc.sweepManagedArea(stackName, { + repoUrl: REPO.repo_url, + branch: REPO.branch, + stackExists: true, + candidateClaims: NO_CANDIDATE_CLAIMS, + })).rejects.toThrow(/generations directory unavailable/); + } finally { + readdirSpy.mockRestore(); + } + }); + it('restores the previous applied generation when the marker matches the stack dir', async () => { const svc = GitProjectManifestService.getInstance(); const stackName = 'sweep-restore'; @@ -763,7 +817,7 @@ describe('sweepManagedArea (crash recovery)', () => { affected: ['app.env', 'compose.yaml'], }); - await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true }); + await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true, candidateClaims: NO_CANDIDATE_CLAIMS }); expect(readStackFile(stackName, 'compose.yaml')).toBe('PRIOR\n'); expect(fs.existsSync(path.join(tmpDir, 'git-managed', '1', stackName, PROMOTION_MARKER))).toBe(false); @@ -817,7 +871,7 @@ describe('sweepManagedArea (crash recovery)', () => { affected: ['compose.yaml'], }); - await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true }); + await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true, candidateClaims: NO_CANDIDATE_CLAIMS }); expect(readStackFile(stackName, 'compose.yaml')).toBe('OPERATOR FIXED ME\n'); expect(fs.existsSync(path.join(tmpDir, 'git-managed', '1', stackName, PROMOTION_MARKER))).toBe(false); @@ -852,7 +906,7 @@ describe('sweepManagedArea (crash recovery)', () => { affected: ['compose.yaml'], }); - await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true }); + await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true, candidateClaims: NO_CANDIDATE_CLAIMS }); expect(readStackFile(stackName, 'compose.yaml')).toBe('PRIOR\n'); expect(fs.existsSync(path.join(tmpDir, 'git-managed', '1', stackName, PROMOTION_MARKER))).toBe(false); @@ -887,7 +941,7 @@ describe('sweepManagedArea (crash recovery)', () => { affected: ['compose.yaml'], }); - await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true }); + await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true, candidateClaims: NO_CANDIDATE_CLAIMS }); expect(readStackFile(stackName, 'compose.yaml')).toBe('NEW\n'); expect(fs.existsSync(path.join(tmpDir, 'git-managed', '1', stackName, PROMOTION_MARKER))).toBe(false); @@ -927,7 +981,7 @@ describe('sweepManagedArea (crash recovery)', () => { affected: ['compose.yaml'], }); - await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true }); + await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true, candidateClaims: NO_CANDIDATE_CLAIMS }); expect(readStackFile(stackName, 'compose.yaml')).toBe('PRIOR\n'); const row = DatabaseService.getInstance().getGitSource(stackName); @@ -969,7 +1023,7 @@ describe('sweepManagedArea (crash recovery)', () => { affected: ['app.env', 'compose.yaml'], }); - await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true }); + await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true, candidateClaims: NO_CANDIDATE_CLAIMS }); expect(readStackFile(stackName, 'compose.yaml')).toBe('NEW\n'); expect(readStackFile(stackName, 'app.env')).toBe('OPERATOR\n'); @@ -981,7 +1035,7 @@ describe('sweepManagedArea (crash recovery)', () => { const svc = GitProjectManifestService.getInstance(); const stackName = 'sweep-orphan'; await svc.writeManifest(stackName, buildManifest(stackName, [managedEntry({ materializedPath: 'compose.yaml' })])); - await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: false }); + await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: false, candidateClaims: NO_CANDIDATE_CLAIMS }); expect(await svc.readManifest(stackName, REPO.repo_url, REPO.branch)).toBeNull(); }); }); @@ -1059,7 +1113,7 @@ describe('detach crash recovery', () => { writeStackFile(stackName, 'compose.yaml', 'services:\n web:\n image: nginx:new\n'); expect(await svc.stageManagedAreaForDetach(stackName)).toBe(true); - await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true }); + await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true, candidateClaims: NO_CANDIDATE_CLAIMS }); expect(readStackFile(stackName, 'compose.yaml')).toBe(original.toString('utf8')); const restored = await svc.readManifest(stackName, REPO.repo_url, REPO.branch); @@ -1303,7 +1357,7 @@ describe('promoteGeneration mid-write failure recovery', () => { }); fs.mkdirSync(path.join(tmpDir, 'git-managed', '1', stackName), { recursive: true }); fs.writeFileSync(path.join(tmpDir, 'git-managed', '1', stackName, PROMOTION_MARKER), '{"v":3 torn', 'utf8'); - await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true }); + await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true, candidateClaims: NO_CANDIDATE_CLAIMS }); expect(fs.existsSync(path.join(tmpDir, 'git-managed', '1', stackName, PROMOTION_MARKER))).toBe(false); expect(DatabaseService.getInstance().getGitSource(stackName)?.manifest_state).toBe('migration_required'); }); @@ -1325,7 +1379,7 @@ describe('promoteGeneration mid-write failure recovery', () => { }), 'utf8'); const stateSpy = vi.spyOn(DatabaseService.getInstance(), 'setGitSourceManifestState'); try { - await expect(svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true })).resolves.toBeUndefined(); + await expect(svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true, candidateClaims: NO_CANDIDATE_CLAIMS })).resolves.toBeUndefined(); expect(stateSpy).toHaveBeenCalledWith(stackName, null, 'migration_required', null); } finally { stateSpy.mockRestore(); @@ -1343,7 +1397,7 @@ describe('promoteGeneration mid-write failure recovery', () => { throw new Error('database unavailable'); }); try { - await expect(svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true })).rejects.toThrow(/database unavailable/); + await expect(svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true, candidateClaims: NO_CANDIDATE_CLAIMS })).rejects.toThrow(/database unavailable/); expect(fs.existsSync(markerPath)).toBe(true); } finally { stateSpy.mockRestore(); @@ -1371,7 +1425,7 @@ describe('promoteGeneration mid-write failure recovery', () => { return originalAccess(...args); }); try { - await expect(svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true })).rejects.toThrow(/permission denied/); + await expect(svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true, candidateClaims: NO_CANDIDATE_CLAIMS })).rejects.toThrow(/permission denied/); expect(fs.existsSync(markerPath)).toBe(true); } finally { accessSpy.mockRestore(); diff --git a/backend/src/__tests__/git-source-apply-recovery.test.ts b/backend/src/__tests__/git-source-apply-recovery.test.ts index 3f080b7b..706339d0 100644 --- a/backend/src/__tests__/git-source-apply-recovery.test.ts +++ b/backend/src/__tests__/git-source-apply-recovery.test.ts @@ -13,6 +13,38 @@ const mockMarkReconciling = vi.fn().mockReturnValue(true); const mockMarkImmediateVerified = vi.fn().mockReturnValue(true); const mockGet = vi.fn(); const mockCompensate = vi.fn(); +const mockGitOpsApplication = { + id: 'gitops-app', + lifecycle_status: 'active', + stack_name: 'app', + candidate_generation_id: null, +}; +const mockGitOpsStore = { + getLiveDirectApplication: vi.fn().mockReturnValue(mockGitOpsApplication), + getApplication: vi.fn().mockReturnValue(mockGitOpsApplication), + getGeneration: vi.fn().mockReturnValue(undefined), + getSettledAttempt: vi.fn().mockReturnValue(undefined), +}; +const mockGitOpsTransitions = { + allocateReconcileAttempt: vi.fn().mockReturnValue({ operationId: 'gitops-app:attempt:1', reserved: true }), + settleReconcileAttempt: vi.fn().mockReturnValue({ settled: true }), +}; + +vi.mock('../services/gitops/store', () => ({ + GitOpsStore: { + getInstance: () => mockGitOpsStore, + }, +})); + +vi.mock('../services/gitops/transitions', async () => { + const actual = await vi.importActual('../services/gitops/transitions'); + return { + ...actual, + GitOpsTransitions: { + getInstance: () => mockGitOpsTransitions, + }, + }; +}); vi.mock('../services/StackUpdateRecoveryService', () => ({ StackUpdateRecoveryService: { @@ -129,10 +161,6 @@ vi.mock('../services/DatabaseService', () => ({ setGitSourceLastPlan: mockSetGitSourceLastPlan, addNotificationHistory: mockAddNotificationHistory, getStackProjectEnvFiles: vi.fn().mockReturnValue([]), - // The apply path now asks whether this stack has a GitOps application. - // These fixtures predate the revision-state model, so the lookup finds - // nothing and every GitOps producer stays a no-op, which is exactly the - // behavior an install with pre-existing Git stacks gets. getDb: () => ({ prepare: () => ({ get: () => undefined, all: () => [], run: () => ({ changes: 0 }) }), transaction: (fn: () => unknown) => () => fn(), @@ -197,7 +225,7 @@ describe('git-source apply recovery (R1)', () => { v: 4, files: { 'compose.yaml': 'services:\n web:\n image: nginx\n' }, contextDir: null, - candidateRelPath: 'generations/cand', + candidateRelPath: 'generations/candidate-abc1234deadbeef', inventory: { inputs: [], refusals: [], @@ -246,7 +274,7 @@ describe('git-source apply recovery (R1)', () => { version: 4, files: [{ path: 'compose.yaml', content: 'services:\n web:\n image: nginx\n' }], contextDir: null, - candidateRelPath: 'generations/cand', + candidateRelPath: 'generations/candidate-abc1234deadbeef', inventory: { inputs: [], refusals: [], buildContexts: [] }, planFingerprint: 'fp-test', planSchemaVersion: GIT_CHANGE_PLAN_SCHEMA_VERSION, diff --git a/backend/src/__tests__/git-source-routes.test.ts b/backend/src/__tests__/git-source-routes.test.ts index 52a996ad..5de0ccc8 100644 --- a/backend/src/__tests__/git-source-routes.test.ts +++ b/backend/src/__tests__/git-source-routes.test.ts @@ -26,11 +26,13 @@ import { ComposeService } from '../services/ComposeService'; import { GitSourceService, GitSourceError } from '../services/GitSourceService'; import { GitOpsStore } from '../services/gitops/store'; import { GitOpsTransitions } from '../services/gitops/transitions'; +import { deliveryKey } from '../services/gitops/triggers'; import { insertHistory } from '../services/gitops/history'; import type { GitOpsApplicationRow } from '../services/gitops/types'; import { PROXY_DEPLOY_ACTOR_HEADER, PROXY_DEPLOY_SOURCE_HEADER } from '../services/license-headers'; import { withLoopbackTargetProtection } from './helpers/allowLoopbackTargets'; import { directApplicationFixture } from './helpers/gitopsFixtures'; +import { ROLE_PERMISSIONS } from '../middleware/permissions'; // ── Hoisted mocks (must come before importing the app) ───────────────── @@ -75,8 +77,17 @@ function adminToken(): string { return jwt.sign({ username: TEST_USERNAME, role: 'admin' }, TEST_JWT_SECRET, { expiresIn: '1m' }); } +function viewerToken(): string { + return jwt.sign({ username: 'viewer', role: 'viewer' }, TEST_JWT_SECRET, { expiresIn: '1m' }); +} + +function nodeAdminToken(): string { + return jwt.sign({ username: 'node-admin', role: 'node-admin' }, TEST_JWT_SECRET, { expiresIn: '1m' }); +} + beforeAll(async () => { tmpDir = await setupTestDb(); + DatabaseService.getInstance().addUser({ username: 'node-admin', password_hash: 'test', role: 'node-admin' }); ({ app } = await import('../index')); // Seed a real stack directory so the PUT handler's existence guard is satisfied @@ -646,6 +657,253 @@ describe('POST /api/stacks/:stackName/git-source/webhook-pull status codes', () expect(res.status).toBe(200); pullSpy.mockRestore(); }); + + it('passes a remote webhook delivery id into the durable pull path', async () => { + seedGitSource('webhook-delivery-id'); + const pullSpy = vi.spyOn(GitSourceService.getInstance(), 'handleWebhookPull') + .mockResolvedValue({ status: 'success', message: 'Pending update ready at abc1234.' }); + const res = await request(app) + .post('/api/stacks/webhook-delivery-id/git-source/webhook-pull') + .set('Authorization', `Bearer ${adminToken()}`) + .send({ deliveryId: 'webhook:42:provider-delivery-1' }); + + expect(res.status).toBe(200); + expect(pullSpy).toHaveBeenCalledWith('webhook-delivery-id', true, 'webhook:42:provider-delivery-1'); + pullSpy.mockRestore(); + }); + + it('requires stack:deploy when a redelivery carries a persisted deploy intent', async () => { + const stackName = 'webhook-delivery-deploy-auth'; + const applicationId = 'webhook-delivery-deploy-auth-app'; + const deliveryId = 'webhook:control:42:provider-delivery-deploy'; + seedGitSource(stackName); + GitOpsStore.getInstance().insertApplication(directApplicationFixture(applicationId, stackName)); + GitOpsTransitions.getInstance().reserveReconcileAttempt( + applicationId, + { + operationId: deliveryKey('webhook', 'fetch', deliveryId), + actor: 'system:webhook', + trigger: 'webhook', + at: Date.now(), + }, + undefined, + { autoApply: true, deploy: true }, + ); + const pullSpy = vi.spyOn(GitSourceService.getInstance(), 'handleWebhookPull'); + const originalPermissions = ROLE_PERMISSIONS['node-admin']; + ROLE_PERMISSIONS['node-admin'] = originalPermissions.filter((permission) => permission !== 'stack:deploy'); + + try { + const res = await request(app) + .post(`/api/stacks/${stackName}/git-source/webhook-pull`) + .set('Authorization', `Bearer ${nodeAdminToken()}`) + .send({ deliveryId }); + + expect(res.status).toBe(403); + expect(res.body.code).toBe('PERMISSION_DENIED'); + expect(pullSpy).not.toHaveBeenCalled(); + } finally { + ROLE_PERMISSIONS['node-admin'] = originalPermissions; + pullSpy.mockRestore(); + } + }); + + it('requires stack:deploy when a first delivery is configured to auto-apply and deploy', async () => { + const stackName = 'webhook-first-delivery-deploy-auth'; + seedGitSource(stackName); + DatabaseService.getInstance().getDb() + .prepare('UPDATE stack_git_sources SET auto_apply_on_webhook = 1, auto_deploy_on_apply = 1 WHERE stack_name = ?') + .run(stackName); + const pullSpy = vi.spyOn(GitSourceService.getInstance(), 'handleWebhookPull'); + const originalPermissions = ROLE_PERMISSIONS['node-admin']; + ROLE_PERMISSIONS['node-admin'] = originalPermissions.filter((permission) => permission !== 'stack:deploy'); + + try { + const res = await request(app) + .post(`/api/stacks/${stackName}/git-source/webhook-pull`) + .set('Authorization', `Bearer ${nodeAdminToken()}`); + + expect(res.status).toBe(403); + expect(res.body.code).toBe('PERMISSION_DENIED'); + expect(pullSpy).not.toHaveBeenCalled(); + } finally { + ROLE_PERMISSIONS['node-admin'] = originalPermissions; + pullSpy.mockRestore(); + } + }); + + it.each([ + ['an object', { nested: true }], + ['a blank string', ' '], + ['a string over 512 characters', 'x'.repeat(513)], + ])('rejects %s as a remote webhook delivery id', async (_caseName, deliveryId) => { + seedGitSource('webhook-delivery-id-invalid'); + const pullSpy = vi.spyOn(GitSourceService.getInstance(), 'handleWebhookPull'); + + try { + const res = await request(app) + .post('/api/stacks/webhook-delivery-id-invalid/git-source/webhook-pull') + .set('Authorization', `Bearer ${adminToken()}`) + .send({ deliveryId }); + + expect(res.status).toBe(400); + expect(pullSpy).not.toHaveBeenCalled(); + } finally { + pullSpy.mockRestore(); + } + }); +}); + +describe('POST /api/stacks/:stackName/git-source/suspend', () => { + it('returns 401 without auth', async () => { + const res = await request(app).post('/api/stacks/existing-stack/git-source/suspend'); + expect(res.status).toBe(401); + }); + + it('returns 400 for an invalid stack name', async () => { + const res = await request(app) + .post('/api/stacks/..%2fescape/git-source/suspend') + .set('Authorization', `Bearer ${adminToken()}`) + .send({}); + expect([400, 404]).toContain(res.status); + }); + + it('passes the reason through and returns the normalized result', async () => { + const suspendSpy = vi.spyOn(GitSourceService.getInstance(), 'suspend') + .mockResolvedValue({ outcome: 'suspended', reason: 'Reconciliation is suspended: maintenance', nextAction: 'resume' }); + const res = await request(app) + .post('/api/stacks/existing-stack/git-source/suspend') + .set('Authorization', `Bearer ${adminToken()}`) + .send({ reason: 'maintenance' }); + expect(res.status).toBe(200); + expect(res.body.outcome).toBe('suspended'); + expect(suspendSpy).toHaveBeenCalledWith('existing-stack', expect.objectContaining({ reason: 'maintenance' })); + suspendSpy.mockRestore(); + }); + + it('omits reason when none is given in the body', async () => { + const suspendSpy = vi.spyOn(GitSourceService.getInstance(), 'suspend') + .mockResolvedValue({ outcome: 'suspended', reason: 'Reconciliation is suspended.', nextAction: 'resume' }); + await request(app) + .post('/api/stacks/existing-stack/git-source/suspend') + .set('Authorization', `Bearer ${adminToken()}`) + .send({}); + expect(suspendSpy).toHaveBeenCalledWith('existing-stack', expect.objectContaining({ reason: undefined })); + suspendSpy.mockRestore(); + }); + + it('maps a refused suspend to 409', async () => { + const suspendSpy = vi.spyOn(GitSourceService.getInstance(), 'suspend') + .mockRejectedValue(new GitSourceError('OPERATION_IN_FLIGHT', 'Cannot suspend existing-stack: source is not live')); + const res = await request(app) + .post('/api/stacks/existing-stack/git-source/suspend') + .set('Authorization', `Bearer ${adminToken()}`) + .send({}); + expect(res.status).toBe(409); + suspendSpy.mockRestore(); + }); + + it('denies without the stack:edit permission', async () => { + const res = await request(app) + .post('/api/stacks/existing-stack/git-source/suspend') + .set('Authorization', `Bearer ${viewerToken()}`) + .send({}); + expect([401, 403]).toContain(res.status); + }); + + it('rejects an oversized reason with 400', async () => { + const suspendSpy = vi.spyOn(GitSourceService.getInstance(), 'suspend'); + const res = await request(app) + .post('/api/stacks/existing-stack/git-source/suspend') + .set('Authorization', `Bearer ${adminToken()}`) + .send({ reason: 'x'.repeat(513) }); + expect(res.status).toBe(400); + expect(suspendSpy).not.toHaveBeenCalled(); + suspendSpy.mockRestore(); + }); +}); + +describe('POST /api/stacks/:stackName/git-source/resume', () => { + it('returns 401 without auth', async () => { + const res = await request(app).post('/api/stacks/existing-stack/git-source/resume'); + expect(res.status).toBe(401); + }); + + it('returns 400 for an invalid stack name', async () => { + const res = await request(app) + .post('/api/stacks/..%2fescape/git-source/resume') + .set('Authorization', `Bearer ${adminToken()}`) + .send({}); + expect([400, 404]).toContain(res.status); + }); + + it('returns the normalized result', async () => { + const resumeSpy = vi.spyOn(GitSourceService.getInstance(), 'resume') + .mockResolvedValue({ outcome: 'no_source_change', reason: 'ok', nextAction: 'none' }); + const res = await request(app) + .post('/api/stacks/existing-stack/git-source/resume') + .set('Authorization', `Bearer ${adminToken()}`) + .send({}); + expect(res.status).toBe(200); + expect(res.body.outcome).toBe('no_source_change'); + expect(resumeSpy).toHaveBeenCalledWith('existing-stack', expect.objectContaining({ actor: expect.any(String) })); + resumeSpy.mockRestore(); + }); + + it('denies without the stack:edit permission', async () => { + const res = await request(app) + .post('/api/stacks/existing-stack/git-source/resume') + .set('Authorization', `Bearer ${viewerToken()}`) + .send({}); + expect([401, 403]).toContain(res.status); + }); +}); + +describe('POST /api/stacks/:stackName/git-source/retry', () => { + it('returns 401 without auth', async () => { + const res = await request(app).post('/api/stacks/existing-stack/git-source/retry'); + expect(res.status).toBe(401); + }); + + it('returns 400 for an invalid stack name', async () => { + const res = await request(app) + .post('/api/stacks/..%2fescape/git-source/retry') + .set('Authorization', `Bearer ${adminToken()}`) + .send({}); + expect([400, 404]).toContain(res.status); + }); + + it('returns the normalized result', async () => { + const retrySpy = vi.spyOn(GitSourceService.getInstance(), 'retry') + .mockResolvedValue({ outcome: 'candidate_already_fetched', reason: 'ok', nextAction: 'none' }); + const res = await request(app) + .post('/api/stacks/existing-stack/git-source/retry') + .set('Authorization', `Bearer ${adminToken()}`) + .send({}); + expect(res.status).toBe(200); + expect(res.body.outcome).toBe('candidate_already_fetched'); + expect(retrySpy).toHaveBeenCalledWith('existing-stack', expect.objectContaining({ actor: expect.any(String) })); + retrySpy.mockRestore(); + }); + + it('maps an unexpected failure to 500', async () => { + const retrySpy = vi.spyOn(GitSourceService.getInstance(), 'retry') + .mockRejectedValue(new Error('unexpected')); + const res = await request(app) + .post('/api/stacks/existing-stack/git-source/retry') + .set('Authorization', `Bearer ${adminToken()}`) + .send({}); + expect(res.status).toBe(500); + retrySpy.mockRestore(); + }); + + it('denies without the stack:edit permission', async () => { + const res = await request(app) + .post('/api/stacks/existing-stack/git-source/retry') + .set('Authorization', `Bearer ${viewerToken()}`) + .send({}); + expect([401, 403]).toContain(res.status); + }); }); describe('DELETE /api/stacks/:stackName/git-source, detach/export contract', () => { diff --git a/backend/src/__tests__/git-source-service.test.ts b/backend/src/__tests__/git-source-service.test.ts index 24f422e5..5e91609d 100644 --- a/backend/src/__tests__/git-source-service.test.ts +++ b/backend/src/__tests__/git-source-service.test.ts @@ -21,10 +21,13 @@ import type { TransportFailure } from '../services/git/errors'; import { GitOpsStore } from '../services/gitops/store'; import { GitOpsTransitions } from '../services/gitops/transitions'; import { StackOpLockService } from '../services/StackOpLockService'; +import { coalesceKey, deliveryKey, type ReconcileRequest, type ReconcileTrigger } from '../services/gitops/triggers'; import { + buildDirectApplicationRow, buildGenerationRow, directSourceIdentity, newGitOpsId, + stackManagedRoot, type DirectSourceConfig, } from '../services/gitops/directApplication'; @@ -239,6 +242,41 @@ function mockSuccessfulClone(options: { return sha; } +/** + * Configure a plain single-file Git source for a stack, without creating the + * stack itself. The caller stages the clone mock first: upsert runs a + * reachability fetch. + */ +async function configureGitSource(stackName: string): Promise { + await GitSourceService.getInstance().upsert({ + stackName, + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); +} + +/** Operation ids of every gitops_history row an application recorded at one stage. */ +function historyOperationIds(applicationId: string, stage: string): string[] { + const rows = DatabaseService.getInstance().getDb() + .prepare('SELECT operation_id FROM gitops_history WHERE application_id = ? AND stage = ?') + .all(applicationId, stage) as { operation_id: string }[]; + return rows.map((r) => r.operation_id); +} + +/** Settled attempt rows for one application, used to compare follower results. */ +function settledAttemptsForApplication(applicationId: string): { operation_id: string; after_json: string }[] { + return DatabaseService.getInstance().getDb() + .prepare("SELECT operation_id, after_json FROM gitops_history WHERE application_id = ? AND stage = 'source_reconcile_settled'") + .all(applicationId) as { operation_id: string; after_json: string }[]; +} + /** Wrap a single compose string in the ComposeFile[] shape the new APIs take. */ function asFiles(content: string): import('../services/GitSourceService').ComposeFile[] { return [{ path: 'compose.yaml', content }]; @@ -258,6 +296,27 @@ async function cleanupStackDir(name: string) { const SKIP_PLAN_FINGERPRINT = { requirePlanFingerprint: false as const }; +describe('GitSourceService.shortOperationId', () => { + function shortOperationId(operationId: string): string { + return (GitSourceService as unknown as { shortOperationId: (id: string) => string }).shortOperationId(operationId); + } + + it('discriminates between reserved attempts on the same application, unlike a fixed-width prefix', () => { + const appId = '4457ddc3-3eb0-444e-902c-7e65d355b36b'; + expect(shortOperationId(`${appId}:attempt:1`)).toBe('1'); + expect(shortOperationId(`${appId}:attempt:2`)).toBe('2'); + }); + + it('uses the delivery-key suffix for a webhook-triggered reservation', () => { + expect(shortOperationId('webhook:fetch:delivery-abc')).toBe('delivery-abc'); + }); + + it('falls back to a prefix for a plain UUID with no colon', () => { + const uuid = '29ec01cd-4129-4c4c-a5f2-4e2368f44490'; + expect(shortOperationId(uuid)).toBe(uuid.slice(0, 8)); + }); +}); + describe('GitSourceService.hashContent', () => { it('produces stable hashes for identical inputs', () => { const svc = GitSourceService.getInstance(); @@ -1488,23 +1547,379 @@ describe('GitSourceService.handleWebhookPull debounce', () => { // Stamp a recent debounce timestamp directly DatabaseService.getInstance().touchGitSourceDebounce('debounce-stack'); - const result = await svc.handleWebhookPull('debounce-stack'); + const result = await svc.handleWebhookPull('debounce-stack', true); expect(result.status).toBe('skipped'); expect(result.message).toMatch(/rate limited/i); }); it('returns error when stack has no Git source configured', async () => { const svc = GitSourceService.getInstance(); - const result = await svc.handleWebhookPull('does-not-exist'); + const result = await svc.handleWebhookPull('does-not-exist', true); expect(result.status).toBe('error'); expect(result.message).toMatch(/no git source/i); }); + it('fails closed and does not clone when reservation itself fails', async () => { + mockSuccessfulClone({ sha: '2'.repeat(40) }); + const svc = GitSourceService.getInstance(); + await configureGitSource('webhook-reservation-fails-closed'); + mockGitClone.mockClear(); + const reserveSpy = vi.spyOn(GitOpsTransitions.prototype, 'allocateReconcileAttempt') + .mockImplementationOnce(() => { throw new Error('simulated reservation failure'); }); + + try { + const result = await svc.handleWebhookPull('webhook-reservation-fails-closed', true); + expect(result.status).toBe('error'); + expect(mockGitClone).not.toHaveBeenCalled(); + } finally { + reserveSpy.mockRestore(); + } + }); + + it('fails closed and does not clone when the application was detached but the source config survives', async () => { + mockSuccessfulClone({ sha: '3'.repeat(40) }); + const svc = GitSourceService.getInstance(); + await configureGitSource('webhook-tombstoned-app'); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('webhook-tombstoned-app')!.id; + GitOpsTransitions.getInstance().applicationTombstoned(applicationId, 'detached', { + operationId: 'op-detach-3', actor: 'tester', trigger: 'test', at: Date.now(), + }); + mockGitClone.mockClear(); + + const result = await svc.handleWebhookPull('webhook-tombstoned-app', true); + + // Skipped, not error: this is a permanent state, and reporting it + // as a delivery failure on every future push risks the Git host + // disabling the webhook for a condition retrying can never fix. + expect(result.status).toBe('skipped'); + expect(result.message).toMatch(/GitOps tracking was removed/); + expect(mockGitClone).not.toHaveBeenCalled(); + }); + + it('fails closed and does not clone when the application was deleted but the source config survives', async () => { + mockSuccessfulClone({ sha: '4'.repeat(40) }); + const svc = GitSourceService.getInstance(); + await configureGitSource('webhook-deleted-app'); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('webhook-deleted-app')!.id; + GitOpsTransitions.getInstance().applicationTombstoned(applicationId, 'deleted', { + operationId: 'op-delete-webhook', actor: 'tester', trigger: 'test', at: Date.now(), + }); + mockGitClone.mockClear(); + + const result = await svc.handleWebhookPull('webhook-deleted-app', true); + + expect(result.status).toBe('skipped'); + expect(result.message).toMatch(/GitOps tracking is unavailable/); + expect(mockGitClone).not.toHaveBeenCalled(); + }); + + it('does not execute a persisted deploy intent when the caller lacks deploy authorization', async () => { + const stackName = 'webhook-persisted-deploy-auth'; + const deliveryId = 'webhook:control:7:deploy-auth'; + mockSuccessfulClone({ sha: '5'.repeat(40) }); + const svc = GitSourceService.getInstance(); + await configureGitSource(stackName); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication(stackName)!.id; + GitOpsTransitions.getInstance().reserveReconcileAttempt( + applicationId, + { + operationId: deliveryKey('webhook', 'fetch', deliveryId), + actor: 'system:webhook', + trigger: 'webhook', + at: Date.now(), + }, + undefined, + { autoApply: true, deploy: true }, + ); + mockGitClone.mockClear(); + + const result = await svc.handleWebhookPull(stackName, false, deliveryId); + + expect(result.status).toBe('error'); + expect(result.message).toMatch(/deploy permission/i); + expect(mockGitClone).not.toHaveBeenCalled(); + }); + + it('does not require deploy authorization when auto-apply is disabled', async () => { + const stackName = 'webhook-fetch-only-deploy-setting'; + const deliveryId = 'delivery-fetch-only-deploy-setting'; + mockSuccessfulClone({ sha: '7'.repeat(40) }); + const svc = GitSourceService.getInstance(); + await svc.upsert({ + stackName, + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + DatabaseService.getInstance().getDb() + .prepare('UPDATE stack_git_sources SET auto_deploy_on_apply = 1 WHERE stack_name = ?') + .run(stackName); + mockGitClone.mockClear(); + + expect(svc.webhookDeliveryRequiresDeploy(stackName, deliveryId)).toBe(false); + const first = await svc.handleWebhookPull(stackName, false, deliveryId); + expect(first.status).toBe('success'); + expect(svc.webhookDeliveryRequiresDeploy(stackName, deliveryId)).toBe(false); + + DatabaseService.getInstance().getDb() + .prepare('UPDATE stack_git_sources SET last_debounce_at = ? WHERE stack_name = ?') + .run(Date.now() - 999_999, stackName); + mockGitClone.mockClear(); + const redelivery = await svc.handleWebhookPull(stackName, false, deliveryId); + + expect(redelivery.status).toBe('success'); + expect(mockGitClone).not.toHaveBeenCalled(); + }); + + it('fails closed on the auto-apply step alone: the fetch settles, the apply reservation fails, and no apply proceeds', async () => { + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: '6'.repeat(40) }); + const svc = GitSourceService.getInstance(); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + try { + await svc.upsert({ + stackName: 'webhook-apply-reservation-fails-closed', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: true, + autoDeployOnApply: false, + }); + } finally { + validateSpy.mockRestore(); + } + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('webhook-apply-reservation-fails-closed')!.id; + + // First call (the fetch) reserves normally; the second (the + // auto-apply) fails, isolating the apply-stage reservation path. + const originalReserve = GitOpsTransitions.prototype.reserveReconcileAttempt; + const reserveSpy = vi.spyOn(GitOpsTransitions.prototype, 'reserveReconcileAttempt') + .mockImplementationOnce(function (this: GitOpsTransitions, ...args: Parameters) { + return originalReserve.apply(this, args); + }) + .mockImplementationOnce(() => { throw new Error('simulated apply reservation failure'); }); + + try { + const result = await svc.handleWebhookPull( + 'webhook-apply-reservation-fails-closed', + true, + 'delivery-apply-reservation-failure', + ); + expect(result.status).toBe('error'); + expect(saveSpy).not.toHaveBeenCalled(); + // The fetch attempt settled normally; only the apply attempt + // never got as far as being reserved at all. + const unsettled = GitOpsStore.getInstance().listUnsettledReconcileAttempts() + .filter((r) => r.application_id === applicationId); + expect(unsettled).toHaveLength(0); + } finally { + reserveSpy.mockRestore(); + } + + DatabaseService.getInstance().getDb() + .prepare('UPDATE stack_git_sources SET auto_apply_on_webhook = 0, last_debounce_at = ? WHERE stack_name = ?') + .run(Date.now() - 999_999, 'webhook-apply-reservation-fails-closed'); + try { + const redelivery = await svc.handleWebhookPull( + 'webhook-apply-reservation-fails-closed', + true, + 'delivery-apply-reservation-failure', + ); + expect(redelivery.status).toBe('success'); + expect(saveSpy).toHaveBeenCalledTimes(1); + } finally { + saveSpy.mockRestore(); + } + }); + + it('reserves and durably settles an attempt for a successful webhook fetch', async () => { + mockSuccessfulClone({ sha: '8'.repeat(40) }); + const svc = GitSourceService.getInstance(); + await configureGitSource('webhook-reserves'); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('webhook-reserves')!.id; + mockGitClone.mockClear(); + mockSuccessfulClone({ sha: '9'.repeat(40) }); + + const result = await svc.handleWebhookPull('webhook-reserves', true); + + expect(result.status).toBe('success'); + expect(historyOperationIds(applicationId, 'source_reconcile_settled').length).toBeGreaterThanOrEqual(1); + expect(GitOpsStore.getInstance().listUnsettledReconcileAttempts().some((r) => r.application_id === applicationId)).toBe(false); + }); + + it('deduplicates a webhook redelivery by its stable delivery id after the debounce window expires', async () => { + const sha = 'a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1'; + mockSuccessfulClone({ sha }); + const svc = GitSourceService.getInstance(); + await configureGitSource('webhook-delivery-recorded'); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('webhook-delivery-recorded')!.id; + + const first = await svc.handleWebhookPull('webhook-delivery-recorded', true, 'delivery-xyz'); + expect(first.status).toBe('success'); + + DatabaseService.getInstance().getDb() + .prepare('UPDATE stack_git_sources SET last_debounce_at = ? WHERE stack_name = ?') + .run(Date.now() - 999_999, 'webhook-delivery-recorded'); + mockGitClone.mockClear(); + + const redelivery = await svc.handleWebhookPull('webhook-delivery-recorded', true, 'delivery-xyz'); + + expect(mockGitClone).not.toHaveBeenCalled(); + expect(redelivery.status).toBe('success'); + expect(historyOperationIds(applicationId, 'source_reconcile_started')).toHaveLength(1); + expect(historyOperationIds(applicationId, 'source_reconcile_settled')).toHaveLength(1); + }); + + it('joins a concurrent redelivery to the whole webhook fetch-and-apply execution', async () => { + const sha = 'a4'.repeat(20); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha }); + const svc = GitSourceService.getInstance(); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + let releaseSave!: () => void; + const saveGate = new Promise((resolve) => { releaseSave = resolve; }); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockImplementation(async () => { await saveGate; }); + + try { + await svc.upsert({ + stackName: 'webhook-whole-delivery-coalesce', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: true, + autoDeployOnApply: false, + }); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('webhook-whole-delivery-coalesce')!.id; + mockGitClone.mockClear(); + + const first = svc.handleWebhookPull('webhook-whole-delivery-coalesce', true, 'delivery-whole-execution'); + await vi.waitFor(() => expect(saveSpy).toHaveBeenCalledTimes(1)); + const redelivery = svc.handleWebhookPull('webhook-whole-delivery-coalesce', true, 'delivery-whole-execution'); + releaseSave(); + const [firstResult, redeliveryResult] = await Promise.all([first, redelivery]); + + expect(redeliveryResult).toEqual(firstResult); + expect(mockGitClone).toHaveBeenCalledTimes(1); + expect(saveSpy).toHaveBeenCalledTimes(1); + expect(historyOperationIds(applicationId, 'source_reconcile_started')).toHaveLength(2); + expect(historyOperationIds(applicationId, 'source_reconcile_settled')).toHaveLength(2); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + } + }); + + it('does not add an apply step to a fetch-only delivery when settings change before redelivery', async () => { + const sha = 'a2'.repeat(20); + mockSuccessfulClone({ sha }); + const svc = GitSourceService.getInstance(); + await configureGitSource('webhook-redelivery-settings-change'); + + const first = await svc.handleWebhookPull('webhook-redelivery-settings-change', true, 'delivery-settings-change'); + expect(first.status).toBe('success'); + + const db = DatabaseService.getInstance().getDb(); + db.prepare('UPDATE stack_git_sources SET auto_apply_on_webhook = 1, last_debounce_at = ? WHERE stack_name = ?') + .run(Date.now() - 999_999, 'webhook-redelivery-settings-change'); + const { FileSystemService } = await import('../services/FileSystemService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + mockGitClone.mockClear(); + + try { + const redelivery = await svc.handleWebhookPull('webhook-redelivery-settings-change', true, 'delivery-settings-change'); + expect(redelivery.status).toBe('success'); + expect(mockGitClone).not.toHaveBeenCalled(); + expect(saveSpy).not.toHaveBeenCalled(); + } finally { + saveSpy.mockRestore(); + } + }); + + it('returns the stored apply failure when auto-apply is disabled before redelivery', async () => { + const sha = 'a3'.repeat(20); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha }); + const svc = GitSourceService.getInstance(); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const { ComposeService } = await import('../services/ComposeService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockRejectedValue(new Error('simulated webhook deploy failure')); + + try { + await svc.upsert({ + stackName: 'webhook-redelivery-apply-failure', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: true, + autoDeployOnApply: true, + }); + + const first = await svc.handleWebhookPull('webhook-redelivery-apply-failure', true, 'delivery-apply-failure'); + expect(first.status).toBe('error'); + expect(first.message).toContain('simulated webhook deploy failure'); + + DatabaseService.getInstance().getDb() + .prepare('UPDATE stack_git_sources SET auto_apply_on_webhook = 0, auto_deploy_on_apply = 0, last_debounce_at = ? WHERE stack_name = ?') + .run(Date.now() - 999_999, 'webhook-redelivery-apply-failure'); + mockGitClone.mockClear(); + saveSpy.mockClear(); + deploySpy.mockClear(); + + const redelivery = await svc.handleWebhookPull('webhook-redelivery-apply-failure', true, 'delivery-apply-failure'); + + expect(redelivery.status).toBe('error'); + expect(redelivery.message).toContain('simulated webhook deploy failure'); + expect(mockGitClone).not.toHaveBeenCalled(); + expect(saveSpy).not.toHaveBeenCalled(); + expect(deploySpy).not.toHaveBeenCalled(); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('logs the recognized delivery id as a traceability breadcrumb when a webhook pull fails', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ sha: '6'.repeat(40) }); + await configureGitSource('webhook-delivery-breadcrumb'); + mockGitClone.mockClear(); + mockGitClone.mockRejectedValueOnce(new Error('simulated network failure')); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + + try { + const result = await svc.handleWebhookPull('webhook-delivery-breadcrumb', true, 'delivery-log-1'); + expect(result.status).toBe('error'); + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('(delivery delivery-log-1)')); + } finally { + errorSpy.mockRestore(); + } + }); + it('runs a single clone for a concurrent webhook fan-out', async () => { - // The original failure: N webhooks for one push each ran a full clone - // because the debounce gate was read before the per-stack lock. The - // gate now lives inside the lock, so the first request stamps the - // window and the rest skip. + // Concurrent deliveries all reserve and join one shared fetch. Every + // caller receives the leader's normalized result, while only the + // leader performs the clone. const sha = 'eeee555eeee555eeee555eeee555eeee555eeee5'; mockSuccessfulClone({ sha }); const svc = GitSourceService.getInstance(); @@ -1526,15 +1941,85 @@ describe('GitSourceService.handleWebhookPull debounce', () => { mockGitClone.mockClear(); const results = await Promise.all( - Array.from({ length: 5 }, () => svc.handleWebhookPull('fanout-stack')), + Array.from({ length: 5 }, () => svc.handleWebhookPull('fanout-stack', true)), ); expect(mockGitClone.mock.calls.length).toBe(1); - expect(results.filter(r => r.status === 'success')).toHaveLength(1); - expect(results.filter(r => r.status === 'skipped')).toHaveLength(4); + expect(results.filter(r => r.status === 'success')).toHaveLength(5); + expect(results.filter(r => r.status === 'skipped')).toHaveLength(0); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('fanout-stack')!.id; + const settled = settledAttemptsForApplication(applicationId); + expect(settled).toHaveLength(5); + expect(new Set(settled.map((row) => row.after_json)).size).toBe(1); validateSpy.mockRestore(); }); + it('coalesces a webhook fetch with a concurrent manual pull', async () => { + const sha = 'ef'.repeat(20); + mockSuccessfulClone({ sha }); + const svc = GitSourceService.getInstance(); + await configureGitSource('webhook-manual-fetch-coalesce'); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('webhook-manual-fetch-coalesce')!.id; + + let releaseClone!: () => void; + const cloneGate = new Promise((resolve) => { releaseClone = resolve; }); + mockGitClone.mockClear(); + mockGitClone.mockImplementation(async (args: { dir: string }) => { + await cloneGate; + const { promises: fsp } = await import('fs'); + const path = await import('path'); + await fsp.writeFile(path.join(args.dir, 'compose.yaml'), 'services:\n x:\n image: alpine\n', 'utf-8'); + }); + mockGitLog.mockResolvedValue([{ oid: sha }]); + + const webhook = svc.handleWebhookPull('webhook-manual-fetch-coalesce', true, 'delivery-cross-producer'); + await vi.waitFor(() => expect(mockGitClone).toHaveBeenCalledTimes(1)); + const manual = svc.pull('webhook-manual-fetch-coalesce'); + releaseClone(); + const [webhookResult, manualResult] = await Promise.all([webhook, manual]); + + expect(webhookResult.status).toBe('success'); + expect(manualResult.commitSha).toBe(sha); + expect(mockGitClone).toHaveBeenCalledTimes(1); + const settled = DatabaseService.getInstance().getDb() + .prepare("SELECT after_json FROM gitops_history WHERE application_id = ? AND stage = 'source_reconcile_settled'") + .all(applicationId) as { after_json: string }[]; + expect(settled).toHaveLength(2); + expect(new Set(settled.map((row) => row.after_json)).size).toBe(1); + }); + + it('coalesces a manual pull with a concurrent webhook fetch', async () => { + const sha = 'f0'.repeat(20); + mockSuccessfulClone({ sha }); + const svc = GitSourceService.getInstance(); + await configureGitSource('manual-webhook-fetch-coalesce'); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('manual-webhook-fetch-coalesce')!.id; + + let releaseClone!: () => void; + const cloneGate = new Promise((resolve) => { releaseClone = resolve; }); + mockGitClone.mockClear(); + mockGitClone.mockImplementation(async (args: { dir: string }) => { + await cloneGate; + const { promises: fsp } = await import('fs'); + const path = await import('path'); + await fsp.writeFile(path.join(args.dir, 'compose.yaml'), 'services:\n x:\n image: alpine\n', 'utf-8'); + }); + mockGitLog.mockResolvedValue([{ oid: sha }]); + + const manual = svc.pull('manual-webhook-fetch-coalesce'); + await vi.waitFor(() => expect(mockGitClone).toHaveBeenCalledTimes(1)); + const webhook = svc.handleWebhookPull('manual-webhook-fetch-coalesce', true, 'delivery-manual-leader'); + releaseClone(); + const [manualResult, webhookResult] = await Promise.all([manual, webhook]); + + expect(manualResult.commitSha).toBe(sha); + expect(webhookResult.status).toBe('success'); + expect(mockGitClone).toHaveBeenCalledTimes(1); + const settled = settledAttemptsForApplication(applicationId); + expect(settled).toHaveLength(2); + expect(new Set(settled.map((row) => row.after_json)).size).toBe(1); + }); + it('returns error when the pulled compose fails validation', async () => { mockSuccessfulClone(); const svc = GitSourceService.getInstance(); @@ -1556,7 +2041,7 @@ describe('GitSourceService.handleWebhookPull debounce', () => { .spyOn(svc as unknown as { runDockerCompose: (a: string[], c: string, t: number) => Promise<{ code: number; stdout: string; stderr: string }> }, 'runDockerCompose') .mockResolvedValue({ code: 1, stdout: '', stderr: 'bad compose' }); - const result = await svc.handleWebhookPull('webhook-validate-fail'); + const result = await svc.handleWebhookPull('webhook-validate-fail', true); expect(result.status).toBe('error'); expect(result.message).toMatch(/validation failed/i); runSpy.mockRestore(); @@ -1588,7 +2073,7 @@ describe('GitSourceService.handleWebhookPull debounce', () => { existing: { action: 'update', actor: 'user:admin', startedAt: Date.now() }, } as never); - const result = await svc.handleWebhookPull('webhook-shared-lock'); + const result = await svc.handleWebhookPull('webhook-shared-lock', true); expect(result.status).toBe('error'); expect(result.message).toMatch(/already in progress/i); expect(runExclusive).toHaveBeenCalledWith( @@ -1789,6 +2274,224 @@ describe('GitSourceService.pull', () => { await expect(svc.pull('does-not-exist')).rejects.toMatchObject({ code: 'GIT_ERROR' }); }); + it('reserves and durably settles an attempt for a successful pull', async () => { + await createFromGit('pull-reserves', '2'.repeat(40)); + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n web:\n image: nginx:2\n', sha: '3'.repeat(40) }); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('pull-reserves')!.id; + + await svc.pull('pull-reserves'); + + expect(historyOperationIds(applicationId, 'source_reconcile_settled').length).toBeGreaterThanOrEqual(1); + }); + + it('stamps the pending fetch record with the same operation id the reserved attempt used, not an independent one', async () => { + await createFromGit('pull-pending-lineage', '2'.repeat(40)); + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n web:\n image: nginx:2\n', sha: '3'.repeat(40) }); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('pull-pending-lineage')!.id; + + await svc.pull('pull-pending-lineage'); + + const [reservedOperationId] = historyOperationIds(applicationId, 'source_reconcile_started'); + expect(reservedOperationId).toBeTruthy(); + const row = DatabaseService.getInstance().getGitSource('pull-pending-lineage'); + const decoded = (svc as unknown as { + decodePendingCompose: (raw: string) => { operationId: string | null }; + }).decodePendingCompose(row!.pending_compose_content!); + expect(decoded.operationId).toBe(reservedOperationId); + }); + + it('coalesces two concurrent pulls for the same stack into one clone', async () => { + const svc = GitSourceService.getInstance(); + await createFromGit('pull-coalesce', '4'.repeat(40)); + + let releaseClone!: () => void; + const gate = new Promise((resolve) => { releaseClone = resolve; }); + mockGitClone.mockClear(); + mockGitClone.mockImplementation(async (args: { dir: string }) => { + await gate; + const { promises: fsp } = await import('fs'); + const path = await import('path'); + const composeAbs = path.join(args.dir, 'compose.yaml'); + await fsp.mkdir(path.dirname(composeAbs), { recursive: true }); + await fsp.writeFile(composeAbs, 'services:\n web:\n image: nginx:3\n', 'utf-8'); + }); + mockGitLog.mockResolvedValue([{ oid: '5'.repeat(40) }]); + + const first = svc.pull('pull-coalesce'); + const second = svc.pull('pull-coalesce'); + releaseClone(); + const [firstResult, secondResult] = await Promise.all([first, second]); + + expect(mockGitClone).toHaveBeenCalledTimes(1); + expect(secondResult).toEqual(firstResult); + }); + + it('fails closed and does not clone when reservation itself fails', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ sha: '1'.repeat(40) }); + await configureGitSource('pull-reservation-fails-closed'); + mockGitClone.mockClear(); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('pull-reservation-fails-closed')!.id; + const reserveSpy = vi.spyOn(GitOpsTransitions.prototype, 'allocateReconcileAttempt') + .mockImplementationOnce(() => { throw new Error('simulated reservation failure'); }); + + try { + await expect(svc.pull('pull-reservation-fails-closed')).rejects.toMatchObject({ code: 'GIT_ERROR' }); + expect(mockGitClone).not.toHaveBeenCalled(); + expect(historyOperationIds(applicationId, 'source_reconcile_started')).toHaveLength(0); + } finally { + reserveSpy.mockRestore(); + } + }); + + it('fails closed and does not clone when the application was detached but the source config survives', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ sha: '1'.repeat(40) }); + await configureGitSource('pull-tombstoned-app'); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('pull-tombstoned-app')!.id; + GitOpsTransitions.getInstance().applicationTombstoned(applicationId, 'detached', { + operationId: 'op-detach-1', actor: 'tester', trigger: 'test', at: Date.now(), + }); + expect(DatabaseService.getInstance().getGitSource('pull-tombstoned-app')).toBeDefined(); + mockGitClone.mockClear(); + const activitySpy = vi.spyOn(DatabaseService.getInstance(), 'addNotificationHistory'); + + try { + await expect(svc.pull('pull-tombstoned-app')).rejects.toMatchObject({ + code: 'GIT_ERROR', + message: expect.stringContaining('GitOps tracking was removed'), + }); + expect(mockGitClone).not.toHaveBeenCalled(); + expect(historyOperationIds(applicationId, 'source_reconcile_started')).toHaveLength(0); + expect(activitySpy).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ + category: 'git_pull_failed', + stack_name: 'pull-tombstoned-app', + })); + } finally { + activitySpy.mockRestore(); + } + }); + + it('fails closed when the application was deleted, then restores tracked pulls after reconfiguration', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ sha: '4'.repeat(40) }); + await configureGitSource('pull-deleted-app-refused'); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('pull-deleted-app-refused')!.id; + GitOpsTransitions.getInstance().applicationTombstoned(applicationId, 'deleted', { + operationId: 'op-delete-1', actor: 'tester', trigger: 'test', at: Date.now(), + }); + mockGitClone.mockClear(); + + await expect(svc.pull('pull-deleted-app-refused')).rejects.toMatchObject({ + code: 'GIT_ERROR', + message: expect.stringContaining('GitOps tracking is unavailable'), + }); + expect(mockGitClone).not.toHaveBeenCalled(); + + await configureGitSource('pull-deleted-app-refused'); + expect(GitOpsStore.getInstance().getLiveDirectApplication('pull-deleted-app-refused')?.id).not.toBe(applicationId); + mockGitClone.mockClear(); + await svc.pull('pull-deleted-app-refused'); + expect(mockGitClone).toHaveBeenCalledTimes(1); + }); + + it('falls through to the ordinary no-source error for a completed detach, not the detach-in-progress message', async () => { + // detach() commits applicationTombstoned('detached') and + // deleteGitSource in one transaction, so a routine, fully + // successful detach leaves exactly this state: a detached + // tombstone with NO surviving source row. The detach-in-progress + // refusal must not fire here, or every previously-detached stack + // name would get a false, unactionable message instead of the + // real, correct "no source configured" error. + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ sha: '5'.repeat(40) }); + await configureGitSource('pull-completed-detach'); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('pull-completed-detach')!.id; + GitOpsTransitions.getInstance().applicationTombstoned(applicationId, 'detached', { + operationId: 'op-detach-4', actor: 'tester', trigger: 'test', at: Date.now(), + }); + DatabaseService.getInstance().deleteGitSource('pull-completed-detach'); + mockGitClone.mockClear(); + + await expect(svc.pull('pull-completed-detach')).rejects.toMatchObject({ + code: 'GIT_ERROR', + message: 'No Git source configured for this stack.', + }); + expect(mockGitClone).not.toHaveBeenCalled(); + }); + + it('does not mask the real pull failure when deriving the settlement result afterward also throws', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ sha: '1'.repeat(40) }); + await configureGitSource('pull-mask-error'); + // A real gitops application exists (so this call reserves an + // attempt), but the source config row is now gone, so pullLocked + // itself throws a specific, truthful error. + DatabaseService.getInstance().deleteGitSource('pull-mask-error'); + const deriveSpy = vi.spyOn(svc as unknown as { deriveReconcileResult: (s: string) => unknown }, 'deriveReconcileResult') + .mockImplementationOnce(() => { throw new Error('derivation boom'); }); + + try { + await expect(svc.pull('pull-mask-error')).rejects.toMatchObject({ + code: 'GIT_ERROR', + message: expect.stringContaining('No Git source configured'), + }); + } finally { + deriveSpy.mockRestore(); + } + }); + + it('fails closed when a reservation collision is forced with no in-process leader', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ sha: '1'.repeat(40) }); + await configureGitSource('pull-forced-collision'); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('pull-forced-collision')!.id; + const nextSeq = GitOpsStore.getInstance().getApplication(applicationId)!.attempt_seq + 1; + const predictedOperationId = `${applicationId}:attempt:${nextSeq}`; + // Force the exact operation id pull() is about to allocate to + // already be reserved, simulating a collision with no in-process + // leader for it. The call must not run untracked work under an + // operation id already owned by another submission. + GitOpsTransitions.getInstance().reserveReconcileAttempt(applicationId, { + operationId: predictedOperationId, actor: 'someone-else', trigger: 'poll', at: Date.now(), + }); + mockSuccessfulClone({ sha: '2'.repeat(40) }); + mockGitClone.mockClear(); + await expect(svc.pull('pull-forced-collision')).rejects.toMatchObject({ + code: 'GIT_ERROR', + message: expect.stringContaining('already recorded'), + }); + expect(mockGitClone).not.toHaveBeenCalled(); + }); + + it('settles a coalesced follower\'s own attempt even when the leader\'s fetch rejects', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ sha: '1'.repeat(40) }); + await configureGitSource('pull-follower-settles-on-reject'); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('pull-follower-settles-on-reject')!.id; + + let releaseClone!: () => void; + const gate = new Promise((resolve) => { releaseClone = resolve; }); + mockGitClone.mockClear(); + mockGitClone.mockImplementation(async () => { + await gate; + throw new Error('simulated clone failure'); + }); + + const first = svc.pull('pull-follower-settles-on-reject'); + const second = svc.pull('pull-follower-settles-on-reject'); + releaseClone(); + await expect(first).rejects.toThrow('simulated clone failure'); + await expect(second).rejects.toThrow('simulated clone failure'); + + // Both the leader's and the follower's own reservations must be + // durably settled; neither may be left open waiting for a crash + // that never happened. + expect(GitOpsStore.getInstance().listUnsettledReconcileAttempts().some((r) => r.application_id === applicationId)).toBe(false); + }); + function generationCount(stackName: string): number { const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName)!; return (DatabaseService.getInstance().getDb() @@ -2236,6 +2939,1701 @@ describe('GitSourceService.apply', () => { return svc; } + function liveApp(stackName: string) { + return GitOpsStore.getInstance().getLiveDirectApplication(stackName); + } + + it('fails closed and does not write or deploy when reservation itself fails, even for a deploying apply', async () => { + const sha = 'df'.repeat(20); + const svc = await seedPending('apply-reservation-fails-closed', 'services:\n x:\n image: alpine\n', sha); + const applicationId = liveApp('apply-reservation-fails-closed')!.id; + const { FileSystemService } = await import('../services/FileSystemService'); + const { ComposeService } = await import('../services/ComposeService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack'); + const startedBefore = historyOperationIds(applicationId, 'source_reconcile_started').length; + const reserveSpy = vi.spyOn(GitOpsTransitions.prototype, 'allocateReconcileAttempt') + .mockImplementationOnce(() => { throw new Error('simulated reservation failure'); }); + + try { + await expect(svc.apply('apply-reservation-fails-closed', sha, { ...SKIP_PLAN_FINGERPRINT, deploy: true })) + .rejects.toMatchObject({ code: 'GIT_ERROR' }); + expect(saveSpy).not.toHaveBeenCalled(); + expect(deploySpy).not.toHaveBeenCalled(); + // No new attempt at all, settled or unsettled: reservation + // itself never landed, so there is nothing new to track. + expect(historyOperationIds(applicationId, 'source_reconcile_started')).toHaveLength(startedBefore); + } finally { + saveSpy.mockRestore(); + deploySpy.mockRestore(); + reserveSpy.mockRestore(); + } + }); + + it('fails closed and does not write when the application was detached but the pending commit survives', async () => { + const sha = 'db'.repeat(20); + const svc = await seedPending('apply-tombstoned-app', 'services:\n x:\n image: alpine\n', sha); + const applicationId = liveApp('apply-tombstoned-app')!.id; + GitOpsTransitions.getInstance().applicationTombstoned(applicationId, 'detached', { + operationId: 'op-detach-2', actor: 'tester', trigger: 'test', at: Date.now(), + }); + expect(DatabaseService.getInstance().getGitSource('apply-tombstoned-app')?.pending_commit_sha).toBe(sha); + const { FileSystemService } = await import('../services/FileSystemService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + const activitySpy = vi.spyOn(DatabaseService.getInstance(), 'addNotificationHistory'); + + try { + await expect(svc.apply('apply-tombstoned-app', sha, SKIP_PLAN_FINGERPRINT)).rejects.toMatchObject({ + code: 'GIT_ERROR', + message: expect.stringContaining('GitOps tracking was removed'), + }); + expect(saveSpy).not.toHaveBeenCalled(); + expect(activitySpy).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ + category: 'git_apply_failed', + stack_name: 'apply-tombstoned-app', + })); + } finally { + saveSpy.mockRestore(); + activitySpy.mockRestore(); + } + }); + + it('fails closed and does not write or deploy when the application was deleted but the pending commit survives', async () => { + const sha = 'dc'.repeat(20); + const svc = await seedPending('apply-deleted-app', 'services:\n x:\n image: alpine\n', sha); + const applicationId = liveApp('apply-deleted-app')!.id; + GitOpsTransitions.getInstance().applicationTombstoned(applicationId, 'deleted', { + operationId: 'op-delete-apply', actor: 'tester', trigger: 'test', at: Date.now(), + }); + const { FileSystemService } = await import('../services/FileSystemService'); + const { ComposeService } = await import('../services/ComposeService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack'); + + try { + await expect(svc.apply('apply-deleted-app', sha, { ...SKIP_PLAN_FINGERPRINT, deploy: true })) + .rejects.toMatchObject({ + code: 'GIT_ERROR', + message: expect.stringContaining('GitOps tracking is unavailable'), + }); + expect(saveSpy).not.toHaveBeenCalled(); + expect(deploySpy).not.toHaveBeenCalled(); + } finally { + saveSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('reserves and durably settles an attempt for a successful apply', async () => { + const sha = '6'.repeat(40); + const svc = await seedPending('apply-reserves', 'services:\n x:\n image: alpine\n', sha); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + + try { + const applicationId = liveApp('apply-reserves')!.id; + await svc.apply('apply-reserves', sha, SKIP_PLAN_FINGERPRINT); + + const settled = historyOperationIds(applicationId, 'source_reconcile_settled'); + const applied = historyOperationIds(applicationId, 'applied'); + expect(settled.length).toBeGreaterThanOrEqual(1); + expect(applied).toHaveLength(1); + expect(settled).toContain(applied[0]); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + } + }); + + it('coalesces two concurrent applies for the same commit into one execution', async () => { + const sha = '7'.repeat(40); + const svc = await seedPending('apply-coalesce', 'services:\n x:\n image: alpine\n', sha); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + let releaseSave!: () => void; + const gate = new Promise((resolve) => { releaseSave = resolve; }); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockImplementation(async () => { await gate; }); + + try { + const first = svc.apply('apply-coalesce', sha, SKIP_PLAN_FINGERPRINT); + const second = svc.apply('apply-coalesce', sha, SKIP_PLAN_FINGERPRINT); + releaseSave(); + const [firstResult, secondResult] = await Promise.all([first, second]); + + expect(saveSpy).toHaveBeenCalledTimes(1); + expect(secondResult).toEqual(firstResult); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + } + }); + + it('does not coalesce two concurrent applies that resolve to different deploy behavior', async () => { + const sha = 'ba'.repeat(20); + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha }); + const seedValidateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + try { + await svc.upsert({ + stackName: 'apply-deploy-mismatch', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: true, + autoDeployOnApply: true, + }); + await svc.pull('apply-deploy-mismatch'); + } finally { + seedValidateSpy.mockRestore(); + } + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + let releaseSave!: () => void; + const gate = new Promise((resolve) => { releaseSave = resolve; }); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockImplementation(async () => { await gate; }); + const { ComposeService } = await import('../services/ComposeService'); + const { HealthGateService } = await import('../services/HealthGateService'); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-git'); + + try { + // The stack has auto_deploy_on_apply: true. The first call + // leaves deploy unresolved (so it resolves to true); the + // second explicitly asks not to deploy. These must never + // share a coalesce key: if they wrongly joined, the second + // call would resolve successfully with the first's deployed + // result instead of running (or failing) on its own terms. + // Since they do not join, and applying clears the pending + // commit, the second genuinely has nothing left to apply + // once the first (which the per-stack lock serializes first) + // completes -- a real, honest failure, not a borrowed result. + const first = svc.apply('apply-deploy-mismatch', sha, SKIP_PLAN_FINGERPRINT); + const second = svc.apply('apply-deploy-mismatch', sha, { ...SKIP_PLAN_FINGERPRINT, deploy: false }); + releaseSave(); + const firstResult = await first; + expect(firstResult.deployed).toBe(true); + await expect(second).rejects.toMatchObject({ code: 'GIT_ERROR' }); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + deploySpy.mockRestore(); + beginSpy.mockRestore(); + } + }); + + describe('reconcile', () => { + it('reports candidate_already_fetched after a fetch-intent reconcile stages a new candidate', async () => { + const sha = 'e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1'; + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha }); + const svc = GitSourceService.getInstance(); + await svc.upsert({ + stackName: 'reconcile-fetch', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + try { + const applicationId = liveApp('reconcile-fetch')!.id; + const result = await svc.reconcile({ + intent: 'fetch', + applicationId, + stackName: 'reconcile-fetch', + trigger: 'manual', + actor: 'tester', + }); + expect(result.outcome).toBe('candidate_already_fetched'); + } finally { + validateSpy.mockRestore(); + } + }); + + it('reports no_source_change after an apply-intent reconcile accepts the candidate', async () => { + const sha = 'e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2'; + const svc = await seedPending('reconcile-apply', 'services:\n x:\n image: alpine\n', sha); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + + try { + const applicationId = liveApp('reconcile-apply')!.id; + const result = await svc.reconcile({ + intent: 'apply', + applicationId, + stackName: 'reconcile-apply', + trigger: 'manual', + actor: 'tester', + commitSha: sha, + planFingerprint: '', + deploy: false, + }); + expect(result.outcome).toBe('no_source_change'); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + } + }); + + it('does not report success when the source applied but the deploy failed', async () => { + const sha = 'e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7'; + const svc = await seedPending('reconcile-apply-deploy-fail', 'services:\n x:\n image: alpine\n', sha); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const { ComposeService } = await import('../services/ComposeService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockRejectedValue( + new Error('compose up failed: docker unavailable'), + ); + + try { + const applicationId = liveApp('reconcile-apply-deploy-fail')!.id; + const result = await svc.reconcile({ + intent: 'apply', + applicationId, + stackName: 'reconcile-apply-deploy-fail', + trigger: 'manual', + actor: 'tester', + commitSha: sha, + planFingerprint: '', + deploy: true, + }); + // The promotion itself succeeded (files landed, generation + // accepted), so the source facet alone reads as converged. + // reconcile must not let that mask the deploy failure, and must + // not claim the previous generation is unchanged either: it isn't. + expect(result.outcome).toBe('recovery_required'); + expect(result.nextAction).toBe('view_target_results'); + expect(result.reason).toMatch(/deploy/i); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('reports a truthful failure, not the stale staged-candidate outcome, when fetch throws before touching the application row', async () => { + const sha = 'e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3'; + const svc = await seedPending('reconcile-fetch-fail', 'services:\n x:\n image: alpine\n', sha); + const applicationId = liveApp('reconcile-fetch-fail')!.id; + // Deleting the config row makes pullLocked throw its `!src` guard, + // which fires before fetchStarted opens any transition: the + // application row is untouched by this failure. + DatabaseService.getInstance().deleteGitSource('reconcile-fetch-fail'); + + const result = await svc.reconcile({ + intent: 'fetch', + applicationId, + stackName: 'reconcile-fetch-fail', + trigger: 'manual', + actor: 'tester', + }); + + expect(result.outcome).not.toBe('candidate_already_fetched'); + expect(result.nextAction).not.toBe('none'); + }); + + it('reports a truthful failure, not the stale staged-candidate outcome, when apply throws on a stale commitSha', async () => { + const sha = 'e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4'; + const svc = await seedPending('reconcile-apply-stale-sha', 'services:\n x:\n image: alpine\n', sha); + const applicationId = liveApp('reconcile-apply-stale-sha')!.id; + + const result = await svc.reconcile({ + intent: 'apply', + applicationId, + stackName: 'reconcile-apply-stale-sha', + trigger: 'manual', + actor: 'tester', + commitSha: 'ffffffffffffffffffffffffffffffffffffffff', + planFingerprint: '', + deploy: false, + }); + + expect(result.outcome).not.toBe('candidate_already_fetched'); + expect(result.nextAction).not.toBe('none'); + }); + + it('fails closed instead of silently reconciling the wrong application when the requested applicationId is stale', async () => { + const sha = 'e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5'; + const svc = await seedPending('reconcile-stale-app-id', 'services:\n x:\n image: alpine\n', sha); + + const result = await svc.reconcile({ + intent: 'fetch', + applicationId: 'no-longer-the-live-application', + stackName: 'reconcile-stale-app-id', + trigger: 'manual', + actor: 'tester', + }); + + expect(result.outcome).toBe('unknown'); + expect(result.nextAction).toBe('none'); + // Fails closed before doing anything: the candidate this stack had + // staged before the call is still exactly as it was. + const stillStaged = liveApp('reconcile-stale-app-id'); + expect(stillStaged?.candidate_generation_id).toBeTruthy(); + }); + + it('fails closed on a stale applicationId even when the live application is stuck in creating, not active', async () => { + const sha = 'e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6'; + const svc = await seedPending('reconcile-creating-app', 'services:\n x:\n image: alpine\n', sha); + const applicationId = liveApp('reconcile-creating-app')!.id; + // gitopsApplicationFor() (used elsewhere to gate transitions) only + // recognizes 'active' rows, but getLiveDirectApplication() (used + // by deriveReconcileResult) recognizes 'active' and 'creating' + // alike. reconcile's identity guard must use the same broad + // definition, or a 'creating' row slips past it entirely. + DatabaseService.getInstance().getDb() + .prepare("UPDATE gitops_applications SET lifecycle_status = 'creating' WHERE id = ?") + .run(applicationId); + + const result = await svc.reconcile({ + intent: 'apply', + applicationId: 'deliberately-mismatched-id', + stackName: 'reconcile-creating-app', + trigger: 'manual', + actor: 'tester', + commitSha: 'ffffffffffffffffffffffffffffffffffffffff', + planFingerprint: '', + deploy: false, + }); + + expect(result.outcome).toBe('unknown'); + expect(result.nextAction).toBe('none'); + // Fails closed before doing anything: the candidate this stack + // had staged before the call is still exactly as it was. + const stillStaged = liveApp('reconcile-creating-app'); + expect(stillStaged?.candidate_generation_id).toBeTruthy(); + }); + + it('fails closed instead of silently applying under the current live application when the requested applicationId still exists but was superseded', async () => { + const sha = 'e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7'; + const svc = await seedPending('reconcile-apply-superseded-id', 'services:\n x:\n image: alpine\n', sha); + const staleApplicationId = liveApp('reconcile-apply-superseded-id')!.id; + // A real row that used to be live for this stack, not a + // fabricated id: this is the exact gap the existing stale-id + // tests (using ids that never existed as any row) do not cover, + // since GitOpsStore.getApplication finds this row just fine. + GitOpsTransitions.getInstance().applicationTombstoned(staleApplicationId, 'detached', { + operationId: 'op-supersede-1', actor: 'tester', trigger: 'test', at: Date.now(), + }); + const config: DirectSourceConfig = { + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + }; + GitOpsStore.getInstance().insertApplication(buildDirectApplicationRow({ + id: newGitOpsId(), + stackName: 'reconcile-apply-superseded-id', + config, + identity: directSourceIdentity(config), + lifecycleStatus: 'active', + at: Date.now(), + })); + const newLiveId = liveApp('reconcile-apply-superseded-id')!.id; + expect(newLiveId).not.toBe(staleApplicationId); + + const applySpy = vi.spyOn( + svc as unknown as { applyWithSharedLock: (...args: unknown[]) => Promise }, + 'applyWithSharedLock', + ); + + try { + const result = await svc.reconcile({ + intent: 'apply', + applicationId: staleApplicationId, + stackName: 'reconcile-apply-superseded-id', + trigger: 'manual', + actor: 'tester', + commitSha: 'ffffffffffffffffffffffffffffffffffffffff', + planFingerprint: '', + deploy: false, + }); + + expect(result.outcome).toBe('unknown'); + expect(result.nextAction).toBe('none'); + expect(applySpy).not.toHaveBeenCalled(); + } finally { + applySpy.mockRestore(); + } + }); + + it('revalidates the application identity after acquiring the fetch lock', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ sha: 'fa'.repeat(20) }); + await configureGitSource('reconcile-fetch-replaced-while-queued'); + const staleApplicationId = liveApp('reconcile-fetch-replaced-while-queued')!.id; + mockGitClone.mockClear(); + + let releaseLock!: () => void; + const lockGate = new Promise((resolve) => { releaseLock = resolve; }); + const lockHolder = (svc as unknown as { + withStackLock: (stackName: string, fn: () => Promise) => Promise; + }).withStackLock('reconcile-fetch-replaced-while-queued', () => lockGate); + + const reconcile = svc.reconcile({ + intent: 'fetch', + applicationId: staleApplicationId, + stackName: 'reconcile-fetch-replaced-while-queued', + trigger: 'poll', + actor: 'system:source-controller', + }); + await vi.waitFor(() => { + expect(historyOperationIds(staleApplicationId, 'source_reconcile_started')).toHaveLength(1); + }); + + GitOpsTransitions.getInstance().applicationTombstoned(staleApplicationId, 'detached', { + operationId: 'op-replace-queued-fetch', actor: 'tester', trigger: 'test', at: Date.now(), + }); + const config: DirectSourceConfig = { + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + }; + GitOpsStore.getInstance().insertApplication(buildDirectApplicationRow({ + id: newGitOpsId(), + stackName: 'reconcile-fetch-replaced-while-queued', + config, + identity: directSourceIdentity(config), + lifecycleStatus: 'active', + at: Date.now(), + })); + + releaseLock(); + await lockHolder; + const result = await reconcile; + + expect(result.outcome).toBe('unknown'); + expect(result.nextAction).toBe('none'); + expect(mockGitClone).not.toHaveBeenCalled(); + expect(settledAttempts(staleApplicationId)).toHaveLength(1); + expect(historyOperationIds(liveApp('reconcile-fetch-replaced-while-queued')!.id, 'fetch_started')).toHaveLength(0); + }); + + it('captures the fetch result before a queued source mutation can change row state', async () => { + const stackName = 'reconcile-settlement-before-queued-suspend'; + const sha = 'f1'.repeat(20); + mockSuccessfulClone({ sha }); + const svc = GitSourceService.getInstance(); + await configureGitSource(stackName); + const applicationId = liveApp(stackName)!.id; + const releaseClone = gatedClone(); + mockGitLog.mockResolvedValue([{ oid: sha }]); + + const fetch = svc.reconcile({ + intent: 'fetch', + applicationId, + stackName, + trigger: 'poll', + actor: 'system:source-controller', + }); + await vi.waitFor(() => expect(mockGitClone).toHaveBeenCalledTimes(1)); + const suspend = svc.suspend(stackName, { actor: 'tester', reason: 'queue behind fetch' }); + releaseClone(); + + const [fetchResult, suspendResult] = await Promise.all([fetch, suspend]); + + expect(fetchResult.outcome).toBe('candidate_already_fetched'); + expect(suspendResult.outcome).toBe('suspended'); + const settled = settledAttempts(applicationId); + expect(settled).toHaveLength(1); + expect(JSON.parse(settled[0].after_json).outcome).toBe('candidate_already_fetched'); + }); + + it('reports unknown for a stack with no GitOps application', async () => { + const svc = GitSourceService.getInstance(); + const result = await svc.reconcile({ + intent: 'fetch', + applicationId: 'unused', + stackName: 'reconcile-no-app', + trigger: 'manual', + actor: 'tester', + }); + expect(result.outcome).toBe('unknown'); + }); + + function settledAttempts(applicationId: string): { operation_id: string; after_json: string }[] { + return DatabaseService.getInstance().getDb() + .prepare("SELECT operation_id, after_json FROM gitops_history WHERE application_id = ? AND stage = 'source_reconcile_settled'") + .all(applicationId) as { operation_id: string; after_json: string }[]; + } + + function unsettledAttempts(applicationId: string): { operation_id: string }[] { + return DatabaseService.getInstance().getDb() + .prepare("SELECT operation_id FROM gitops_history started WHERE started.application_id = ? AND started.stage = 'source_reconcile_started' AND NOT EXISTS (SELECT 1 FROM gitops_history settled WHERE settled.application_id = started.application_id AND settled.operation_id = started.operation_id AND settled.stage = 'source_reconcile_settled')") + .all(applicationId) as { operation_id: string }[]; + } + + /** + * Hold the clone open so a second reconcile submission is guaranteed + * to arrive while the first is still executing. Returns the release + * function; calling it lets the clone finish and write its compose + * file. + */ + function gatedClone(): () => void { + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + mockGitClone.mockClear(); + mockGitClone.mockImplementation(async (args: { dir: string }) => { + await gate; + const { promises: fsp } = await import('fs'); + const path = await import('path'); + const composeAbs = path.join(args.dir, 'compose.yaml'); + await fsp.mkdir(path.dirname(composeAbs), { recursive: true }); + await fsp.writeFile(composeAbs, 'services:\n x:\n image: alpine\n', 'utf-8'); + }); + return release; + } + + it('durably settles a reconcile attempt for a successful fetch, leaving nothing unsettled', async () => { + const sha = 'e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8'; + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha }); + const svc = GitSourceService.getInstance(); + await svc.upsert({ + stackName: 'reconcile-durable', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + try { + const applicationId = liveApp('reconcile-durable')!.id; + const result = await svc.reconcile({ + intent: 'fetch', + applicationId, + stackName: 'reconcile-durable', + trigger: 'manual', + actor: 'tester', + }); + + const settled = settledAttempts(applicationId); + expect(settled).toHaveLength(1); + expect(JSON.parse(settled[0].after_json)).toMatchObject({ outcome: result.outcome, reason: result.reason }); + expect(unsettledAttempts(applicationId)).toHaveLength(0); + } finally { + validateSpy.mockRestore(); + } + }); + + it('reserves and settles an attempt for every normalized trigger kind', async () => { + const stackName = 'reconcile-trigger-matrix'; + const sha = 'e9'.repeat(20); + const triggers: ReconcileTrigger[] = [ + 'manual', + 'api', + 'webhook', + 'poll', + 'retry', + 'config_change', + 'startup', + 'resume', + 'provider_event', + 'schedule', + 'binding_change', + ]; + mockSuccessfulClone({ sha }); + const svc = GitSourceService.getInstance(); + await configureGitSource(stackName); + const applicationId = liveApp(stackName)!.id; + + for (const trigger of triggers) { + await svc.reconcile({ + intent: 'fetch', + applicationId, + stackName, + trigger, + actor: `system:${trigger}`, + ...(trigger === 'webhook' ? { deliveryId: 'trigger-matrix-webhook' } : {}), + }); + } + + const started = DatabaseService.getInstance().getDb() + .prepare("SELECT operation_id, trigger FROM gitops_history WHERE application_id = ? AND stage = 'source_reconcile_started' ORDER BY rowid") + .all(applicationId) as { operation_id: string; trigger: string }[]; + expect(started.map((row) => row.trigger)).toEqual(triggers); + expect(new Set(started.map((row) => row.operation_id)).size).toBe(triggers.length); + expect(started.find((row) => row.trigger === 'webhook')?.operation_id) + .toBe(deliveryKey('webhook', 'fetch', 'trigger-matrix-webhook')); + expect(unsettledAttempts(applicationId)).toHaveLength(0); + }); + + it('threads the reserved attempt\'s operation id into the generation the fetch produces', async () => { + const sha = 'ed'.repeat(20); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha }); + const svc = GitSourceService.getInstance(); + await svc.upsert({ + stackName: 'reconcile-operation-id-threading', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + try { + const applicationId = liveApp('reconcile-operation-id-threading')!.id; + await svc.reconcile({ + intent: 'fetch', + applicationId, + stackName: 'reconcile-operation-id-threading', + trigger: 'manual', + actor: 'tester', + }); + + const settled = settledAttempts(applicationId); + expect(settled).toHaveLength(1); + const candidateGenerationId = liveApp('reconcile-operation-id-threading')!.candidate_generation_id; + expect(candidateGenerationId).toBeTruthy(); + const generation = GitOpsStore.getInstance().getGeneration(candidateGenerationId!); + expect(generation?.operation_id).toBe(settled[0].operation_id); + } finally { + validateSpy.mockRestore(); + } + }); + + it('threads the reserved attempt\'s operation id into the apply-side transition an apply-intent reconcile produces', async () => { + const sha = 'ee'.repeat(20); + const svc = await seedPending('reconcile-apply-operation-id-threading', 'services:\n x:\n image: alpine\n', sha); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + + try { + const applicationId = liveApp('reconcile-apply-operation-id-threading')!.id; + await svc.reconcile({ + intent: 'apply', + applicationId, + stackName: 'reconcile-apply-operation-id-threading', + trigger: 'manual', + actor: 'tester', + commitSha: sha, + planFingerprint: '', + deploy: false, + }); + + // seedPending's own pull() reserves and settles its own fetch + // attempt now that pull() is wired through reservation too, so + // more than one settled row is expected here; only the + // apply-intent one needs to match the applied-stage transition. + const settled = settledAttempts(applicationId).map((r) => r.operation_id); + const applied = historyOperationIds(applicationId, 'applied'); + expect(applied).toHaveLength(1); + expect(settled).toContain(applied[0]); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + } + }); + + it('coalesces two concurrent fetch-intent reconciles into one execution, each settling its own durable attempt', async () => { + const newSha = 'e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9'; + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0' }); + const svc = GitSourceService.getInstance(); + await svc.upsert({ + stackName: 'reconcile-coalesce', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + const releaseClone = gatedClone(); + mockGitLog.mockResolvedValue([{ oid: newSha }]); + + try { + const applicationId = liveApp('reconcile-coalesce')!.id; + const first = svc.reconcile({ + intent: 'fetch', applicationId, stackName: 'reconcile-coalesce', trigger: 'manual', actor: 'tester-a', + }); + const second = svc.reconcile({ + intent: 'fetch', applicationId, stackName: 'reconcile-coalesce', trigger: 'manual', actor: 'tester-b', + }); + releaseClone(); + const [firstResult, secondResult] = await Promise.all([first, second]); + + expect(mockGitClone).toHaveBeenCalledTimes(1); + expect(firstResult).toEqual(secondResult); + + const settled = settledAttempts(applicationId); + expect(settled).toHaveLength(2); + expect(new Set(settled.map((r) => r.operation_id)).size).toBe(2); + expect(unsettledAttempts(applicationId)).toHaveLength(0); + } finally { + validateSpy.mockRestore(); + } + }); + + it('coalesces a concurrent manual pull and a controller-triggered reconcile into one clone, each with its own complete settled history and a follower-to-leader link', async () => { + const newSha = 'f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1'; + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0' }); + const svc = GitSourceService.getInstance(); + await configureGitSource('pull-reconcile-coalesce'); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + const releaseClone = gatedClone(); + mockGitLog.mockResolvedValue([{ oid: newSha }]); + + try { + const applicationId = liveApp('pull-reconcile-coalesce')!.id; + const generationCountBefore = (DatabaseService.getInstance().getDb() + .prepare('SELECT COUNT(*) AS count FROM gitops_generations WHERE application_id = ?') + .get(applicationId) as { count: number }).count; + const candidateReadyCountBefore = historyOperationIds(applicationId, 'candidate_ready').length; + // Two different producers, one a manual pull() and the other + // a controller poll driving reconcile(), submitting for the + // exact same live application: coalesceKey() does not vary + // by trigger or producer, so this must join into one clone + // rather than each running its own. + const manualPull = svc.pull('pull-reconcile-coalesce'); + const controllerReconcile = svc.reconcile({ + intent: 'fetch', + applicationId, + stackName: 'pull-reconcile-coalesce', + trigger: 'poll', + actor: 'system:source-controller', + }); + releaseClone(); + const [pullResult, reconcileResult] = await Promise.all([manualPull, controllerReconcile]); + + expect(mockGitClone).toHaveBeenCalledTimes(1); + expect(pullResult.commitSha).toBe(newSha); + expect(pullResult.candidateReady).toBe(true); + expect(reconcileResult.outcome).toBe('candidate_already_fetched'); + const generationCountAfter = (DatabaseService.getInstance().getDb() + .prepare('SELECT COUNT(*) AS count FROM gitops_generations WHERE application_id = ?') + .get(applicationId) as { count: number }).count; + expect(generationCountAfter).toBe(generationCountBefore + 1); + expect(historyOperationIds(applicationId, 'candidate_ready')).toHaveLength(candidateReadyCountBefore + 1); + + // Two complete histories: each caller reserved and settled + // its own durable attempt, neither left dangling. + const settled = settledAttempts(applicationId); + expect(settled).toHaveLength(2); + const settledOperationIds = settled.map((r) => r.operation_id); + expect(new Set(settledOperationIds).size).toBe(2); + expect(unsettledAttempts(applicationId)).toHaveLength(0); + + // A follower-to-leader link: exactly one of the two + // reservations recorded that it was made on behalf of the + // other. + const started = DatabaseService.getInstance().getDb() + .prepare("SELECT operation_id, after_json FROM gitops_history WHERE application_id = ? AND stage = 'source_reconcile_started'") + .all(applicationId) as { operation_id: string; after_json: string }[]; + const followerLinks = started + .map((r) => (JSON.parse(r.after_json) as { followerOf?: string }).followerOf) + .filter((followerOf): followerOf is string => followerOf !== undefined); + expect(followerLinks).toHaveLength(1); + expect(settledOperationIds).toContain(followerLinks[0]); + } finally { + validateSpy.mockRestore(); + } + }); + + it('leaves every coalesced attempt unsettled when the leader result is not durable, so recovery gives them one result', async () => { + const stackName = 'reconcile-shared-settlement-failure'; + const sha = 'c7'.repeat(20); + mockSuccessfulClone({ sha }); + const svc = GitSourceService.getInstance(); + await configureGitSource(stackName); + const applicationId = liveApp(stackName)!.id; + const priorStarted = new Set(historyOperationIds(applicationId, 'source_reconcile_started')); + const releaseClone = gatedClone(); + mockGitLog.mockResolvedValue([{ oid: sha }]); + const originalSettle = GitOpsTransitions.prototype.settleReconcileAttempt; + const settleSpy = vi.spyOn(GitOpsTransitions.prototype, 'settleReconcileAttempt') + .mockImplementationOnce(() => { throw new Error('simulated leader settlement failure'); }) + .mockImplementation(function (this: GitOpsTransitions, ...args: Parameters) { + return originalSettle.apply(this, args); + }); + + try { + const manual = svc.pull(stackName); + const controller = svc.reconcile({ + intent: 'fetch', + applicationId, + stackName, + trigger: 'poll', + actor: 'system:source-controller', + }); + releaseClone(); + await Promise.all([manual, controller]); + + const operationIds = historyOperationIds(applicationId, 'source_reconcile_started') + .filter((operationId) => !priorStarted.has(operationId)); + expect(operationIds).toHaveLength(2); + expect(unsettledAttempts(applicationId).map((row) => row.operation_id).sort()) + .toEqual([...operationIds].sort()); + + await svc.suspend(stackName, { actor: 'tester', reason: 'settle through recovery' }); + await svc.recoverUnsettledReconcileAttempts(); + + const recovered = settledAttempts(applicationId) + .filter((row) => operationIds.includes(row.operation_id)); + expect(recovered).toHaveLength(2); + expect(new Set(recovered.map((row) => row.after_json)).size).toBe(1); + expect(JSON.parse(recovered[0].after_json).outcome).toBe('suspended'); + } finally { + settleSpy.mockRestore(); + } + }); + + it.each([ + ['manual pull', true], + ['controller reconcile', false], + ] as const)('settles both callers from one failed shared fetch when %s leads', async (_leader, manualLeads) => { + const stackName = manualLeads + ? 'pull-reconcile-failure-manual-leads' + : 'pull-reconcile-failure-controller-leads'; + mockSuccessfulClone({ sha: 'f2'.repeat(20) }); + const svc = GitSourceService.getInstance(); + await configureGitSource(stackName); + const applicationId = liveApp(stackName)!.id; + + let releaseClone!: () => void; + const cloneGate = new Promise((resolve) => { releaseClone = resolve; }); + mockGitClone.mockClear(); + mockGitClone.mockImplementation(async () => { + await cloneGate; + throw new Error('shared fetch failed'); + }); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const request = { + intent: 'fetch' as const, + applicationId, + stackName, + trigger: 'poll' as const, + actor: 'system:source-controller', + }; + + try { + const first = manualLeads ? svc.pull(stackName) : svc.reconcile(request); + const second = manualLeads ? svc.reconcile(request) : svc.pull(stackName); + releaseClone(); + const [firstResult, secondResult] = await Promise.allSettled([first, second]); + + const reconcileResult = manualLeads ? secondResult : firstResult; + expect(reconcileResult.status).toBe('fulfilled'); + if (reconcileResult.status !== 'fulfilled') throw reconcileResult.reason; + const settled = settledAttempts(applicationId).map((row) => JSON.parse(row.after_json)); + expect(settled).toHaveLength(2); + expect(settled).toEqual([reconcileResult.value, reconcileResult.value]); + expect(mockGitClone).toHaveBeenCalledTimes(1); + expect(unsettledAttempts(applicationId)).toHaveLength(0); + } finally { + errorSpy.mockRestore(); + } + }); + + it('settles a fetch-intent reconcile durably with the same classified result it returns, even for a pre-transition failure the row does not yet reflect', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ sha: '3'.repeat(40) }); + await configureGitSource('reconcile-pretransition-failure'); + const applicationId = liveApp('reconcile-pretransition-failure')!.id; + // Deletes the config row pullLocked itself checks for, before + // any transition table write, so the row state alone (a + // generic row-derivation, with no notion of this failure) would + // misreport the outcome as an unremarkable "never reconciled" + // rather than the real, classified failure the caller receives. + DatabaseService.getInstance().deleteGitSource('reconcile-pretransition-failure'); + + const result = await svc.reconcile({ + intent: 'fetch', + applicationId, + stackName: 'reconcile-pretransition-failure', + trigger: 'poll', + actor: 'system:source-controller', + }); + + expect(result.outcome).not.toBe('unknown'); + const settled = settledAttempts(applicationId); + expect(settled).toHaveLength(1); + expect(JSON.parse(settled[0].after_json)).toEqual(result); + }); + + it('does not re-execute a redelivered request whose original attempt was reserved but never settled', async () => { + const sha = 'dededededededededededededededededededede'; + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha }); + const svc = GitSourceService.getInstance(); + await configureGitSource('reconcile-orphaned-redelivery'); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + try { + const applicationId = liveApp('reconcile-orphaned-redelivery')!.id; + const request = { + intent: 'fetch' as const, + applicationId, + stackName: 'reconcile-orphaned-redelivery', + trigger: 'webhook' as const, + actor: 'tester', + deliveryId: 'delivery-orphaned', + }; + await svc.reconcile(request); + // Simulate a crash between reservation and settlement: the + // original attempt's reservation survives, but its + // settlement row never got written, and no in-process + // leader remains for it in this fresh call. + DatabaseService.getInstance().getDb() + .prepare("DELETE FROM gitops_history WHERE application_id = ? AND stage = 'source_reconcile_settled'") + .run(applicationId); + mockGitClone.mockClear(); + + const redeliveryResult = await svc.reconcile(request); + + expect(mockGitClone).not.toHaveBeenCalled(); + expect(redeliveryResult.outcome).not.toBe('unknown'); + } finally { + validateSpy.mockRestore(); + } + }); + + it('reports an unknown redelivery result when durable settlement fails', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ sha: 'd1'.repeat(20) }); + await configureGitSource('reconcile-redelivery-settlement-failure'); + const applicationId = liveApp('reconcile-redelivery-settlement-failure')!.id; + const request: ReconcileRequest & { intent: 'fetch'; deliveryId: string } = { + intent: 'fetch', + applicationId, + stackName: 'reconcile-redelivery-settlement-failure', + trigger: 'webhook', + actor: 'tester', + deliveryId: 'delivery-settlement-failure', + }; + GitOpsTransitions.getInstance().reserveReconcileAttempt(applicationId, { + operationId: deliveryKey('webhook', 'fetch', request.deliveryId), + actor: request.actor, + trigger: request.trigger, + at: Date.now(), + }); + mockGitClone.mockClear(); + const settleSpy = vi.spyOn(GitOpsTransitions.prototype, 'settleReconcileAttempt') + .mockImplementationOnce(() => { throw new Error('simulated redelivery settlement failure'); }); + + try { + const result = await svc.reconcile(request); + + expect(mockGitClone).not.toHaveBeenCalled(); + expect(result).toEqual({ + outcome: 'unknown', + reason: 'This attempt could not be durably resolved.', + nextAction: 'none', + }); + expect(unsettledAttempts(applicationId)).toHaveLength(1); + } finally { + settleSpy.mockRestore(); + } + }); + + it('resolves a redelivery from its own settled history rather than an unrelated leader that happens to be running under the shared coalesce key', async () => { + const sha = 'cececececececececececececececececececece'; + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha }); + const svc = GitSourceService.getInstance(); + await configureGitSource('reconcile-redelivery-vs-unrelated-leader'); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + try { + const applicationId = liveApp('reconcile-redelivery-vs-unrelated-leader')!.id; + const request = { + intent: 'fetch' as const, + applicationId, + stackName: 'reconcile-redelivery-vs-unrelated-leader', + trigger: 'webhook' as const, + actor: 'tester', + deliveryId: 'delivery-vs-unrelated-leader', + }; + const first = await svc.reconcile(request); + + // A completely unrelated fetch (a plain manual pull, no + // deliveryId) becomes the in-process leader registered + // under this application's shared fetch coalesce key, + // which does not vary by deliveryId or trigger. + const releaseClone = gatedClone(); + mockGitLog.mockResolvedValue([{ oid: 'dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf' }]); + const unrelatedPull = svc.pull('reconcile-redelivery-vs-unrelated-leader'); + + // A redelivery of the original event arrives while that + // unrelated pull is still running: it must resolve from its + // own settled history, not from the unrelated in-flight + // leader it happens to find under the shared key. + const redelivery = await svc.reconcile(request); + releaseClone(); + await unrelatedPull; + + expect(redelivery).toEqual(first); + } finally { + validateSpy.mockRestore(); + } + }); + + it('does not re-execute a redelivered request carrying the same deliveryId, and returns the original settled result', async () => { + const sha = 'eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea'; + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha }); + const svc = GitSourceService.getInstance(); + await svc.upsert({ + stackName: 'reconcile-dedupe', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + try { + const applicationId = liveApp('reconcile-dedupe')!.id; + const request = { + intent: 'fetch' as const, + applicationId, + stackName: 'reconcile-dedupe', + trigger: 'webhook' as const, + actor: 'tester', + deliveryId: 'delivery-1', + }; + const first = await svc.reconcile(request); + mockGitClone.mockClear(); + const second = await svc.reconcile(request); + + expect(mockGitClone).not.toHaveBeenCalled(); + expect(second).toEqual(first); + expect(settledAttempts(applicationId)).toHaveLength(1); + } finally { + validateSpy.mockRestore(); + } + }); + + it('joins a concurrent redelivery of the same deliveryId to the in-flight leader, returning the leader\'s real result rather than a stale snapshot', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'e0'.repeat(20) }); + await svc.upsert({ + stackName: 'reconcile-concurrent-redelivery', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + const releaseClone = gatedClone(); + mockGitLog.mockResolvedValue([{ oid: 'e1'.repeat(20) }]); + + try { + const applicationId = liveApp('reconcile-concurrent-redelivery')!.id; + const request = { + intent: 'fetch' as const, + applicationId, + stackName: 'reconcile-concurrent-redelivery', + trigger: 'webhook' as const, + actor: 'tester', + deliveryId: 'delivery-race', + }; + const first = svc.reconcile(request); + const redelivery = svc.reconcile(request); + releaseClone(); + const [firstResult, redeliveryResult] = await Promise.all([first, redelivery]); + + expect(mockGitClone).toHaveBeenCalledTimes(1); + // The redelivery must report the leader's real post-fetch + // outcome, not a snapshot of the row from before the fetch + // ran (which would still show no candidate staged). + expect(redeliveryResult).toEqual(firstResult); + expect(firstResult.outcome).toBe('candidate_already_fetched'); + } finally { + validateSpy.mockRestore(); + } + }); + + it('logs and reports unknown rather than throwing when a settled attempt\'s stored result is corrupted', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'ef'.repeat(20) }); + await svc.upsert({ + stackName: 'reconcile-corrupt-settled', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + try { + const applicationId = liveApp('reconcile-corrupt-settled')!.id; + const request = { + intent: 'fetch' as const, + applicationId, + stackName: 'reconcile-corrupt-settled', + trigger: 'webhook' as const, + actor: 'tester', + deliveryId: 'delivery-corrupt', + }; + await svc.reconcile(request); + DatabaseService.getInstance().getDb() + .prepare("UPDATE gitops_history SET after_json = ? WHERE application_id = ? AND stage = 'source_reconcile_settled'") + .run('not valid json{{{', applicationId); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + + const result = await svc.reconcile(request); + + expect(result.outcome).toBe('unknown'); + expect(errorSpy).toHaveBeenCalled(); + errorSpy.mockRestore(); + } finally { + validateSpy.mockRestore(); + } + }); + + it('joins a same-delivery apply under a different coalesce key to the real in-flight leader rather than settling a stale snapshot', async () => { + const svc = GitSourceService.getInstance(); + const sha = 'a6'.repeat(20); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha }); + await svc.upsert({ + stackName: 'reconcile-apply-delivery-race', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + await svc.pull('reconcile-apply-delivery-race'); + const applicationId = liveApp('reconcile-apply-delivery-race')!.id; + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + let releaseSave!: () => void; + const gate = new Promise((resolve) => { releaseSave = resolve; }); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockImplementation(async () => { await gate; }); + + try { + // Two apply requests sharing a delivery id (so their + // operation ids collide) but differing in commitSha, which + // coalesceKey includes for an apply intent -- so they run + // under different coalesce keys despite the shared + // operation id. + const first = svc.reconcile({ + intent: 'apply', + applicationId, + stackName: 'reconcile-apply-delivery-race', + trigger: 'webhook', + actor: 'tester', + commitSha: sha, + planFingerprint: '', + deploy: false, + deliveryId: 'shared-delivery', + }); + const second = svc.reconcile({ + intent: 'apply', + applicationId, + stackName: 'reconcile-apply-delivery-race', + trigger: 'webhook', + actor: 'tester', + commitSha: 'ff'.repeat(20), + planFingerprint: 'different-fingerprint', + deploy: true, + deliveryId: 'shared-delivery', + }); + releaseSave(); + const [firstResult, secondResult] = await Promise.all([first, second]); + + // The second request must never have run its own apply + // (a different, unstaged commitSha would fail on its own + // terms); it must instead have joined the first's real + // execution and returned its actual result. + expect(secondResult).toEqual(firstResult); + expect(saveSpy).toHaveBeenCalledTimes(1); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + } + }); + + it('coalesces a concurrent manual apply and webhook apply into one promotion and one normalized result', async () => { + const sha = 'a7'.repeat(20); + const svc = await seedPending('apply-reconcile-coalesce', 'services:\n x:\n image: alpine\n', sha); + const applicationId = liveApp('apply-reconcile-coalesce')!.id; + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + let releaseSave!: () => void; + const saveGate = new Promise((resolve) => { releaseSave = resolve; }); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockImplementation(async () => { await saveGate; }); + const priorSettlements = new Set(settledAttempts(applicationId).map((row) => row.operation_id)); + + try { + const manualApply = svc.apply('apply-reconcile-coalesce', sha, SKIP_PLAN_FINGERPRINT); + const controllerApply = svc.reconcile({ + intent: 'apply', + applicationId, + stackName: 'apply-reconcile-coalesce', + trigger: 'webhook', + actor: 'system:webhook', + commitSha: sha, + planFingerprint: '', + deploy: false, + deliveryId: 'apply-cross-producer', + }); + releaseSave(); + const [manualResult, reconcileResult] = await Promise.all([manualApply, controllerApply]); + + expect(manualResult.applied).toBe(true); + expect(saveSpy).toHaveBeenCalledTimes(1); + const settled = settledAttempts(applicationId) + .filter((row) => !priorSettlements.has(row.operation_id)) + .map((row) => JSON.parse(row.after_json)); + expect(settled).toHaveLength(2); + expect(settled).toEqual([reconcileResult, reconcileResult]); + expect(unsettledAttempts(applicationId)).toHaveLength(0); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + } + }); + + it('does not let a fingerprint-enforcing manual apply borrow an internal apply that bypasses the fingerprint check', async () => { + const sha = 'b7'.repeat(20); + const svc = await seedPending('apply-fingerprint-mode-isolation', 'services:\n x:\n image: alpine\n', sha); + const applicationId = liveApp('apply-fingerprint-mode-isolation')!.id; + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + let releaseSave!: () => void; + const saveGate = new Promise((resolve) => { releaseSave = resolve; }); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockImplementation(async () => { await saveGate; }); + + try { + const internalApply = svc.reconcile({ + intent: 'apply', + applicationId, + stackName: 'apply-fingerprint-mode-isolation', + trigger: 'webhook', + actor: 'system:webhook', + commitSha: sha, + planFingerprint: 'stale-fingerprint', + deploy: false, + deliveryId: 'apply-fingerprint-mode-isolation', + }); + await vi.waitFor(() => expect(saveSpy).toHaveBeenCalledTimes(1)); + const manualApply = svc.apply('apply-fingerprint-mode-isolation', sha, { + planFingerprint: 'stale-fingerprint', + deploy: false, + }); + releaseSave(); + + await expect(internalApply).resolves.toMatchObject({ outcome: expect.any(String) }); + await expect(manualApply).rejects.toMatchObject({ code: expect.any(String) }); + expect(saveSpy).toHaveBeenCalledTimes(1); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + } + }); + + it('resolves an apply redelivery from its own history while an unrelated matching apply is running', async () => { + const sha = 'a8'.repeat(20); + const svc = await seedPending('apply-redelivery-vs-unrelated-leader', 'services:\n x:\n image: alpine\n', sha); + const applicationId = liveApp('apply-redelivery-vs-unrelated-leader')!.id; + const originalRequest: ReconcileRequest & { intent: 'apply' } = { + intent: 'apply' as const, + applicationId, + stackName: 'apply-redelivery-vs-unrelated-leader', + trigger: 'webhook' as const, + actor: 'tester', + commitSha: sha, + planFingerprint: '', + deploy: true, + deliveryId: 'original-delivery', + }; + const originalOperationId = deliveryKey('webhook', 'apply', 'original-delivery'); + const originalResult = { + outcome: 'recovery_required' as const, + reason: 'The source applied, but the deploy failed: first delivery deploy failed', + nextAction: 'view_target_results' as const, + }; + const tx = GitOpsTransitions.getInstance(); + const envelope = { operationId: originalOperationId, actor: 'tester', trigger: 'webhook', at: Date.now() }; + tx.reserveReconcileAttempt(applicationId, envelope); + tx.settleReconcileAttempt(applicationId, envelope, originalResult); + + type ApplyExecution = { + status: 'fulfilled'; + value: { applied: boolean; deployed: boolean }; + result: { outcome: 'no_source_change'; reason: string; nextAction: 'none' }; + }; + type ApplyCompletion = { execution: ApplyExecution; settled: boolean }; + let releaseLeader!: (completion: ApplyCompletion) => void; + const leaderPromise = new Promise((resolve) => { releaseLeader = resolve; }); + const inFlightApplies = (svc as unknown as { + inFlightApplies: Map }>; + }).inFlightApplies; + const executionKey = `${coalesceKey(originalRequest)}:fingerprint-optional`; + inFlightApplies.set(executionKey, { + operationId: 'unrelated-operation', + promise: leaderPromise, + }); + + try { + const redeliveryResult = await svc.reconcile(originalRequest); + expect(redeliveryResult).toEqual(originalResult); + } finally { + inFlightApplies.delete(executionKey); + releaseLeader({ + settled: true, + execution: { + status: 'fulfilled', + value: { applied: true, deployed: true }, + result: { outcome: 'no_source_change', reason: 'Unrelated leader finished.', nextAction: 'none' }, + }, + }); + } + }); + }); + + describe('dispatchAcceptedGeneration', () => { + const directContext = { targetMode: 'direct', nodeId: null, bindingRevision: null } as const; + const manualDispatch = { trigger: 'manual', actor: 'tester' } as const; + + async function acceptedGenerationFor(stackName: string) { + const { buildAcceptedGeneration } = await import('../services/gitops/handoff'); + const app = liveApp(stackName)!; + const row = GitOpsStore.getInstance().getGeneration(app.candidate_generation_id!)!; + return buildAcceptedGeneration(row); + } + + it('dispatches a direct-mode generation by driving the existing apply path', async () => { + const sha = 'd1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1'; + const svc = await seedPending('dispatch-direct', 'services:\n x:\n image: alpine\n', sha); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + + try { + const generation = await acceptedGenerationFor('dispatch-direct'); + const result = await svc.dispatchAcceptedGeneration(generation, directContext, manualDispatch); + expect(result).toEqual({ status: 'dispatched' }); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + } + }); + + it('blocks a blueprint-mode generation by delegating to BlueprintTargetAdapter', async () => { + const sha = 'd2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2'; + const svc = await seedPending('dispatch-blueprint', 'services:\n x:\n image: alpine\n', sha); + const generation = await acceptedGenerationFor('dispatch-blueprint'); + + const result = await svc.dispatchAcceptedGeneration( + generation, + { targetMode: 'blueprint', nodeId: 1, bindingRevision: 'rev-1' }, + manualDispatch, + ); + + expect(result).toEqual({ + status: 'blocked', + reason: 'Blueprint rollout orchestration is not yet implemented.', + }); + }); + + it('blocks and forwards the reason when the underlying apply fails', async () => { + const sha = 'd3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3'; + const svc = await seedPending('dispatch-apply-fails', 'services:\n x:\n image: alpine\n', sha); + const generation = await acceptedGenerationFor('dispatch-apply-fails'); + const staleGeneration = { ...generation, commitSha: 'ffffffffffffffffffffffffffffffffffffffff' }; + + const result = await svc.dispatchAcceptedGeneration(staleGeneration, directContext, manualDispatch); + + expect(result).toEqual({ + status: 'blocked', + reason: expect.stringMatching(/pending commit has changed/i), + }); + }); + + it('blocks a direct-mode dispatch when the generation names an application that no longer exists', async () => { + const sha = 'd4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4'; + const svc = await seedPending('dispatch-no-stack', 'services:\n x:\n image: alpine\n', sha); + const generation = await acceptedGenerationFor('dispatch-no-stack'); + const orphanGeneration = { ...generation, applicationId: 'no-such-application' }; + + const result = await svc.dispatchAcceptedGeneration(orphanGeneration, directContext, manualDispatch); + + expect(result).toEqual({ + status: 'blocked', + reason: expect.stringMatching(/no direct stack is bound/i), + }); + }); + + it('honors an auto_deploy_on_apply source setting by requesting a deploy on dispatch', async () => { + const sha = 'd5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5'; + const svc = await seedPending('dispatch-auto-deploy', 'services:\n x:\n image: alpine\n', sha); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const { ComposeService } = await import('../services/ComposeService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); + DatabaseService.getInstance().getDb() + .prepare('UPDATE stack_git_sources SET auto_deploy_on_apply = 1 WHERE stack_name = ?') + .run('dispatch-auto-deploy'); + + try { + const generation = await acceptedGenerationFor('dispatch-auto-deploy'); + const result = await svc.dispatchAcceptedGeneration(generation, directContext, manualDispatch); + expect(result).toEqual({ status: 'dispatched' }); + expect(deploySpy).toHaveBeenCalled(); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('blocks, rather than reporting dispatched, when an auto_deploy_on_apply dispatch applies but the deploy fails', async () => { + const sha = 'd6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6'; + const svc = await seedPending('dispatch-auto-deploy-fails', 'services:\n x:\n image: alpine\n', sha); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const { ComposeService } = await import('../services/ComposeService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockRejectedValue( + new Error('compose up failed: docker unavailable'), + ); + DatabaseService.getInstance().getDb() + .prepare('UPDATE stack_git_sources SET auto_deploy_on_apply = 1 WHERE stack_name = ?') + .run('dispatch-auto-deploy-fails'); + + try { + const generation = await acceptedGenerationFor('dispatch-auto-deploy-fails'); + const result = await svc.dispatchAcceptedGeneration(generation, directContext, manualDispatch); + expect(result).toEqual({ status: 'blocked', reason: expect.stringMatching(/deploy/i) }); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + }); + + describe('suspend / resume / retry', () => { + it('suspends an active source and reflects it in the reconcile result', async () => { + const svc = await seedPending('suspend-basic', 'services:\n x:\n image: alpine\n', 's1s1s1s1s1s1s1s1s1s1s1s1s1s1s1s1s1s1s1s1'); + + const result = await svc.suspend('suspend-basic', { actor: 'tester', reason: 'maintenance window' }); + + expect(result.outcome).toBe('suspended'); + expect(result.reason).toMatch(/maintenance window/i); + const app = liveApp('suspend-basic'); + expect(app?.suspended_at).toBeTruthy(); + expect(app?.source_suspended_reason).toBe('maintenance window'); + }); + + it('suspends with a default reason when none is given', async () => { + const svc = await seedPending('suspend-default-reason', 'services:\n x:\n image: alpine\n', 's2s2s2s2s2s2s2s2s2s2s2s2s2s2s2s2s2s2s2s2'); + + const result = await svc.suspend('suspend-default-reason', { actor: 'tester' }); + + expect(result.outcome).toBe('suspended'); + expect(liveApp('suspend-default-reason')?.source_suspended_reason).toBe('Suspended by operator.'); + }); + + it('falls back to the default reason when only whitespace is given', async () => { + const svc = await seedPending('suspend-whitespace-reason', 'services:\n x:\n image: alpine\n', 's9s9s9s9s9s9s9s9s9s9s9s9s9s9s9s9s9s9s9s9'); + + await svc.suspend('suspend-whitespace-reason', { actor: 'tester', reason: ' ' }); + + expect(liveApp('suspend-whitespace-reason')?.source_suspended_reason).toBe('Suspended by operator.'); + }); + + it('reports unknown when suspending a stack with no GitOps application', async () => { + const result = await GitSourceService.getInstance().suspend('suspend-no-app', { actor: 'tester' }); + + expect(result.outcome).toBe('unknown'); + }); + + it('surfaces a real error, rather than a silent no-op, when suspending an application that is not live', async () => { + const config: DirectSourceConfig = { + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + }; + GitOpsStore.getInstance().insertApplication(buildDirectApplicationRow({ + id: newGitOpsId(), + stackName: 'suspend-creating-app', + config, + identity: directSourceIdentity(config), + lifecycleStatus: 'creating', + at: Date.now(), + })); + + await expect(GitSourceService.getInstance().suspend('suspend-creating-app', { actor: 'tester' })) + .rejects.toMatchObject({ code: 'OPERATION_IN_FLIGHT' }); + }); + + it('actually stops a fetch, not just the projected outcome, once suspended', async () => { + const svc = await seedPending('suspend-blocks-fetch', 'services:\n x:\n image: alpine\n', 's6s6s6s6s6s6s6s6s6s6s6s6s6s6s6s6s6s6s6s6'); + await svc.suspend('suspend-blocks-fetch', { actor: 'tester', reason: 'pausing' }); + mockGitClone.mockClear(); + + await expect(svc.pull('suspend-blocks-fetch')).rejects.toMatchObject({ code: 'OPERATION_IN_FLIGHT' }); + + expect(mockGitClone).not.toHaveBeenCalled(); + }); + + it('actually stops an apply once suspended, even for a pending commit fetched before suspension', async () => { + const sha = 's7s7s7s7s7s7s7s7s7s7s7s7s7s7s7s7s7s7s7s7'; + const svc = await seedPending('suspend-blocks-apply', 'services:\n x:\n image: alpine\n', sha); + await svc.suspend('suspend-blocks-apply', { actor: 'tester', reason: 'pausing' }); + const { FileSystemService } = await import('../services/FileSystemService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + + try { + await expect(svc.apply('suspend-blocks-apply', sha, SKIP_PLAN_FINGERPRINT)) + .rejects.toMatchObject({ code: 'OPERATION_IN_FLIGHT' }); + expect(saveSpy).not.toHaveBeenCalled(); + } finally { + saveSpy.mockRestore(); + } + }); + + it('a webhook delivery to a suspended source is skipped, not reported as a failed pull', async () => { + const svc = await seedPending('suspend-webhook', 'services:\n x:\n image: alpine\n', 'scscscscscscscscscscscscscscscscscscscsc'); + await svc.suspend('suspend-webhook', { actor: 'tester', reason: 'pausing' }); + mockGitClone.mockClear(); + const activitySpy = vi.spyOn(DatabaseService.getInstance(), 'addNotificationHistory'); + + try { + const result = await svc.handleWebhookPull('suspend-webhook', true); + + expect(result.status).toBe('skipped'); + expect(mockGitClone).not.toHaveBeenCalled(); + expect(activitySpy).not.toHaveBeenCalled(); + } finally { + activitySpy.mockRestore(); + } + }); + + it('resumes a suspended source', async () => { + const svc = await seedPending('resume-basic', 'services:\n x:\n image: alpine\n', 's3s3s3s3s3s3s3s3s3s3s3s3s3s3s3s3s3s3s3s3'); + await svc.suspend('resume-basic', { actor: 'tester', reason: 'pausing' }); + + const result = await svc.resume('resume-basic', { actor: 'tester' }); + + expect(result.outcome).not.toBe('suspended'); + const app = liveApp('resume-basic'); + expect(app?.suspended_at).toBeNull(); + expect(app?.source_suspended_reason).toBeNull(); + }); + + it('resuming a source that is not suspended is a harmless no-op, not an error', async () => { + const svc = await seedPending('resume-noop', 'services:\n x:\n image: alpine\n', 's8s8s8s8s8s8s8s8s8s8s8s8s8s8s8s8s8s8s8s8'); + + const result = await svc.resume('resume-noop', { actor: 'tester' }); + + expect(result.outcome).toBe('candidate_already_fetched'); + }); + + it('pulling and applying again succeeds once a suspended source is resumed', async () => { + const sha = 'sbsbsbsbsbsbsbsbsbsbsbsbsbsbsbsbsbsbsbsb'; + const svc = await seedPending('suspend-resume-roundtrip', 'services:\n x:\n image: alpine\n', sha); + await svc.suspend('suspend-resume-roundtrip', { actor: 'tester', reason: 'pausing' }); + await expect(svc.pull('suspend-resume-roundtrip')).rejects.toMatchObject({ code: 'OPERATION_IN_FLIGHT' }); + await expect(svc.apply('suspend-resume-roundtrip', sha, SKIP_PLAN_FINGERPRINT)) + .rejects.toMatchObject({ code: 'OPERATION_IN_FLIGHT' }); + + await svc.resume('suspend-resume-roundtrip', { actor: 'tester' }); + const pullResult = await svc.pull('suspend-resume-roundtrip'); + expect(pullResult.candidateReady).toBe(true); + + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + try { + const applyResult = await svc.apply('suspend-resume-roundtrip', sha, SKIP_PLAN_FINGERPRINT); + expect(applyResult.applied).toBe(true); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + } + }); + + it('retries by driving a fresh fetch-intent reconcile', async () => { + const svc = await seedPending('retry-basic', 'services:\n x:\n image: alpine\n', 's5s5s5s5s5s5s5s5s5s5s5s5s5s5s5s5s5s5s5s5'); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + + try { + const result = await svc.retry('retry-basic', { actor: 'tester' }); + expect(result.outcome).toBe('candidate_already_fetched'); + expect(reconcileSpy).toHaveBeenCalledWith(expect.objectContaining({ trigger: 'retry', intent: 'fetch' })); + } finally { + validateSpy.mockRestore(); + reconcileSpy.mockRestore(); + } + }); + + it('reports unknown when retrying a stack with no GitOps application', async () => { + const result = await GitSourceService.getInstance().retry('retry-no-app', { actor: 'tester' }); + + expect(result.outcome).toBe('unknown'); + }); + + it('retrying a suspended source reports suspended, not a generic unknown', async () => { + const svc = await seedPending('retry-while-suspended', 'services:\n x:\n image: alpine\n', 'sasasasasasasasasasasasasasasasasasasasa'); + await svc.suspend('retry-while-suspended', { actor: 'tester', reason: 'pausing' }); + + const result = await svc.retry('retry-while-suspended', { actor: 'tester' }); + + expect(result.outcome).toBe('suspended'); + expect(result.nextAction).toBe('resume'); + }); + }); + it('throws when pending has been cleared between pull and apply', async () => { const svc = await seedPending('apply-cleared', 'services:\n x:\n image: alpine\n', 'aaaa111aaaa111aaaa111aaaa111aaaa111aaaa1'); DatabaseService.getInstance().clearGitSourcePending('apply-cleared'); @@ -2292,6 +4690,13 @@ describe('GitSourceService.apply', () => { const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockRejectedValue( new Error('compose up failed: docker unavailable'), ); + const applicationId = liveApp('apply-deploy-fail')!.id; + const priorSettlements = new Set( + DatabaseService.getInstance().getDb() + .prepare("SELECT operation_id FROM gitops_history WHERE application_id = ? AND stage = 'source_reconcile_settled'") + .all(applicationId) + .map((row) => (row as { operation_id: string }).operation_id), + ); try { // Assert the return SHAPE: apply must not throw, deployError must @@ -2306,6 +4711,16 @@ describe('GitSourceService.apply', () => { const row = DatabaseService.getInstance().getGitSource('apply-deploy-fail'); expect(row?.last_applied_commit_sha).toBe(sha); expect(row?.pending_commit_sha).toBeNull(); + + const settled = DatabaseService.getInstance().getDb() + .prepare("SELECT operation_id, after_json FROM gitops_history WHERE application_id = ? AND stage = 'source_reconcile_settled'") + .all(applicationId) + .filter((item) => !priorSettlements.has((item as { operation_id: string }).operation_id)) as { after_json: string }[]; + expect(settled).toHaveLength(1); + expect(JSON.parse(settled[0].after_json)).toMatchObject({ + outcome: 'recovery_required', + nextAction: 'view_target_results', + }); } finally { validateSpy.mockRestore(); saveSpy.mockRestore(); @@ -2502,6 +4917,306 @@ describe('GitSourceService.apply', () => { }); }); +describe('GitSourceService.recoverUnsettledReconcileAttempts', () => { + it('settles a follower from its leader\'s stored result rather than deriving independently, when only the follower is unsettled', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'a1'.repeat(20) }); + await svc.upsert({ + stackName: 'recover-leader-follower', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('recover-leader-follower')!.id; + const tx = GitOpsTransitions.getInstance(); + + tx.reserveReconcileAttempt(applicationId, { operationId: 'leader-op', actor: 'tester', trigger: 'manual', at: Date.now() }); + tx.reserveReconcileAttempt( + applicationId, + { operationId: 'follower-op', actor: 'tester', trigger: 'manual', at: Date.now() + 1 }, + 'leader-op', + ); + // The leader settled with a specific result before the crash that + // orphaned only the follower. This must not match whatever + // independent derivation from current row state would produce, or + // the test cannot tell a real leader-link recovery from a + // coincidence. + tx.settleReconcileAttempt(applicationId, { operationId: 'leader-op', actor: 'tester', trigger: 'manual', at: Date.now() }, { + outcome: 'blocked', + reason: 'a specific reason only the leader would know', + nextAction: 'resolve_conflict', + }); + + await svc.recoverUnsettledReconcileAttempts(); + + const followerSettled = GitOpsStore.getInstance().getSettledAttempt(applicationId, 'follower-op'); + expect(followerSettled).toBeDefined(); + expect(JSON.parse(followerSettled!.after_json)).toMatchObject({ + outcome: 'blocked', + reason: 'a specific reason only the leader would know', + nextAction: 'resolve_conflict', + }); + }); + + it('leaves no unsettled follower after restart when both leader and follower crashed before settling', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'a2'.repeat(20) }); + await svc.upsert({ + stackName: 'recover-both-unsettled', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('recover-both-unsettled')!.id; + const tx = GitOpsTransitions.getInstance(); + tx.reserveReconcileAttempt(applicationId, { operationId: 'leader-op-2', actor: 'tester', trigger: 'manual', at: Date.now() }); + tx.reserveReconcileAttempt( + applicationId, + { operationId: 'follower-op-2', actor: 'tester', trigger: 'manual', at: Date.now() + 1 }, + 'leader-op-2', + ); + + await svc.recoverUnsettledReconcileAttempts(); + + expect(GitOpsStore.getInstance().getSettledAttempt(applicationId, 'leader-op-2')).toBeDefined(); + expect(GitOpsStore.getInstance().getSettledAttempt(applicationId, 'follower-op-2')).toBeDefined(); + expect(GitOpsStore.getInstance().listUnsettledReconcileAttempts().some((r) => r.application_id === applicationId)).toBe(false); + }); + + it('does not settle a follower independently when its leader also fails to settle in this same recovery pass', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'a5'.repeat(20) }); + await svc.upsert({ + stackName: 'recover-leader-also-fails', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('recover-leader-also-fails')!.id; + const tx = GitOpsTransitions.getInstance(); + tx.reserveReconcileAttempt(applicationId, { operationId: 'leader-op-3', actor: 'tester', trigger: 'manual', at: Date.now() }); + tx.reserveReconcileAttempt( + applicationId, + { operationId: 'follower-op-3', actor: 'tester', trigger: 'manual', at: Date.now() + 1 }, + 'leader-op-3', + ); + // The application vanishes before recovery runs, so the leader's + // own settlement (pass 1, independent branch) will itself throw + // and fail, not just "not yet have happened." + DatabaseService.getInstance().getDb().prepare('DELETE FROM gitops_applications WHERE id = ?').run(applicationId); + + await svc.recoverUnsettledReconcileAttempts(); + + // Neither settles: the follower must not be given an independently + // guessed result while its leader's own fate is still unresolved, + // even though the leader failed rather than merely being deferred. + expect(GitOpsStore.getInstance().getSettledAttempt(applicationId, 'leader-op-3')).toBeUndefined(); + expect(GitOpsStore.getInstance().getSettledAttempt(applicationId, 'follower-op-3')).toBeUndefined(); + }); + + it.each([ + ['malformed JSON', '{invalid'], + ['a non-string follower link', JSON.stringify({ followerOf: 123 })], + ])('leaves a follower unsettled when its reservation contains %s', async (_caseName, corruptAfterJson) => { + const svc = GitSourceService.getInstance(); + const stackName = `recover-corrupt-follower-${crypto.randomUUID()}`; + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'a6'.repeat(20) }); + await svc.upsert({ + stackName, + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication(stackName)!.id; + const tx = GitOpsTransitions.getInstance(); + tx.reserveReconcileAttempt(applicationId, { operationId: 'corrupt-leader', actor: 'tester', trigger: 'manual', at: Date.now() }); + tx.reserveReconcileAttempt( + applicationId, + { operationId: 'corrupt-follower', actor: 'tester', trigger: 'manual', at: Date.now() + 1 }, + 'corrupt-leader', + ); + DatabaseService.getInstance().getDb() + .prepare("UPDATE gitops_history SET after_json = ? WHERE application_id = ? AND operation_id = ? AND stage = 'source_reconcile_started'") + .run(corruptAfterJson, applicationId, 'corrupt-follower'); + const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + try { + await svc.recoverUnsettledReconcileAttempts(); + + expect(GitOpsStore.getInstance().getSettledAttempt(applicationId, 'corrupt-leader')).toBeDefined(); + expect(GitOpsStore.getInstance().getSettledAttempt(applicationId, 'corrupt-follower')).toBeUndefined(); + const unsettled = GitOpsStore.getInstance().listUnsettledReconcileAttempts(); + expect(unsettled.some((row) => row.application_id === applicationId && row.operation_id === 'corrupt-follower')).toBe(true); + } finally { + consoleSpy.mockRestore(); + } + }); + + it('drains the full backlog across multiple pages even when an earlier row can never be recovered', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'a3'.repeat(20) }); + await svc.upsert({ + stackName: 'recover-paginated-unrecoverable', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const unrecoverableAppId = GitOpsStore.getInstance().getLiveDirectApplication('recover-paginated-unrecoverable')!.id; + const tx = GitOpsTransitions.getInstance(); + // The oldest unsettled row's application is gone, so it can never + // settle. With a page size of 1, a query that keeps returning "the + // oldest still-unsettled row" would return only this one forever. + tx.reserveReconcileAttempt(unrecoverableAppId, { operationId: 'op-unrecoverable', actor: 'tester', trigger: 'manual', at: Date.now() }); + DatabaseService.getInstance().getDb().prepare('DELETE FROM gitops_applications WHERE id = ?').run(unrecoverableAppId); + + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'a4'.repeat(20) }); + await svc.upsert({ + stackName: 'recover-paginated-recoverable', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const recoverableAppId = GitOpsStore.getInstance().getLiveDirectApplication('recover-paginated-recoverable')!.id; + tx.reserveReconcileAttempt(recoverableAppId, { operationId: 'op-recoverable', actor: 'tester', trigger: 'manual', at: Date.now() + 1 }); + + await svc.recoverUnsettledReconcileAttempts(1); + + expect(GitOpsStore.getInstance().getSettledAttempt(recoverableAppId, 'op-recoverable')).toBeDefined(); + }); + + it('settles an attempt left unsettled by a crash, without re-executing a fetch', async () => { + const sha = 'eb'.repeat(20); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha }); + const svc = GitSourceService.getInstance(); + await svc.upsert({ + stackName: 'recover-unsettled', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('recover-unsettled')!.id; + GitOpsTransitions.getInstance().reserveReconcileAttempt(applicationId, { + operationId: 'orphaned-op-1', actor: 'tester', trigger: 'poll', at: Date.now(), + }); + mockGitClone.mockClear(); + + await svc.recoverUnsettledReconcileAttempts(); + + expect(mockGitClone).not.toHaveBeenCalled(); + const settled = GitOpsStore.getInstance().getSettledAttempt(applicationId, 'orphaned-op-1'); + expect(settled).toBeDefined(); + }); + + it('leaves a different application unaffected when only one has an unsettled attempt', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'ec'.repeat(20) }); + await svc.upsert({ + stackName: 'recover-clean', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('recover-clean')!.id; + + await expect(svc.recoverUnsettledReconcileAttempts()).resolves.toBeUndefined(); + + expect(GitOpsStore.getInstance().listUnsettledReconcileAttempts().some((r) => r.application_id === applicationId)).toBe(false); + }); + + it('does not let one attempt whose application vanished block recovery of another, older, still-real attempt', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'ed'.repeat(20) }); + await svc.upsert({ + stackName: 'recover-poisoned', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const poisonedAppId = GitOpsStore.getInstance().getLiveDirectApplication('recover-poisoned')!.id; + GitOpsTransitions.getInstance().reserveReconcileAttempt(poisonedAppId, { + operationId: 'poisoned-op-1', actor: 'tester', trigger: 'poll', at: Date.now(), + }); + // Simulate the application row vanishing between listing and processing. + DatabaseService.getInstance().getDb().prepare('DELETE FROM gitops_applications WHERE id = ?').run(poisonedAppId); + + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'ee'.repeat(20) }); + await svc.upsert({ + stackName: 'recover-healthy', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const healthyAppId = GitOpsStore.getInstance().getLiveDirectApplication('recover-healthy')!.id; + GitOpsTransitions.getInstance().reserveReconcileAttempt(healthyAppId, { + operationId: 'healthy-op-1', actor: 'tester', trigger: 'poll', at: Date.now(), + }); + + await expect(svc.recoverUnsettledReconcileAttempts()).resolves.toBeUndefined(); + + expect(GitOpsStore.getInstance().getSettledAttempt(healthyAppId, 'healthy-op-1')).toBeDefined(); + }); +}); + describe('GitSourceService DB normalization (compose_paths back-compat)', () => { it('reads back [compose_path] when a row stores compose_paths as null (legacy)', async () => { mockSuccessfulClone({ composePath: 'stacks/web/compose.yaml' }); @@ -3123,28 +5838,8 @@ describe('GitSourceService legacy pending apply (migration path)', () => { const { FileSystemService } = await import('../services/FileSystemService'); const fsSvc = FileSystemService.getInstance(); await fsSvc.createStack('legacy-apply'); - db.upsertGitSource({ - stack_name: 'legacy-apply', - repo_url: 'https://github.com/example/repo.git', - branch: 'main', - compose_path: 'compose.yaml', - compose_paths: ['compose.yaml'], - context_dir: null, - sync_env: false, - env_path: null, - auth_type: 'none', - encrypted_token: null, encrypted_deploy_key: null, ssh_known_hosts_entry: null, ssh_host_key_fingerprint: null, - encrypted_ca_bundle: null, - auto_apply_on_webhook: false, - auto_deploy_on_apply: false, - last_applied_commit_sha: null, - last_applied_content_hash: null, - pending_commit_sha: sha, - pending_compose_content: null, - pending_env_content: null, - pending_fetched_at: null, - last_debounce_at: null, - }); + mockSuccessfulClone({ sha }); + await configureGitSource('legacy-apply'); // Seed the v2 blob directly, as a pre-upgrade row would carry it. const svcPriv = svc as unknown as { crypto: { encrypt(s: string): string } }; db.setGitSourcePending('legacy-apply', sha, svcPriv.crypto.encrypt(JSON.stringify({ v: 2, files: [{ path: 'compose.yaml', content: 'services:\n web:\n image: nginx\n' }], contextDir: null })), null); @@ -3402,28 +6097,8 @@ describe('GitSourceService classified plan fingerprint', () => { const db = DatabaseService.getInstance(); const { FileSystemService } = await import('../services/FileSystemService'); await FileSystemService.getInstance().createStack('plan-unavail'); - db.upsertGitSource({ - stack_name: 'plan-unavail', - repo_url: 'https://github.com/example/repo.git', - branch: 'main', - compose_path: 'compose.yaml', - compose_paths: ['compose.yaml'], - context_dir: null, - sync_env: false, - env_path: null, - auth_type: 'none', - encrypted_token: null, encrypted_deploy_key: null, ssh_known_hosts_entry: null, ssh_host_key_fingerprint: null, - encrypted_ca_bundle: null, - auto_apply_on_webhook: false, - auto_deploy_on_apply: false, - last_applied_commit_sha: null, - last_applied_content_hash: null, - pending_commit_sha: sha, - pending_compose_content: null, - pending_env_content: null, - pending_fetched_at: null, - last_debounce_at: null, - }); + mockSuccessfulClone({ sha }); + await configureGitSource('plan-unavail'); const svcPriv = svc as unknown as { crypto: { encrypt(s: string): string } }; db.setGitSourcePending( 'plan-unavail', @@ -3494,3 +6169,180 @@ function seedDirectCandidate(stackName: string): { appId: string; generationId: GitOpsTransitions.getInstance().candidateReady(appId, generationId, false, testEnvelope()); return { appId, generationId }; } + +// ── sweepOrphans claimant fixtures ───────────────────────────────────── + +/** A deterministic 40-char hex sha derived from a short seed. */ +function shaFromSeed(seed: string): string { + return seed.repeat(40).slice(0, 40); +} + +/** + * Create a Git-backed stack, pull one update into it, and backdate the + * resulting candidate directory past the orphan-candidate age threshold so a + * claimant-blind sweep would reap it as stale. Each test then arranges only + * the claimant pointers it exercises before running the sweep. + */ +async function stageStaleCandidate( + stackName: string, + shaSeed: string, +): Promise<{ appId: string; generationId: string; candidateAbs: string }> { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: shaFromSeed(shaSeed) }); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + try { + await svc.createStackFromGit({ + stackName, + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + token: null, + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine:2\n', sha: shaFromSeed(`${shaSeed}a`) }); + await svc.pull(stackName); + } finally { + validateSpy.mockRestore(); + } + const store = GitOpsStore.getInstance(); + const app = store.getLiveDirectApplication(stackName)!; + expect(app.candidate_generation_id).toBeTruthy(); + const generation = store.getGeneration(app.candidate_generation_id!)!; + const candidateAbs = path.join(stackManagedRoot(stackName), generation.candidate_dir); + expect(fs.existsSync(candidateAbs)).toBe(true); + const staleMtime = (Date.now() - 25 * 60 * 60 * 1000) / 1000; + fs.utimesSync(candidateAbs, staleMtime, staleMtime); + return { appId: app.id, generationId: generation.id, candidateAbs }; +} + +/** Drop the application row's own pointer at its staged candidate. */ +function clearCandidatePointer(appId: string): void { + DatabaseService.getInstance().getDb() + .prepare('UPDATE gitops_applications SET candidate_generation_id = NULL WHERE id = ?') + .run(appId); +} + +/** Drop the pending fetch record, the claimant that is independent of the application row. */ +function clearPendingFetch(stackName: string): void { + DatabaseService.getInstance().getDb() + .prepare('UPDATE stack_git_sources SET pending_commit_sha = NULL, pending_compose_content = NULL WHERE stack_name = ?') + .run(stackName); +} + +describe('GitSourceService.sweepOrphans candidate claimant preservation', () => { + it('preserves a stale, complete candidate directory still referenced by the live application\'s candidate_generation_id', async () => { + const { candidateAbs } = await stageStaleCandidate('sweep-claims-candidate', 'c1'); + + await GitSourceService.getInstance().sweepOrphans(); + + expect(fs.existsSync(candidateAbs)).toBe(true); + }); + + it('still reaps a stale, complete candidate directory nothing on the application row references', async () => { + const { appId, candidateAbs } = await stageStaleCandidate('sweep-reaps-unclaimed', 'c2'); + // Nothing on the row, and nothing pending, points at this generation + // any more: the exact "leftover from an earlier attempt" case the + // sweep exists to clean up. + clearCandidatePointer(appId); + clearPendingFetch('sweep-reaps-unclaimed'); + + await GitSourceService.getInstance().sweepOrphans(); + + expect(fs.existsSync(candidateAbs)).toBe(false); + }); + + it('preserves a stale, complete candidate directory referenced only by the pending fetch record, with no generation row yet', async () => { + const { appId, candidateAbs } = await stageStaleCandidate('sweep-claims-pending-only', 'c3'); + // Simulate the row-level pointer being gone (the exact + // fetchedInvalid/no-live-application gap where a generation may never + // have existed at all) while the pending fetch record, written + // independently, still names this candidate. + clearCandidatePointer(appId); + + await GitSourceService.getInstance().sweepOrphans(); + + expect(fs.existsSync(candidateAbs)).toBe(true); + }); + + it('preserves a stale, complete candidate directory referenced only by accepted_generation_id', async () => { + const { appId, generationId, candidateAbs } = await stageStaleCandidate('sweep-claims-accepted-only', 'c4'); + // Simulate the sourceAccepted-committed-but-not-yet-promoted window: + // accepted_generation_id names this candidate, the row's own candidate + // pointer is gone, and the pending record no longer references it + // either, so this pointer alone must be what preserves the directory. + DatabaseService.getInstance().getDb() + .prepare('UPDATE gitops_applications SET accepted_generation_id = ? WHERE id = ?') + .run(generationId, appId); + clearCandidatePointer(appId); + clearPendingFetch('sweep-claims-accepted-only'); + + await GitSourceService.getInstance().sweepOrphans(); + + expect(fs.existsSync(candidateAbs)).toBe(true); + }); + + it('preserves a stale candidate referenced only by an unsettled attempt', async () => { + const { appId, generationId, candidateAbs } = await stageStaleCandidate('sweep-claims-unsettled', 'c5'); + const generation = GitOpsStore.getInstance().getGeneration(generationId)!; + clearCandidatePointer(appId); + clearPendingFetch('sweep-claims-unsettled'); + DatabaseService.getInstance().getDb() + .prepare("DELETE FROM gitops_history WHERE application_id = ? AND operation_id = ? AND stage = 'source_reconcile_settled'") + .run(appId, generation.operation_id); + + await GitSourceService.getInstance().sweepOrphans(); + + expect(fs.existsSync(candidateAbs)).toBe(true); + }); + + it('preserves stale candidates when pending claimant metadata is unreadable', async () => { + const stackName = 'sweep-preserves-unreadable-claims'; + const { appId, candidateAbs } = await stageStaleCandidate(stackName, 'c6'); + clearCandidatePointer(appId); + DatabaseService.getInstance().getDb() + .prepare('UPDATE stack_git_sources SET pending_compose_content = ? WHERE stack_name = ?') + .run('{"v":4 invalid', stackName); + + await GitSourceService.getInstance().sweepOrphans(); + + expect(fs.existsSync(candidateAbs)).toBe(true); + expect(DatabaseService.getInstance().getGitSource(stackName)?.manifest_state).toBe('migration_required'); + }); + + it('preserves stale candidates when a generation claimant pointer is dangling', async () => { + const stackName = 'sweep-preserves-dangling-claim'; + const { appId, candidateAbs } = await stageStaleCandidate(stackName, 'c7'); + clearPendingFetch(stackName); + DatabaseService.getInstance().getDb() + .prepare('UPDATE gitops_applications SET candidate_generation_id = ? WHERE id = ?') + .run('missing-generation', appId); + + await GitSourceService.getInstance().sweepOrphans(); + + expect(fs.existsSync(candidateAbs)).toBe(true); + expect(DatabaseService.getInstance().getGitSource(stackName)?.manifest_state).toBe('migration_required'); + }); + + it('preserves stale candidates when unsettled-attempt claimant lookup fails', async () => { + const stackName = 'sweep-preserves-claim-query-failure'; + const { appId, candidateAbs } = await stageStaleCandidate(stackName, 'c8'); + clearCandidatePointer(appId); + clearPendingFetch(stackName); + const claimantSpy = vi.spyOn(GitOpsStore.prototype, 'listGenerationsClaimedByUnsettledAttempts') + .mockImplementationOnce(() => { throw new Error('simulated claimant query failure'); }); + + try { + await GitSourceService.getInstance().sweepOrphans(); + + expect(fs.existsSync(candidateAbs)).toBe(true); + expect(DatabaseService.getInstance().getGitSource(stackName)?.manifest_state).toBe('migration_required'); + } finally { + claimantSpy.mockRestore(); + } + }); +}); diff --git a/backend/src/__tests__/gitops-approvals.test.ts b/backend/src/__tests__/gitops-approvals.test.ts index 75b0d132..1ab89bab 100644 --- a/backend/src/__tests__/gitops-approvals.test.ts +++ b/backend/src/__tests__/gitops-approvals.test.ts @@ -309,6 +309,10 @@ function directApp(id: string, stackName: string): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, @@ -351,6 +355,12 @@ function generation(id: string, applicationId: string): GitOpsGenerationRow { actor: 'tester', previous_generation_id: null, redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, created_at: 1, }; } diff --git a/backend/src/__tests__/gitops-backoff.test.ts b/backend/src/__tests__/gitops-backoff.test.ts new file mode 100644 index 00000000..79a84f82 --- /dev/null +++ b/backend/src/__tests__/gitops-backoff.test.ts @@ -0,0 +1,157 @@ +/** + * Failure classification and retry-delay coverage for the GitOps source + * controller. classifyFailure is a compile-enforced total map: adding a new + * TransportFailureReason or GitSourceErrorCode without updating the lookup + * tables here fails the build, not just these tests. + */ +import { describe, it, expect } from 'vitest'; +import { + classifyFailure, + nextRetryAt, + DEFAULT_TRANSIENT_CEILING, + LOW_TRANSIENT_CEILING, + type FailureEvidence, +} from '../services/gitops/backoff'; + +function gitSourceError(code: string, transportReason?: string): FailureEvidence { + return { kind: 'git_source_error', code: code as never, transportReason: transportReason as never }; +} + +describe('classifyFailure', () => { + it('classifies a tip-changed race as supersession, not backoff', () => { + expect(classifyFailure(gitSourceError('GIT_ERROR', 'tip-changed'))).toEqual({ class: 'supersession' }); + }); + + it('classifies a standalone timeout reason as transient', () => { + expect(classifyFailure(gitSourceError('NETWORK_TIMEOUT', 'timeout'))) + .toEqual({ class: 'transient', retryCeiling: DEFAULT_TRANSIENT_CEILING }); + }); + + it('classifies DNS resolution failure (target-unresolved) as transient', () => { + expect(classifyFailure(gitSourceError('NETWORK_TIMEOUT', 'target-unresolved'))) + .toEqual({ class: 'transient', retryCeiling: DEFAULT_TRANSIENT_CEILING }); + }); + + it('classifies an exit-coded network timeout as transient', () => { + expect(classifyFailure(gitSourceError('NETWORK_TIMEOUT', 'exit'))) + .toEqual({ class: 'transient', retryCeiling: DEFAULT_TRANSIENT_CEILING }); + }); + + it('classifies an exit-coded rate limit as transient', () => { + expect(classifyFailure(gitSourceError('RATE_LIMITED', 'exit'))) + .toEqual({ class: 'transient', retryCeiling: DEFAULT_TRANSIENT_CEILING }); + }); + + it('classifies an exit-coded unrecognized git error with a low retry ceiling', () => { + expect(classifyFailure(gitSourceError('GIT_ERROR', 'exit'))) + .toEqual({ class: 'transient', retryCeiling: LOW_TRANSIENT_CEILING }); + }); + + it.each(['invalid-url', 'unsafe-target', 'invalid-ref', 'redirect-scope'])( + 'classifies %s as permanent configuration', + (reason) => { + expect(classifyFailure(gitSourceError('GIT_ERROR', reason))).toEqual({ class: 'permanent' }); + }, + ); + + it.each(['git-missing', 'git-old'])('classifies %s as permanent environment', (reason) => { + expect(classifyFailure(gitSourceError('GIT_ERROR', reason))).toEqual({ class: 'permanent' }); + }); + + it('classifies a repository over the size cap as permanent', () => { + expect(classifyFailure(gitSourceError('GIT_ERROR', 'size'))).toEqual({ class: 'permanent' }); + }); + + it.each(['ssh-auth-required'])('classifies %s as permanent authorization', (reason) => { + expect(classifyFailure(gitSourceError('GIT_ERROR', reason))).toEqual({ class: 'permanent' }); + }); + + it.each(['AUTH_FAILED', 'SSH_HOST_KEY_FAILED'])('classifies %s (no transport reason) as permanent', (code) => { + expect(classifyFailure(gitSourceError(code))).toEqual({ class: 'permanent' }); + }); + + it.each(['ref-not-found', 'unsupported-ref'])('classifies %s as permanent configuration', (reason) => { + expect(classifyFailure(gitSourceError('GIT_ERROR', reason))).toEqual({ class: 'permanent' }); + }); + + it.each(['REPO_NOT_FOUND', 'REF_NOT_FOUND', 'REF_DELETED', 'UNSUPPORTED_REF'])( + 'classifies %s (no transport reason) as permanent', + (code) => { + expect(classifyFailure(gitSourceError(code))).toEqual({ class: 'permanent' }); + }, + ); + + it.each(['STALE_PLAN', 'PLAN_BLOCKED', 'PLAN_FINGERPRINT_REQUIRED', 'LEGACY_PENDING', 'PLAN_UNAVAILABLE', 'FILE_NOT_FOUND'])( + 'classifies %s as requiring operator action', + (code) => { + expect(classifyFailure(gitSourceError(code))).toEqual({ class: 'operator_action_required' }); + }, + ); + + it('classifies a conflicting in-flight operation for reconciliation, not blind retry', () => { + expect(classifyFailure(gitSourceError('OPERATION_IN_FLIGHT'))).toEqual({ class: 'reconcile' }); + }); + + it('classifies an unavailable policy scanner as degraded', () => { + expect(classifyFailure({ kind: 'policy_unavailable' })).toEqual({ class: 'degraded' }); + }); + + it('classifies unavailable persistence as transient with no source-stage progress', () => { + expect(classifyFailure({ kind: 'persistence_unavailable' })) + .toEqual({ class: 'transient', retryCeiling: DEFAULT_TRANSIENT_CEILING }); + }); + + it('classifies an invalid target binding as permanent at the target level', () => { + expect(classifyFailure({ kind: 'target_binding_invalid' })).toEqual({ class: 'target_permanent' }); + }); + + it('classifies a temporarily unavailable target as transient at the target level', () => { + expect(classifyFailure({ kind: 'target_unavailable' })).toEqual({ class: 'target_transient' }); + }); + + it('classifies a deploy/health failure after a successful apply as its own class, never refetch or reapply', () => { + expect(classifyFailure({ kind: 'target_mutation_failed' })).toEqual({ class: 'target_mutation_failed' }); + }); + + it('classifies unavailable Blueprint evaluation as blocked, not retried', () => { + expect(classifyFailure({ kind: 'blueprint_unavailable' })).toEqual({ class: 'blocked' }); + }); + + it('classifies an interrupted or unknown-completion operation for reconciliation', () => { + expect(classifyFailure({ kind: 'interrupted' })).toEqual({ class: 'reconcile' }); + }); +}); + +describe('nextRetryAt', () => { + it('computes the base delay with up to +-10% jitter on the first attempt', () => { + const now = 1_000_000; + const at = nextRetryAt(now, 0); + expect(at).toBeGreaterThanOrEqual(now + 54_000); + expect(at).toBeLessThanOrEqual(now + 66_000); + }); + + it('doubles the delay per retry count', () => { + const now = 1_000_000; + const at = nextRetryAt(now, 3); // 60s * 2^3 = 480s + expect(at).toBeGreaterThanOrEqual(now + 432_000); + expect(at).toBeLessThanOrEqual(now + 528_000); + }); + + it('caps the delay at one hour regardless of retry count', () => { + const now = 1_000_000; + const at = nextRetryAt(now, 20); + expect(at).toBeLessThanOrEqual(now + 3_600_000 * 1.1); + }); + + it('honors a provider retry floor larger than the computed delay', () => { + const now = 1_000_000; + const at = nextRetryAt(now, 0, 10_000_000); + expect(at).toBe(now + 10_000_000); + }); + + it('ignores a provider retry floor smaller than the computed delay', () => { + const now = 1_000_000; + const at = nextRetryAt(now, 5, 1_000); + expect(at).toBeGreaterThan(now + 1_000); + }); +}); diff --git a/backend/src/__tests__/gitops-blueprint-transitions.test.ts b/backend/src/__tests__/gitops-blueprint-transitions.test.ts index 9c5b6632..ff13ab72 100644 --- a/backend/src/__tests__/gitops-blueprint-transitions.test.ts +++ b/backend/src/__tests__/gitops-blueprint-transitions.test.ts @@ -609,6 +609,10 @@ function inlineApp(id: string, blueprintId: number): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/__tests__/gitops-create-recovery.test.ts b/backend/src/__tests__/gitops-create-recovery.test.ts index 85f4c614..71dde879 100644 --- a/backend/src/__tests__/gitops-create-recovery.test.ts +++ b/backend/src/__tests__/gitops-create-recovery.test.ts @@ -441,6 +441,10 @@ function creatingApp(id: string, stackName: string): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, @@ -483,6 +487,12 @@ function gen(id: string, applicationId: string): GitOpsGenerationRow { actor: 'tester', previous_generation_id: null, redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, created_at: 1, }; } diff --git a/backend/src/__tests__/gitops-create.test.ts b/backend/src/__tests__/gitops-create.test.ts index 0fdee147..1eb9e824 100644 --- a/backend/src/__tests__/gitops-create.test.ts +++ b/backend/src/__tests__/gitops-create.test.ts @@ -707,6 +707,10 @@ function creatingApp(id: string, stackName: string): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, @@ -749,6 +753,12 @@ function gen(id: string, applicationId: string): GitOpsGenerationRow { actor: 'tester', previous_generation_id: null, redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, created_at: 1, }; } diff --git a/backend/src/__tests__/gitops-deferred.test.ts b/backend/src/__tests__/gitops-deferred.test.ts index b901e0e0..565712da 100644 --- a/backend/src/__tests__/gitops-deferred.test.ts +++ b/backend/src/__tests__/gitops-deferred.test.ts @@ -340,6 +340,10 @@ function app(id: string, stackName: string): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, @@ -382,6 +386,12 @@ function gen(id: string, applicationId: string): GitOpsGenerationRow { actor: 'tester', previous_generation_id: null, redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, created_at: 1, }; } diff --git a/backend/src/__tests__/gitops-derive.test.ts b/backend/src/__tests__/gitops-derive.test.ts index 7b03c86d..dd0f18b6 100644 --- a/backend/src/__tests__/gitops-derive.test.ts +++ b/backend/src/__tests__/gitops-derive.test.ts @@ -963,6 +963,10 @@ function app(id: string, stackName: string): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, @@ -1010,6 +1014,12 @@ function gen(id: string, applicationId: string): GitOpsGenerationRow { actor: 'tester', previous_generation_id: null, redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, created_at: 1, }; } diff --git a/backend/src/__tests__/gitops-direct-producers.test.ts b/backend/src/__tests__/gitops-direct-producers.test.ts index 42690eff..936a30a1 100644 --- a/backend/src/__tests__/gitops-direct-producers.test.ts +++ b/backend/src/__tests__/gitops-direct-producers.test.ts @@ -644,7 +644,7 @@ describe('Direct Git producers drive the revision state', () => { expect(recovered.active_operation_stage).toBeNull(); }); - it('leaves a stack with no GitOps application untouched', async () => { + it('refuses to fetch when a configured stack has no GitOps application', async () => { const svc = GitSourceService.getInstance(); const store = GitOpsStore.getInstance(); const stackName = 'producers-legacy'; @@ -681,9 +681,12 @@ describe('Direct Git producers drive the revision state', () => { expect(store.getLiveDirectApplication(stackName)).toBeUndefined(); stageRepo(COMPOSE_V2, 'fffffff6'); - await svc.pull(stackName, { actor: 'tester' }); + await expect(svc.pull(stackName, { actor: 'tester' })).rejects.toMatchObject({ + code: 'GIT_ERROR', + message: expect.stringContaining('GitOps tracking is unavailable'), + }); - // The pull succeeded operationally and wrote no GitOps rows. + // No untracked fetch or GitOps history was written. expect(store.getLiveDirectApplication(stackName)).toBeUndefined(); const historyRows = (await import('../services/DatabaseService')).DatabaseService .getInstance().getDb() diff --git a/backend/src/__tests__/gitops-handoff.test.ts b/backend/src/__tests__/gitops-handoff.test.ts new file mode 100644 index 00000000..7956af58 --- /dev/null +++ b/backend/src/__tests__/gitops-handoff.test.ts @@ -0,0 +1,115 @@ +/** + * The accepted-generation contract: a portable, content-only projection of + * a gitops_generations row, plus the target-dispatch boundary. + */ +import { describe, it, expect } from 'vitest'; +import { + buildAcceptedGeneration, + BlueprintTargetAdapter, + type AcceptedGeneration, +} from '../services/gitops/handoff'; +import type { GitOpsGenerationRow } from '../services/gitops/types'; + +function baseRow(overrides: Partial = {}): GitOpsGenerationRow { + return { + id: 'gen-1', + application_id: 'app-1', + commit_sha: 'a'.repeat(40), + repo_url: 'https://github.com/example/repo.git', + configured_ref: 'main', + resolved_ref_kind: 'branch', + repo_identity_json: '{"host":"github.com","pathname":"/example/repo.git"}', + manifest_version: 4, + candidate_dir: 'generations/candidate-a', + applied_dir: 'generations/applied-a-0', + expected_invocation_json: '{}', + materialization_fingerprint: 'f'.repeat(64), + validation_ok: 1, + plan_blocked: 0, + change_plan_fingerprint: 'fp-1', + operation_id: 'op-1', + trigger: 'manual', + actor: 'tester', + previous_generation_id: null, + redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, + created_at: 1, + ...overrides, + }; +} + +describe('buildAcceptedGeneration', () => { + it('decodes identity and lineage fields directly from the row', () => { + const gen = buildAcceptedGeneration(baseRow()); + expect(gen.contractVersion).toBe(1); + expect(gen.generationId).toBe('gen-1'); + expect(gen.applicationId).toBe('app-1'); + expect(gen.repoIdentity).toEqual({ host: 'github.com', pathname: '/example/repo.git' }); + expect(gen.configuredRef).toBe('main'); + expect(gen.commitSha).toBe('a'.repeat(40)); + expect(gen.resolvedRefKind).toBe('branch'); + expect(gen.validationOk).toBe(true); + expect(gen.trigger).toBe('manual'); + expect(gen.operationId).toBe('op-1'); + }); + + it('records an explicit limitation for each missing portable field on a legacy row, never inventing evidence', () => { + const gen = buildAcceptedGeneration(baseRow()); + expect(gen.portableManifest).toBeNull(); + expect(gen.composeInputs).toBeNull(); + expect(gen.sourcePolicyEvidence).toBeNull(); + expect(gen.limitations).toEqual(expect.arrayContaining([ + 'portable_manifest_missing', + 'compose_inputs_missing', + 'source_policy_evidence_missing', + 'security_policy_evidence_missing', + 'support_requirements_missing', + 'compatibility_requirements_missing', + ])); + }); + + it('decodes real evidence when the row carries it, recording no limitation for that field', () => { + const gen = buildAcceptedGeneration(baseRow({ + portable_manifest_json: '{"files":[]}', + compose_inputs_json: '{"composeFileOrder":["compose.yaml"]}', + })); + expect(gen.portableManifest).toEqual({ files: [] }); + expect(gen.composeInputs).toEqual({ composeFileOrder: ['compose.yaml'] }); + expect(gen.limitations).not.toContain('portable_manifest_missing'); + expect(gen.limitations).not.toContain('compose_inputs_missing'); + }); + + it('refuses to build a contract from an unparseable repo identity', () => { + expect(() => buildAcceptedGeneration(baseRow({ repo_identity_json: 'not json' }))).toThrow(); + }); + + it('never populates secretCapability with a value, only its absence as capability metadata', () => { + const gen = buildAcceptedGeneration(baseRow()); + expect(gen.secretCapability).toBeNull(); + }); +}); + +// A future field on AcceptedGeneration named like a target-mode concept +// (selector, frozen target set, node id, rollout batch, target project +// name, local path, or secret value) must fail this compile, not merely +// this test run: the contract stays structurally content-only. +type AssertNoTargetModeFields = T extends Record< + 'selector' | 'targetSet' | 'nodeId' | 'nodeIds' | 'rolloutBatch' | 'projectName' | 'candidateDir' | 'secretValue', + unknown +> ? never : true; +const _structurallyContentOnly: AssertNoTargetModeFields = true; +void _structurallyContentOnly; + +describe('BlueprintTargetAdapter', () => { + it('always returns a durable blocked result, never inspecting selectors or placement', async () => { + const adapter = new BlueprintTargetAdapter(); + const gen = buildAcceptedGeneration(baseRow()); + const result = await adapter.dispatch(gen, { targetMode: 'blueprint', nodeId: null, bindingRevision: null }); + expect(result.status).toBe('blocked'); + }); +}); diff --git a/backend/src/__tests__/gitops-history-read.test.ts b/backend/src/__tests__/gitops-history-read.test.ts index 8417dbcc..d1f10aef 100644 --- a/backend/src/__tests__/gitops-history-read.test.ts +++ b/backend/src/__tests__/gitops-history-read.test.ts @@ -531,6 +531,10 @@ function application(): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/__tests__/gitops-managed-sweep.test.ts b/backend/src/__tests__/gitops-managed-sweep.test.ts index 87d62944..32f104ed 100644 --- a/backend/src/__tests__/gitops-managed-sweep.test.ts +++ b/backend/src/__tests__/gitops-managed-sweep.test.ts @@ -167,6 +167,10 @@ function creatingApp(id: string, stackName: string): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/__tests__/gitops-outcomes.test.ts b/backend/src/__tests__/gitops-outcomes.test.ts new file mode 100644 index 00000000..48eb975b --- /dev/null +++ b/backend/src/__tests__/gitops-outcomes.test.ts @@ -0,0 +1,152 @@ +/** + * Normalized reconcile-outcome coverage. outcomeFromSourceFacet derives from + * the existing SourceFacet projection rather than inventing a second status + * source, so "no source change" and "converged" cannot silently collapse + * into the same result. + */ +import { describe, it, expect } from 'vitest'; +import { outcomeFromSourceFacet } from '../services/gitops/outcomes'; +import type { SourceFacet } from '../services/gitops/types'; + +const identity = { + configuredRepoUrl: 'https://github.com/example/repo.git', + repoIdentity: { host: 'github.com', pathname: '/example/repo.git' }, + configuredRef: 'main', + desiredCommitSha: 'a'.repeat(40), + fetchedCommitSha: 'a'.repeat(40), + candidateGenerationId: null, + acceptedGenerationId: 'gen-1', +}; + +describe('outcomeFromSourceFacet', () => { + it('reports no_source_change for an accepted generation, never converged on SHA alone', () => { + const facet: SourceFacet = { ...identity, status: 'application_generation_accepted' }; + const result = outcomeFromSourceFacet(facet); + expect(result.outcome).toBe('no_source_change'); + expect(result.commitSha).toBe('a'.repeat(40)); + }); + + it('reports candidate_already_fetched for a ready candidate', () => { + const facet: SourceFacet = { ...identity, status: 'candidate_ready' }; + expect(outcomeFromSourceFacet(facet).outcome).toBe('candidate_already_fetched'); + }); + + it('reports pending_review with a review next action', () => { + const facet: SourceFacet = { ...identity, status: 'source_review_pending' }; + const result = outcomeFromSourceFacet(facet); + expect(result.outcome).toBe('pending_review'); + expect(result.nextAction).toBe('review'); + }); + + it('reports blocked with a resolve_conflict next action for a source conflict', () => { + const facet: SourceFacet = { ...identity, status: 'source_conflict_blocker' }; + const result = outcomeFromSourceFacet(facet); + expect(result.outcome).toBe('blocked'); + expect(result.nextAction).toBe('resolve_conflict'); + }); + + it('reports superseded for a candidate a newer revision replaced', () => { + const facet: SourceFacet = { ...identity, status: 'source_superseded', supersededGenerationId: 'gen-old' }; + expect(outcomeFromSourceFacet(facet).outcome).toBe('superseded'); + }); + + it('reports retry_scheduled with the retry time surfaced', () => { + const facet: SourceFacet = { ...identity, status: 'source_retry_scheduled', retryAt: 12345, retryCount: 2 }; + const result = outcomeFromSourceFacet(facet); + expect(result.outcome).toBe('retry_scheduled'); + expect(result.retryAt).toBe(12345); + expect(result.nextAction).toBe('none'); + }); + + it('reports suspended with a resume next action', () => { + const facet: SourceFacet = { ...identity, status: 'source_suspended', suspendedAt: 999, suspendedReason: 'operator paused sync' }; + const result = outcomeFromSourceFacet(facet); + expect(result.outcome).toBe('suspended'); + expect(result.nextAction).toBe('resume'); + expect(result.reason).toContain('operator paused sync'); + }); + + it('reports failed_previous_intact for a source failure, with a retry next action when a retry is scheduled', () => { + const facet: SourceFacet = { + ...identity, + status: 'source_failed', + failureStage: 'fetch', + failureClass: 'permanent', + failureAt: 100, + retryAt: 200, + retryCount: 1, + }; + const result = outcomeFromSourceFacet(facet); + expect(result.outcome).toBe('failed_previous_intact'); + expect(result.nextAction).toBe('retry'); + expect(result.retryAt).toBe(200); + }); + + it('reports failed_previous_intact with no retry next action when no retry is scheduled', () => { + const facet: SourceFacet = { + ...identity, + status: 'source_failed', + failureStage: 'fetch', + failureClass: 'permanent', + failureAt: 100, + retryAt: null, + retryCount: 0, + }; + const result = outcomeFromSourceFacet(facet); + expect(result.outcome).toBe('failed_previous_intact'); + expect(result.nextAction).toBe('configure_credentials'); + }); + + it('reports recovery_required for an interrupted operation', () => { + const facet: SourceFacet = { + ...identity, + status: 'source_unknown', + interruptedStage: 'fetch_started', + interruptedAt: 100, + interruptedOperationId: 'op-1', + interruptedGenerationId: null, + }; + expect(outcomeFromSourceFacet(facet).outcome).toBe('recovery_required'); + }); + + it('reports recovery_required with a view_target_results next action when recovery is outstanding', () => { + const facet: SourceFacet = { ...identity, status: 'recovery_required', recoveryRef: 'rec-1', recoveryGenerationId: 'gen-1' }; + const result = outcomeFromSourceFacet(facet); + expect(result.outcome).toBe('recovery_required'); + expect(result.nextAction).toBe('view_target_results'); + }); + + it('reports recovery_required when recovery itself failed, distinguishing that in the reason', () => { + const facet: SourceFacet = { + ...identity, + status: 'recovery_failed', + recoveryRef: 'rec-1', + recoveryGenerationId: 'gen-1', + failureClass: 'io_error', + failureAt: 100, + }; + const result = outcomeFromSourceFacet(facet); + expect(result.outcome).toBe('recovery_required'); + expect(result.reason).toMatch(/recovery/i); + }); + + it('reports unknown for an application that is no longer live', () => { + const facet: SourceFacet = { ...identity, status: 'not_live', lifecycleStatus: 'detached' }; + expect(outcomeFromSourceFacet(facet).outcome).toBe('unknown'); + }); + + it.each(['not_applicable', 'never_reconciled', 'checking_fetching', 'source_reconcile_required'] as const)( + 'reports unknown for %s, which has no settled outcome yet', + (status) => { + const facet = status === 'not_applicable' + ? ({ status } as SourceFacet) + : ({ ...identity, status } as SourceFacet); + expect(outcomeFromSourceFacet(facet).outcome).toBe('unknown'); + }, + ); + + it('reports unknown while an operation is in flight (applying)', () => { + const facet: SourceFacet = { ...identity, status: 'applying', activeOperationId: 'op-1', activeGenerationId: 'gen-1' }; + expect(outcomeFromSourceFacet(facet).outcome).toBe('unknown'); + }); +}); diff --git a/backend/src/__tests__/gitops-reconcile-attempts.test.ts b/backend/src/__tests__/gitops-reconcile-attempts.test.ts new file mode 100644 index 00000000..9f1699ed --- /dev/null +++ b/backend/src/__tests__/gitops-reconcile-attempts.test.ts @@ -0,0 +1,387 @@ +/** + * Durable reconcile-attempt reservation and settlement: a bare history + * insert in its own transaction, never through mutateApp, so a reservation + * writes no application-row state and can be safely repeated. + */ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { GitOpsStore } from '../services/gitops/store'; +import { GitOpsTransitions, type EventEnvelope } from '../services/gitops/transitions'; +import { DatabaseService } from '../services/DatabaseService'; +import type { GitOpsApplicationRow } from '../services/gitops/types'; + +let tmpDir: string; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + GitOpsStore.resetForTests(); + GitOpsTransitions.resetForTests(); +}); + +afterAll(() => { + cleanupTestDb(tmpDir); +}); + +describe('reconcile attempt reservation and settlement', () => { + it('reserves an attempt without touching application state', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-res', 'res-web'), nodeId: 1, envelope: env('op-act-res') }); + const before = store.getApplication('app-res')!; + + const result = tx.reserveReconcileAttempt('app-res', env('op-res-1')); + + expect(result.reserved).toBe(true); + const after = store.getApplication('app-res')!; + expect(after.updated_at).toBe(before.updated_at); + expect(after.desired_commit_sha).toBe(before.desired_commit_sha); + }); + + it('returns reserved: false on a repeated reservation for the same operation', () => { + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-res2', 'res2-web'), nodeId: 1, envelope: env('op-act-res2') }); + + const first = tx.reserveReconcileAttempt('app-res2', env('op-res2-1')); + const second = tx.reserveReconcileAttempt('app-res2', env('op-res2-1')); + + expect(first.reserved).toBe(true); + expect(second.reserved).toBe(false); + }); + + it('allows two different operations to each reserve their own attempt', () => { + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-res3', 'res3-web'), nodeId: 1, envelope: env('op-act-res3') }); + + const first = tx.reserveReconcileAttempt('app-res3', env('op-res3-a')); + const second = tx.reserveReconcileAttempt('app-res3', env('op-res3-b')); + + expect(first.reserved).toBe(true); + expect(second.reserved).toBe(true); + }); + + it('settles a reserved attempt and finds it by operation id afterward', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-settle', 'settle-web'), nodeId: 1, envelope: env('op-act-settle') }); + tx.reserveReconcileAttempt('app-settle', env('op-settle-1')); + + const settleResult = tx.settleReconcileAttempt('app-settle', env('op-settle-1'), { + outcome: 'no_source_change', + reason: 'Nothing new to fetch.', + nextAction: 'none', + }); + + expect(settleResult.settled).toBe(true); + const settled = store.getSettledAttempt('app-settle', 'op-settle-1'); + expect(settled).toBeDefined(); + }); + + it('settling twice for the same operation is a no-op the second time', () => { + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-settle2', 'settle2-web'), nodeId: 1, envelope: env('op-act-settle2') }); + tx.reserveReconcileAttempt('app-settle2', env('op-settle2-1')); + + const first = tx.settleReconcileAttempt('app-settle2', env('op-settle2-1'), { + outcome: 'no_source_change', + reason: 'first', + nextAction: 'none', + }); + const second = tx.settleReconcileAttempt('app-settle2', env('op-settle2-1'), { + outcome: 'no_source_change', + reason: 'second, must not overwrite', + nextAction: 'none', + }); + + expect(first.settled).toBe(true); + expect(second.settled).toBe(false); + }); + + it('has no settled attempt for a reservation that was never settled', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-orphan', 'orphan-web'), nodeId: 1, envelope: env('op-act-orphan') }); + tx.reserveReconcileAttempt('app-orphan', env('op-orphan-1')); + + expect(store.getSettledAttempt('app-orphan', 'op-orphan-1')).toBeUndefined(); + }); + + it('lists an unsettled reservation but not one that has settled', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-unsettled', 'unsettled-web'), nodeId: 1, envelope: env('op-act-unsettled') }); + tx.reserveReconcileAttempt('app-unsettled', env('op-unsettled-orphan')); + tx.reserveReconcileAttempt('app-unsettled', env('op-unsettled-done')); + tx.settleReconcileAttempt('app-unsettled', env('op-unsettled-done'), { + outcome: 'no_source_change', + reason: 'done', + nextAction: 'none', + }); + + const unsettled = store.listUnsettledReconcileAttempts(); + const operationIds = unsettled.filter((r) => r.application_id === 'app-unsettled').map((r) => r.operation_id); + expect(operationIds).toContain('op-unsettled-orphan'); + expect(operationIds).not.toContain('op-unsettled-done'); + }); + + it('gets the started row for one exact attempt, or undefined when it was never reserved', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-started', 'started-web'), nodeId: 1, envelope: env('op-act-started') }); + tx.reserveReconcileAttempt('app-started', env('op-started-1')); + + expect(store.getStartedAttempt('app-started', 'op-started-1')).toBeDefined(); + expect(store.getStartedAttempt('app-started', 'op-never-reserved')).toBeUndefined(); + }); + + it('pages past a permanently unsettled row instead of returning it forever on every call with the same cursor', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-cursor', 'cursor-web'), nodeId: 1, envelope: env('op-act-cursor') }); + tx.reserveReconcileAttempt('app-cursor', { operationId: 'op-cursor-stuck', actor: 'tester', trigger: 'manual', at: 1_000 }); + tx.reserveReconcileAttempt('app-cursor', { operationId: 'op-cursor-next', actor: 'tester', trigger: 'manual', at: 2_000 }); + + const firstPage = store.listUnsettledReconcileAttempts(1); + expect(firstPage.map((r) => r.operation_id)).toEqual(['op-cursor-stuck']); + const cursor = { createdAt: firstPage[0].created_at, id: firstPage[0].id }; + // Simulate op-cursor-stuck being permanently unrecoverable: it is never + // settled, so a caller must page past it using the cursor rather than + // seeing it again on the next call. + const secondPage = store.listUnsettledReconcileAttempts(1, cursor); + expect(secondPage.map((r) => r.operation_id)).toEqual(['op-cursor-next']); + }); + + it('allocates a fresh attemptSeq-derived operation id and reserves it in one transaction', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-alloc', 'alloc-web'), nodeId: 1, envelope: env('op-act-alloc') }); + const before = store.getApplication('app-alloc')!; + + const first = tx.allocateReconcileAttempt('app-alloc', 'tester', 'manual', Date.now()); + const second = tx.allocateReconcileAttempt('app-alloc', 'tester', 'manual', Date.now()); + + expect(first.reserved).toBe(true); + expect(second.reserved).toBe(true); + expect(first.operationId).not.toBe(second.operationId); + const after = store.getApplication('app-alloc')!; + expect(after.attempt_seq).toBe(before.attempt_seq + 2); + }); + + it('rolls back the allocated sequence when reservation insertion fails', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-alloc-rollback', 'alloc-rollback-web'), nodeId: 1, envelope: env('op-act-alloc-rollback') }); + const db = DatabaseService.getInstance().getDb(); + const before = store.getApplication('app-alloc-rollback')!; + db.exec(` + CREATE TRIGGER fail_reconcile_reservation + BEFORE INSERT ON gitops_history + WHEN NEW.stage = 'source_reconcile_started' + BEGIN + SELECT RAISE(ABORT, 'simulated reservation insert failure'); + END + `); + + try { + expect(() => tx.allocateReconcileAttempt('app-alloc-rollback', 'tester', 'manual', Date.now())) + .toThrow('simulated reservation insert failure'); + expect(store.getApplication('app-alloc-rollback')!.attempt_seq).toBe(before.attempt_seq); + expect(store.listUnsettledReconcileAttempts().some((row) => row.application_id === 'app-alloc-rollback')).toBe(false); + } finally { + db.exec('DROP TRIGGER fail_reconcile_reservation'); + } + }); + + it('records a follower link on a reservation made on behalf of a coalesced request', () => { + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-follower', 'follower-web'), nodeId: 1, envelope: env('op-act-follower') }); + + const leader = tx.allocateReconcileAttempt('app-follower', 'tester', 'manual', Date.now()); + const follower = tx.allocateReconcileAttempt('app-follower', 'tester', 'manual', Date.now(), leader.operationId); + + expect(follower.reserved).toBe(true); + const started = DatabaseService.getInstance().getDb() + .prepare("SELECT after_json FROM gitops_history WHERE application_id = ? AND operation_id = ? AND stage = 'source_reconcile_started'") + .get('app-follower', follower.operationId) as { after_json: string }; + expect(JSON.parse(started.after_json)).toEqual({ followerOf: leader.operationId }); + }); + + it('records the original webhook delivery intent on its stable reservation', () => { + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-delivery-intent', 'delivery-intent-web'), nodeId: 1, envelope: env('op-act-delivery-intent') }); + + tx.reserveReconcileAttempt( + 'app-delivery-intent', + env('webhook:fetch:delivery-intent'), + undefined, + { autoApply: true, deploy: false }, + ); + + const started = GitOpsStore.getInstance().getStartedAttempt('app-delivery-intent', 'webhook:fetch:delivery-intent')!; + expect(JSON.parse(started.after_json)).toEqual({ + deliveryIntent: { autoApply: true, deploy: false }, + }); + }); + + it('reports the most recently settled attempt even when both share the same millisecond timestamp', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-latest', 'latest-web'), nodeId: 1, envelope: env('op-act-latest') }); + // Same `at` on purpose: created_at alone cannot tell these two apart, + // so the query must break the tie by insertion order (rowid), not the + // id column, which is a random UUID unrelated to recency. + const sameInstant = 5_000; + tx.reserveReconcileAttempt('app-latest', { operationId: 'op-latest-1', actor: 'tester', trigger: 'manual', at: sameInstant }); + tx.settleReconcileAttempt( + 'app-latest', + { operationId: 'op-latest-1', actor: 'tester', trigger: 'manual', at: sameInstant }, + { outcome: 'no_source_change', reason: 'first', nextAction: 'none' }, + ); + tx.reserveReconcileAttempt('app-latest', { operationId: 'op-latest-2', actor: 'tester', trigger: 'manual', at: sameInstant }); + tx.settleReconcileAttempt( + 'app-latest', + { operationId: 'op-latest-2', actor: 'tester', trigger: 'manual', at: sameInstant }, + { outcome: 'retry_scheduled', reason: 'second', nextAction: 'none' }, + ); + + const latest = store.latestSettledAttempt('app-latest'); + expect(latest?.operation_id).toBe('op-latest-2'); + }); +}); + +describe('poll and retry eligibility queries', () => { + it('lists a source whose next_poll_at has arrived', () => { + const store = GitOpsStore.getInstance(); + store.insertApplication({ ...app('app-poll-due', 'poll-due-web'), next_poll_at: 1_000 }); + const due = store.listSourcesDueForPoll(1_000); + expect(due.map((a) => a.id)).toContain('app-poll-due'); + }); + + it('excludes a source whose next_poll_at has not arrived yet', () => { + const store = GitOpsStore.getInstance(); + store.insertApplication({ ...app('app-poll-future', 'poll-future-web'), next_poll_at: 5_000 }); + const due = store.listSourcesDueForPoll(1_000); + expect(due.map((a) => a.id)).not.toContain('app-poll-future'); + }); + + it('excludes a suspended source even when its poll time has arrived', () => { + const store = GitOpsStore.getInstance(); + store.insertApplication({ ...app('app-poll-susp', 'poll-susp-web'), next_poll_at: 1_000, suspended_at: 500 }); + const due = store.listSourcesDueForPoll(1_000); + expect(due.map((a) => a.id)).not.toContain('app-poll-susp'); + }); + + it('excludes a source with an operation already in flight', () => { + const store = GitOpsStore.getInstance(); + store.insertApplication({ + ...app('app-poll-busy', 'poll-busy-web'), + next_poll_at: 1_000, + active_operation_stage: 'fetch_started', + }); + const due = store.listSourcesDueForPoll(1_000); + expect(due.map((a) => a.id)).not.toContain('app-poll-busy'); + }); + + it('excludes a Blueprint-mode application from polling', () => { + const store = GitOpsStore.getInstance(); + store.insertApplication({ + ...app('app-poll-bp', 'unused-bp'), + stack_name: null, + blueprint_id: 42, + target_mode: 'blueprint', + configured_repo_url: 'https://github.com/org/repo.git', + next_poll_at: 1_000, + }); + const due = store.listSourcesDueForPoll(1_000); + expect(due.map((a) => a.id)).not.toContain('app-poll-bp'); + }); + + it('lists an application whose retry_at has arrived', () => { + const store = GitOpsStore.getInstance(); + store.insertApplication({ ...app('app-retry-due', 'retry-due-web'), retry_at: 1_000 }); + const due = store.listApplicationsDueForRetry(1_000); + expect(due.map((a) => a.id)).toContain('app-retry-due'); + }); + + it('excludes an application with no retry scheduled', () => { + const store = GitOpsStore.getInstance(); + store.insertApplication(app('app-retry-none', 'retry-none-web')); + const due = store.listApplicationsDueForRetry(1_000); + expect(due.map((a) => a.id)).not.toContain('app-retry-none'); + }); + + it('excludes a suspended application even when its retry time has arrived', () => { + const store = GitOpsStore.getInstance(); + store.insertApplication({ ...app('app-retry-susp', 'retry-susp-web'), retry_at: 1_000, suspended_at: 500 }); + const due = store.listApplicationsDueForRetry(1_000); + expect(due.map((a) => a.id)).not.toContain('app-retry-susp'); + }); +}); + +function env(operationId: string): EventEnvelope { + return { operationId, actor: 'tester', trigger: 'manual', at: Date.now() }; +} + +function app(id: string, stackName: string): GitOpsApplicationRow { + return { + id, + lifecycle_key: `direct:${stackName}`, + lifecycle_status: 'active', + target_mode: 'direct', + stack_name: stackName, + blueprint_id: null, + configured_repo_url: 'https://github.com/org/repo.git', + repo_identity_json: '{"host":"github.com","pathname":"/org/repo.git"}', + configured_ref: 'main', + compose_paths_json: '["compose.yml"]', + context_dir: null, + sync_env: 0, + env_path: null, + materialization_fingerprint: 'a'.repeat(64), + desired_commit_sha: null, + fetched_commit_sha: null, + fetched_resolved_ref_kind: null, + candidate_generation_id: null, + accepted_generation_id: null, + candidate_plan_blocked: 0, + review_required: 0, + artifact_set_id: null, + latest_artifact_set_id: null, + intent_revision_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + rollout_authorization_ref: null, + legacy_combined_approval_ref: null, + preflight_fingerprint: null, + latest_operation_id: null, + active_operation_id: null, + active_operation_stage: null, + active_operation_at: null, + active_generation_id: null, + pause_at: null, + pause_reason: null, + source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, + partial_json: null, + failure_stage: null, + failure_class: null, + failure_at: null, + retry_at: null, + retry_count: 0, + suspended_at: null, + recovery_ref: null, + recovery_phase: null, + interruption_stage: null, + interruption_at: null, + interruption_operation_id: null, + interruption_generation_id: null, + evidence_fresh_at: null, + evidence_limitations_json: null, + created_at: 1, + updated_at: 1, + }; +} diff --git a/backend/src/__tests__/gitops-recovery-capture.test.ts b/backend/src/__tests__/gitops-recovery-capture.test.ts index ee8b2957..ef76d704 100644 --- a/backend/src/__tests__/gitops-recovery-capture.test.ts +++ b/backend/src/__tests__/gitops-recovery-capture.test.ts @@ -147,6 +147,10 @@ function app(id: string, stackName: string): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, @@ -189,6 +193,12 @@ function gen(id: string, applicationId: string): GitOpsGenerationRow { actor: 'tester', previous_generation_id: null, redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, created_at: 1, }; } diff --git a/backend/src/__tests__/gitops-recovery.test.ts b/backend/src/__tests__/gitops-recovery.test.ts index eeb3a510..e754cf71 100644 --- a/backend/src/__tests__/gitops-recovery.test.ts +++ b/backend/src/__tests__/gitops-recovery.test.ts @@ -461,6 +461,10 @@ function app(id: string, stackName: string): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, @@ -503,6 +507,12 @@ function gen(id: string, applicationId: string): GitOpsGenerationRow { actor: 'tester', previous_generation_id: null, redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, created_at: 1, }; } diff --git a/backend/src/__tests__/gitops-schema.test.ts b/backend/src/__tests__/gitops-schema.test.ts index ca2fa446..11dbcd45 100644 --- a/backend/src/__tests__/gitops-schema.test.ts +++ b/backend/src/__tests__/gitops-schema.test.ts @@ -239,6 +239,61 @@ describe('gitops schema', () => { expect(row?.configured_ref).toBe('v1'); }); + it('round-trips the portable generation contract fields, defaulting to null for legacy rows', async () => { + const store = GitOpsStore.getInstance(); + store.insertApplication(directApp('app-portable', 'portable-web')); + store.insertGeneration({ + ...generation('gen-portable-legacy', 'app-portable'), + }); + const legacy = store.getGeneration('gen-portable-legacy'); + expect(legacy?.portable_manifest_json).toBeNull(); + expect(legacy?.compose_inputs_json).toBeNull(); + expect(legacy?.source_policy_evidence_json).toBeNull(); + expect(legacy?.security_policy_evidence_json).toBeNull(); + expect(legacy?.support_requirements_json).toBeNull(); + expect(legacy?.compatibility_requirements_json).toBeNull(); + + store.insertGeneration({ + ...generation('gen-portable-new', 'app-portable'), + portable_manifest_json: '{"files":[]}', + compose_inputs_json: '{"composeFileOrder":["compose.yaml"]}', + source_policy_evidence_json: '{"policy":"manual"}', + security_policy_evidence_json: '{"status":"allowed"}', + support_requirements_json: '{}', + compatibility_requirements_json: '{}', + }); + const populated = store.getGeneration('gen-portable-new'); + expect(populated?.portable_manifest_json).toBe('{"files":[]}'); + expect(populated?.compose_inputs_json).toBe('{"composeFileOrder":["compose.yaml"]}'); + expect(populated?.source_policy_evidence_json).toBe('{"policy":"manual"}'); + }); + + it('defaults controller-owned columns to manual, off, and zero on a fresh application', async () => { + const store = GitOpsStore.getInstance(); + store.insertApplication(directApp('app-ctrl', 'ctrl-web')); + const app = store.getApplication('app-ctrl'); + expect(app?.source_policy).toBe('manual'); + expect(app?.poll_interval_secs).toBeNull(); + expect(app?.next_poll_at).toBeNull(); + expect(app?.attempt_seq).toBe(0); + }); + + it('round-trips a configured poll interval and policy', async () => { + const store = GitOpsStore.getInstance(); + store.insertApplication({ + ...directApp('app-ctrl2', 'ctrl2-web'), + source_policy: 'automatic', + poll_interval_secs: 120, + next_poll_at: 5000, + attempt_seq: 3, + }); + const app = store.getApplication('app-ctrl2'); + expect(app?.source_policy).toBe('automatic'); + expect(app?.poll_interval_secs).toBe(120); + expect(app?.next_poll_at).toBe(5000); + expect(app?.attempt_seq).toBe(3); + }); + it('round-trips fetched_resolved_ref_kind on application fetch transitions', async () => { const store = GitOpsStore.getInstance(); store.insertApplication(directApp('app-fetch-kind', 'fetch-web')); @@ -294,6 +349,10 @@ function directApp(id: string, stackName: string): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, @@ -362,6 +421,12 @@ function generation(id: string, applicationId: string): GitOpsGenerationRow { actor: 'tester', previous_generation_id: null, redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, created_at: 1, }; } diff --git a/backend/src/__tests__/gitops-transitions.test.ts b/backend/src/__tests__/gitops-transitions.test.ts index 0d65150d..8cc883ae 100644 --- a/backend/src/__tests__/gitops-transitions.test.ts +++ b/backend/src/__tests__/gitops-transitions.test.ts @@ -813,6 +813,10 @@ function app(id: string, stackName: string): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, @@ -855,6 +859,12 @@ function gen(id: string, applicationId: string): GitOpsGenerationRow { actor: 'tester', previous_generation_id: null, redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, created_at: 1, }; } diff --git a/backend/src/__tests__/gitops-triggers.test.ts b/backend/src/__tests__/gitops-triggers.test.ts new file mode 100644 index 00000000..7577e5db --- /dev/null +++ b/backend/src/__tests__/gitops-triggers.test.ts @@ -0,0 +1,85 @@ +/** + * Pure trigger normalization and coalescing-key coverage for the GitOps + * source controller. No DB, no store: these are the identity/joining rules + * a controller submission goes through before anything durable happens. + */ +import { describe, it, expect } from 'vitest'; +import { coalesceKey, deliveryKey, type ReconcileRequest } from '../services/gitops/triggers'; + +function fetchRequest(overrides: Partial> = {}): ReconcileRequest { + return { + intent: 'fetch', + applicationId: 'app-1', + stackName: 'web', + trigger: 'manual', + actor: 'tester', + ...overrides, + }; +} + +function applyRequest(overrides: Partial> = {}): ReconcileRequest { + return { + intent: 'apply', + applicationId: 'app-1', + stackName: 'web', + trigger: 'manual', + actor: 'tester', + commitSha: 'a'.repeat(40), + planFingerprint: 'fp-1', + deploy: false, + ...overrides, + }; +} + +describe('coalesceKey', () => { + it('joins two fetch requests for the same application', () => { + expect(coalesceKey(fetchRequest())).toBe(coalesceKey(fetchRequest({ trigger: 'poll' }))); + }); + + it('does not join fetch requests for different applications', () => { + expect(coalesceKey(fetchRequest({ applicationId: 'app-1' }))) + .not.toBe(coalesceKey(fetchRequest({ applicationId: 'app-2' }))); + }); + + it('joins two apply requests with identical commit, fingerprint, and deploy flag', () => { + expect(coalesceKey(applyRequest())).toBe(coalesceKey(applyRequest({ trigger: 'webhook' }))); + }); + + it('does not join two applies with different plan fingerprints', () => { + expect(coalesceKey(applyRequest({ planFingerprint: 'fp-1' }))) + .not.toBe(coalesceKey(applyRequest({ planFingerprint: 'fp-2' }))); + }); + + it('does not join two applies with different commits', () => { + expect(coalesceKey(applyRequest({ commitSha: 'a'.repeat(40) }))) + .not.toBe(coalesceKey(applyRequest({ commitSha: 'b'.repeat(40) }))); + }); + + it('does not join two applies that differ only in deploy', () => { + expect(coalesceKey(applyRequest({ deploy: false }))) + .not.toBe(coalesceKey(applyRequest({ deploy: true }))); + }); + + it('never joins a fetch and an apply for the same application', () => { + expect(coalesceKey(fetchRequest())).not.toBe(coalesceKey(applyRequest())); + }); + + it('does not join two fetches for the same applicationId but different stack names', () => { + expect(coalesceKey(fetchRequest({ stackName: 'web' }))) + .not.toBe(coalesceKey(fetchRequest({ stackName: 'other-stack' }))); + }); +}); + +describe('deliveryKey', () => { + it('namespaces the same delivery id differently per trigger', () => { + expect(deliveryKey('webhook', 'fetch', 'delivery-1')).not.toBe(deliveryKey('api', 'fetch', 'delivery-1')); + }); + + it('namespaces the same delivery id differently per intent', () => { + expect(deliveryKey('webhook', 'fetch', 'delivery-1')).not.toBe(deliveryKey('webhook', 'apply', 'delivery-1')); + }); + + it('is stable for the same trigger, intent, and delivery id', () => { + expect(deliveryKey('webhook', 'fetch', 'delivery-1')).toBe(deliveryKey('webhook', 'fetch', 'delivery-1')); + }); +}); diff --git a/backend/src/__tests__/helpers/gitopsFixtures.ts b/backend/src/__tests__/helpers/gitopsFixtures.ts index a80878ed..71d08ebc 100644 --- a/backend/src/__tests__/helpers/gitopsFixtures.ts +++ b/backend/src/__tests__/helpers/gitopsFixtures.ts @@ -57,6 +57,10 @@ export function directApplicationFixture(id: string, stackName: string): GitOpsA pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/__tests__/source-controller.test.ts b/backend/src/__tests__/source-controller.test.ts new file mode 100644 index 00000000..1d9b204e --- /dev/null +++ b/backend/src/__tests__/source-controller.test.ts @@ -0,0 +1,207 @@ +/** + * SourceController: the background driver for unattended reconciliation. + * GitOpsStore's due-queries and GitSourceService.reconcile() are mocked so + * these tests exercise only the timer/coalescing behavior, not real fetch + * or apply mechanics (already covered by git-source-service.test.ts). + */ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { directApplicationFixture } from './helpers/gitopsFixtures'; +import { GitOpsStore } from '../services/gitops/store'; +import { GitSourceService } from '../services/GitSourceService'; +import { SourceController } from '../services/gitops/SourceController'; +import type { GitOpsApplicationRow } from '../services/gitops/types'; +import type { ReconcileResult } from '../services/gitops/outcomes'; + +const TICK_MS = 60_000; +const okResult: ReconcileResult = { outcome: 'no_source_change', reason: 'ok', nextAction: 'none' }; + +let tmpDir: string; +let controller: SourceController; + +/** Point both due-queries at fixed rows; the scan reads nothing else. */ +function mockDue(duePoll: GitOpsApplicationRow[], dueRetry: GitOpsApplicationRow[] = []): void { + vi.spyOn(GitOpsStore.getInstance(), 'listSourcesDueForPoll').mockReturnValue(duePoll); + vi.spyOn(GitOpsStore.getInstance(), 'listApplicationsDueForRetry').mockReturnValue(dueRetry); +} + +function spyOnReconcile() { + return vi.spyOn(GitSourceService.getInstance(), 'reconcile'); +} + +/** Run the next scheduled tick and let the evaluations it fires settle. */ +async function advanceOneTick(): Promise { + await vi.advanceTimersByTimeAsync(TICK_MS); +} + +beforeAll(async () => { + tmpDir = await setupTestDb(); + GitOpsStore.resetForTests(); +}); + +afterAll(() => { + cleanupTestDb(tmpDir); +}); + +beforeEach(() => { + vi.useFakeTimers(); + SourceController.resetForTests(); + controller = SourceController.getInstance(); +}); + +afterEach(() => { + controller.stop(); + vi.restoreAllMocks(); + vi.useRealTimers(); +}); + +describe('SourceController', () => { + it('evaluates a source whose poll interval is due', async () => { + mockDue([directApplicationFixture('app-poll', 'poll-web')]); + const reconcile = spyOnReconcile().mockResolvedValue(okResult); + + controller.start(); + await advanceOneTick(); + + expect(reconcile).toHaveBeenCalledWith(expect.objectContaining({ + intent: 'fetch', + applicationId: 'app-poll', + stackName: 'poll-web', + trigger: 'poll', + })); + }); + + it('evaluates an application whose retry_at has arrived, tagged as a retry trigger', async () => { + const app = { ...directApplicationFixture('app-retry', 'retry-web'), retry_at: Date.now() - 1_000 }; + mockDue([], [app]); + const reconcile = spyOnReconcile().mockResolvedValue(okResult); + + controller.start(); + await advanceOneTick(); + + expect(reconcile).toHaveBeenCalledWith(expect.objectContaining({ + applicationId: 'app-retry', + trigger: 'retry', + })); + }); + + it('evaluates an application due for both poll and retry exactly once', async () => { + const app = { ...directApplicationFixture('app-both', 'both-web'), retry_at: Date.now() - 1_000 }; + mockDue([app], [app]); + const reconcile = spyOnReconcile().mockResolvedValue(okResult); + + controller.start(); + await advanceOneTick(); + + expect(reconcile).toHaveBeenCalledTimes(1); + }); + + it('does not re-evaluate an application still in flight from a previous tick', async () => { + mockDue([directApplicationFixture('app-slow', 'slow-web')]); + let settleFirstCall!: (result: ReconcileResult) => void; + const firstCall = new Promise((resolve) => { settleFirstCall = resolve; }); + const reconcile = spyOnReconcile().mockReturnValue(firstCall); + + controller.start(); + await advanceOneTick(); + expect(reconcile).toHaveBeenCalledTimes(1); + + // A second tick fires while the first evaluation is still pending. + await advanceOneTick(); + expect(reconcile).toHaveBeenCalledTimes(1); + + settleFirstCall(okResult); + await Promise.resolve(); + await Promise.resolve(); + + // Now that the first evaluation has settled, a later tick may pick it up again. + await advanceOneTick(); + expect(reconcile).toHaveBeenCalledTimes(2); + }); + + it('recovers on the next tick after a store query throws, rather than dying permanently', async () => { + mockDue([directApplicationFixture('app-recovers', 'recovers-web')]); + vi.spyOn(GitOpsStore.getInstance(), 'listSourcesDueForPoll').mockImplementationOnce(() => { + throw new Error('database is locked'); + }); + const reconcile = spyOnReconcile().mockResolvedValue(okResult); + vi.spyOn(console, 'error').mockImplementation(() => {}); + + controller.start(); + await advanceOneTick(); + expect(reconcile).not.toHaveBeenCalled(); + + await advanceOneTick(); + expect(reconcile).toHaveBeenCalledTimes(1); + }); + + it('releases the in-flight slot for an application whose reconcile rejects', async () => { + mockDue([directApplicationFixture('app-rejects', 'rejects-web')]); + const reconcile = spyOnReconcile().mockRejectedValue(new Error('boom')); + + controller.start(); + await advanceOneTick(); + await advanceOneTick(); + + expect(reconcile).toHaveBeenCalledTimes(2); + }); + + it('does not evaluate anything after stop', async () => { + mockDue([directApplicationFixture('app-stopped', 'stopped-web')]); + const reconcile = spyOnReconcile().mockResolvedValue(okResult); + + controller.start(); + controller.stop(); + await advanceOneTick(); + await advanceOneTick(); + + expect(reconcile).not.toHaveBeenCalled(); + }); + + it('does not double-arm when start() is called reentrantly from within an in-flight evaluation', async () => { + mockDue([directApplicationFixture('app-reentrant-start', 'reentrant-start-web')]); + // tick() nulls `timer` before scanning, so a start() call landing + // synchronously during that scan must not see a false "not running" + // reading and arm a second timer. + spyOnReconcile().mockImplementation(() => { + controller.start(); + return Promise.resolve(okResult); + }); + + controller.start(); + await advanceOneTick(); + + expect(vi.getTimerCount()).toBe(1); + }); + + it('logs rather than silently skipping an application with no stack_name', async () => { + mockDue([{ ...directApplicationFixture('app-no-stack', 'no-stack-web'), stack_name: null }]); + const reconcile = spyOnReconcile().mockResolvedValue(okResult); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + + controller.start(); + await advanceOneTick(); + + expect(reconcile).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('app-no-stack')); + }); + + it('restartPolling never leaves two timers running', () => { + controller.start(); + controller.restartPolling(); + controller.restartPolling(); + + expect(vi.getTimerCount()).toBe(1); + }); + + it('start is a no-op when already running', async () => { + mockDue([directApplicationFixture('app-double-start', 'double-start-web')]); + const reconcile = spyOnReconcile().mockResolvedValue(okResult); + + controller.start(); + controller.start(); + await advanceOneTick(); + + expect(reconcile).toHaveBeenCalledTimes(1); + }); +}); diff --git a/backend/src/__tests__/webhooks-git-source.test.ts b/backend/src/__tests__/webhooks-git-source.test.ts index 5ce0832c..b0bd2a7a 100644 --- a/backend/src/__tests__/webhooks-git-source.test.ts +++ b/backend/src/__tests__/webhooks-git-source.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; import request from 'supertest'; import jwt from 'jsonwebtoken'; +import crypto from 'crypto'; import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb'; let tmpDir: string; @@ -237,4 +238,142 @@ describe('node-aware Git source webhooks', () => { expect(history[0].status).toBe('success'); expect(history[0].error).toMatch(/debounced/i); }); + + it('forwards a provider delivery id to a remote node under the webhook namespace', async () => { + const db = DatabaseService.getInstance(); + const remoteNodeId = db.addNode({ + name: 'remote-delivery-id-webhook', + type: 'remote', + compose_dir: '/tmp', + is_default: false, + api_url: 'http://remote-delivery.example', + api_token: 'remote-token', + }); + const webhookId = db.addWebhook({ + node_id: remoteNodeId, + name: 'delivery id remote git', + stack_name: 'remote-stack', + action: 'git-pull', + secret: WebhookService.getInstance().generateSecret(), + enabled: true, + }); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response(JSON.stringify({ status: 'success', message: 'Fetched.' }), { status: 200 }), + ); + + const webhook = db.getWebhook(webhookId)!; + const deliverySourceId = db.getGlobalSettings().delivery_source_id; + const result = await WebhookService.getInstance().execute( + webhook, + 'git-pull', + 'test', + undefined, + 'provider-delivery-1', + ); + + expect(result.success).toBe(true); + expect(fetchSpy).toHaveBeenCalledWith( + 'http://remote-delivery.example/api/stacks/remote-stack/git-source/webhook-pull', + expect.objectContaining({ + method: 'POST', + body: expect.stringContaining(`"deliveryId":"webhook:${deliverySourceId}:${webhookId}:provider-delivery-1"`), + }), + ); + }); + + it('passes the same producer-scoped delivery identity to a local Git source', async () => { + const db = DatabaseService.getInstance(); + const nodeId = db.getDefaultNode()!.id; + const webhookId = db.addWebhook({ + node_id: nodeId, + name: 'delivery id local git', + stack_name: 'local-delivery-stack', + action: 'git-pull', + secret: WebhookService.getInstance().generateSecret(), + enabled: true, + }); + const webhook = db.getWebhook(webhookId)!; + const deliverySourceId = db.getGlobalSettings().delivery_source_id; + const { FileSystemService } = await import('../services/FileSystemService'); + const { GitSourceService } = await import('../services/GitSourceService'); + const stacksSpy = vi.spyOn(FileSystemService.prototype, 'getStacks') + .mockResolvedValue(['local-delivery-stack']); + const pullSpy = vi.spyOn(GitSourceService.getInstance(), 'handleWebhookPull') + .mockResolvedValue({ status: 'success', message: 'Fetched.' }); + + try { + const result = await WebhookService.getInstance().execute( + webhook, + 'git-pull', + 'test', + undefined, + 'provider-delivery-2', + ); + + expect(result).toEqual({ success: true, duration_ms: expect.any(Number) }); + expect(pullSpy).toHaveBeenCalledWith( + 'local-delivery-stack', + true, + `webhook:${deliverySourceId}:${webhookId}:provider-delivery-2`, + ); + + const secondWebhookId = db.addWebhook({ + node_id: nodeId, + name: 'second delivery id local git', + stack_name: 'local-delivery-stack', + action: 'git-pull', + secret: webhook.secret, + enabled: true, + }); + pullSpy.mockClear(); + await WebhookService.getInstance().execute( + db.getWebhook(secondWebhookId)!, + 'git-pull', + 'test', + undefined, + 'provider-delivery-2', + ); + expect(pullSpy).toHaveBeenCalledWith( + 'local-delivery-stack', + true, + `webhook:${deliverySourceId}:${secondWebhookId}:provider-delivery-2`, + ); + + db.updateGlobalSetting('delivery_source_id', 'second-control-source'); + pullSpy.mockClear(); + await WebhookService.getInstance().execute( + webhook, + 'git-pull', + 'test', + undefined, + 'provider-delivery-2', + ); + expect(pullSpy).toHaveBeenCalledWith( + 'local-delivery-stack', + true, + `webhook:second-control-source:${webhookId}:provider-delivery-2`, + ); + db.updateGlobalSetting('delivery_source_id', deliverySourceId!); + + const oversizedDeliveryId = 'x'.repeat(300); + const boundedId = crypto.createHash('sha256').update(oversizedDeliveryId).digest('hex'); + pullSpy.mockClear(); + await WebhookService.getInstance().execute( + webhook, + 'git-pull', + 'test', + undefined, + oversizedDeliveryId, + ); + expect(pullSpy).toHaveBeenCalledWith( + 'local-delivery-stack', + true, + `webhook:${deliverySourceId}:${webhookId}:sha256:${boundedId}`, + ); + } finally { + if (deliverySourceId) db.updateGlobalSetting('delivery_source_id', deliverySourceId); + stacksSpy.mockRestore(); + pullSpy.mockRestore(); + } + }); }); diff --git a/backend/src/__tests__/webhooks-trigger.test.ts b/backend/src/__tests__/webhooks-trigger.test.ts index 977e1b8c..a03dd246 100644 --- a/backend/src/__tests__/webhooks-trigger.test.ts +++ b/backend/src/__tests__/webhooks-trigger.test.ts @@ -251,6 +251,55 @@ describe('POST /api/webhooks/:id/trigger: authenticated happy path', () => { expect(res.body).toMatchObject({ action: 'start' }); }); + it('extracts a recognized provider delivery header and passes it through to execute', async () => { + const { id, secret } = createWebhook({ action: 'stop' }); + const body = '{}'; + const executeSpy = vi.spyOn(WebhookService.getInstance(), 'execute').mockResolvedValue({ success: true, duration_ms: 0 }); + + try { + await request(app) + .post(`/api/webhooks/${id}/trigger`) + .set('Content-Type', 'application/json') + .set('X-Webhook-Signature', sign(body, secret)) + .set('X-GitHub-Delivery', 'gh-delivery-123') + .send(body); + + expect(executeSpy).toHaveBeenCalledWith( + expect.objectContaining({ id }), + 'stop', + expect.anything(), + true, + 'gh-delivery-123', + ); + } finally { + executeSpy.mockRestore(); + } + }); + + it('passes no delivery id through when the caller sends no recognized header', async () => { + const { id, secret } = createWebhook({ action: 'stop' }); + const body = '{}'; + const executeSpy = vi.spyOn(WebhookService.getInstance(), 'execute').mockResolvedValue({ success: true, duration_ms: 0 }); + + try { + await request(app) + .post(`/api/webhooks/${id}/trigger`) + .set('Content-Type', 'application/json') + .set('X-Webhook-Signature', sign(body, secret)) + .send(body); + + expect(executeSpy).toHaveBeenCalledWith( + expect.objectContaining({ id }), + 'stop', + expect.anything(), + true, + undefined, + ); + } finally { + executeSpy.mockRestore(); + } + }); + it('rejects an unknown action override with 400 after the signature passes (L2)', async () => { const { id, secret } = createWebhook(); const body = '{"action":"nuke-the-cluster"}'; diff --git a/backend/src/bootstrap/shutdown.ts b/backend/src/bootstrap/shutdown.ts index 82c62a03..f5951cbd 100644 --- a/backend/src/bootstrap/shutdown.ts +++ b/backend/src/bootstrap/shutdown.ts @@ -9,6 +9,7 @@ import { FleetSyncRetryService } from '../services/FleetSyncRetryService'; import { SuppressionRetractionRetryService } from '../services/SuppressionRetractionRetryService'; import { DockerEventManager } from '../services/DockerEventManager'; import { ImageUpdateService } from '../services/ImageUpdateService'; +import { SourceController } from '../services/gitops/SourceController'; import { SchedulerService } from '../services/SchedulerService'; import { MfaService } from '../services/MfaService'; import { MeshService } from '../services/MeshService'; @@ -49,6 +50,9 @@ export function installShutdownHandlers(server: Server): void { try { ImageUpdateService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] ImageUpdateService cleanup failed:', (e as Error).message); } + try { SourceController.getInstance().stop(); } catch (e) { + console.warn('[Shutdown] SourceController cleanup failed:', (e as Error).message); + } try { SchedulerService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] SchedulerService cleanup failed:', (e as Error).message); } diff --git a/backend/src/bootstrap/startup.ts b/backend/src/bootstrap/startup.ts index e9453541..d7b97608 100644 --- a/backend/src/bootstrap/startup.ts +++ b/backend/src/bootstrap/startup.ts @@ -28,10 +28,11 @@ import { applyPilotModeCapabilityFilter } from '../services/CapabilityRegistry'; import { PilotTunnelManager } from '../services/PilotTunnelManager'; import { PilotMetrics } from '../services/PilotMetrics'; import { invalidateRemoteMetaCache } from '../helpers/cacheInvalidation'; -import { sweepStaleTempDirs as sweepStaleGitTempDirs, sweepGitManifestOrphans } from '../services/GitSourceService'; +import { GitSourceService, sweepStaleTempDirs as sweepStaleGitTempDirs, sweepGitManifestOrphans } from '../services/GitSourceService'; import { assertCreatesSettled, reclassifyInterruptedOperations, resolveInterruptedCreates } from '../services/gitops/createRecovery'; import { loadMigrationManifests, migrateDirectGitStacks, migrateInlineBlueprints } from '../services/gitops/migrate'; import { setGitOpsEventSink } from '../services/gitops/publish'; +import { SourceController } from '../services/gitops/SourceController'; import { NotificationService } from '../services/NotificationService'; import { sanitizeForLog } from '../utils/safeLog'; import { PORT } from '../helpers/constants'; @@ -98,6 +99,33 @@ function clearSelfContainerNotificationRouting(): void { } } +/** + * Reconcile-attempt recovery, then the managed-area sweep, in that fixed + * order: an attempt reserved but never settled (a crash between the two) + * must be resolved from durable state before the sweep or + * SourceController's own timer (started later in startServer, after + * registry delivery recovery settles per AUD-36) can race a recovery pass + * over the same attempts. Exported so this ordering is directly testable + * without driving the rest of startServer's unrelated service + * initialization. + */ +export async function runGitOpsSourceRecovery(): Promise { + try { + await GitSourceService.getInstance().recoverUnsettledReconcileAttempts(); + } catch (err) { + console.error('[GitSource] Reconcile-attempt recovery failed:', err instanceof Error ? err.stack ?? err.message : String(err)); + } + + // The managed-area sweep follows. It preserves anything whose ownership it + // cannot prove, so a failure here can only leave files behind, never remove + // the wrong ones, and retrying next boot is safe. + try { + await sweepGitManifestOrphans(); + } catch (err) { + console.warn('[GitManifest] Managed-area sweep failed:', err instanceof Error ? err.message : String(err)); + } +} + /** * Run the startup sequence: stack-directory migration, service initialization, * background watchdogs, then bind the HTTP server. The caller passes the @@ -233,14 +261,9 @@ export async function startServer(server: Server): Promise { console.error('[GitOps] Migration of pre-existing blueprints failed:', err instanceof Error ? err.stack ?? err.message : String(err)); } - // The managed-area sweep follows. It preserves anything whose ownership it - // cannot prove, so a failure here can only leave files behind, never remove - // the wrong ones, and retrying next boot is safe. - try { - await sweepGitManifestOrphans(); - } catch (err) { - console.warn('[GitManifest] Managed-area sweep failed:', err instanceof Error ? err.message : String(err)); - } + // Both steps must settle before SourceController starts below; see that + // function's own doc comment for why. + await runGitOpsSourceRecovery(); // Registry delivery recovery sweeps must settle before any mutation-capable // producer starts (AUD-36). @@ -287,6 +310,7 @@ export async function startServer(server: Server): Promise { FleetSyncRetryService.getInstance().start(); SuppressionRetractionRetryService.getInstance().start(); ImageUpdateService.getInstance().start(); + SourceController.getInstance().start(); SchedulerService.getInstance().start(); MfaService.getInstance().start(); MeshService.getInstance().start().catch((err) => { diff --git a/backend/src/routes/gitSources.ts b/backend/src/routes/gitSources.ts index cef891f0..0e6f9054 100644 --- a/backend/src/routes/gitSources.ts +++ b/backend/src/routes/gitSources.ts @@ -5,7 +5,7 @@ import { GitProjectManifestService } from '../services/GitProjectManifestService import { FileSystemService } from '../services/FileSystemService'; import { DatabaseService } from '../services/DatabaseService'; import { CryptoService } from '../services/CryptoService'; -import { requirePermission } from '../middleware/permissions'; +import { checkPermission, requirePermission } from '../middleware/permissions'; import { classifySourceRow, satisfiesGitOpsRead } from '../services/gitops/readAuth'; import { NOT_APPLICABLE_REVISION, projectStackRevision, stackResourceSet } from '../helpers/gitopsResponse'; import { respondWithHistory } from '../helpers/gitopsHistoryPage'; @@ -27,6 +27,8 @@ import { assertSafeOutboundHostname, resolveSafeOutboundHostname, UnsafeOutbound const MAX_BRANCH_LENGTH = REF_MAX_LEN; const MAX_ENV_PATH_LENGTH = 1024; const MAX_TOKEN_LENGTH = 8192; +const MAX_SUSPEND_REASON_LENGTH = 512; +const MAX_WEBHOOK_DELIVERY_ID_LENGTH = 512; /** * Shared handler for the "browse repository" compose-file picker: validate the @@ -584,14 +586,28 @@ stackGitSourceRouter.post('/:stackName/git-source/webhook-pull', async (req: Req return; } if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return; + const deliveryId = req.body?.deliveryId; + if ( + deliveryId !== undefined + && (typeof deliveryId !== 'string' || !deliveryId.trim() || deliveryId.length > MAX_WEBHOOK_DELIVERY_ID_LENGTH) + ) { + res.status(400).json({ error: 'deliveryId must be a non-empty string of at most 512 characters' }); + return; + } try { - const source = GitSourceService.getInstance().get(stackName); + const service = GitSourceService.getInstance(); + const source = service.get(stackName); if (!source) { res.status(404).json({ error: 'No Git source configured for this stack', status: 'error' }); return; } - if (source.auto_apply_on_webhook && source.auto_deploy_on_apply && !requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return; - const result = await GitSourceService.getInstance().handleWebhookPull(stackName); + const normalizedDeliveryId = deliveryId?.trim(); + const deployAuthorized = checkPermission(req, 'stack:deploy', 'stack', stackName); + if (service.webhookDeliveryRequiresDeploy(stackName, normalizedDeliveryId) && !deployAuthorized) { + requirePermission(req, res, 'stack:deploy', 'stack', stackName); + return; + } + const result = await service.handleWebhookPull(stackName, deployAuthorized, normalizedDeliveryId); // Map the outcome to a real HTTP status so a Git provider sees a 4xx on // failure instead of a 200 with an error body (which it would read as // "delivered fine, stop retrying"). @@ -616,6 +632,64 @@ stackGitSourceRouter.post('/:stackName/git-source/dismiss-pending', async (req: } }); +stackGitSourceRouter.post('/:stackName/git-source/suspend', async (req: Request, res: Response): Promise => { + const stackName = req.params.stackName as string; + if (!isValidStackName(stackName)) { + res.status(400).json({ error: 'Invalid stack name' }); + return; + } + if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return; + const { reason: rawReason } = req.body ?? {}; + const reason = typeof rawReason === 'string' ? rawReason : undefined; + if (reason !== undefined && reason.length > MAX_SUSPEND_REASON_LENGTH) { + res.status(400).json({ error: 'reason is too long' }); + return; + } + try { + const result = await GitSourceService.getInstance().suspend(stackName, { + actor: req.user?.username ?? 'unknown', + reason, + }); + res.json(result); + } catch (error) { + sendGitSourceError(res, error); + } +}); + +stackGitSourceRouter.post('/:stackName/git-source/resume', async (req: Request, res: Response): Promise => { + const stackName = req.params.stackName as string; + if (!isValidStackName(stackName)) { + res.status(400).json({ error: 'Invalid stack name' }); + return; + } + if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return; + try { + const result = await GitSourceService.getInstance().resume(stackName, { + actor: req.user?.username ?? 'unknown', + }); + res.json(result); + } catch (error) { + sendGitSourceError(res, error); + } +}); + +stackGitSourceRouter.post('/:stackName/git-source/retry', async (req: Request, res: Response): Promise => { + const stackName = req.params.stackName as string; + if (!isValidStackName(stackName)) { + res.status(400).json({ error: 'Invalid stack name' }); + return; + } + if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return; + try { + const result = await GitSourceService.getInstance().retry(stackName, { + actor: req.user?.username ?? 'unknown', + }); + res.json(result); + } catch (error) { + sendGitSourceError(res, error); + } +}); + // Edit-mode repo browse for an existing stack: gated by stack:edit so a user who // can edit (but not create) stacks can re-pick files, and reuses the stored token // when the request omits one. diff --git a/backend/src/routes/webhooks.ts b/backend/src/routes/webhooks.ts index f9b97d62..42f32e3c 100644 --- a/backend/src/routes/webhooks.ts +++ b/backend/src/routes/webhooks.ts @@ -12,6 +12,34 @@ function isWebhookAction(value: unknown): value is WebhookAction { return typeof value === 'string' && (VALID_WEBHOOK_ACTIONS as readonly string[]).includes(value); } +// Recognized per-delivery identity headers, in priority order. This +// endpoint is a generic HMAC-signed trigger, not a provider-specific +// receiver, so a well-known provider header is the only delivery identity +// available, and only when the caller happens to send one. The value is a +// stable delivery identity available. WebhookService namespaces the value by +// control instance and configured webhook before it reaches GitSourceService, +// so two producers cannot collide on the same provider-assigned id. +// +// Each header is still the provider's actual per-delivery identity rather +// than a webhook- or connection-level id that stays constant across every +// delivery from that source: GitHub's X-GitHub-Delivery GUID changes per +// delivery (it is stable only across redeliveries of the same delivery); +// GitLab's is Webhook-ID, the modern name for its Idempotency-Key, not +// X-Gitlab-Event-UUID, which tracks recursive-trigger chains and can +// repeat across genuinely distinct events; Bitbucket's is X-Request-UUID, +// not X-Hook-UUID, which identifies the webhook configuration itself. +// Picking the wrong one would deduplicate genuinely distinct pushes. +const DELIVERY_ID_HEADERS = ['x-github-delivery', 'webhook-id', 'idempotency-key', 'x-request-uuid', 'x-webhook-delivery-id'] as const; + +function deliveryIdFromHeaders(headers: Request['headers']): string | undefined { + for (const name of DELIVERY_ID_HEADERS) { + const value = headers[name]; + const first = Array.isArray(value) ? value[0] : value; + if (first) return first; + } + return undefined; +} + export const webhooksRouter = Router(); webhooksRouter.get('/', authMiddleware, async (req: Request, res: Response): Promise => { @@ -193,6 +221,7 @@ webhooksRouter.post('/:id/trigger', webhookTriggerLimiter, async (req: Request, action = overrideAction; } const triggerSource = req.headers['user-agent'] || req.ip || null; + const deliveryId = deliveryIdFromHeaders(req.headers); // Execute asynchronously; return 202 immediately. res.status(202).json({ message: 'Webhook accepted', action }); @@ -202,7 +231,7 @@ webhooksRouter.post('/:id/trigger', webhookTriggerLimiter, async (req: Request, // dispatch the action still completes and recordExecution swallows the // FK error from the CASCADE. atomic is unconditionally true, so the // deploy/pull paths always run in atomic mode here. - svc.execute(webhook, action, triggerSource, true).catch(err => { + svc.execute(webhook, action, triggerSource, true, deliveryId).catch(err => { console.error(`[Webhooks] Execution error for webhook ${id}:`, err); }); } catch (error) { diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index d999877d..c406e6b0 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -1976,6 +1976,24 @@ export class DatabaseService { // existing installs already have. New installs get it from the // CREATE TABLE; older DBs need the additive column here. maybeAddCol('gitops_applications', 'source_suspended_reason', 'TEXT NULL'); + // Portable accepted-generation contract fields. Additive and + // nullable: existing generation rows decode these as an explicit + // limitation rather than invented evidence. + maybeAddCol('gitops_generations', 'portable_manifest_json', 'TEXT NULL'); + maybeAddCol('gitops_generations', 'compose_inputs_json', 'TEXT NULL'); + maybeAddCol('gitops_generations', 'source_policy_evidence_json', 'TEXT NULL'); + maybeAddCol('gitops_generations', 'security_policy_evidence_json', 'TEXT NULL'); + maybeAddCol('gitops_generations', 'support_requirements_json', 'TEXT NULL'); + maybeAddCol('gitops_generations', 'compatibility_requirements_json', 'TEXT NULL'); + // Controller-owned bookkeeping (source policy, poll cadence, attempt + // sequence). New installs get these from the CREATE TABLE; older DBs + // need the additive columns here. Existing installations must not + // start unattended polling, so poll_interval_secs and next_poll_at + // stay NULL until an operator (or the migration below) sets one. + maybeAddCol('gitops_applications', 'source_policy', "TEXT NOT NULL DEFAULT 'manual' CHECK (source_policy IN ('manual','review','automatic'))"); + maybeAddCol('gitops_applications', 'poll_interval_secs', 'INTEGER NULL'); + maybeAddCol('gitops_applications', 'next_poll_at', 'INTEGER NULL'); + maybeAddCol('gitops_applications', 'attempt_seq', 'INTEGER NOT NULL DEFAULT 0'); // Distributed API model columns maybeAddCol('nodes', 'api_url', "TEXT DEFAULT ''"); diff --git a/backend/src/services/GitProjectManifestService.ts b/backend/src/services/GitProjectManifestService.ts index df4138eb..e38f8d1e 100644 --- a/backend/src/services/GitProjectManifestService.ts +++ b/backend/src/services/GitProjectManifestService.ts @@ -51,6 +51,10 @@ export const MANAGED_ROOT_NAME = 'git-managed'; export const MANIFEST_FILENAME = 'manifest.v1.json'; export const PROMOTION_MARKER = 'promotion.json'; export const CANDIDATE_COMPLETE_MARKER = '.candidate-complete'; + +type CandidateClaims = + | { complete: true; dirs: ReadonlySet } + | { complete: false }; export const GENERATIONS_DIR = 'generations'; const DETACH_RECOVERY_MARKER = 'detach-recovery.v1.json'; @@ -421,10 +425,19 @@ export class GitProjectManifestService { /** Atomic manifest write (tmp + rename). */ async writeManifest(stackName: string, manifest: GitProjectManifest): Promise { - const dir = this.managedRoot(stackName); - await fs.promises.mkdir(dir, { recursive: true }); - const target = path.join(dir, MANIFEST_FILENAME); - const tmp = path.join(dir, `${MANIFEST_FILENAME}.tmp`); + // Inline barrier at the mkdir/write/rename sinks (CodeQL path-injection): + // confine the resolved managed directory to the managed area before any + // filesystem call touches it, then confine the filenames joined onto it. + const root = path.resolve(this.managedRoot(stackName)); + if (!root.startsWith(managedAreaBase() + path.sep)) { + throw Object.assign(new Error('Path escapes the managed area'), { code: 'INVALID_PATH' }); + } + const target = path.resolve(root, MANIFEST_FILENAME); + const tmp = path.resolve(root, `${MANIFEST_FILENAME}.tmp`); + if (!target.startsWith(root + path.sep) || !tmp.startsWith(root + path.sep)) { + throw Object.assign(new Error('Path escapes managed project directory'), { code: 'INVALID_PATH' }); + } + await fs.promises.mkdir(root, { recursive: true }); await fs.promises.writeFile(tmp, JSON.stringify(manifest, null, 2), 'utf8'); await fs.promises.rename(tmp, target); } @@ -1268,12 +1281,20 @@ export class GitProjectManifestService { * is finalized; an uncommitted promotion restores the prior generation. * A third state is treated as an operator edit, so recovery declines and * flags migration_required. Interrupted detach snapshots are restored first. + * Complete candidate claims hold the directory basenames that durable state + * still references. Incomplete claims preserve every candidate because + * ownership is uncertain. */ async sweepManagedArea( stackName: string, - opts: { repoUrl: string; branch: string; stackExists: boolean }, + opts: { + repoUrl: string; + branch: string; + stackExists: boolean; + candidateClaims: CandidateClaims; + }, ): Promise { - const { repoUrl, branch, stackExists } = opts; + const { repoUrl, branch, stackExists, candidateClaims } = opts; if (!stackExists) { await this.deleteManagedArea(stackName); return; @@ -1359,44 +1380,64 @@ export class GitProjectManifestService { } } - // Orphan candidates: incomplete or stale. + if (!candidateClaims.complete) { + await this.flagRecoveryRequired( + stackName, + `candidate ownership for ${sanitizeForLog(stackName)} could not be established`, + ); + return; + } + + // With a complete claim inventory, reap candidates that are incomplete + // or stale and unclaimed. const dir = this.generationsDir(stackName); + let entries: fs.Dirent[]; try { - const entries = await fs.promises.readdir(dir, { withFileTypes: true }); - const now = Date.now(); - const areaBase = managedAreaBase(); - for (const entry of entries) { - if (!entry.isDirectory() || !entry.name.startsWith('candidate-')) continue; - const abs = path.resolve(dir, entry.name); - // Inline containment barrier at the removal sink (see - // `managedAreaBase`): the analyzer credits this literal - // comparison, not the positional check below it. - if (!abs.startsWith(areaBase + path.sep)) { - console.warn(`[GitManifest] refusing to reap orphan candidate ${sanitizeForLog(entry.name)} for ${sanitizeForLog(stackName)}: it resolves outside the managed area`); - continue; - } - // Same positional barrier as generation pruning: the boot sweep - // reaps candidate directories nobody claims, which is precisely - // the kind of unattended delete a planted link would steer. - if (!await isRealPathAtManagedLocation(abs)) { - console.warn(`[GitManifest] refusing to reap orphan candidate ${sanitizeForLog(entry.name)} for ${sanitizeForLog(stackName)}: it is not at its own location in the managed area`); - continue; - } - const complete = await fs.promises - .access(path.join(abs, CANDIDATE_COMPLETE_MARKER)) - .then(() => true) - .catch(() => false); - if (!complete) { - await fs.promises.rm(abs, { recursive: true, force: true }); - continue; - } - const st = await fs.promises.stat(abs); - if (now - st.mtimeMs > ORPHAN_CANDIDATE_AGE_MS) { - await fs.promises.rm(abs, { recursive: true, force: true }); - } + entries = await fs.promises.readdir(dir, { withFileTypes: true }); + } catch (e) { + if ((e as NodeJS.ErrnoException).code === 'ENOENT') return; + throw e; + } + const now = Date.now(); + const areaBase = managedAreaBase(); + for (const entry of entries) { + if (!entry.isDirectory() || !entry.name.startsWith('candidate-')) continue; + // A claimed candidate is still needed regardless of age or + // completeness. Claims come from application pointers, the + // source pending record, and generation rows linked to + // unsettled attempts. Together they are the durable ownership + // record that makes recovery before cleanup safe. + if (candidateClaims.dirs.has(entry.name)) continue; + const abs = path.resolve(dir, entry.name); + // Inline containment barrier at the removal sink (see + // `managedAreaBase`): the analyzer credits this literal + // comparison, not the positional check below it. + if (!abs.startsWith(areaBase + path.sep)) { + console.warn(`[GitManifest] refusing to reap orphan candidate ${sanitizeForLog(entry.name)} for ${sanitizeForLog(stackName)}: it resolves outside the managed area`); + continue; + } + // Same positional barrier as generation pruning: the boot sweep + // reaps candidate directories nobody claims, which is precisely + // the kind of unattended delete a planted link would steer. + if (!await isRealPathAtManagedLocation(abs)) { + console.warn(`[GitManifest] refusing to reap orphan candidate ${sanitizeForLog(entry.name)} for ${sanitizeForLog(stackName)}: it is not at its own location in the managed area`); + continue; + } + let complete = true; + try { + await fs.promises.access(path.join(abs, CANDIDATE_COMPLETE_MARKER)); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== 'ENOENT') throw e; + complete = false; + } + if (!complete) { + await fs.promises.rm(abs, { recursive: true, force: true }); + continue; + } + const st = await fs.promises.stat(abs); + if (now - st.mtimeMs > ORPHAN_CANDIDATE_AGE_MS) { + await fs.promises.rm(abs, { recursive: true, force: true }); } - } catch { - // no generations dir yet } } diff --git a/backend/src/services/GitSourceService.ts b/backend/src/services/GitSourceService.ts index 345e2560..07e23eca 100644 --- a/backend/src/services/GitSourceService.ts +++ b/backend/src/services/GitSourceService.ts @@ -34,7 +34,19 @@ import { nativeGitTransport, verifyFastForward } from './git/nativeGitTransport' import { fingerprintFromKnownHostsLine } from './git/sshTrust'; import { validateCaBundlePem } from './git/caBundle'; import { GitOpsStore } from './gitops/store'; -import { GitOpsTransitions, GitOpsTransitionError } from './gitops/transitions'; +import type { GitOpsHistoryCursor } from './gitops/history'; +import { projectApplication } from './gitops/derive'; +import { outcomeFromSourceFacet, isNextAction, isReconcileOutcome, type ReconcileOutcome, type ReconcileResult } from './gitops/outcomes'; +import { coalesceKey, deliveryKey, type ReconcileRequest, type ReconcileTrigger } from './gitops/triggers'; +import { classifyFailure } from './gitops/backoff'; +import { BlueprintTargetAdapter, type AcceptedGeneration, type DispatchContext, type DispatchResult } from './gitops/handoff'; +import { + GitOpsTransitions, + GitOpsTransitionError, + type EventEnvelope, + type ReconcileDeliveryIntent, +} from './gitops/transitions'; +import { decodeGitOpsJson, isRecord, GitOpsJsonError } from './gitops/json'; import { buildCreateCheckpointRow, buildDirectApplicationRow, @@ -43,7 +55,7 @@ import { newGitOpsId, stackManagedRoot, } from './gitops/directApplication'; -import type { GitOpsApplicationRow } from './gitops/types'; +import type { GitOpsApplicationRow, GitOpsHistoryRow } from './gitops/types'; import { appliedRelPathFor, candidateRelPathForSha, deleteStagingMarker, readStagingMarker, validateCandidateRelPath, writeStagingMarker } from './gitops/createStagingMarker'; import { cleanupUnclaimedManagedRoot, removeOperationOwnedPaths } from './gitops/createCleanup'; import { managedAreaBase } from './gitops/managedPaths'; @@ -291,6 +303,37 @@ export interface GitApplyOpts { requirePlanFingerprint?: boolean; } +type GitApplyResult = { + applied: boolean; + deployed: boolean; + deployError?: string; + recoveryId?: string; +}; + +type WorkOutcome = + | { status: 'fulfilled'; value: T } + | { status: 'rejected'; reason: unknown }; + +type SharedExecution = WorkOutcome & { result: ReconcileResult }; + +type LeaderCompletion = { + execution: SharedExecution; + settled: boolean; +}; + +/** + * In-process executions keyed by a request-derived execution key: the + * operation id each reservation minted, and the promise a later joiner awaits + * instead of repeating the work. + */ +type InFlightMap = Map> }>; + +type ExecutionSubmission = + | { kind: 'executed'; execution: SharedExecution } + | { kind: 'replayed'; result: ReconcileResult }; + +type WebhookPullResult = { status: 'success' | 'skipped' | 'error'; message: string }; + // ─── Constants ─────────────────────────────────────────────────────────────── const TEMP_DIR_PREFIX = 'sencho-git-'; @@ -878,13 +921,13 @@ export class GitSourceService { last_debounce_at: existing?.last_debounce_at ?? null, }); - if (configChanged) { + const app = this.gitopsApplicationFor(input.stackName); + if (configChanged || !app) { db.clearGitSourcePending(input.stackName); } - const app = this.gitopsApplicationFor(input.stackName); const envelope = this.gitopsEnvelope(crypto.randomUUID(), 'system:git-source', 'configure'); - if (!app && !existing && !this.gitopsNameHeld(input.stackName)) { + if (!app && !this.gitopsNameHeld(input.stackName)) { // Linking a stack that already exists. Nothing is fetched or // accepted yet, so the application starts live with no desired // commit and the projection asks for a fetch. @@ -1885,29 +1928,68 @@ export class GitSourceService { // ─── Pull / apply ──────────────────────────────────────────────────────── + /** + * A short, log-friendly token for an operation id. A reserved id + * (`:attempt:` or `::`) + * repeats the same prefix across every attempt on one stack, so only the + * suffix after the last colon tells two attempts apart. A plain UUID has + * no such suffix, so its leading characters are used instead. + */ + private static shortOperationId(operationId: string): string { + const lastColon = operationId.lastIndexOf(':'); + return lastColon === -1 ? operationId.slice(0, 8) : operationId.slice(lastColon + 1); + } + public async pull(stackName: string, opts: { actor?: string } = {}): Promise { // Guarded by the per-stack mutex (see withStackLock). Without this, a // concurrent delete-source + pull can land a pending row on a stack // whose config row has just been removed. const actor = opts.actor ?? 'unknown'; - return this.withStackLock(stackName, async () => { + + // Reservation and coalescing need a real gitops application to attach a + // durable attempt to, the same definition pullLocked itself uses to + // decide whether it has any gitops bookkeeping to do at all. + const gitopsApp = this.gitopsApplicationFor(stackName); + const doPull = this.doPullWork(stackName, actor, gitopsApp?.id); + if (!gitopsApp) { + this.refuseUntrackedSource(stackName, actor, 'fetch', true); + return doPull(); + } + + const request: ReconcileRequest = { intent: 'fetch', applicationId: gitopsApp.id, stackName, trigger: 'manual', actor }; + const submission = await this.submitExecution( + this.inFlightFetches, + request, + doPull, + (outcome) => this.fetchExecutionResult(stackName, outcome), + ); + return GitSourceService.valueFromSubmission(submission); + } + + /** + * The shared fetch work closure: the per-stack lock plus pullLocked + * itself, reporting a real failure to the server console and to the + * stack's activity feed here rather than in each producer, so it is + * recorded exactly once no matter which fetch-intent producer (pull(), + * reconcile()) owns the reservation that ends up leading. Kept as one + * factory so pull() and reconcile()'s fetch path run the literal same + * closure shape, which is what lets them join the same coalescing map + * (inFlightFetches) in the first place: two producers can only coalesce + * onto one execution if that execution really is the same work. + */ + private doPullWork(stackName: string, actor: string, applicationId?: string): (operationId?: string) => Promise { + return (operationId?: string) => this.withStackLock(stackName, async () => { try { - return await this.pullLocked(stackName, actor); + if (applicationId) this.assertLiveApplication(stackName, applicationId); + return await this.pullLocked(stackName, actor, operationId); } catch (e) { + console.error(`[GitSource] fetch failed for ${sanitizeForLog(stackName)}:`, e instanceof Error ? e.message : String(e)); this.recordGitActivity(stackName, 'git_pull_failed', `Git pull failed for ${stackName}`, actor, 'error'); throw e; } }); } - /** - * Body of pull(); assumes the caller already holds the per-stack lock. - * handleWebhookPull calls this directly so that its debounce re-check, - * this fetch, and the apply all run inside the single lock that - * handleWebhookPull holds. Without that, a concurrent webhook fan-out - * reads last_debounce_at while it is still unset on every request, slips - * past the gate, and clones once per request. - */ /** * The GitOps application tracking this stack, or null when there is none. * @@ -1977,12 +2059,23 @@ export class GitSourceService { }); } - private async pullLocked(stackName: string, actor: string): Promise { + private async pullLocked(stackName: string, actor: string, operationId?: string): Promise { const db = DatabaseService.getInstance(); const src = db.getGitSource(stackName); if (!src) throw new GitSourceError('GIT_ERROR', 'No Git source configured for this stack.'); const gitopsApp = this.gitopsApplicationFor(stackName); - const gitopsOperationId = crypto.randomUUID(); + // fetchStarted has its own suspension guard, but recordGitOps swallows + // its rejection (so a fetch that already touched the filesystem is never + // failed out from under itself), which would let a suspended source keep + // cloning and staging pending updates. Stop before any of that starts. + if (gitopsApp?.suspended_at) { + throw new GitSourceError('OPERATION_IN_FLIGHT', `Reconciliation is suspended for ${stackName}.`); + } + // A caller that reserved a durable attempt for this fetch passes its own + // operation id in, so the attempt and every stage of gitops evidence it + // produces share one identity. A direct low-level call that reserved + // nothing still gets an id of its own. + const gitopsOperationId = operationId ?? crypto.randomUUID(); const gitopsEnv = this.gitopsEnvelope(gitopsOperationId, actor, 'pull'); // A fetch that starts and never terminates is worse than one that is // never recorded: fetchStarted refuses to open a second operation, so @@ -2078,7 +2171,6 @@ export class GitSourceService { const prior = priorRead; if (prior) manifestSummary = manifestSvc.summaryFrom(prior); - const operationId = crypto.randomUUID(); let plan: GitChangePlan | null = null; if (materialization.value?.inventory) { plan = await this.computeChangePlan({ @@ -2180,7 +2272,13 @@ export class GitSourceService { { fingerprint: plan?.fingerprint ?? '', schemaVersion: GIT_CHANGE_PLAN_SCHEMA_VERSION, - operationId, + // This fetch attempt's own id, not an independent one: + // applyLockedBody falls back to pending.operationId for + // its gitops transitions when its caller reserved no + // attempt, so that fallback has to inherit the real fetch + // attempt's lineage rather than an identity nothing else + // knows about. + operationId: gitopsOperationId, reviewedLive: plan ? this.reviewedLiveFromPlan(plan) : [], }, ), @@ -2196,7 +2294,7 @@ export class GitSourceService { this.recordGitActivity( stackName, 'git_plan_blocked', - `Git plan blocked for ${stackName} (${shortSha}, op ${operationId.slice(0, 8)}, plan ${fpPrefix})`, + `Git plan blocked for ${stackName} (${shortSha}, op ${GitSourceService.shortOperationId(gitopsOperationId)}, plan ${fpPrefix})`, actor, 'warning', ); @@ -2204,7 +2302,7 @@ export class GitSourceService { this.recordGitActivity( stackName, 'git_pull_ready', - `Git pull ready for ${stackName} (${shortSha}, op ${operationId.slice(0, 8)}, plan ${fpPrefix})`, + `Git pull ready for ${stackName} (${shortSha}, op ${GitSourceService.shortOperationId(gitopsOperationId)}, plan ${fpPrefix})`, actor, ); } @@ -2243,11 +2341,892 @@ export class GitSourceService { stackName: string, commitSha: string, opts: GitApplyOpts = {}, - ): Promise<{ applied: boolean; deployed: boolean; deployError?: string; recoveryId?: string }> { - return this.withStackLock(stackName, () => this.applyWithSharedLock(stackName, commitSha, { - ...opts, - requirePlanFingerprint: opts.requirePlanFingerprint !== false, - })); + ): Promise { + // Resolved once, with applyLockedBody's own formula (its shouldDeploy), + // so the coalesce key below and the deploy behavior actually executed + // can never disagree: two calls that resolve to different deploy + // behavior must never share a coalesce key, or one could silently + // receive the other's deployed/not-deployed result. + const resolvedDeploy = opts.deploy ?? DatabaseService.getInstance().getGitSource(stackName)?.auto_deploy_on_apply ?? false; + const finalOpts: GitApplyOpts = { ...opts, deploy: resolvedDeploy, requirePlanFingerprint: opts.requirePlanFingerprint !== false }; + const doApply = (applicationId?: string, operationId?: string) => this.withStackLock( + stackName, + async () => { + if (applicationId) this.assertLiveApplication(stackName, applicationId); + return this.applyWithSharedLock(stackName, commitSha, finalOpts, operationId); + }, + ); + + // Same reasoning as pull(): reservation needs a real gitops + // application to attach a durable attempt to. + const gitopsApp = this.gitopsApplicationFor(stackName); + if (!gitopsApp) { + this.refuseUntrackedSource(stackName, opts.actor ?? 'unknown', 'apply', true); + return doApply(); + } + + const request: ReconcileRequest = { + intent: 'apply', + applicationId: gitopsApp.id, + stackName, + trigger: 'manual', + actor: opts.actor ?? 'unknown', + commitSha, + planFingerprint: opts.planFingerprint ?? '', + deploy: resolvedDeploy, + }; + // A policy-bypassing apply must never coalesce with anything else: + // bypassPolicy changes behavior but is not part of the coalesce key (a + // plain policy-gate bypass has no natural identity to key on), so + // joining could hand a non-bypassing caller someone else's bypassed + // result, or silently drop an admin's explicit bypass onto a request + // that never asked for one. A call-local key suffix guarantees this + // call can neither join an existing leader nor be joined by a later one. + const baseKey = GitSourceService.applyExecutionKey(request, finalOpts.requirePlanFingerprint === true); + const key = opts.bypassPolicy + ? `${baseKey}:policy-bypass:${crypto.randomUUID()}` + : baseKey; + const submission = await this.submitExecution( + this.inFlightApplies, + request, + (operationId) => doApply(gitopsApp.id, operationId), + (outcome) => this.applyExecutionResult(stackName, outcome), + key, + ); + return GitSourceService.valueFromSubmission(submission); + } + + /** + * Outcomes the application row already reflects truthfully, so reconcile + * trusts the derived result over a classified fallback. Most throw sites + * in pullLocked/applyLockedBody fire before any transition opens (missing + * config, stale commitSha, lock contention) and leave the row saying + * nothing about the failure, which is what the fallback covers; + * 'suspended' belongs here because its guards throw only when the row + * already, correctly, says so. + */ + private static readonly FAILURE_REFLECTED_OUTCOMES: ReadonlySet = new Set([ + 'failed_previous_intact', + 'retry_scheduled', + 'recovery_required', + 'blocked', + 'suspended', + ]); + + /** The settled result for a stack that carries no GitOps application at all. */ + private static noApplicationResult(): ReconcileResult { + return { outcome: 'unknown', reason: 'No GitOps application exists for this stack.', nextAction: 'none' }; + } + + /** + * The settled result for a request naming an application id that is no + * longer the live one for its stack. Failing closed here (rather than + * proceeding against whatever application now holds the stack name) + * keeps a request from settling against an application it never named. + */ + private static staleApplicationResult(): ReconcileResult { + return { + outcome: 'unknown', + reason: 'The live application for this stack no longer matches the requested application id.', + nextAction: 'none', + }; + } + + /** Refuse work when the application resolved before locking is no longer live. */ + private assertLiveApplication(stackName: string, applicationId: string): void { + if (GitOpsStore.getInstance().getLiveDirectApplication(stackName)?.id === applicationId) return; + throw new GitSourceError('OPERATION_IN_FLIGHT', GitSourceService.staleApplicationResult().reason); + } + + /** + * Shared fetch and apply executions. Manual routes, controller requests, + * and webhooks all use these maps so producer choice cannot create a + * duplicate side effect or a different normalized result. + */ + private readonly inFlightFetches: InFlightMap = new Map(); + private readonly inFlightApplies: InFlightMap = new Map(); + private readonly inFlightWebhookDeliveries = new Map }>(); + + /** + * The controller-facing entry point: one normalized submission in, + * one normalized result out, for any trigger (manual, poll, retry, + * API, config change, startup, resume). + * + * Every submission against a real application row gets a durable + * attempt, reserved before any side effect and settled with the + * normalized result once execution finishes -- reserved but never + * settled is exactly what startup recovery looks for after a crash. + * A submission naming an application id that does not exist at all + * (a fabricated id, or a stack with no GitOps application) reserves + * nothing: there is no row to attach a durable attempt to, and the + * identity/no-application guards below already produce a truthful + * result for it without doing any work worth protecting. + */ + public async reconcile(request: ReconcileRequest): Promise { + if (!GitOpsStore.getInstance().getApplication(request.applicationId)) { + // request.applicationId does not exist as any row, so it can + // never equal a real live application's id: the identity guard + // alone already produces the truthful result, without a stack + // lock or any work worth protecting. + const liveApp = GitOpsStore.getInstance().getLiveDirectApplication(request.stackName); + return liveApp ? GitSourceService.staleApplicationResult() : GitSourceService.noApplicationResult(); + } + + if (request.intent === 'fetch') { + return this.reconcileFetch(request); + } + + const submission = await this.submitExecution( + this.inFlightApplies, + request, + (operationId) => this.withStackLock(request.stackName, async () => { + this.assertLiveApplication(request.stackName, request.applicationId); + return this.applyWithSharedLock(request.stackName, request.commitSha, { + actor: request.actor, + deploy: request.deploy, + planFingerprint: request.planFingerprint, + requirePlanFingerprint: false, + }, operationId); + }), + (outcome) => this.applyExecutionResult(request.stackName, outcome), + GitSourceService.applyExecutionKey(request, false), + ); + return GitSourceService.resultFromSubmission(submission); + } + + private static applyExecutionKey( + request: ReconcileRequest & { intent: 'apply' }, + requirePlanFingerprint: boolean, + ): string { + return `${coalesceKey(request)}:fingerprint-${requirePlanFingerprint ? 'required' : 'optional'}`; + } + + /** + * The in-flight execution in `map` whose reservation minted exactly this + * operation id, regardless of which coalesce key it is running under. + * Coalesce keys and operation ids are not co-extensive (an apply's key + * includes its commitSha/planFingerprint/deploy, which a shared external + * delivery id does not carry), so two submissions can collide on operation + * id while running under different keys. + */ + private static findByOperationId(map: InFlightMap, operationId: string): Promise> | undefined { + for (const entry of map.values()) { + if (entry.operationId === operationId) return entry.promise; + } + return undefined; + } + + private static resultFromSubmission(submission: ExecutionSubmission): ReconcileResult { + return submission.kind === 'replayed' ? submission.result : submission.execution.result; + } + + /** Preserve manual route return and throw behavior after settlement is attempted. */ + private static valueFromSubmission(submission: ExecutionSubmission): T { + if (submission.kind === 'replayed') { + throw new GitSourceError('GIT_ERROR', 'This operation was already recorded and cannot be replayed as a new manual request.'); + } + if (submission.execution.status === 'rejected') throw submission.execution.reason; + return submission.execution.value; + } + + /** + * Reserve one submission, join equivalent work when possible, and attempt + * to settle each reservation from the leader's single normalized result. + * A failed leader settlement leaves its followers unsettled for recovery. + */ + private async submitExecution( + map: InFlightMap, + request: ReconcileRequest, + work: (operationId: string) => Promise, + normalize: (outcome: WorkOutcome) => ReconcileResult, + key = coalesceKey(request), + deliveryIntent?: ReconcileDeliveryIntent, + ): Promise> { + const leader = map.get(key); + const { envelope, reserved } = this.reserveOwnAttemptOrFailClosed(request, leader?.operationId, deliveryIntent); + + if (leader && (reserved || leader.operationId === envelope.operationId)) { + const completion = await leader.promise; + if (reserved && completion.settled) { + this.settleAttempt(request.applicationId, envelope, completion.execution.result); + } + return { kind: 'executed', execution: completion.execution }; + } + + if (!reserved) { + const byOperationId = GitSourceService.findByOperationId(map, envelope.operationId); + if (byOperationId) { + return { kind: 'executed', execution: (await byOperationId).execution }; + } + return { + kind: 'replayed', + result: this.resolveAlreadyReservedAttempt( + request.applicationId, + envelope.operationId, + request.actor, + request.trigger, + ), + }; + } + + const promise = this.captureExecution(request, envelope, work, normalize) + .then((execution): LeaderCompletion => ({ + execution, + settled: this.settleAttempt(request.applicationId, envelope, execution.result), + })); + const entry = { operationId: envelope.operationId, promise }; + map.set(key, entry); + try { + const completion = await promise; + return { kind: 'executed', execution: completion.execution }; + } finally { + if (map.get(key) === entry) map.delete(key); + } + } + + /** Capture raw producer behavior and compute one normalized result for all followers. */ + private async captureExecution( + request: ReconcileRequest, + envelope: EventEnvelope, + work: (operationId: string) => Promise, + normalize: (outcome: WorkOutcome) => ReconcileResult, + ): Promise> { + let outcome: WorkOutcome; + try { + outcome = { status: 'fulfilled', value: await work(envelope.operationId) }; + } catch (reason) { + outcome = { status: 'rejected', reason: reason ?? new Error('Rejected with no reason.') }; + } + try { + return { ...outcome, result: normalize(outcome) }; + } catch (e) { + console.error( + '[GitSource] Failed to derive a settlement result for attempt %s on application %s:', + sanitizeForLog(envelope.operationId), + sanitizeForLog(request.applicationId), + e instanceof Error ? e.message : String(e), + ); + return { + ...outcome, + result: { outcome: 'unknown', reason: 'This attempt could not be resolved.', nextAction: 'none' }, + }; + } + } + + /** + * Reserve this request's durable attempt, failing closed when the + * reservation bookkeeping itself fails: no fetch, apply, promotion, or + * deploy may run without a durable record of it, so a failure here stops + * the operation rather than letting it proceed as an untracked side + * effect. Two failure shapes, reported differently: an application torn + * down in the window between resolving it and reserving against it (a + * GitOpsTransitionError from requireApp) can never succeed on retry, so + * it gets its own message; anything else (a transient DB error) is worth + * retrying. The refusal is recorded to the stack's own activity history + * as well as the server console, since it is itself an event an operator + * needs to see later. Callers (an HTTP route, or handleWebhookPull's own + * try/catch) already handle a thrown GitSourceError the same way they + * handle any other failure from the work itself. + */ + private reserveOwnAttemptOrFailClosed( + request: ReconcileRequest, + followerOf: string | undefined, + deliveryIntent?: ReconcileDeliveryIntent, + ): { envelope: EventEnvelope; reserved: boolean } { + try { + return this.reserveOwnAttempt(request, followerOf, deliveryIntent); + } catch (e) { + console.error( + `[GitSource] Failed to reserve a durable attempt for application ${sanitizeForLog(request.applicationId)}; refusing to proceed without one:`, + e instanceof Error ? e.stack ?? e.message : String(e), + ); + const isFetch = request.intent === 'fetch'; + this.recordGitActivity( + request.stackName, + isFetch ? 'git_pull_failed' : 'git_apply_failed', + `Git ${request.intent} for ${request.stackName} was refused: could not durably record the attempt.`, + request.actor, + 'error', + ); + if (e instanceof GitOpsTransitionError) { + throw new GitSourceError( + 'GIT_ERROR', + `This stack's GitOps tracking is unavailable; reconfigure the source before ${isFetch ? 'pulling' : 'applying'} again.`, + ); + } + throw new GitSourceError('GIT_ERROR', 'Could not durably record this operation. Please try again.'); + } + } + + /** Refuse any configured source whose work cannot receive a durable attempt. */ + private refuseUntrackedSource(stackName: string, actor: string, intent: 'fetch' | 'apply', recordActivity: boolean): void { + if (!DatabaseService.getInstance().getGitSource(stackName)) return; + const wasDetached = GitOpsStore.getInstance().hasDetachedDirectApplication(stackName); + const message = wasDetached + ? `This stack's GitOps tracking was removed but its Git source configuration still exists; delete the Git source configuration to finish detaching before ${intent === 'fetch' ? 'pulling' : 'applying'} again.` + : `This stack's GitOps tracking is unavailable; reconfigure the source before ${intent === 'fetch' ? 'pulling' : 'applying'} again.`; + if (recordActivity) { + this.recordGitActivity(stackName, intent === 'fetch' ? 'git_pull_failed' : 'git_apply_failed', message, actor, 'error'); + } + throw new GitSourceError('GIT_ERROR', message); + } + + /** + * Reserve this submission's own durable attempt. A request carrying a + * stable external delivery id (webhook redelivery) reserves under a + * producer-namespaced key derived from it, so a redelivery of the same + * event reuses the same operation id and reports `reserved: false` + * rather than minting a second attempt. Any other submission has no + * such stable identity, so its operation id is freshly allocated from + * the row's own attemptSeq, which is always a first-time reservation. + */ + private reserveOwnAttempt( + request: ReconcileRequest, + followerOf: string | undefined, + deliveryIntent?: ReconcileDeliveryIntent, + ): { envelope: EventEnvelope; reserved: boolean } { + const tx = GitOpsTransitions.getInstance(); + if (request.deliveryId) { + const operationId = deliveryKey(request.trigger, request.intent, request.deliveryId); + const envelope = this.gitopsEnvelope(operationId, request.actor, request.trigger); + const { reserved } = tx.reserveReconcileAttempt(request.applicationId, envelope, followerOf, deliveryIntent); + return { envelope, reserved }; + } + const allocated = tx.allocateReconcileAttempt(request.applicationId, request.actor, request.trigger, Date.now(), followerOf); + return { + envelope: this.gitopsEnvelope(allocated.operationId, request.actor, request.trigger), + reserved: allocated.reserved, + }; + } + + /** + * Settle a durable attempt with its already-computed result, tolerating + * a settlement failure rather than letting it turn a correctly-computed + * result (up to and including a real fetch or apply that already + * touched the filesystem) into a thrown error for the caller. The + * attempt is left unsettled on this path, which is exactly the signal + * startup recovery looks for, so nothing here is lost, only deferred. + */ + private settleAttempt(applicationId: string, envelope: EventEnvelope, result: ReconcileResult): boolean { + try { + const { settled } = GitOpsTransitions.getInstance().settleReconcileAttempt(applicationId, envelope, result); + return settled || !!GitOpsStore.getInstance().getSettledAttempt(applicationId, envelope.operationId); + } catch (e) { + console.error( + '[GitSource] Failed to settle reconcile attempt %s for application %s:', + sanitizeForLog(envelope.operationId), + sanitizeForLog(applicationId), + e instanceof Error ? e.message : String(e), + ); + return false; + } + } + + /** + * A submission whose operation id was already reserved elsewhere, with + * no leader for it running in this process: a settled row means a + * duplicate delivery arrived after its original attempt finished, so + * its stored result is returned rather than repeating the work. No + * settled row means the original attempt was orphaned by a crash (in + * this process or another); either way this call must not re-execute a + * fetch or apply someone else may already have run, so it resolves + * from whatever is already durably recorded, settling when that + * yields a real answer and otherwise reporting truthfully that the + * outcome is not yet known rather than guessing one. + */ + private resolveAlreadyReservedAttempt(applicationId: string, operationId: string, actor: string | null, trigger: string): ReconcileResult { + try { + const store = GitOpsStore.getInstance(); + const settled = store.getSettledAttempt(applicationId, operationId); + if (settled) return GitSourceService.resultFromSettledAttempt(settled); + return this.settleFromDurableState(applicationId, operationId, actor, trigger); + } catch (e) { + console.error( + '[GitSource] Failed to resolve already-reserved attempt %s for application %s:', + sanitizeForLog(operationId), + sanitizeForLog(applicationId), + e instanceof Error ? e.message : String(e), + ); + return { outcome: 'unknown', reason: 'This attempt could not be resolved from durable state.', nextAction: 'none' }; + } + } + + /** + * Resolve one reconcile attempt purely from what is already recorded, + * never by re-executing a fetch or apply, settling it durably when that + * yields a real answer. A follower is settled from its leader; anything + * else is derived from the application's current row state. A follower + * whose leader is still unresolved is left unsettled for a later call + * (a future recovery pass, or the leader itself finally settling) to + * resolve, for the reason resolveFollowerOutcome states. + */ + private settleFromDurableState( + applicationId: string, + operationId: string, + actor: string | null, + trigger: string, + ): ReconcileResult { + const started = GitOpsStore.getInstance().getStartedAttempt(applicationId, operationId); + // Prefer the reservation's own recorded actor/trigger over this + // call's, so the settled row's audit trail reflects who and what + // actually reserved the attempt rather than whoever happened to + // resolve it later. + const envelope: EventEnvelope = { + operationId, + actor: started?.actor ?? actor, + trigger: started?.trigger ?? trigger, + at: Date.now(), + }; + const followerOf = started ? GitSourceService.followerOfFromRow(started) : undefined; + if (followerOf) { + const outcome = this.resolveFollowerOutcome(applicationId, followerOf); + if (!outcome.known) { + return { outcome: 'unknown', reason: 'This attempt is waiting on its leader to settle.', nextAction: 'none' }; + } + if (!this.settleAttempt(applicationId, envelope, outcome.result)) { + return GitSourceService.durableResolutionFailureResult(); + } + return outcome.result; + } + const result = this.deriveResultForApplication(applicationId); + if (!this.settleAttempt(applicationId, envelope, result)) { + return GitSourceService.durableResolutionFailureResult(); + } + return result; + } + + private static durableResolutionFailureResult(): ReconcileResult { + return { outcome: 'unknown', reason: 'This attempt could not be durably resolved.', nextAction: 'none' }; + } + + /** + * A follower's outcome from its leader alone: the leader's settled + * result when it has one. When the leader has no settled row but its + * own reservation genuinely exists, its fate is still unresolved + * (`known: false`) and must not be guessed at independently, since it + * could settle to something else later and the follower would then + * durably disagree with it. Only when the leader's own reservation + * cannot be found at all (nothing durable to ever wait for) does + * independent derivation apply, logged distinctly since it means the + * leader/follower agreement invariant could not be honored here. + */ + private resolveFollowerOutcome( + applicationId: string, + leaderOperationId: string, + ): { known: true; result: ReconcileResult } | { known: false } { + const store = GitOpsStore.getInstance(); + const leaderSettled = store.getSettledAttempt(applicationId, leaderOperationId); + if (leaderSettled) return { known: true, result: GitSourceService.resultFromSettledAttempt(leaderSettled) }; + if (store.getStartedAttempt(applicationId, leaderOperationId)) return { known: false }; + console.error( + `[GitSource] follower's leader ${sanitizeForLog(leaderOperationId)} has no recorded reservation for application ${sanitizeForLog(applicationId)}; deriving independently`, + ); + return { known: true, result: this.deriveResultForApplication(applicationId) }; + } + + /** + * The current truthful result for an application, independent of any + * specific attempt. Fails closed on a superseded application id for the + * same reason every live execution revalidates identity under its lock. + */ + private deriveResultForApplication(applicationId: string): ReconcileResult { + const store = GitOpsStore.getInstance(); + const app = store.getApplication(applicationId); + if (!app?.stack_name) return GitSourceService.noApplicationResult(); + if (store.getLiveDirectApplication(app.stack_name)?.id !== applicationId) { + return GitSourceService.staleApplicationResult(); + } + return this.deriveReconcileResult(app.stack_name); + } + + /** The follower-link operation id recorded on a reservation, if any. */ + private static followerOfFromRow(row: GitOpsHistoryRow): string | undefined { + const decoded = decodeGitOpsJson(row.after_json); + if (!isRecord(decoded)) throw new GitOpsJsonError('reserved attempt metadata must be an object'); + if (!('followerOf' in decoded)) return undefined; + if (typeof decoded.followerOf !== 'string') { + throw new GitOpsJsonError('reserved attempt followerOf must be a string'); + } + return decoded.followerOf; + } + + private static deliveryIntentFromStartedAttempt(row: GitOpsHistoryRow): ReconcileDeliveryIntent { + const decoded = decodeGitOpsJson(row.after_json); + if (!isRecord(decoded) || !isRecord(decoded.deliveryIntent)) { + throw new GitOpsJsonError('reserved webhook attempt has no delivery intent'); + } + const { autoApply, deploy } = decoded.deliveryIntent; + if (typeof autoApply !== 'boolean' || typeof deploy !== 'boolean') { + throw new GitOpsJsonError('reserved webhook attempt has an invalid delivery intent'); + } + return GitSourceService.deliveryIntent(autoApply, deploy); + } + + private static deliveryIntent(autoApply: boolean, deploy: boolean): ReconcileDeliveryIntent { + return autoApply ? { autoApply: true, deploy } : { autoApply: false, deploy: false }; + } + + /** + * Decode a settled attempt's recorded result back into a + * ReconcileResult. Unreadable JSON and a well-formed-but-wrong-shaped + * payload are both logged: a corrupt or unexpected settled row is a + * storage or encoding bug an operator needs to see, not a routine + * response variation, matching decodeHistoryDelta's own rule for this + * exact column. + */ + private static resultFromSettledAttempt(row: GitOpsHistoryRow): ReconcileResult { + const unreadable: ReconcileResult = { + outcome: 'unknown', + reason: 'The settled attempt result could not be read.', + nextAction: 'none', + }; + + let decoded: unknown; + try { + decoded = decodeGitOpsJson(row.after_json); + } catch (e) { + if (!(e instanceof GitOpsJsonError)) throw e; + console.error(`[GitSource] settled attempt ${sanitizeForLog(row.operation_id)} is not decodable JSON: ${e.message}`); + return unreadable; + } + if ( + !isRecord(decoded) + || !isReconcileOutcome(decoded.outcome) + || typeof decoded.reason !== 'string' + || !isNextAction(decoded.nextAction) + ) { + console.error(`[GitSource] settled attempt ${sanitizeForLog(row.operation_id)} decoded to an unexpected shape`); + return unreadable; + } + return { + outcome: decoded.outcome, + reason: decoded.reason, + nextAction: decoded.nextAction, + retryAt: typeof decoded.retryAt === 'number' ? decoded.retryAt : undefined, + commitSha: typeof decoded.commitSha === 'string' ? decoded.commitSha : undefined, + }; + } + + /** + * Startup recovery: settle every reconcile attempt that reserved but + * never settled, most likely because the process crashed between the + * two. Never re-executes a fetch or apply. Must run before + * SourceController starts, so no live poll or retry tick can race a + * recovery pass over the same attempts. + * + * Pages by cursor rather than by "still unsettled" status, so a row + * this run cannot recover never blocks the rest of the backlog; see + * listUnsettledReconcileAttempts for why that matters. + * + * Two passes. Pass 1 settles every independent (non-follower) attempt + * by deriving the application's current truthful state, and defers + * every follower rather than settling it yet, so its leader (which + * can only be earlier in this same backlog, since a follower's own + * reservation records that its leader was already in flight) gets a + * chance to settle first. Pass 2 then settles each deferred follower + * from its leader's now-settled result, so a leader and its followers + * always agree; a follower whose leader is still unresolved is left + * for a later recovery run, per resolveFollowerOutcome. + * + * One row failing to recover (a transient DB error, an application + * deleted between listing and processing) is isolated: logged and + * counted, never allowed to block any other row. + */ + public async recoverUnsettledReconcileAttempts(pageSize = 200): Promise { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + let recovered = 0; + let failed = 0; + let stillWaiting = 0; + const deferredFollowers: { row: GitOpsHistoryRow; followerOf: string }[] = []; + + const settle = (row: GitOpsHistoryRow, result: ReconcileResult): void => { + tx.settleReconcileAttempt( + row.application_id, + { operationId: row.operation_id, actor: row.actor, trigger: row.trigger, at: Date.now() }, + result, + ); + recovered++; + }; + const noteFailure = (row: GitOpsHistoryRow, e: unknown): void => { + failed++; + console.error( + `[GitSource] Failed to recover reconcile attempt ${sanitizeForLog(row.operation_id)} for application ${sanitizeForLog(row.application_id)}:`, + e instanceof Error ? e.message : String(e), + ); + }; + + // Last-resort guard: the cursor advances strictly past every page, so + // the loop is already bounded by the size of the backlog itself. + const MAX_PAGES = 10_000; + let cursor: GitOpsHistoryCursor | undefined; + let pagesRead = 0; + for (; pagesRead < MAX_PAGES; pagesRead++) { + const page = store.listUnsettledReconcileAttempts(pageSize, cursor); + if (page.length === 0) break; + const last = page[page.length - 1]; + cursor = { createdAt: last.created_at, id: last.id }; + for (const row of page) { + try { + const followerOf = GitSourceService.followerOfFromRow(row); + if (followerOf) { + deferredFollowers.push({ row, followerOf }); + continue; + } + settle(row, this.deriveResultForApplication(row.application_id)); + } catch (e) { + noteFailure(row, e); + } + } + if (page.length < pageSize) break; + } + if (pagesRead === MAX_PAGES) { + console.warn(`[GitSource] Reconcile-attempt recovery stopped at its per-run page cap (${MAX_PAGES} pages); remaining rows will be retried on the next startup.`); + } + + for (const { row, followerOf } of deferredFollowers) { + try { + const outcome = this.resolveFollowerOutcome(row.application_id, followerOf); + if (!outcome.known) { + stillWaiting++; + continue; + } + settle(row, outcome.result); + } catch (e) { + noteFailure(row, e); + } + } + + if (recovered > 0 || failed > 0 || stillWaiting > 0) { + console.log(`[GitSource] Reconcile-attempt recovery: ${recovered} settled, ${failed} could not be recovered, ${stillWaiting} still waiting on their leader.`); + } + } + + /** Run fetch-intent reconcile through the same durable execution as pull(). */ + private async reconcileFetch(request: ReconcileRequest & { intent: 'fetch' }): Promise { + const liveApp = GitOpsStore.getInstance().getLiveDirectApplication(request.stackName); + if (!liveApp) return GitSourceService.noApplicationResult(); + if (liveApp.id !== request.applicationId) return GitSourceService.staleApplicationResult(); + + const submission = await this.submitExecution( + this.inFlightFetches, + request, + this.doPullWork(request.stackName, request.actor, request.applicationId), + (outcome) => this.fetchExecutionResult(request.stackName, outcome), + ); + return GitSourceService.resultFromSubmission(submission); + } + + /** Normalize one shared fetch execution for its leader and all followers. */ + private fetchExecutionResult(stackName: string, outcome: WorkOutcome): ReconcileResult { + return this.finalizeReconcileOutcome(stackName, outcome.status === 'rejected' ? outcome.reason : undefined); + } + + /** Normalize one shared apply execution for its leader and all followers. */ + private applyExecutionResult(stackName: string, outcome: WorkOutcome): ReconcileResult { + if (outcome.status === 'fulfilled' && outcome.value.deployError) { + return { + outcome: 'recovery_required', + reason: `The source applied, but the deploy failed: ${outcome.value.deployError}`, + nextAction: 'view_target_results', + }; + } + return this.finalizeReconcileOutcome(stackName, outcome.status === 'rejected' ? outcome.reason : undefined); + } + + /** + * The result derived from the application's own row state, unless a + * failure occurred that the derived state does not already reflect, in + * which case the failure itself is classified instead. + */ + private finalizeReconcileOutcome(stackName: string, failure: unknown): ReconcileResult { + const derived = this.deriveReconcileResult(stackName); + if (failure === undefined || GitSourceService.FAILURE_REFLECTED_OUTCOMES.has(derived.outcome)) { + return derived; + } + return this.reconcileFailureResult(failure); + } + + /** + * A truthful fallback for a reconcile failure the application row does + * not yet reflect. Routes through the same classifyFailure disposition + * table the controller's own retry/backoff logic uses, so an unretryable + * failure is never reported with nextAction: 'retry'. + */ + private reconcileFailureResult(failure: unknown): ReconcileResult { + if (!(failure instanceof GitSourceError)) { + return { + outcome: 'failed_previous_intact', + reason: 'The reconcile attempt failed unexpectedly.', + nextAction: 'retry', + }; + } + const disposition = classifyFailure({ + kind: 'git_source_error', + code: failure.code, + transportReason: failure.extras?.transportReason, + }); + switch (disposition.class) { + case 'supersession': + return { outcome: 'superseded', reason: failure.message, nextAction: 'none' }; + case 'permanent': + return { outcome: 'failed_previous_intact', reason: failure.message, nextAction: 'configure_credentials' }; + case 'operator_action_required': + return { outcome: 'blocked', reason: failure.message, nextAction: 'resolve_conflict' }; + case 'reconcile': + return { outcome: 'unknown', reason: failure.message, nextAction: 'none' }; + // 'degraded'/'target_*'/'blocked' are not reachable from a + // git_source_error classification today, but are grouped with + // 'transient' so this switch stays exhaustive if that changes. + case 'transient': + case 'degraded': + case 'target_permanent': + case 'target_transient': + case 'target_mutation_failed': + case 'blocked': + return { outcome: 'failed_previous_intact', reason: failure.message, nextAction: 'retry' }; + } + } + + private deriveReconcileResult(stackName: string): ReconcileResult { + const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName); + if (!app) { + return GitSourceService.noApplicationResult(); + } + const projection = projectApplication(app.id, false); + if (projection.targetMode === 'not_applicable') { + if (projection.limitations.some((l) => l.code === 'application_row_missing')) { + return { + outcome: 'recovery_required', + reason: 'The application this reconcile was resolved from is no longer present.', + nextAction: 'view_target_results', + }; + } + return GitSourceService.noApplicationResult(); + } + return outcomeFromSourceFacet(projection.facets.source); + } + + /** + * Stop acting on a source without forgetting anything about it: no new + * fetch, acceptance, or dispatch until resumed (enforced by the + * suspended_at checks at the top of pullLocked/applyLockedBody, not by + * this method itself). Takes the per-stack mutex not to reject a + * concurrent fetch or apply, but because sourceSuspended interrupts and + * clears any in-flight operation's active state, which would corrupt a + * genuinely running apply's own terminal transition; a suspend queued + * behind one instead takes effect once that work settles. + * + * A refused suspend is surfaced as a real error rather than swallowed: + * silently no-op'ing here would leave an operator believing a source is + * suspended when it is not, which is the same false-safety failure this + * method exists to prevent. + */ + public async suspend(stackName: string, opts: { actor: string; reason?: string }): Promise { + return this.withStackLock(stackName, async () => { + const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName); + if (!app) return GitSourceService.noApplicationResult(); + const envelope = this.gitopsEnvelope(crypto.randomUUID(), opts.actor, 'suspend'); + const reason = opts.reason?.trim() || 'Suspended by operator.'; + try { + GitOpsTransitions.getInstance().sourceSuspended(app.id, reason, envelope); + } catch (error) { + if (error instanceof GitOpsTransitionError) { + throw new GitSourceError('OPERATION_IN_FLIGHT', `Cannot suspend ${stackName}: ${error.message}`); + } + throw error; + } + return this.deriveReconcileResult(stackName); + }); + } + + /** + * Resume acting on a source. Does not itself fetch; the next scheduled + * poll, retry, or manual reconcile picks the source back up. + * + * Unlike suspend(), a refused resume is tolerated rather than surfaced. + * The result is read back from the row after the attempted write, so a + * resume that did not take (already not suspended, the application + * vanished, a transient persistence failure) still truthfully reports + * {outcome:'suspended', nextAction:'resume'} rather than a false + * "resumed". The caller cannot be told the source is unsuspended when it + * is not, so there is no false-safety risk to mirror suspend()'s rethrow. + */ + public async resume(stackName: string, opts: { actor: string }): Promise { + return this.withStackLock(stackName, async () => { + const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName); + if (!app) return GitSourceService.noApplicationResult(); + const envelope = this.gitopsEnvelope(crypto.randomUUID(), opts.actor, 'resume'); + this.recordGitOps(stackName, 'source resume', () => { + GitOpsTransitions.getInstance().sourceUnsuspended(app.id, envelope); + }); + return this.deriveReconcileResult(stackName); + }); + } + + /** + * An explicit, operator-initiated re-evaluation: resolves the live + * application server-side rather than trusting a caller-supplied id, for + * callers that hold only a stack name, and drives a fresh fetch-intent + * reconcile through it. + * + * The 'retry' trigger is recorded on the durable attempt for audit and + * recovery. It does not yet change fetch behavior; a later permanent- + * failure gate can use it to authorize an explicit operator retry. + */ + public async retry(stackName: string, opts: { actor: string }): Promise { + const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName); + if (!app) return GitSourceService.noApplicationResult(); + return this.reconcile({ + intent: 'fetch', + applicationId: app.id, + stackName, + trigger: 'retry', + actor: opts.actor, + }); + } + + /** + * Route an accepted generation to its target. Blueprint mode always + * blocks (BlueprintTargetAdapter; rollout orchestration does not exist + * yet). Direct mode has no separate generation-based promotion pipeline + * today, so it dispatches by driving the same reconcile()/apply path a + * manual or webhook apply already uses, translating the normalized + * ReconcileResult into the narrower dispatched/blocked shape a target + * adapter reports. + */ + public async dispatchAcceptedGeneration( + generation: AcceptedGeneration, + context: DispatchContext, + opts: { trigger: ReconcileTrigger; actor: string }, + ): Promise { + if (context.targetMode === 'blueprint') { + return new BlueprintTargetAdapter().dispatch(generation, context); + } + const app = GitOpsStore.getInstance().getApplication(generation.applicationId); + if (!app?.stack_name) { + return { status: 'blocked', reason: 'No Direct stack is bound to this application.' }; + } + const source = DatabaseService.getInstance().getGitSource(app.stack_name); + const result = await this.reconcile({ + intent: 'apply', + applicationId: generation.applicationId, + stackName: app.stack_name, + trigger: opts.trigger, + actor: opts.actor, + commitSha: generation.commitSha, + planFingerprint: generation.changePlanFingerprint ?? '', + deploy: source?.auto_deploy_on_apply ?? false, + }); + // 'converged' is not produced by reconcile() today (it requires + // target + health evidence this source-only path does not have), + // but it is a declared success member of ReconcileOutcome; treating + // only 'no_source_change' as success would silently misreport it as + // blocked the day a broader derivation starts emitting it. + if (result.outcome === 'no_source_change' || result.outcome === 'converged') { + return { status: 'dispatched' }; + } + return { status: 'blocked', reason: result.reason }; } /** @@ -2260,6 +3239,7 @@ export class GitSourceService { stackName: string, commitSha: string, opts: GitApplyOpts, + operationId?: string, ): Promise<{ applied: boolean; deployed: boolean; deployError?: string; recoveryId?: string }> { const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); const lock = await StackOpLockService.getInstance().runExclusive( @@ -2267,7 +3247,7 @@ export class GitSourceService { stackName, 'git_apply', opts.actor ?? 'system:git-source', - () => this.applyLocked(stackName, commitSha, opts), + () => this.applyLocked(stackName, commitSha, opts, operationId), getRegistryDeliveryLockContext(), ); if (!lock.ran) { @@ -2292,6 +3272,7 @@ export class GitSourceService { stackName: string, commitSha: string, opts: GitApplyOpts, + operationId?: string, ): Promise<{ applied: boolean; deployed: boolean; deployError?: string; recoveryId?: string }> { const started: { app: GitOpsApplicationRow | null; env: ReturnType | null; settled: boolean } = { app: null, @@ -2299,7 +3280,7 @@ export class GitSourceService { settled: false, }; try { - return await this.applyLockedBody(stackName, commitSha, opts, started); + return await this.applyLockedBody(stackName, commitSha, opts, started, operationId); } catch (e) { if (started.app && started.env && !started.settled) { const app = started.app; @@ -2321,11 +3302,18 @@ export class GitSourceService { commitSha: string, opts: GitApplyOpts, started: { app: GitOpsApplicationRow | null; env: ReturnType | null; settled: boolean }, + operationId?: string, ): Promise<{ applied: boolean; deployed: boolean; deployError?: string; recoveryId?: string }> { const diag = isDebugEnabled(); const db = DatabaseService.getInstance(); const src = db.getGitSource(stackName); if (!src) throw new GitSourceError('GIT_ERROR', 'No Git source configured for this stack.'); + // Same reasoning as pullLocked's guard: applyStarted's own suspension + // check is swallowed by recordGitOps once promotion is already + // underway, so stop the promotion before it starts, not after. + if (this.gitopsApplicationFor(stackName)?.suspended_at) { + throw new GitSourceError('OPERATION_IN_FLIGHT', `Reconciliation is suspended for ${stackName}.`); + } if (!src.pending_commit_sha || !src.pending_compose_content) { throw new GitSourceError('GIT_ERROR', 'No pending pull to apply. Fetch the source again.'); @@ -2383,7 +3371,16 @@ export class GitSourceService { sanitizeForLog(stackName), sanitizeForLog(commitSha.slice(0, 7)), ); } - const gitopsEnv = this.gitopsEnvelope(pending.operationId, actor, 'apply'); + // A caller that reserved a durable attempt for this apply passes its own + // operation id in, so the attempt and its gitops transitions + // (applyStarted/applied/applyFailed) share one identity. A caller that + // reserved none falls back to the fetch-time pending.operationId, as + // every caller did before reservation existed. The activity messages + // below reuse this same value, not pending.operationId directly, so + // they report the id the durable evidence for this apply actually + // carries rather than the earlier fetch attempt's. + const applyOperationId = operationId ?? pending.operationId; + const gitopsEnv = this.gitopsEnvelope(applyOperationId, actor, 'apply'); if (gitopsApp && gitopsGenerationId) { this.recordGitOps(stackName, 'apply start', () => { GitOpsTransitions.getInstance().applyStarted(gitopsApp.id, gitopsGenerationId, gitopsEnv); @@ -2421,8 +3418,14 @@ export class GitSourceService { pending.candidateRelPath, ); } - const dataDir = process.env.DATA_DIR || path.join(process.cwd(), 'data'); - const candidateAbs = path.join(dataDir, 'git-managed', String(nodeId), stackName, pending.candidateRelPath); + const managedRoot = path.resolve(stackManagedRoot(stackName)); + const pathReason = validateCandidateRelPath(pending.candidateRelPath, managedRoot); + if (pathReason) throw new GitSourceError('GIT_ERROR', pathReason); + // Inline barrier at the access sink (CodeQL path-injection). + const candidateAbs = path.resolve(managedRoot, pending.candidateRelPath); + if (!candidateAbs.startsWith(managedRoot + path.sep)) { + throw new GitSourceError('GIT_ERROR', 'candidateRelPath escapes the managed root'); + } try { await fsPromises.access(candidateAbs); } catch (accessErr: unknown) { @@ -2490,7 +3493,7 @@ export class GitSourceService { { plan: publicPlan, planFingerprint: plan.fingerprint }, ); } - const blockedPlanActivity = `Git plan blocked for ${stackName} (${commitSha.slice(0, 7)}, op ${pending.operationId.slice(0, 8)}, plan ${plan.fingerprint.slice(0, 12)})`; + const blockedPlanActivity = `Git plan blocked for ${stackName} (${commitSha.slice(0, 7)}, op ${GitSourceService.shortOperationId(applyOperationId)}, plan ${plan.fingerprint.slice(0, 12)})`; if (plan.blocked) { this.upsertGitPlanDrift(stackName, plan); db.setGitSourceLastPlan(stackName, plan.fingerprint, 'blocked'); @@ -2629,7 +3632,7 @@ export class GitSourceService { this.recordGitActivity( stackName, 'git_apply_rolled_back', - `Git apply rolled back for ${stackName} (${commitSha.slice(0, 7)}, op ${pending.operationId.slice(0, 8)}, plan ${plan.fingerprint.slice(0, 12)})`, + `Git apply rolled back for ${stackName} (${commitSha.slice(0, 7)}, op ${GitSourceService.shortOperationId(applyOperationId)}, plan ${plan.fingerprint.slice(0, 12)})`, actor, 'warning', ); @@ -2638,7 +3641,7 @@ export class GitSourceService { this.recordGitActivity( stackName, 'git_apply_failed', - `Git apply failed for ${stackName} (${commitSha.slice(0, 7)}, op ${pending.operationId.slice(0, 8)}, plan ${plan.fingerprint.slice(0, 12)})`, + `Git apply failed for ${stackName} (${commitSha.slice(0, 7)}, op ${GitSourceService.shortOperationId(applyOperationId)}, plan ${plan.fingerprint.slice(0, 12)})`, actor, 'error', ); @@ -2651,7 +3654,7 @@ export class GitSourceService { this.recordGitActivity( stackName, 'git_apply', - `Git apply succeeded for ${stackName} (${commitSha.slice(0, 7)}, op ${pending.operationId.slice(0, 8)}, plan ${plan.fingerprint.slice(0, 12)})`, + `Git apply succeeded for ${stackName} (${commitSha.slice(0, 7)}, op ${GitSourceService.shortOperationId(applyOperationId)}, plan ${plan.fingerprint.slice(0, 12)})`, actor, ); // Promotion has committed and rewritten the authoritative Compose @@ -3259,20 +4262,19 @@ export class GitSourceService { } rowInserted = true; - const operationId = crypto.randomUUID(); if (completeProjectManifest && materialization.value && recordedCreatePlan) { db.setGitSourceLastPlan(input.stackName, recordedCreatePlan.fingerprint, 'applied'); this.recordGitActivity( input.stackName, 'git_create', - `Git create succeeded for ${input.stackName} (${fetched.commitSha.slice(0, 7)}, op ${operationId.slice(0, 8)}, plan ${recordedCreatePlan.fingerprint.slice(0, 12)})`, + `Git create succeeded for ${input.stackName} (${fetched.commitSha.slice(0, 7)}, op ${GitSourceService.shortOperationId(gitopsOperationId)}, plan ${recordedCreatePlan.fingerprint.slice(0, 12)})`, 'system:git-source', ); } else { this.recordGitActivity( input.stackName, 'git_create', - `Git create succeeded for ${input.stackName} (${fetched.commitSha.slice(0, 7)}, op ${operationId.slice(0, 8)})`, + `Git create succeeded for ${input.stackName} (${fetched.commitSha.slice(0, 7)}, op ${GitSourceService.shortOperationId(gitopsOperationId)})`, 'system:git-source', ); } @@ -3445,6 +4447,75 @@ export class GitSourceService { } } + /** + * The candidate directory basenames (e.g. "candidate-") the boot + * sweep must not reap for this stack: a row still points at each one, so + * it is still needed no matter how old or how incomplete it looks on + * disk. Direct-mode only, matching the candidate/generation model itself. + */ + private claimedCandidateDirsFor(stackName: string): { dirs: Set; complete: boolean } { + const store = GitOpsStore.getInstance(); + const claimed = new Set(); + let complete = true; + const app = store.getLiveDirectApplication(stackName); + if (app) { + // candidate_generation_id names the currently staged candidate. + // accepted_generation_id is set by applySourceAcceptanceMutation + // and never cleared, so after an ordinary apply it names an + // already-promoted generation whose candidate directory has been + // moved away (harmless to check, just not load-bearing). It earns + // its place for the sourceAccepted-committed-but-targetApplied- + // not-yet-committed window, where the generation is accepted and + // genuinely still unpromoted on disk. Nothing calls sourceAccepted + // or targetApplied yet, so that is forward-looking coverage rather + // than dead code. + for (const generationId of [app.candidate_generation_id, app.accepted_generation_id]) { + if (!generationId) continue; + const generation = store.getGeneration(generationId); + if (generation) { + claimed.add(path.basename(generation.candidate_dir)); + } else { + complete = false; + console.warn( + `[GitSource] Generation claimant ${sanitizeForLog(generationId)} for ${sanitizeForLog(stackName)} could not be resolved during the sweep.`, + ); + } + } + try { + for (const generation of store.listGenerationsClaimedByUnsettledAttempts(app.id)) { + claimed.add(path.basename(generation.candidate_dir)); + } + } catch (e) { + complete = false; + console.warn( + `[GitSource] Could not read unsettled-attempt candidate claims for ${sanitizeForLog(stackName)} during the sweep:`, + e instanceof Error ? e.message : String(e), + ); + } + } + // The pending blob's own candidateRelPath is a third, independent + // claimant: it is written outside the transaction that mints a + // generation, so a candidate can be staged and recorded as pending + // with no generation row at all (fetchedInvalid) or with no live + // application to read a pointer from (a stack whose boot migration + // failed). A decode failure must not abort the sweep; it only means + // this extra claim is unavailable. + try { + const src = DatabaseService.getInstance().getGitSource(stackName); + if (src?.pending_compose_content) { + const pending = this.decodePendingCompose(src.pending_compose_content); + if (pending.candidateRelPath) claimed.add(path.basename(pending.candidateRelPath)); + } + } catch (e) { + complete = false; + console.warn( + `[GitSource] Could not read the pending candidate reference for ${sanitizeForLog(stackName)} while computing sweep claimants:`, + e instanceof Error ? e.message : String(e), + ); + } + return { dirs: claimed, complete }; + } + public async sweepOrphans(): Promise { const fsSvc = FileSystemService.getInstance(); const manifestSvc = GitProjectManifestService.getInstance(); @@ -3482,9 +4553,17 @@ export class GitSourceService { }); continue; } - await this.withStackLock(row.stack_name, () => - manifestSvc.sweepManagedArea(row.stack_name, { repoUrl: row.repo_url, branch: row.branch, stackExists: true }), - ); + await this.withStackLock(row.stack_name, () => { + const claims = this.claimedCandidateDirsFor(row.stack_name); + return manifestSvc.sweepManagedArea(row.stack_name, { + repoUrl: row.repo_url, + branch: row.branch, + stackExists: true, + candidateClaims: claims.complete + ? { complete: true, dirs: claims.dirs } + : { complete: false }, + }); + }); } catch (e) { console.error(`[GitManifest] sweep failed for ${row.stack_name}:`, (e as Error).message); } @@ -3566,81 +4645,221 @@ export class GitSourceService { // ─── Webhook-triggered pull ────────────────────────────────────────────── + /** Whether this delivery's effective intent requires deploy permission. */ + public webhookDeliveryRequiresDeploy(stackName: string, deliveryId?: string): boolean { + const source = DatabaseService.getInstance().getGitSource(stackName); + if (!source) return false; + const app = this.gitopsApplicationFor(stackName); + const started = app && deliveryId + ? GitOpsStore.getInstance().getStartedAttempt( + app.id, + deliveryKey('webhook', 'fetch', deliveryId), + ) + : undefined; + if (started) return GitSourceService.deliveryIntentFromStartedAttempt(started).deploy; + return source.auto_apply_on_webhook && source.auto_deploy_on_apply; + } + /** - * Invoked by the webhook dispatcher. Returns a short status string to - * record in webhook_executions. Enforces the per-source debounce. + * Invoked by the webhook dispatcher. A provider-scoped delivery id resolves + * redeliveries durably. Debounce still rate-limits new deliveries and is + * the only deduplication mechanism when no stable id is available. The + * caller must explicitly pass whether the principal is authorized to + * execute a requested deploy. */ - public async handleWebhookPull(stackName: string): Promise<{ status: 'success' | 'skipped' | 'error'; message: string }> { - // Run the whole critical section under a single lock acquisition so a - // concurrent fan-out (N webhooks for one push) serializes AND re-reads - // last_debounce_at after acquiring the lock. The first request stamps - // the window; every queued duplicate then sees the stamp and skips - // instead of cloning again. The debounce is still stamped only after a - // successful fetch, so a transient failure stays immediately retriable. - return this.withStackLock<{ status: 'success' | 'skipped' | 'error'; message: string }>(stackName, async () => { - const diag = isDebugEnabled(); - const db = DatabaseService.getInstance(); - const src = db.getGitSource(stackName); - if (!src) { - return { status: 'error', message: 'No Git source configured for this stack.' }; - } + public async handleWebhookPull( + stackName: string, + deployAuthorized: boolean, + deliveryId?: string, + ): Promise { + if (!deliveryId) return this.handleWebhookPullOnce(stackName, undefined, deployAuthorized); + const key = `${stackName}:${deliveryId}`; + const leader = this.inFlightWebhookDeliveries.get(key); + if (leader) return leader.promise; - const now = Date.now(); - if (src.last_debounce_at !== null && (now - src.last_debounce_at) < WEBHOOK_DEBOUNCE_MS) { - if (diag) console.log(`[GitSource:diag] webhook debounced stack=${stackName} age=${now - src.last_debounce_at}ms`); - return { status: 'skipped', message: 'Rate limited (debounced).' }; + const promise = this.handleWebhookPullOnce(stackName, deliveryId, deployAuthorized); + const entry = { promise }; + this.inFlightWebhookDeliveries.set(key, entry); + try { + return await promise; + } finally { + if (this.inFlightWebhookDeliveries.get(key) === entry) { + this.inFlightWebhookDeliveries.delete(key); } + } + } - let pullResult: PullResult; + private async handleWebhookPullOnce( + stackName: string, + deliveryId: string | undefined, + deployAuthorized: boolean, + ): Promise { + const actor = 'system:webhook'; + const diag = isDebugEnabled(); + const deliverySuffix = deliveryId ? ` (delivery ${sanitizeForLog(deliveryId)})` : ''; + const db = DatabaseService.getInstance(); + const src = db.getGitSource(stackName); + if (!src) return { status: 'error', message: 'No Git source configured for this stack.' }; + + const gitopsApp = this.gitopsApplicationFor(stackName); + const startedDelivery = gitopsApp && deliveryId + ? GitOpsStore.getInstance().getStartedAttempt( + gitopsApp.id, + deliveryKey('webhook', 'fetch', deliveryId), + ) + : undefined; + const existingDelivery = !!startedDelivery; + const now = Date.now(); + if (!existingDelivery && src.last_debounce_at !== null && (now - src.last_debounce_at) < WEBHOOK_DEBOUNCE_MS) { + if (diag) console.log(`[GitSource:diag] webhook debounced stack=${stackName} age=${now - src.last_debounce_at}ms`); + return { status: 'skipped', message: 'Rate limited (debounced).' }; + } + if (!existingDelivery && gitopsApp?.suspended_at) { + return { status: 'skipped', message: 'Reconciliation is suspended for this source.' }; + } + if (!gitopsApp) { try { - pullResult = await this.pullLocked(stackName, 'system:webhook'); + this.refuseUntrackedSource(stackName, actor, 'fetch', false); } catch (e) { - const msg = e instanceof GitSourceError ? `${e.code}: ${e.message}` : (e as Error).message; - const scrubbed = scrubCredentials(msg); - this.recordGitActivity(stackName, 'git_pull_failed', `Git pull failed for ${stackName}`, 'system:webhook', 'error'); - console.error(`[GitSource] Webhook pull failed for ${sanitizeForLog(stackName)}: ${sanitizeForLog(scrubbed)}`); - return { status: 'error', message: scrubbed }; + const msg = e instanceof GitSourceError ? e.message : (e as Error).message; + console.warn(`[GitSource] Webhook delivery skipped for ${sanitizeForLog(stackName)}: ${sanitizeForLog(msg)}`); + return { status: 'skipped', message: msg }; } + return { status: 'error', message: 'GitOps tracking is unavailable for this source.' }; + } + + let deliveryIntent = GitSourceService.deliveryIntent( + src.auto_apply_on_webhook, + src.auto_deploy_on_apply, + ); + if (startedDelivery) { try { - // Only burn the debounce window once the fetch actually produced - // something. A transient network failure should be retriable - // immediately rather than locked out for the debounce interval. - db.touchGitSourceDebounce(stackName); - if (!pullResult.validation.ok) { - // Webhooks are unattended, so always leave a server-side - // breadcrumb; the caller only sees the HTTP status. - console.warn(`[GitSource] Webhook pull validation failed for ${sanitizeForLog(stackName)}: ${sanitizeForLog(pullResult.validation.error ?? 'unknown')}`); - return { status: 'error', message: `Validation failed: ${pullResult.validation.error}` }; - } - - if (!src.auto_apply_on_webhook) { - if (diag) console.log(`[GitSource:diag] webhook pending-only stack=${stackName} sha=${pullResult.commitSha.slice(0, 7)}`); - return { status: 'success', message: `Pending update ready at ${pullResult.commitSha.slice(0, 7)}.` }; - } - - const applied = await this.applyWithSharedLock(stackName, pullResult.commitSha, { - deploy: src.auto_deploy_on_apply, - actor: 'system:webhook', - requirePlanFingerprint: false, - }); - if (applied.deployError) { - // Apply wrote to disk but deploy failed. Surface it so the - // webhook_executions row records a degraded outcome instead - // of a clean success. - return { status: 'error', message: `Applied commit ${pullResult.commitSha.slice(0, 7)} but deploy failed: ${applied.deployError}` }; - } - const suffix = applied.deployed ? ' and deployed' : ''; - return { status: 'success', message: `Applied commit ${pullResult.commitSha.slice(0, 7)}${suffix}.` }; + deliveryIntent = GitSourceService.deliveryIntentFromStartedAttempt(startedDelivery); } catch (e) { - const msg = e instanceof GitSourceError ? `${e.code}: ${e.message}` : (e as Error).message; - const scrubbed = scrubCredentials(msg); - // Unattended path: record the failure server-side so an operator - // can diagnose without diag mode, since the Git provider only - // logs the HTTP status. - console.error(`[GitSource] Webhook pull failed for ${sanitizeForLog(stackName)}: ${sanitizeForLog(scrubbed)}`); - return { status: 'error', message: scrubbed }; + console.error( + '[GitSource] Could not recover webhook delivery intent for %s%s:', + sanitizeForLog(stackName), + deliverySuffix, + e instanceof Error ? e.message : String(e), + ); + return { status: 'error', message: 'Could not recover the original webhook delivery intent.' }; } - }); + } + if (deliveryIntent.deploy && !deployAuthorized) { + return { status: 'error', message: 'Deploy permission is required for this webhook delivery.' }; + } + + let pullResult: PullResult | undefined; + let fetchResult: ReconcileResult | undefined; + let replayedFetch: boolean; + try { + const request: ReconcileRequest = { + intent: 'fetch', + applicationId: gitopsApp.id, + stackName, + trigger: 'webhook', + actor, + deliveryId, + }; + const submission = await this.submitExecution( + this.inFlightFetches, + request, + this.doPullWork(stackName, actor, gitopsApp.id), + (outcome) => this.fetchExecutionResult(stackName, outcome), + coalesceKey(request), + deliveryId ? deliveryIntent : undefined, + ); + fetchResult = GitSourceService.resultFromSubmission(submission); + replayedFetch = submission.kind === 'replayed'; + if (submission.kind === 'executed') { + if (submission.execution.status === 'rejected') throw submission.execution.reason; + pullResult = submission.execution.value; + } + } catch (e) { + return this.webhookFailure(stackName, deliverySuffix, 'pull', e); + } + + if (fetchResult && !pullResult) { + const replayResult = GitSourceService.webhookResultFromReconcile(fetchResult); + if (replayResult.status !== 'success') return replayResult; + } + if (!replayedFetch) db.touchGitSourceDebounce(stackName); + if (pullResult && !pullResult.validation.ok) { + console.warn(`[GitSource] Webhook pull validation failed for ${sanitizeForLog(stackName)}: ${sanitizeForLog(pullResult.validation.error ?? 'unknown')}`); + return { status: 'error', message: `Validation failed: ${pullResult.validation.error}` }; + } + + const commitSha = pullResult?.commitSha ?? fetchResult?.commitSha; + if (!commitSha) { + return { status: 'error', message: fetchResult?.reason ?? 'The fetch completed without a candidate commit.' }; + } + const currentSource = db.getGitSource(stackName); + if (!currentSource) return { status: 'error', message: 'The Git source configuration was removed during reconciliation.' }; + if (!deliveryIntent.autoApply) { + if (diag) console.log(`[GitSource:diag] webhook pending-only stack=${stackName} sha=${commitSha.slice(0, 7)}`); + return { status: 'success', message: `Pending update ready at ${commitSha.slice(0, 7)}.` }; + } + + try { + const request: ReconcileRequest = { + intent: 'apply', + applicationId: gitopsApp.id, + stackName, + trigger: 'webhook', + actor, + deliveryId, + commitSha, + planFingerprint: '', + deploy: deliveryIntent.deploy, + }; + const submission = await this.submitExecution( + this.inFlightApplies, + request, + (operationId) => this.withStackLock(stackName, async () => { + this.assertLiveApplication(stackName, gitopsApp.id); + return this.applyWithSharedLock(stackName, commitSha, { + deploy: deliveryIntent.deploy, + actor, + requirePlanFingerprint: false, + }, operationId); + }), + (outcome) => this.applyExecutionResult(stackName, outcome), + GitSourceService.applyExecutionKey(request, false), + ); + if (submission.kind === 'replayed') { + return GitSourceService.webhookResultFromReconcile(submission.result); + } + if (submission.execution.status === 'rejected') throw submission.execution.reason; + const applied = submission.execution.value; + if (applied.deployError) { + return { status: 'error', message: `Applied commit ${commitSha.slice(0, 7)} but deploy failed: ${applied.deployError}` }; + } + const suffix = applied.deployed ? ' and deployed' : ''; + return { status: 'success', message: `Applied commit ${commitSha.slice(0, 7)}${suffix}.` }; + } catch (e) { + return this.webhookFailure(stackName, deliverySuffix, 'apply', e); + } + } + + private webhookFailure(stackName: string, deliverySuffix: string, phase: 'pull' | 'apply', error: unknown): WebhookPullResult { + const msg = error instanceof GitSourceError ? `${error.code}: ${error.message}` : error instanceof Error ? error.message : String(error); + const scrubbed = scrubCredentials(msg); + console.error(`[GitSource] Webhook ${phase} failed for ${sanitizeForLog(stackName)}${deliverySuffix}: ${sanitizeForLog(scrubbed)}`); + return { status: 'error', message: scrubbed }; + } + + private static webhookResultFromReconcile(result: ReconcileResult): WebhookPullResult { + switch (result.outcome) { + case 'converged': + case 'no_source_change': + case 'candidate_already_fetched': + case 'pending_review': + return { status: 'success', message: result.reason }; + case 'suspended': + return { status: 'skipped', message: result.reason }; + default: + return { status: 'error', message: result.reason }; + } } // ─── Change plan helpers ───────────────────────────────────────────────── diff --git a/backend/src/services/WebhookService.ts b/backend/src/services/WebhookService.ts index e462a606..1ecf50b5 100644 --- a/backend/src/services/WebhookService.ts +++ b/backend/src/services/WebhookService.ts @@ -20,6 +20,7 @@ type ExecutionResult = { success: boolean; error?: string; duration_ms: number } type ExecutionStatus = 'success' | 'failure'; const REMOTE_WEBHOOK_REQUEST_TIMEOUT_MS = 30_000; +const MAX_PROVIDER_DELIVERY_ID_LENGTH = 256; // Maps a webhook lifecycle action to the per-stack lock action. 'pull' updates, // so it locks as 'update'; 'git-pull' is excluded (it locks inside GitSourceService). @@ -96,6 +97,7 @@ export class WebhookService { action: string, triggerSource: string | null, atomic?: boolean, + deliveryId?: string, ): Promise { if (webhook.id === undefined) { throw new Error('Webhook must be loaded from the database before execution'); @@ -110,11 +112,34 @@ export class WebhookService { return { success: false, error, duration_ms: 0 }; } + const scopedDeliveryId = action === 'git-pull' + ? WebhookService.scopedDeliveryId( + DatabaseService.getInstance().getGlobalSettings().delivery_source_id, + webhookId, + deliveryId, + ) + : undefined; + if (node.type === 'remote') { - return this.executeRemote(webhookId, nodeId, webhook.stack_name, action, triggerSource, atomic); + return this.executeRemote(webhookId, nodeId, webhook.stack_name, action, triggerSource, atomic, scopedDeliveryId); } - return this.executeLocal(webhookId, nodeId, webhook.stack_name, action, triggerSource, atomic); + return this.executeLocal(webhookId, nodeId, webhook.stack_name, action, triggerSource, atomic, scopedDeliveryId); + } + + /** Stable external identity, isolated to one configured webhook producer. */ + private static scopedDeliveryId( + deliverySourceId: string | undefined, + webhookId: number, + deliveryId: string | undefined, + ): string | undefined { + const normalized = deliveryId?.trim(); + if (!normalized) return undefined; + if (!deliverySourceId) throw new Error('Webhook delivery source identity is not configured'); + const bounded = normalized.length <= MAX_PROVIDER_DELIVERY_ID_LENGTH + ? normalized + : `sha256:${crypto.createHash('sha256').update(normalized).digest('hex')}`; + return `webhook:${deliverySourceId}:${webhookId}:${bounded}`; } public maskSecret(secret: string): string { @@ -129,6 +154,7 @@ export class WebhookService { action: string, triggerSource: string | null, atomic?: boolean, + deliveryId?: string, ): Promise { const stacks = await FileSystemService.getInstance(nodeId).getStacks(); if (!stacks.includes(stackName)) { @@ -142,7 +168,7 @@ export class WebhookService { // git-pull pulls then deploys through GitSourceService, which holds // the per-stack lock itself; locking here too would self-conflict. if (action === 'git-pull') { - return this.executeLocalGitPull(webhookId, stackName, action, triggerSource, startTime); + return this.executeLocalGitPull(webhookId, stackName, action, triggerSource, startTime, deliveryId); } const lockAction = WEBHOOK_LOCK_ACTION[action]; if (!lockAction) throw new Error(`Unknown action: ${action}`); @@ -225,8 +251,9 @@ export class WebhookService { action: string, triggerSource: string | null, startTime: number, + deliveryId?: string, ): Promise { - const result = await GitSourceService.getInstance().handleWebhookPull(stackName); + const result = await GitSourceService.getInstance().handleWebhookPull(stackName, true, deliveryId); const durationMs = Date.now() - startTime; if (result.status === 'error') { this.recordExecution(webhookId, action, 'failure', triggerSource, durationMs, result.message); @@ -255,6 +282,7 @@ export class WebhookService { action: string, triggerSource: string | null, atomic?: boolean, + deliveryId?: string, ): Promise { const startTime = Date.now(); try { @@ -263,7 +291,9 @@ export class WebhookService { : action === 'pull' ? 'update' : action; - const body = atomic === undefined ? undefined : { atomic }; + const body = action === 'git-pull' + ? { ...(atomic === undefined ? {} : { atomic }), ...(deliveryId ? { deliveryId } : {}) } + : atomic === undefined ? undefined : { atomic }; const response = await this.remoteStackRequest(nodeId, stackName, endpoint, 'POST', body); const durationMs = Date.now() - startTime; const payload = await response.json().catch(() => ({})) as { error?: string; message?: string; status?: string }; diff --git a/backend/src/services/gitops/SourceController.ts b/backend/src/services/gitops/SourceController.ts new file mode 100644 index 00000000..a551b96f --- /dev/null +++ b/backend/src/services/gitops/SourceController.ts @@ -0,0 +1,158 @@ +import { GitOpsStore } from './store'; +import { GitSourceService } from '../GitSourceService'; +import type { GitOpsApplicationRow } from './types'; +import { sanitizeForLog } from '../../utils/safeLog'; + +/** + * Background driver for unattended GitOps reconciliation: polls sources on + * their configured interval and re-evaluates applications whose retry_at + * has arrived, driving each through GitSourceService.reconcile(). + * + * Scope for this delivery: a tick only issues a fetch-intent reconcile, the + * same "detect and stage a candidate" step a manual pull performs. It does + * not evaluate source policy or drive automatic acceptance/dispatch for a + * newly staged candidate, and it does not yet resume a failure at the + * specific stage it failed at (fetch vs. dispatch); every retry re-issues + * a fetch. Both are real gaps against the source policy design, not silent + * omissions: automatic-policy acceptance and stage-aware retry are follow-on + * work once dispatchAcceptedGeneration() has a caller that decides when a + * staged candidate should be accepted. + * + * One self-rescheduling timer drives the scan, matching ImageUpdateService. + * The re-arm always runs, even when a scan throws, so one bad tick (a + * locked database, a transient store error) never permanently stops the + * driver. The per-application in-flight set, not the timer, is what keeps + * one busy application from blocking another: the tick never awaits any + * evaluation before rescheduling, so a slow application only pauses itself. + */ +export class SourceController { + private static instance: SourceController; + + private static readonly TICK_INTERVAL_MS = 60_000; + + private timer: NodeJS.Timeout | null = null; + private polling = false; + // Bumped by cancelPending(), so by stop() and restartPolling(). tick() has + // no internal await point today, so nothing can currently call either one + // mid-tick; this is a second, currently-redundant line of defense against a + // stale timer firing, kept cheap on purpose for when stage-aware retry (see + // above) gives evaluate() a real yield point. + private scheduleGeneration = 0; + private readonly inFlight = new Set(); + + private constructor() { } + + static getInstance(): SourceController { + if (!SourceController.instance) { + SourceController.instance = new SourceController(); + } + return SourceController.instance; + } + + /** Test-only: replace the singleton so timer/in-flight state never leaks between tests. */ + static resetForTests(): void { + SourceController.instance = new SourceController(); + } + + start(): void { + // Guards on `polling`, not `timer`: tick() nulls `timer` before it + // scans (so a stale timer reference can never block a restart), which + // would otherwise let a start() call landing during that scan see a + // false "not running" reading and arm a second timer. + if (this.polling) return; + this.polling = true; + this.armNext(); + } + + stop(): void { + this.cancelPending(); + this.polling = false; + } + + /** + * Reschedule the next tick without restarting: always clears any pending + * timer first, so calling this any number of times in a row never leaves + * more than one timer armed. + */ + restartPolling(): void { + this.cancelPending(); + if (this.polling) { + this.armNext(); + } + } + + isPolling(): boolean { + return this.polling; + } + + /** Clear any armed timer and invalidate the tick it would have run. */ + private cancelPending(): void { + this.scheduleGeneration++; + if (this.timer) { + clearTimeout(this.timer); + this.timer = null; + } + } + + private armNext(): void { + const gen = this.scheduleGeneration; + this.timer = setTimeout(() => { void this.tick(gen); }, SourceController.TICK_INTERVAL_MS); + this.timer.unref(); + } + + private async tick(gen: number): Promise { + if (!this.polling || gen !== this.scheduleGeneration) return; + this.timer = null; + try { + this.scan(); + } catch (e) { + console.error('[SourceController] scan failed:', e instanceof Error ? e.message : String(e)); + } finally { + if (this.polling && gen === this.scheduleGeneration) { + this.armNext(); + } + } + } + + /** + * Fire an evaluation for every due application without waiting for any + * of them. An application already in the in-flight set is left for a + * later tick instead of being queued behind its own still-running + * evaluation. A row due for both poll and retry is evaluated once. + */ + private scan(): void { + const now = Date.now(); + const store = GitOpsStore.getInstance(); + const due = new Map(); + for (const app of store.listSourcesDueForPoll(now)) due.set(app.id, app); + for (const app of store.listApplicationsDueForRetry(now)) due.set(app.id, app); + + for (const app of due.values()) { + if (this.inFlight.has(app.id)) continue; + this.inFlight.add(app.id); + this.evaluate(app).finally(() => this.inFlight.delete(app.id)); + } + } + + private async evaluate(app: GitOpsApplicationRow): Promise { + if (!app.stack_name) { + console.warn(`[SourceController] Skipping ${sanitizeForLog(app.id)}: direct-mode application has no stack_name.`); + return; + } + const isRetry = app.retry_at !== null && app.retry_at <= Date.now(); + try { + await GitSourceService.getInstance().reconcile({ + intent: 'fetch', + applicationId: app.id, + stackName: app.stack_name, + trigger: isRetry ? 'retry' : 'poll', + actor: 'system:source-controller', + }); + } catch (e) { + console.error( + `[SourceController] evaluation failed for ${sanitizeForLog(app.id)}:`, + e instanceof Error ? e.message : String(e), + ); + } + } +} diff --git a/backend/src/services/gitops/backoff.ts b/backend/src/services/gitops/backoff.ts new file mode 100644 index 00000000..a1e2ff22 --- /dev/null +++ b/backend/src/services/gitops/backoff.ts @@ -0,0 +1,140 @@ +import type { GitSourceErrorCode } from '../GitSourceService'; +import type { TransportFailureReason } from '../git/errors'; + +/** + * Everything a controller attempt can fail with. `git_source_error` covers + * every GitSourceError the fetch/apply path can throw, transport-classified + * or not. The remaining kinds cover dispatch-stage failures that have no + * GitSourceError at all (target binding, target availability, a deploy or + * health failure after a successful apply, Blueprint evaluation, and an + * interrupted or unknown-completion operation). + */ +export type FailureEvidence = + | { kind: 'git_source_error'; code: GitSourceErrorCode; transportReason?: TransportFailureReason } + | { kind: 'policy_unavailable' } + | { kind: 'persistence_unavailable' } + | { kind: 'target_binding_invalid' } + | { kind: 'target_unavailable' } + | { kind: 'target_mutation_failed' } + | { kind: 'blueprint_unavailable' } + | { kind: 'interrupted' }; + +export type FailureDisposition = + /** A newer revision replaced the one being worked on; re-resolve, not backoff. */ + | { class: 'supersession' } + /** Retryable. retryCeiling bounds how many attempts before it escalates to permanent. */ + | { class: 'transient'; retryCeiling: number } + /** Will never succeed by retrying; needs a configuration, environment, or credential change. */ + | { class: 'permanent' } + /** A human decision is required (review a plan, resolve a conflict); not retried automatically. */ + | { class: 'operator_action_required' } + /** Evidence could not be produced (e.g. scanner unavailable); held for review, not retried blind. */ + | { class: 'degraded' } + /** The target itself will never accept this generation without a configuration change. */ + | { class: 'target_permanent' } + /** The target is temporarily unreachable; retryable at the target/dispatch stage only. */ + | { class: 'target_transient' } + /** Applied but deploy or health failed: never refetch or reapply, only redeploy. */ + | { class: 'target_mutation_failed' } + /** Blocked pending a capability this program does not yet provide (e.g. Blueprint rollout). */ + | { class: 'blocked' } + /** Ambiguous or interrupted; reconcile from durable state, never blind retry. */ + | { class: 'reconcile' }; + +export const DEFAULT_TRANSIENT_CEILING = 8; +export const LOW_TRANSIENT_CEILING = 3; + +const TRANSIENT_DEFAULT: FailureDisposition = { class: 'transient', retryCeiling: DEFAULT_TRANSIENT_CEILING }; +const TRANSIENT_LOW: FailureDisposition = { class: 'transient', retryCeiling: LOW_TRANSIENT_CEILING }; +const PERMANENT: FailureDisposition = { class: 'permanent' }; + +/** + * Total over every TransportFailureReason except `exit`, which is generic + * and needs the classified GitSourceErrorCode (see CODE_DISPOSITION) to + * tell a transient network condition from a rate limit from an + * unrecognized error. Adding a new reason to the source union without + * adding it here fails the build. + */ +const REASON_DISPOSITION: Record, FailureDisposition> = { + 'tip-changed': { class: 'supersession' }, + timeout: TRANSIENT_DEFAULT, + 'target-unresolved': TRANSIENT_DEFAULT, + 'invalid-url': PERMANENT, + 'unsafe-target': PERMANENT, + 'invalid-ref': PERMANENT, + 'redirect-scope': PERMANENT, + 'git-missing': PERMANENT, + 'git-old': PERMANENT, + size: PERMANENT, + 'ssh-auth-required': PERMANENT, + 'ref-not-found': PERMANENT, + 'unsupported-ref': PERMANENT, +}; + +/** + * Total over every GitSourceErrorCode. Used directly when there is no + * transport reason (a plan/validation/file/operation-conflict error), and + * as the exit-reason fallback (RATE_LIMITED, NETWORK_TIMEOUT, and GIT_ERROR + * only ever arise from an `exit` transport reason). Adding a new code + * without adding it here fails the build. + */ +const CODE_DISPOSITION: Record = { + REPO_NOT_FOUND: PERMANENT, + AUTH_FAILED: PERMANENT, + REF_NOT_FOUND: PERMANENT, + REF_DELETED: PERMANENT, + UNSUPPORTED_REF: PERMANENT, + SSH_HOST_KEY_FAILED: PERMANENT, + FILE_NOT_FOUND: { class: 'operator_action_required' }, + RATE_LIMITED: TRANSIENT_DEFAULT, + NETWORK_TIMEOUT: TRANSIENT_DEFAULT, + GIT_ERROR: TRANSIENT_LOW, + STALE_PLAN: { class: 'operator_action_required' }, + PLAN_FINGERPRINT_REQUIRED: { class: 'operator_action_required' }, + PLAN_BLOCKED: { class: 'operator_action_required' }, + LEGACY_PENDING: { class: 'operator_action_required' }, + PLAN_UNAVAILABLE: { class: 'operator_action_required' }, + OPERATION_IN_FLIGHT: { class: 'reconcile' }, +}; + +export function classifyFailure(evidence: FailureEvidence): FailureDisposition { + switch (evidence.kind) { + case 'git_source_error': + if (evidence.transportReason && evidence.transportReason !== 'exit') { + return REASON_DISPOSITION[evidence.transportReason]; + } + return CODE_DISPOSITION[evidence.code]; + case 'policy_unavailable': + return { class: 'degraded' }; + case 'persistence_unavailable': + return TRANSIENT_DEFAULT; + case 'target_binding_invalid': + return { class: 'target_permanent' }; + case 'target_unavailable': + return { class: 'target_transient' }; + case 'target_mutation_failed': + return { class: 'target_mutation_failed' }; + case 'blueprint_unavailable': + return { class: 'blocked' }; + case 'interrupted': + return { class: 'reconcile' }; + } +} + +const BASE_DELAY_MS = 60_000; +const MAX_DELAY_MS = 3_600_000; +const JITTER_RATIO = 0.1; + +/** + * Bounded exponential backoff with jitter: 60s * 2^retryCount, capped at one + * hour, with up to +-10% jitter so many sources retrying at once do not + * all land on the same second. A provider-supplied retry floor (e.g. a + * rate-limit Retry-After) takes precedence whenever it is larger than the + * computed delay. + */ +export function nextRetryAt(now: number, retryCount: number, providerFloorMs?: number): number { + const capped = Math.min(BASE_DELAY_MS * 2 ** retryCount, MAX_DELAY_MS); + const jittered = capped + capped * JITTER_RATIO * (Math.random() * 2 - 1); + const delay = providerFloorMs !== undefined ? Math.max(jittered, providerFloorMs) : jittered; + return now + delay; +} diff --git a/backend/src/services/gitops/blueprintProducers.ts b/backend/src/services/gitops/blueprintProducers.ts index 7dfc5f17..8dfc5013 100644 --- a/backend/src/services/gitops/blueprintProducers.ts +++ b/backend/src/services/gitops/blueprintProducers.ts @@ -341,7 +341,7 @@ export function commitBlueprintDelete(blueprintId: number, actor: string | null) } /** A Blueprint application before anything has been asked of it. */ -export function blankInlineApplication(id: string, blueprintId: number, at: number) { +export function blankInlineApplication(id: string, blueprintId: number, at: number): GitOpsApplicationRow { return { id, lifecycle_key: `blueprint:${blueprintId}`, @@ -382,6 +382,10 @@ export function blankInlineApplication(id: string, blueprintId: number, at: numb pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/services/gitops/directApplication.ts b/backend/src/services/gitops/directApplication.ts index c3693dd9..2ad5d303 100644 --- a/backend/src/services/gitops/directApplication.ts +++ b/backend/src/services/gitops/directApplication.ts @@ -157,6 +157,10 @@ export function buildDirectApplicationRow(args: { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, @@ -217,6 +221,12 @@ export function buildGenerationRow(args: { actor: args.actor, previous_generation_id: null, redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, created_at: args.at, }; } diff --git a/backend/src/services/gitops/handoff.ts b/backend/src/services/gitops/handoff.ts new file mode 100644 index 00000000..cc8e0f8f --- /dev/null +++ b/backend/src/services/gitops/handoff.ts @@ -0,0 +1,149 @@ +import type { RepoIdentity } from './repoIdentity'; +import type { RefKind } from '../git/types'; +import type { GitOpsGenerationRow } from './types'; + +/** + * A portable projection of the managed-project manifest: authored file set + * and per-file content digests, without node id, stack name, or generation + * directory. The real GitProjectManifest carries those identity fields; + * this is deliberately narrower. + */ +export type PortableManifest = { + files: Array<{ path: string; role: string; contentSha256?: string | null }>; +}; + +/** Authored Compose invocation shape, without a target project name. */ +export type ComposeInputs = { + composeFileOrder: string[]; + profiles?: string[]; + contextDir?: string | null; +}; + +/** + * One accepted generation, described purely by what it contains. Direct and + * Blueprint dispatch consume the exact same shape; current target mode, + * binding revision, and any execution-local path travel separately in + * DispatchContext, re-read under the dispatch lock rather than carried + * here, so a stale acceptance can never authorize a routing decision made + * after it. + */ +export type AcceptedGeneration = { + contractVersion: 1; + generationId: string; + applicationId: string; + repoIdentity: RepoIdentity; + configuredRef: string; + commitSha: string; + resolvedRefKind: RefKind | null; + manifestVersion: number; + portableManifest: PortableManifest | null; + composeInputs: ComposeInputs | null; + materializationFingerprint: string; + changePlanFingerprint: string | null; + validationOk: boolean; + sourcePolicyEvidence: unknown | null; + securityPolicyEvidence: unknown | null; + supportRequirements: unknown | null; + compatibilityRequirements: unknown | null; + /** Capability metadata only; never a secret value. Not yet populated by any producer. */ + secretCapability: unknown | null; + trigger: string; + actor: string | null; + operationId: string; + previousGenerationId: string | null; + /** Why some field above could not be proven, recorded honestly rather than guessed. */ + limitations: string[]; +}; + +function parseOptionalJson(raw: string | null, limitationLabel: string, limitations: string[]): T | null { + if (raw === null) { + limitations.push(limitationLabel); + return null; + } + try { + return JSON.parse(raw) as T; + } catch { + limitations.push(`${limitationLabel}_unparseable`); + return null; + } +} + +/** + * Build the portable accepted-generation contract from a persisted + * generation row. A legacy row predating one of the portable-contract + * columns decodes that field as null with an explicit limitation recorded, + * never as invented evidence. + */ +export function buildAcceptedGeneration(row: GitOpsGenerationRow): AcceptedGeneration { + let repoIdentity: RepoIdentity; + try { + repoIdentity = JSON.parse(row.repo_identity_json) as RepoIdentity; + } catch { + throw new Error(`Generation ${row.id} has an unparseable repo_identity_json; refusing to build an accepted-generation contract from corrupt evidence.`); + } + + const limitations: string[] = JSON.parse(row.redacted_limitations_json) as string[]; + const portableManifest = parseOptionalJson(row.portable_manifest_json, 'portable_manifest_missing', limitations); + const composeInputs = parseOptionalJson(row.compose_inputs_json, 'compose_inputs_missing', limitations); + const sourcePolicyEvidence = parseOptionalJson(row.source_policy_evidence_json, 'source_policy_evidence_missing', limitations); + const securityPolicyEvidence = parseOptionalJson(row.security_policy_evidence_json, 'security_policy_evidence_missing', limitations); + const supportRequirements = parseOptionalJson(row.support_requirements_json, 'support_requirements_missing', limitations); + const compatibilityRequirements = parseOptionalJson(row.compatibility_requirements_json, 'compatibility_requirements_missing', limitations); + + return { + contractVersion: 1, + generationId: row.id, + applicationId: row.application_id, + repoIdentity, + configuredRef: row.configured_ref, + commitSha: row.commit_sha, + resolvedRefKind: row.resolved_ref_kind, + manifestVersion: row.manifest_version, + portableManifest, + composeInputs, + materializationFingerprint: row.materialization_fingerprint, + changePlanFingerprint: row.change_plan_fingerprint, + validationOk: row.validation_ok === 1, + sourcePolicyEvidence, + securityPolicyEvidence, + supportRequirements, + compatibilityRequirements, + secretCapability: null, + trigger: row.trigger, + actor: row.actor, + operationId: row.operation_id, + previousGenerationId: row.previous_generation_id, + limitations, + }; +} + +/** + * Current target mode and binding, re-read under the dispatch lock rather + * than carried on AcceptedGeneration, so a routing decision is always made + * from the current state, never a value an earlier acceptance froze. + */ +export type DispatchContext = { + targetMode: 'direct' | 'blueprint'; + nodeId: number | null; + bindingRevision: string | null; +}; + +export type DispatchResult = + | { status: 'dispatched' } + | { status: 'blocked'; reason: string }; + +export interface TargetAdapter { + dispatch(generation: AcceptedGeneration, context: DispatchContext): Promise; +} + +/** + * Fails closed until Blueprint rollout orchestration exists. Never + * inspects selectors, target sets, or placement: an accepted generation + * for a Blueprint-mode application is evaluated the same as Direct, but + * dispatch stops here. + */ +export class BlueprintTargetAdapter implements TargetAdapter { + async dispatch(_generation: AcceptedGeneration, _context: DispatchContext): Promise { + return { status: 'blocked', reason: 'Blueprint rollout orchestration is not yet implemented.' }; + } +} diff --git a/backend/src/services/gitops/history.ts b/backend/src/services/gitops/history.ts index bee1642f..65ef7005 100644 --- a/backend/src/services/gitops/history.ts +++ b/backend/src/services/gitops/history.ts @@ -70,6 +70,8 @@ export type GitOpsHistoryStage = | 'rollout_unpaused' | 'source_accepted' | 'source_conflict_blocker' + | 'source_reconcile_started' + | 'source_reconcile_settled' | 'source_retry_scheduled' | 'source_suspended' | 'source_unsuspended' diff --git a/backend/src/services/gitops/migrate.ts b/backend/src/services/gitops/migrate.ts index 002a3652..aa7a6e6e 100644 --- a/backend/src/services/gitops/migrate.ts +++ b/backend/src/services/gitops/migrate.ts @@ -226,6 +226,12 @@ function migrateAccepted( actor: envelope.actor, previous_generation_id: null, redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, created_at: envelope.at, }; store.insertGeneration(generation); diff --git a/backend/src/services/gitops/outcomes.ts b/backend/src/services/gitops/outcomes.ts new file mode 100644 index 00000000..0b72babb --- /dev/null +++ b/backend/src/services/gitops/outcomes.ts @@ -0,0 +1,211 @@ +import type { SourceFacet } from './types'; + +/** + * Normalized reconcile outcomes. Silence is not an acceptable GitOps + * result: every attempt settles into exactly one of these, never a bare + * success/failure boolean. + * + * `converged` is deliberately never produced by outcomeFromSourceFacet: it + * requires target and health evidence this source-only projection does not + * have, and "no source change" is not proof of full convergence. A later + * composition over source + target + health facets is what may report it. + */ +export type ReconcileOutcome = + | 'converged' + | 'no_source_change' + | 'candidate_already_fetched' + | 'pending_review' + | 'suspended' + | 'retry_scheduled' + | 'blocked' + | 'superseded' + | 'failed_previous_intact' + | 'recovery_required' + | 'unknown'; + +export type NextAction = + | 'none' + | 'review' + | 'resume' + | 'retry' + | 'resolve_conflict' + | 'configure_credentials' + | 'view_target_results'; + +export type ReconcileResult = { + outcome: ReconcileOutcome; + reason: string; + nextAction: NextAction; + retryAt?: number; + commitSha?: string; +}; + +/** Every ReconcileOutcome member, for runtime validation of a value read back from storage. */ +const RECONCILE_OUTCOMES: ReadonlySet = new Set([ + 'converged', + 'no_source_change', + 'candidate_already_fetched', + 'pending_review', + 'suspended', + 'retry_scheduled', + 'blocked', + 'superseded', + 'failed_previous_intact', + 'recovery_required', + 'unknown', +]); + +/** Every NextAction member, for runtime validation of a value read back from storage. */ +const NEXT_ACTIONS: ReadonlySet = new Set([ + 'none', + 'review', + 'resume', + 'retry', + 'resolve_conflict', + 'configure_credentials', + 'view_target_results', +]); + +export function isReconcileOutcome(value: unknown): value is ReconcileOutcome { + return typeof value === 'string' && RECONCILE_OUTCOMES.has(value); +} + +export function isNextAction(value: unknown): value is NextAction { + return typeof value === 'string' && NEXT_ACTIONS.has(value); +} + +function commitShaOf(facet: Extract): string | undefined { + return facet.desiredCommitSha ?? facet.fetchedCommitSha ?? undefined; +} + +/** + * Derive the normalized outcome of a settled source reconcile attempt from + * the existing source-facet projection, rather than re-deriving status from + * raw application-row fields. Keeps the outcome vocabulary and the + * projection's own status vocabulary from silently drifting apart. + */ +export function outcomeFromSourceFacet(facet: SourceFacet): ReconcileResult { + switch (facet.status) { + case 'not_applicable': + return { outcome: 'unknown', reason: 'No GitOps application exists for this stack.', nextAction: 'none' }; + + case 'never_reconciled': + return { outcome: 'unknown', reason: 'The source has never been reconciled.', nextAction: 'none' }; + + case 'checking_fetching': + case 'applying': + return { + outcome: 'unknown', + reason: 'A reconcile operation is currently in flight; no settled result yet.', + nextAction: 'none', + commitSha: commitShaOf(facet), + }; + + case 'source_reconcile_required': + return { + outcome: 'unknown', + reason: 'The source has advanced but reconciliation has not evaluated it yet.', + nextAction: 'none', + commitSha: commitShaOf(facet), + }; + + case 'application_generation_accepted': + return { + outcome: 'no_source_change', + reason: 'The configured ref still resolves to the accepted generation. This is not proof of full convergence.', + nextAction: 'none', + commitSha: commitShaOf(facet), + }; + + case 'candidate_ready': + return { + outcome: 'candidate_already_fetched', + reason: 'A candidate generation is already staged and awaiting acceptance.', + nextAction: 'none', + commitSha: commitShaOf(facet), + }; + + case 'source_review_pending': + return { + outcome: 'pending_review', + reason: 'A candidate is staged and requires explicit review before acceptance.', + nextAction: 'review', + commitSha: commitShaOf(facet), + }; + + case 'source_conflict_blocker': + return { + outcome: 'blocked', + reason: 'A local conflict is blocking the candidate from being accepted.', + nextAction: 'resolve_conflict', + commitSha: commitShaOf(facet), + }; + + case 'source_superseded': + return { + outcome: 'superseded', + reason: 'A newer revision superseded this candidate before it was accepted.', + nextAction: 'none', + commitSha: commitShaOf(facet), + }; + + case 'source_retry_scheduled': + return { + outcome: 'retry_scheduled', + reason: `A previous attempt failed transiently; retry ${facet.retryCount + 1} is scheduled.`, + nextAction: 'none', + retryAt: facet.retryAt, + commitSha: commitShaOf(facet), + }; + + case 'source_suspended': + return { + outcome: 'suspended', + reason: facet.suspendedReason + ? `Reconciliation is suspended: ${facet.suspendedReason}` + : 'Reconciliation is suspended.', + nextAction: 'resume', + commitSha: commitShaOf(facet), + }; + + case 'source_failed': + return { + outcome: 'failed_previous_intact', + reason: `The ${facet.failureStage} stage failed (${facet.failureClass}). The previously accepted generation is unchanged.`, + nextAction: facet.retryAt !== null ? 'retry' : 'configure_credentials', + retryAt: facet.retryAt ?? undefined, + commitSha: commitShaOf(facet), + }; + + case 'source_unknown': + return { + outcome: 'recovery_required', + reason: `An operation was interrupted at ${facet.interruptedStage} and its outcome is unproven.`, + nextAction: 'view_target_results', + commitSha: commitShaOf(facet), + }; + + case 'recovery_required': + return { + outcome: 'recovery_required', + reason: 'Recovery from an earlier failed mutation is still outstanding.', + nextAction: 'view_target_results', + commitSha: commitShaOf(facet), + }; + + case 'recovery_failed': + return { + outcome: 'recovery_required', + reason: `Recovery itself failed (${facet.failureClass}); this needs operator attention.`, + nextAction: 'view_target_results', + commitSha: commitShaOf(facet), + }; + + case 'not_live': + return { + outcome: 'unknown', + reason: `The application is ${facet.lifecycleStatus}, not live; there is nothing to reconcile.`, + nextAction: 'none', + }; + } +} diff --git a/backend/src/services/gitops/schema.ts b/backend/src/services/gitops/schema.ts index 1c311e24..d881ed3e 100644 --- a/backend/src/services/gitops/schema.ts +++ b/backend/src/services/gitops/schema.ts @@ -100,6 +100,16 @@ CREATE TABLE IF NOT EXISTS gitops_applications ( -- target rows, so suspending a source can never clobber an unrelated -- rollout pause reason (or the reverse). source_suspended_reason TEXT NULL, + -- Controller-owned bookkeeping. NULL poll_interval_secs inherits the + -- global default; 0 disables polling for this application. next_poll_at + -- is the durable scheduling cursor. attempt_seq is allocated + -- transactionally per submission lacking a stable external delivery id. + source_policy TEXT NOT NULL DEFAULT 'manual' CHECK ( + source_policy IN ('manual','review','automatic') + ), + poll_interval_secs INTEGER NULL, + next_poll_at INTEGER NULL, + attempt_seq INTEGER NOT NULL DEFAULT 0, partial_json TEXT NULL, failure_stage TEXT NULL CHECK ( failure_stage IS NULL OR failure_stage IN ( @@ -177,6 +187,17 @@ CREATE TABLE IF NOT EXISTS gitops_generations ( actor TEXT NULL, previous_generation_id TEXT NULL, redacted_limitations_json TEXT NOT NULL DEFAULT '[]', + -- Portable accepted-generation contract (content only: no node id, local + -- path, target mode, or secret value). Additive and nullable so existing + -- rows decode as an explicit limitation rather than invented evidence; a + -- legacy pending candidate lacking these must be re-evaluated before it + -- can be accepted or dispatched. + portable_manifest_json TEXT NULL, + compose_inputs_json TEXT NULL, + source_policy_evidence_json TEXT NULL, + security_policy_evidence_json TEXT NULL, + support_requirements_json TEXT NULL, + compatibility_requirements_json TEXT NULL, created_at INTEGER NOT NULL ); CREATE INDEX IF NOT EXISTS idx_gitops_gen_app_created @@ -453,6 +474,10 @@ CREATE INDEX IF NOT EXISTS idx_gitops_history_node ON gitops_history(node_id); CREATE INDEX IF NOT EXISTS idx_gitops_history_trigger ON gitops_history(trigger); CREATE INDEX IF NOT EXISTS idx_gitops_history_actor ON gitops_history(actor); CREATE INDEX IF NOT EXISTS idx_gitops_history_outcome ON gitops_history(outcome); +-- listUnsettledReconcileAttempts filters on stage and orders by created_at; +-- without this, that query (run on every startup, ahead of the server +-- listening) scans and sorts the whole table. +CREATE INDEX IF NOT EXISTS idx_gitops_history_stage_created ON gitops_history(stage, created_at); CREATE INDEX IF NOT EXISTS idx_gitops_history_repo_ref ON gitops_history(repo_url, configured_ref); CREATE INDEX IF NOT EXISTS idx_gitops_history_stack_created diff --git a/backend/src/services/gitops/store.ts b/backend/src/services/gitops/store.ts index 4069d6a2..f5199f92 100644 --- a/backend/src/services/gitops/store.ts +++ b/backend/src/services/gitops/store.ts @@ -1,5 +1,6 @@ import type Database from 'better-sqlite3'; import { DatabaseService } from '../DatabaseService'; +import type { GitOpsHistoryCursor } from './history'; import { decodeArtifactEvidenceJson, decodeGitOpsApprovedTargetEffectJson, @@ -16,6 +17,7 @@ import type { GitOpsCreateCheckpointRow, GitOpsCreatePhase, GitOpsGenerationRow, + GitOpsHistoryRow, GitOpsIntentRevisionRow, GitOpsRolloutCandidateRow, GitOpsTargetCurrentRow, @@ -131,6 +133,29 @@ export class GitOpsStore { ).get(stackName) as GitOpsApplicationRow | undefined; } + /** + * Whether a detached Direct application exists for this stack name, + * distinct from "no application was ever created" (the legitimate + * pre-migration case, where a stack's git-source config predates + * GitOps tracking entirely). `detached` only, matching + * getDetachedDirectApplication above and for the same reason: + * `deleted` is not a safe signal here. Two production paths tombstone + * an application as `deleted` while deliberately preserving its + * git-source row so a future upsert or migration can rebuild from it + * (`gitops/createRecovery.ts`'s checkpointless-create sweep, and + * `gitops/migrate.ts`'s `tombstoned_missing_stack` outcome) -- + * treating that state as a refusal would be permanent and + * unrecoverable, since neither upsert() nor migration currently mints + * a fresh application once a matching migration checkpoint exists. + * `detach()` itself deletes the git-source row in the same transaction + * as tombstoning (`detached`), so the window this method exists to + * catch (tracking removed, config surviving) is a crash between those + * two writes, not routine operation. + */ + hasDetachedDirectApplication(stackName: string): boolean { + return this.getDetachedDirectApplication(stackName) !== undefined; + } + /** Direct applications that never reached their success boundary. */ listCreatingDirectApplications(): GitOpsApplicationRow[] { return this.db().prepare( @@ -144,6 +169,25 @@ export class GitOpsStore { return this.db().prepare('SELECT * FROM gitops_generations WHERE id = ?').get(id) as GitOpsGenerationRow | undefined; } + /** Generations whose creating reconcile attempt has not durably settled. */ + listGenerationsClaimedByUnsettledAttempts(applicationId: string): GitOpsGenerationRow[] { + return this.db().prepare( + `SELECT DISTINCT generation.* + FROM gitops_generations generation + JOIN gitops_history started + ON started.application_id = generation.application_id + AND started.operation_id = generation.operation_id + AND started.stage = 'source_reconcile_started' + WHERE generation.application_id = ? + AND NOT EXISTS ( + SELECT 1 FROM gitops_history settled + WHERE settled.application_id = started.application_id + AND settled.operation_id = started.operation_id + AND settled.stage = 'source_reconcile_settled' + )`, + ).all(applicationId) as GitOpsGenerationRow[]; + } + getArtifactSet(id: string): GitOpsArtifactSetRow | undefined { return this.db().prepare('SELECT * FROM gitops_artifact_sets WHERE id = ?').get(id) as GitOpsArtifactSetRow | undefined; } @@ -195,6 +239,126 @@ export class GitOpsStore { ).all() as GitOpsApplicationRow[]; } + /** + * The settled result for one exact reconcile attempt, or undefined when + * that attempt has not settled (or never existed). Used to recover a + * reservation that already completed rather than repeating the work. + */ + getSettledAttempt(applicationId: string, operationId: string): GitOpsHistoryRow | undefined { + return this.db().prepare( + `SELECT * FROM gitops_history + WHERE application_id = ? AND operation_id = ? AND stage = 'source_reconcile_settled' + LIMIT 1`, + ).get(applicationId, operationId) as GitOpsHistoryRow | undefined; + } + + /** + * The reservation row for one exact reconcile attempt, or undefined when + * it was never reserved. Used to recover this attempt's own recorded + * follower link (if any), so a follower can be settled from its + * leader's actual result rather than derived independently of it. + */ + getStartedAttempt(applicationId: string, operationId: string): GitOpsHistoryRow | undefined { + return this.db().prepare( + `SELECT * FROM gitops_history + WHERE application_id = ? AND operation_id = ? AND stage = 'source_reconcile_started' + LIMIT 1`, + ).get(applicationId, operationId) as GitOpsHistoryRow | undefined; + } + + /** + * Every reservation with no matching settled row, oldest first: an + * attempt that started but never recorded a result, most likely because + * the process crashed between reservation and settlement. Startup + * recovery reconciles these from durable stage evidence rather than + * leaving them silently open forever. + * + * `after` pages strictly forward by (created_at, id), the same cursor + * shape `queryHistoryRows` uses, and is load-bearing rather than + * cosmetic: a row a caller cannot settle (its application vanished, a DB + * error) stays unsettled forever by definition, so without a cursor it + * would occupy the same "oldest N" window on every future call and hide + * every genuinely recoverable row behind it once the backlog exceeds one + * page. + */ + listUnsettledReconcileAttempts(limit = 200, after?: GitOpsHistoryCursor): GitOpsHistoryRow[] { + const clauses = ["started.stage = 'source_reconcile_started'"]; + const params: Array = []; + if (after) { + clauses.push('(started.created_at > ? OR (started.created_at = ? AND started.id > ?))'); + params.push(after.createdAt, after.createdAt, after.id); + } + params.push(limit); + return this.db().prepare( + `SELECT started.* FROM gitops_history started + WHERE ${clauses.join(' AND ')} + AND NOT EXISTS ( + SELECT 1 FROM gitops_history settled + WHERE settled.application_id = started.application_id + AND settled.operation_id = started.operation_id + AND settled.stage = 'source_reconcile_settled' + ) + ORDER BY started.created_at ASC, started.id ASC + LIMIT ?`, + ).all(...params) as GitOpsHistoryRow[]; + } + + /** + * The most recently settled attempt for an application, for API and UI + * projection. Distinct from getSettledAttempt, which looks up one exact + * operation rather than the newest one. + */ + latestSettledAttempt(applicationId: string): GitOpsHistoryRow | undefined { + // rowid (SQLite's implicit insertion-order key), not the id column: id + // is a random UUID and does not sort by recency the way rowid does, so + // it cannot break a created_at tie between two attempts settled within + // the same millisecond. + return this.db().prepare( + `SELECT * FROM gitops_history + WHERE application_id = ? AND stage = 'source_reconcile_settled' + ORDER BY created_at DESC, rowid DESC + LIMIT 1`, + ).get(applicationId) as GitOpsHistoryRow | undefined; + } + + /** + * Direct sources whose poll time has arrived: active, not suspended, no + * operation in flight. Blueprint-mode applications are never polled here + * -- source evaluation for them is blocked at the evaluation boundary + * until an application-keyed source engine exists for that mode. + */ + listSourcesDueForPoll(now: number, limit = 200): GitOpsApplicationRow[] { + return this.db().prepare( + `SELECT * FROM gitops_applications + WHERE target_mode = 'direct' + AND lifecycle_status = 'active' + AND suspended_at IS NULL + AND active_operation_stage IS NULL + AND next_poll_at IS NOT NULL + AND next_poll_at <= ? + ORDER BY next_poll_at ASC + LIMIT ?`, + ).all(now, limit) as GitOpsApplicationRow[]; + } + + /** + * Applications with a scheduled retry that has come due: not suspended, + * no operation in flight. Poll eligibility and retry eligibility are + * deliberately separate queries, since a retry can be due on an + * application whose poll cadence would not otherwise select it yet. + */ + listApplicationsDueForRetry(now: number, limit = 200): GitOpsApplicationRow[] { + return this.db().prepare( + `SELECT * FROM gitops_applications + WHERE retry_at IS NOT NULL + AND retry_at <= ? + AND suspended_at IS NULL + AND active_operation_stage IS NULL + ORDER BY retry_at ASC + LIMIT ?`, + ).all(now, limit) as GitOpsApplicationRow[]; + } + /** Every live target on one node, across all applications. */ listActiveTargetsForNode(nodeId: number): GitOpsTargetCurrentRow[] { return this.db().prepare( @@ -365,12 +529,13 @@ export class GitOpsStore { intent_revision_id, rollout_candidate_id, rollout_generation_id, source_acceptance_ref, placement_approval_ref, rollout_authorization_ref, legacy_combined_approval_ref, preflight_fingerprint, latest_operation_id, active_operation_id, active_operation_stage, - active_operation_at, active_generation_id, pause_at, pause_reason, source_suspended_reason, partial_json, + active_operation_at, active_generation_id, pause_at, pause_reason, source_suspended_reason, + source_policy, poll_interval_secs, next_poll_at, attempt_seq, partial_json, failure_stage, failure_class, failure_at, retry_at, retry_count, suspended_at, recovery_ref, recovery_phase, interruption_stage, interruption_at, interruption_operation_id, interruption_generation_id, evidence_fresh_at, evidence_limitations_json, created_at, updated_at - ) VALUES (${Array(56).fill('?').join(', ')})`, + ) VALUES (${Array(60).fill('?').join(', ')})`, ).run( row.id, row.lifecycle_key, row.lifecycle_status, row.target_mode, row.stack_name, row.blueprint_id, row.configured_repo_url, row.repo_identity_json, row.configured_ref, row.compose_paths_json, @@ -380,7 +545,8 @@ export class GitOpsStore { row.intent_revision_id, row.rollout_candidate_id, row.rollout_generation_id, row.source_acceptance_ref, row.placement_approval_ref, row.rollout_authorization_ref, row.legacy_combined_approval_ref, row.preflight_fingerprint, row.latest_operation_id, row.active_operation_id, row.active_operation_stage, - row.active_operation_at, row.active_generation_id, row.pause_at, row.pause_reason, row.source_suspended_reason, row.partial_json, + row.active_operation_at, row.active_generation_id, row.pause_at, row.pause_reason, row.source_suspended_reason, + row.source_policy, row.poll_interval_secs, row.next_poll_at, row.attempt_seq, row.partial_json, row.failure_stage, row.failure_class, row.failure_at, row.retry_at, row.retry_count, row.suspended_at, row.recovery_ref, row.recovery_phase, row.interruption_stage, row.interruption_at, row.interruption_operation_id, row.interruption_generation_id, row.evidence_fresh_at, @@ -394,13 +560,18 @@ export class GitOpsStore { id, application_id, commit_sha, repo_url, configured_ref, resolved_ref_kind, repo_identity_json, manifest_version, candidate_dir, applied_dir, expected_invocation_json, materialization_fingerprint, validation_ok, plan_blocked, change_plan_fingerprint, - operation_id, trigger, actor, previous_generation_id, redacted_limitations_json, created_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + operation_id, trigger, actor, previous_generation_id, redacted_limitations_json, + portable_manifest_json, compose_inputs_json, source_policy_evidence_json, + security_policy_evidence_json, support_requirements_json, compatibility_requirements_json, + created_at + ) VALUES (${Array(27).fill('?').join(', ')})`, ).run( row.id, row.application_id, row.commit_sha, row.repo_url, row.configured_ref, row.resolved_ref_kind, row.repo_identity_json, row.manifest_version, row.candidate_dir, row.applied_dir, row.expected_invocation_json, row.materialization_fingerprint, row.validation_ok, row.plan_blocked, row.change_plan_fingerprint, row.operation_id, row.trigger, row.actor, row.previous_generation_id, row.redacted_limitations_json, + row.portable_manifest_json, row.compose_inputs_json, row.source_policy_evidence_json, + row.security_policy_evidence_json, row.support_requirements_json, row.compatibility_requirements_json, row.created_at, ); } diff --git a/backend/src/services/gitops/transitions.ts b/backend/src/services/gitops/transitions.ts index 5ebf736a..91fb286e 100644 --- a/backend/src/services/gitops/transitions.ts +++ b/backend/src/services/gitops/transitions.ts @@ -28,6 +28,10 @@ export type EventEnvelope = { at: number; }; +export type ReconcileDeliveryIntent = + | { autoApply: false; deploy: false } + | { autoApply: true; deploy: boolean }; + export type AppliedArgs = { applicationId: string; generationId: string; @@ -893,6 +897,102 @@ export class GitOpsTransitions { }); } + /** + * Reserve a durable attempt before any side effect: a bare history + * insert in its own transaction, deliberately not through mutateApp, so + * nothing about the application row changes. A `reserved: false` return + * means this exact (application, operation) already reserved -- the + * caller reconstructs from durable state rather than repeating work. + * + * `followerOf` records that this reservation joined another running + * attempt. Recovery uses the link to settle the follower from its leader's + * result. `deliveryIntent` preserves the original webhook apply and deploy + * decision so redelivery cannot change behavior with later settings. + */ + reserveReconcileAttempt( + applicationId: string, + envelope: EventEnvelope, + followerOf?: string, + deliveryIntent?: ReconcileDeliveryIntent, + ): { reserved: boolean } { + return this.raw().transaction(() => ({ + reserved: this.insertReconcileReservation(this.requireApp(applicationId), envelope, followerOf, deliveryIntent), + }))(); + } + + /** + * Allocate the next attemptSeq for a submission with no stable external + * delivery identity, and reserve its durable attempt in the same + * transaction, so two concurrent submissions can never mint the same + * operation id. Unlike reserveReconcileAttempt, this does write one + * column of application state (attempt_seq) -- allocation is the one + * thing here that is not a bare history insert, since a fresh id has to + * come from somewhere durable. Only the allocated id's uniqueness is + * load-bearing; its embedded sequence number is for traceability. + */ + allocateReconcileAttempt( + applicationId: string, + actor: string | null, + trigger: string, + at: number, + followerOf?: string, + ): { operationId: string; reserved: boolean } { + return this.raw().transaction(() => { + const app = this.requireApp(applicationId); + const seq = app.attempt_seq + 1; + this.raw().prepare('UPDATE gitops_applications SET attempt_seq = ? WHERE id = ?').run(seq, applicationId); + const operationId = `${applicationId}:attempt:${seq}`; + const envelope: EventEnvelope = { operationId, actor, trigger, at }; + return { operationId, reserved: this.insertReconcileReservation(app, envelope, followerOf) }; + })(); + } + + /** + * The reservation row itself, shared by reserveReconcileAttempt and + * allocateReconcileAttempt: a bare history insert whose dedupe index is + * what makes a repeat reservation report false rather than recording a + * second attempt. + */ + private insertReconcileReservation( + app: GitOpsApplicationRow, + envelope: EventEnvelope, + followerOf: string | undefined, + deliveryIntent?: ReconcileDeliveryIntent, + ): boolean { + return this.history(app, envelope, { + stage: 'source_reconcile_started', + outcome: 'committed', + before: {}, + after: { + ...(followerOf ? { followerOf } : {}), + ...(deliveryIntent ? { deliveryIntent } : {}), + }, + }) !== null; + } + + /** + * Settle a reserved attempt with its normalized outcome, the same way: + * a bare history insert, not a state mutation. Safe to call more than + * once for the same operation; a repeat is a no-op via the history + * dedupe index, so the first settled result is never overwritten. + */ + settleReconcileAttempt( + applicationId: string, + envelope: EventEnvelope, + result: { outcome: string; reason: string; nextAction: string; retryAt?: number; commitSha?: string }, + ): { settled: boolean } { + return this.raw().transaction(() => { + const app = this.requireApp(applicationId); + const historyId = this.history(app, envelope, { + stage: 'source_reconcile_settled', + outcome: 'committed', + before: {}, + after: { ...result }, + }); + return { settled: historyId !== null }; + })(); + } + /** * Pause a rollout, application-wide or on one target. * @@ -2319,7 +2419,8 @@ export class GitOpsTransitions { legacy_combined_approval_ref=?, preflight_fingerprint=?, latest_operation_id=?, active_operation_id=?, active_operation_stage=?, active_operation_at=?, active_generation_id=?, - pause_at=?, pause_reason=?, source_suspended_reason=?, partial_json=?, + pause_at=?, pause_reason=?, source_suspended_reason=?, + source_policy=?, poll_interval_secs=?, next_poll_at=?, attempt_seq=?, partial_json=?, failure_stage=?, failure_class=?, failure_at=?, retry_at=?, retry_count=?, suspended_at=?, recovery_ref=?, recovery_phase=?, interruption_stage=?, interruption_at=?, interruption_operation_id=?, @@ -2336,7 +2437,8 @@ export class GitOpsTransitions { app.legacy_combined_approval_ref, app.preflight_fingerprint, app.latest_operation_id, app.active_operation_id, app.active_operation_stage, app.active_operation_at, app.active_generation_id, - app.pause_at, app.pause_reason, app.source_suspended_reason, app.partial_json, + app.pause_at, app.pause_reason, app.source_suspended_reason, + app.source_policy, app.poll_interval_secs, app.next_poll_at, app.attempt_seq, app.partial_json, app.failure_stage, app.failure_class, app.failure_at, app.retry_at, app.retry_count, app.suspended_at, app.recovery_ref, app.recovery_phase, app.interruption_stage, app.interruption_at, app.interruption_operation_id, diff --git a/backend/src/services/gitops/triggers.ts b/backend/src/services/gitops/triggers.ts new file mode 100644 index 00000000..16c7c512 --- /dev/null +++ b/backend/src/services/gitops/triggers.ts @@ -0,0 +1,77 @@ +/** + * Normalized reconciliation triggers for the GitOps source controller. + * + * A trigger only authorizes evaluation; it is not proof anything changed. + * `manual`, `webhook`, `poll`, and `retry` have current execution producers. + * The remaining values are typed ahead of later deliveries so those callers + * extend this union instead of inventing a parallel one. + */ +export type ReconcileTrigger = + | 'manual' + | 'api' + | 'webhook' + | 'poll' + | 'retry' + | 'config_change' + | 'startup' + | 'resume' + | 'provider_event' + | 'schedule' + | 'binding_change'; + +/** + * One normalized submission to the controller. `dismiss` is deliberately + * not a reconcile intent: it changes candidate state but does not + * authorize source evaluation. + */ +export type ReconcileRequest = + | { + intent: 'fetch'; + applicationId: string; + stackName: string; + trigger: ReconcileTrigger; + actor: string; + deliveryId?: string; + } + | { + intent: 'apply'; + applicationId: string; + stackName: string; + trigger: ReconcileTrigger; + actor: string; + commitSha: string; + planFingerprint: string; + deploy: boolean; + deliveryId?: string; + }; + +/** + * The in-process joining key for concurrent evaluations of the same work. + * A fetch has only one live outcome per application regardless of trigger, + * so any two fetch submissions for the same application and stack join. An + * apply is identified by exactly what it would do: two applies join only + * when they target the same commit, the same plan fingerprint, and the + * same deploy choice. Two applies that differ in any of those must never + * join, or one request could silently receive another request's result. + * + * Both the fetch and the apply form carry the stack name alongside the + * applicationId, so a caller that pairs a live applicationId with the + * wrong stackName can never join a leader evaluating the right one. + */ +export function coalesceKey(request: ReconcileRequest): string { + if (request.intent === 'fetch') { + return `${request.applicationId}:${request.stackName}:fetch`; + } + return `${request.applicationId}:${request.stackName}:apply:${request.commitSha}:${request.planFingerprint}:${request.deploy}`; +} + +/** + * A producer-namespaced key for an external delivery, so the same delivery + * id from two different trigger sources is never treated as one delivery. + * Also namespaced by intent: a webhook that both fetches and applies under + * one delivery id must reserve two distinct attempts, not have the apply's + * reservation collide with the fetch's and silently never run. + */ +export function deliveryKey(trigger: ReconcileTrigger, intent: ReconcileRequest['intent'], deliveryId: string): string { + return `${trigger}:${intent}:${deliveryId}`; +} diff --git a/backend/src/services/gitops/types.ts b/backend/src/services/gitops/types.ts index c9f1ffa8..3d3790dc 100644 --- a/backend/src/services/gitops/types.ts +++ b/backend/src/services/gitops/types.ts @@ -73,6 +73,11 @@ export type GitOpsApplicationRow = { pause_reason: string | null; /** sourceSuspended/sourceUnsuspended's own reason field; independent of pause_reason. */ source_suspended_reason: string | null; + /** Controller-owned. See gitops/SourceController.ts. */ + source_policy: 'manual' | 'review' | 'automatic'; + poll_interval_secs: number | null; + next_poll_at: number | null; + attempt_seq: number; partial_json: string | null; failure_stage: ApplicationFailureStage | null; failure_class: string | null; @@ -161,6 +166,13 @@ export type GitOpsGenerationRow = { actor: string | null; previous_generation_id: string | null; redacted_limitations_json: string; + /** Portable accepted-generation contract fields. See gitops/handoff.ts. */ + portable_manifest_json: string | null; + compose_inputs_json: string | null; + source_policy_evidence_json: string | null; + security_policy_evidence_json: string | null; + support_requirements_json: string | null; + compatibility_requirements_json: string | null; created_at: number; }; From 72fd455346ff0436adce2f61390eabaf543d22bf Mon Sep 17 00:00:00 2001 From: Anso Date: Wed, 9 Sep 2026 08:57:35 -0400 Subject: [PATCH 3/3] fix(deps): bump grpc-go to v1.83.2 --- Dockerfile | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3f8a3733..65d8bd9e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -93,7 +93,7 @@ RUN if [ "$TARGETARCH" = "$BUILDARCH" ]; then \ # Stage 4a: Build Docker CLI from source against Go 1.26.3 # # CLI v29.4.1 ships otel/sdk v1.43.0, resolving CVE-2026-39883 (BSD kenv) and -# CVE-2026-39882 (OTLP response OOM). Building against grpc v1.83.1 below +# CVE-2026-39882 (OTLP response OOM). Building against grpc v1.83.2 below # transitively resolves otel to v1.44.0, which additionally clears # CVE-2026-41178 (baggage header parsing dropped its raw-length cap, allowing # resource exhaustion via an oversized header, present through v1.43.0). It @@ -105,8 +105,8 @@ RUN if [ "$TARGETARCH" = "$BUILDARCH" ]; then \ # TARGET. The fetch pulls only the v29.4.1 commit, minimising transfer size. # docker/cli uses CalVer and ships vendor.mod instead of go.mod to avoid # SemVer compliance requirements. We copy vendor.mod -> go.mod, drop the -# committed vendor tree, bump golang.org/x/net to v0.56.0, golang.org/x/text -# to v0.39.0, google.golang.org/grpc to v1.83.1, and +# committed vendor tree, bump golang.org/x/net to v0.58.0, golang.org/x/text +# to v0.41.0, google.golang.org/grpc to v1.83.2, and # github.com/moby/go-archive to v0.3.0, and build with -mod=mod so the patched # modules are resolved from the module proxy. x/net v0.53.0 is flagged for six # HIGH advisories (CVE-2026-25680, -25681, -27136, -39821, -42502, -42506; @@ -144,9 +144,9 @@ RUN mkdir -p /build RUN cp vendor.mod go.mod && cp vendor.sum go.sum && \ rm -rf vendor && \ - go get golang.org/x/net@v0.56.0 \ - golang.org/x/text@v0.39.0 \ - google.golang.org/grpc@v1.83.1 \ + go get golang.org/x/net@v0.58.0 \ + golang.org/x/text@v0.41.0 \ + google.golang.org/grpc@v1.83.2 \ github.com/moby/go-archive@v0.3.0 && \ CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \ -mod=mod \ @@ -175,7 +175,7 @@ RUN cp vendor.mod go.mod && cp vendor.sum go.sum && \ # CVE-2026-41178 (baggage header parsing dropped its raw-length cap, allowing # resource exhaustion via an oversized header, present through v1.43.0) so # that the compose binary scans completely clean; v1.44.0 is also grpc -# v1.83.1's minimum below. It also bumps github.com/moby/go-archive to v0.3.0 +# v1.83.2's minimum below. It also bumps github.com/moby/go-archive to v0.3.0 # for CVE-2026-17106 (HIGH), where a crafted tar archive can write outside the # extraction directory; compose pulls the same archive code in transitively # through buildkit. @@ -190,14 +190,14 @@ RUN cp vendor.mod go.mod && cp vendor.sum go.sum && \ # daemon-side (containerd's CRI service) and is not reached by compose at all, # so this is defense-in-depth rather than a live exposure. # -# The same go get also bumps google.golang.org/grpc from v1.80.0 to v1.83.1 to +# The same go get also bumps google.golang.org/grpc from v1.80.0 to v1.83.2 to # clear GHSA-hrxh-6v49-42gf (xDS RBAC fail-open and HTTP/2 transport issues) # and CVE-2026-84304 (unauthenticated peer OOM via fragmented HTTP/2 DATA # frames buffered per-message), golang.org/x/text from v0.38.0 to v0.41.0 to # clear CVE-2026-56852 (norm.Iter infinite loop on crafted input) and satisfy -# x/crypto's minimum, golang.org/x/net from v0.55.0 to v0.57.0 to clear -# CVE-2026-46600 (dnsmessage denial of service) and satisfy x/crypto's minimum -# version, and golang.org/x/crypto from v0.53.0 to v0.55.0 to clear +# x/crypto's minimum, golang.org/x/net from v0.55.0 to v0.58.0 to clear +# CVE-2026-46600 (dnsmessage denial of service) and satisfy grpc v1.83.2's +# minimum version, and golang.org/x/crypto from v0.53.0 to v0.55.0 to clear # CVE-2026-56854 (SSH host-key verification bypass). # Base image pinned by digest (same image as cli-builder above) so both # source builds share an identical, immutable Go toolchain. @@ -222,10 +222,11 @@ RUN mkdir -p /build # Patch otel/sdk and exporters from v1.42.0 → v1.44.0 to clear CVE-2026-39883, # CVE-2026-39882, and CVE-2026-41178 (present through v1.43.0; v1.44.0 is also -# grpc v1.83.1's minimum below), bump containerd/v2 from v2.2.3 → v2.2.5 to +# grpc v1.83.2's minimum below), bump containerd/v2 from v2.2.3 → v2.2.5 to # clear CVE-2026-46680 plus the CVE-2026-53488 / 53489 / 53492 cluster, bump -# google.golang.org/grpc to v1.83.1 to clear GHSA-hrxh-6v49-42gf and -# CVE-2026-84304, bump golang.org/x/net to v0.57.0 to clear CVE-2026-46600, +# google.golang.org/grpc to v1.83.2 to clear GHSA-hrxh-6v49-42gf, +# CVE-2026-84304, and CVE-2026-84445, bump golang.org/x/net to v0.58.0 to clear +# CVE-2026-46600, # and bump golang.org/x/crypto to v0.55.0 to clear CVE-2026-56854. The # containerd bump is patch-level; the otel, grpc, x/net, and x/crypto bumps # are minor security releases. None introduce breaking API changes. @@ -241,9 +242,9 @@ RUN --mount=type=cache,id=go-mod,sharing=locked,target=/go/pkg/mod \ go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc@v1.44.0 \ go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp@v1.44.0 \ github.com/containerd/containerd/v2@v2.2.5 \ - google.golang.org/grpc@v1.83.1 \ + google.golang.org/grpc@v1.83.2 \ golang.org/x/text@v0.41.0 \ - golang.org/x/net@v0.57.0 \ + golang.org/x/net@v0.58.0 \ golang.org/x/crypto@v0.55.0 \ github.com/moby/go-archive@v0.3.0 && \ go mod tidy