diff --git a/Dockerfile b/Dockerfile index 3f8a3733..65d8bd9e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -93,7 +93,7 @@ RUN if [ "$TARGETARCH" = "$BUILDARCH" ]; then \ # Stage 4a: Build Docker CLI from source against Go 1.26.3 # # CLI v29.4.1 ships otel/sdk v1.43.0, resolving CVE-2026-39883 (BSD kenv) and -# CVE-2026-39882 (OTLP response OOM). Building against grpc v1.83.1 below +# CVE-2026-39882 (OTLP response OOM). Building against grpc v1.83.2 below # transitively resolves otel to v1.44.0, which additionally clears # CVE-2026-41178 (baggage header parsing dropped its raw-length cap, allowing # resource exhaustion via an oversized header, present through v1.43.0). It @@ -105,8 +105,8 @@ RUN if [ "$TARGETARCH" = "$BUILDARCH" ]; then \ # TARGET. The fetch pulls only the v29.4.1 commit, minimising transfer size. # docker/cli uses CalVer and ships vendor.mod instead of go.mod to avoid # SemVer compliance requirements. We copy vendor.mod -> go.mod, drop the -# committed vendor tree, bump golang.org/x/net to v0.56.0, golang.org/x/text -# to v0.39.0, google.golang.org/grpc to v1.83.1, and +# committed vendor tree, bump golang.org/x/net to v0.58.0, golang.org/x/text +# to v0.41.0, google.golang.org/grpc to v1.83.2, and # github.com/moby/go-archive to v0.3.0, and build with -mod=mod so the patched # modules are resolved from the module proxy. x/net v0.53.0 is flagged for six # HIGH advisories (CVE-2026-25680, -25681, -27136, -39821, -42502, -42506; @@ -144,9 +144,9 @@ RUN mkdir -p /build RUN cp vendor.mod go.mod && cp vendor.sum go.sum && \ rm -rf vendor && \ - go get golang.org/x/net@v0.56.0 \ - golang.org/x/text@v0.39.0 \ - google.golang.org/grpc@v1.83.1 \ + go get golang.org/x/net@v0.58.0 \ + golang.org/x/text@v0.41.0 \ + google.golang.org/grpc@v1.83.2 \ github.com/moby/go-archive@v0.3.0 && \ CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \ -mod=mod \ @@ -175,7 +175,7 @@ RUN cp vendor.mod go.mod && cp vendor.sum go.sum && \ # CVE-2026-41178 (baggage header parsing dropped its raw-length cap, allowing # resource exhaustion via an oversized header, present through v1.43.0) so # that the compose binary scans completely clean; v1.44.0 is also grpc -# v1.83.1's minimum below. It also bumps github.com/moby/go-archive to v0.3.0 +# v1.83.2's minimum below. It also bumps github.com/moby/go-archive to v0.3.0 # for CVE-2026-17106 (HIGH), where a crafted tar archive can write outside the # extraction directory; compose pulls the same archive code in transitively # through buildkit. @@ -190,14 +190,14 @@ RUN cp vendor.mod go.mod && cp vendor.sum go.sum && \ # daemon-side (containerd's CRI service) and is not reached by compose at all, # so this is defense-in-depth rather than a live exposure. # -# The same go get also bumps google.golang.org/grpc from v1.80.0 to v1.83.1 to +# The same go get also bumps google.golang.org/grpc from v1.80.0 to v1.83.2 to # clear GHSA-hrxh-6v49-42gf (xDS RBAC fail-open and HTTP/2 transport issues) # and CVE-2026-84304 (unauthenticated peer OOM via fragmented HTTP/2 DATA # frames buffered per-message), golang.org/x/text from v0.38.0 to v0.41.0 to # clear CVE-2026-56852 (norm.Iter infinite loop on crafted input) and satisfy -# x/crypto's minimum, golang.org/x/net from v0.55.0 to v0.57.0 to clear -# CVE-2026-46600 (dnsmessage denial of service) and satisfy x/crypto's minimum -# version, and golang.org/x/crypto from v0.53.0 to v0.55.0 to clear +# x/crypto's minimum, golang.org/x/net from v0.55.0 to v0.58.0 to clear +# CVE-2026-46600 (dnsmessage denial of service) and satisfy grpc v1.83.2's +# minimum version, and golang.org/x/crypto from v0.53.0 to v0.55.0 to clear # CVE-2026-56854 (SSH host-key verification bypass). # Base image pinned by digest (same image as cli-builder above) so both # source builds share an identical, immutable Go toolchain. @@ -222,10 +222,11 @@ RUN mkdir -p /build # Patch otel/sdk and exporters from v1.42.0 → v1.44.0 to clear CVE-2026-39883, # CVE-2026-39882, and CVE-2026-41178 (present through v1.43.0; v1.44.0 is also -# grpc v1.83.1's minimum below), bump containerd/v2 from v2.2.3 → v2.2.5 to +# grpc v1.83.2's minimum below), bump containerd/v2 from v2.2.3 → v2.2.5 to # clear CVE-2026-46680 plus the CVE-2026-53488 / 53489 / 53492 cluster, bump -# google.golang.org/grpc to v1.83.1 to clear GHSA-hrxh-6v49-42gf and -# CVE-2026-84304, bump golang.org/x/net to v0.57.0 to clear CVE-2026-46600, +# google.golang.org/grpc to v1.83.2 to clear GHSA-hrxh-6v49-42gf, +# CVE-2026-84304, and CVE-2026-84445, bump golang.org/x/net to v0.58.0 to clear +# CVE-2026-46600, # and bump golang.org/x/crypto to v0.55.0 to clear CVE-2026-56854. The # containerd bump is patch-level; the otel, grpc, x/net, and x/crypto bumps # are minor security releases. None introduce breaking API changes. @@ -241,9 +242,9 @@ RUN --mount=type=cache,id=go-mod,sharing=locked,target=/go/pkg/mod \ go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc@v1.44.0 \ go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp@v1.44.0 \ github.com/containerd/containerd/v2@v2.2.5 \ - google.golang.org/grpc@v1.83.1 \ + google.golang.org/grpc@v1.83.2 \ golang.org/x/text@v0.41.0 \ - golang.org/x/net@v0.57.0 \ + golang.org/x/net@v0.58.0 \ golang.org/x/crypto@v0.55.0 \ github.com/moby/go-archive@v0.3.0 && \ go mod tidy diff --git a/backend/src/__tests__/bootstrap-startup-gitops-order.test.ts b/backend/src/__tests__/bootstrap-startup-gitops-order.test.ts new file mode 100644 index 00000000..6ccf2e57 --- /dev/null +++ b/backend/src/__tests__/bootstrap-startup-gitops-order.test.ts @@ -0,0 +1,99 @@ +import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest'; +import fs from 'fs'; +import path from 'path'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; + +let tmpDir: string; +let runGitOpsSourceRecovery: typeof import('../bootstrap/startup').runGitOpsSourceRecovery; +let GitSourceService: typeof import('../services/GitSourceService').GitSourceService; +let gitSourceServiceModule: typeof import('../services/GitSourceService'); + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ runGitOpsSourceRecovery } = await import('../bootstrap/startup')); + gitSourceServiceModule = await import('../services/GitSourceService'); + ({ GitSourceService } = gitSourceServiceModule); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +afterAll(() => { + cleanupTestDb(tmpDir); +}); + +describe('runGitOpsSourceRecovery', () => { + it('recovers unsettled reconcile attempts before sweeping the managed area', async () => { + const order: string[] = []; + // Recovery yields before recording itself: if the sweep were ever + // started concurrently instead of strictly after recovery resolves, + // the sweep's own synchronous push would land first and this would + // catch it, rather than merely proving call order at invocation + // time. + vi.spyOn(GitSourceService.getInstance(), 'recoverUnsettledReconcileAttempts') + .mockImplementation(async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + order.push('recover'); + }); + vi.spyOn(gitSourceServiceModule, 'sweepGitManifestOrphans') + .mockImplementation(async () => { order.push('sweep'); }); + + await runGitOpsSourceRecovery(); + + expect(order).toEqual(['recover', 'sweep']); + }); + + it('still runs the sweep when recovery itself throws, tolerating the failure', async () => { + const order: string[] = []; + vi.spyOn(GitSourceService.getInstance(), 'recoverUnsettledReconcileAttempts') + .mockImplementation(async () => { throw new Error('simulated recovery failure'); }); + vi.spyOn(gitSourceServiceModule, 'sweepGitManifestOrphans') + .mockImplementation(async () => { order.push('sweep'); }); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + + await expect(runGitOpsSourceRecovery()).resolves.toBeUndefined(); + + expect(order).toEqual(['sweep']); + expect(errorSpy).toHaveBeenCalled(); + }); + + it('still resolves when the sweep itself throws, tolerating the failure rather than aborting startup', async () => { + const recoverSpy = vi.spyOn(GitSourceService.getInstance(), 'recoverUnsettledReconcileAttempts') + .mockImplementation(async () => {}); + vi.spyOn(gitSourceServiceModule, 'sweepGitManifestOrphans') + .mockImplementation(async () => { throw new Error('simulated sweep failure'); }); + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + + await expect(runGitOpsSourceRecovery()).resolves.toBeUndefined(); + + expect(recoverSpy).toHaveBeenCalled(); + expect(warnSpy).toHaveBeenCalled(); + }); +}); + +describe('startServer source ordering', () => { + it('calls runGitOpsSourceRecovery before starting SourceController, so recovery and the sweep always precede the controller poll loop', () => { + // A structural check, not a behavioral one: startServer drives ~25 + // unrelated services and isn't practical to run end to end in a + // test. What matters here is the one property a future edit could + // silently break: SourceController must never start before + // recovery and the sweep have both awaited to completion. Comments + // are stripped first so a mention of either symbol in prose can't + // satisfy the match, and matching is whitespace/chaining-tolerant + // so a harmless reformat (line-wrapped method chain, a `const` + // extracted for the controller instance) doesn't false-fail this. + const source = fs.readFileSync(path.join(__dirname, '../bootstrap/startup.ts'), 'utf-8'); + const startServerStart = source.indexOf('export async function startServer'); + // startServer is the last top-level declaration in this file today; + // if that ever changes, bound this slice to its closing brace + // instead of running to end of file. + const startServerBody = source.slice(startServerStart); + const code = startServerBody.replace(/\/\*[\s\S]*?\*\//g, '').replace(/\/\/[^\n]*/g, ''); + const reconcileCallIndex = code.search(/await\s+runGitOpsSourceRecovery\s*\(/); + const controllerStartIndex = code.search(/SourceController\s*\.\s*getInstance\s*\(\s*\)\s*\.\s*start\s*\(/); + expect(reconcileCallIndex).toBeGreaterThan(-1); + expect(controllerStartIndex).toBeGreaterThan(-1); + expect(reconcileCallIndex).toBeLessThan(controllerStartIndex); + }); +}); diff --git a/backend/src/__tests__/build-info-route.test.ts b/backend/src/__tests__/build-info-route.test.ts new file mode 100644 index 00000000..1b6a5bbc --- /dev/null +++ b/backend/src/__tests__/build-info-route.test.ts @@ -0,0 +1,225 @@ +/** + * Route coverage for the canonical build identity: + * + * - GET /api/build-info is proxy-exempt (always served by the control + * instance), requires a signed-in human session (rejects machine / API-token + * credentials), redacts hardened image references to non-admins via + * `restricted: true`, and never mislabels a redacted field "Unknown". + * - GET /api/meta exposes only the bounded `buildChannel` enum on the public + * surface and never leaks the running image reference. + */ +import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest'; +import request from 'supertest'; +import jwt from 'jsonwebtoken'; +import bcrypt from 'bcrypt'; +import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb'; +import { withLoopbackTargetProtection } from './helpers/allowLoopbackTargets'; + +let tmpDir: string; +let app: import('express').Express; +let adminAuth: string; +let viewerAuth: string; +let machineAuth: string; +let remoteNodeId: number; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; +let SelfIdentityService: typeof import('../services/SelfIdentityService').default; + +const IMAGE_ID = 'b'.repeat(64); +const DIGEST = 'a'.repeat(64); + +function mockBuildInfo(over: Record = {}) { + const svc = SelfIdentityService.getInstance(); + vi.spyOn(svc, 'getBuildInfo').mockReturnValue({ + version: '0.97.1', + channel: 'dev', + imageRef: 'ghcr.io/studio-saelix/sencho-dev:dev-abc1234', + imageId: IMAGE_ID, + revision: 'dev-abc1234', + ...over, + }); +} + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ app } = await import('../index')); + ({ DatabaseService } = await import('../services/DatabaseService')); + SelfIdentityService = (await import('../services/SelfIdentityService')).default; + + const db = DatabaseService.getInstance(); + remoteNodeId = db.addNode({ + name: 'build-info-remote', + type: 'remote', + compose_dir: '/tmp', + is_default: false, + api_url: 'http://127.0.0.1:1', + api_token: 'build-info-remote-token', + }); + // A signed-in non-admin human session (role resolved from the DB row). + db.addUser({ + username: 'build-info-viewer', + password_hash: await bcrypt.hash('pw', 1), + role: 'viewer', + }); + + adminAuth = `Bearer ${jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' })}`; + viewerAuth = `Bearer ${jwt.sign({ username: 'build-info-viewer' }, TEST_JWT_SECRET, { expiresIn: '1m' })}`; + // node_proxy machine credential: authMiddleware maps it to role admin with + // userId 0, so requireUserSession must reject it as not a human session. + machineAuth = `Bearer ${jwt.sign({ scope: 'node_proxy' }, TEST_JWT_SECRET, { expiresIn: '1m' })}`; +}); + +afterAll(() => cleanupTestDb(tmpDir)); + +afterEach(() => vi.restoreAllMocks()); + +describe('GET /api/build-info auth', () => { + it('requires authentication', async () => { + mockBuildInfo(); + const res = await request(app).get('/api/build-info'); + expect(res.status).toBe(401); + }); + + it('rejects node_proxy machine credentials (not a human session)', async () => { + mockBuildInfo(); + const res = await request(app).get('/api/build-info').set('Authorization', machineAuth); + expect(res.status).toBe(403); + expect(res.body.code).toBe('SESSION_REQUIRED'); + }); +}); + +describe('GET /api/build-info is proxy-exempt', () => { + it('serves locally even when x-node-id targets a remote node', async () => { + mockBuildInfo(); + // The remote node's api_url is a closed loopback port. A 502 would mean the + // proxy intercepted the request; anything else proves the local handler + // matched, exactly as the existing /api/nodes proxy-exempt test asserts. + const res = await withLoopbackTargetProtection(() => request(app) + .get('/api/build-info') + .set('Authorization', adminAuth) + .set('x-node-id', String(remoteNodeId))); + expect(res.status).not.toBe(502); + expect(res.body.channel).toBe('dev'); + }); +}); + +describe('GET /api/build-info as admin', () => { + it('returns the full dev identity for a dev image (regression: semver still previous stable)', async () => { + mockBuildInfo(); + const res = await request(app).get('/api/build-info').set('Authorization', adminAuth); + expect(res.status).toBe(200); + expect(res.body.version).toBe('0.97.1'); + expect(res.body.channel).toBe('dev'); + expect(res.body.imageChannel).toBe('community'); + expect(res.body.imageRef).toBe('ghcr.io/studio-saelix/sencho-dev:dev-abc1234'); + expect(res.body.imageId).toBe(IMAGE_ID); + expect(res.body.revision).toBe('dev-abc1234'); + expect(res.body.restricted).toBe(false); + }); + + it('returns the bounded imageChannel for a hardened image', async () => { + mockBuildInfo({ + channel: 'stable', + imageRef: 'ghcr.io/studio-saelix/sencho-hardened:0.97.1', + }); + const res = await request(app).get('/api/build-info').set('Authorization', adminAuth); + expect(res.status).toBe(200); + expect(res.body.channel).toBe('stable'); + expect(res.body.imageChannel).toBe('hardened'); + expect(res.body.imageRef).toBe('ghcr.io/studio-saelix/sencho-hardened:0.97.1'); + expect(res.body.restricted).toBe(false); + }); +}); + +describe('GET /api/build-info as a non-admin', () => { + it('redacts a hardened image reference and revision to a non-admin via restricted:true', async () => { + mockBuildInfo({ + channel: 'stable', + imageRef: 'ghcr.io/studio-saelix/sencho-hardened:0.97.1', + revision: `sha256:${DIGEST}`, + }); + const res = await request(app).get('/api/build-info').set('Authorization', viewerAuth); + expect(res.status).toBe(200); + // The build channel stays stable; the procurement channel is what gates + // redaction. Both reference fields are nulled with restricted:true so the + // UI can label them "Restricted", never "Unknown". + expect(res.body.channel).toBe('stable'); + expect(res.body.imageChannel).toBe('hardened'); + expect(res.body.imageRef).toBeNull(); + expect(res.body.revision).toBeNull(); + expect(res.body.restricted).toBe(true); + // The image ID is not a registry reference and is always returned. + expect(res.body.imageId).toBe(IMAGE_ID); + }); + + it('does not redact a community image for a non-admin', async () => { + mockBuildInfo(); + const res = await request(app).get('/api/build-info').set('Authorization', viewerAuth); + expect(res.status).toBe(200); + expect(res.body.imageRef).toBe('ghcr.io/studio-saelix/sencho-dev:dev-abc1234'); + expect(res.body.revision).toBe('dev-abc1234'); + expect(res.body.restricted).toBe(false); + }); + + it('reports unknown procurement channel on bare metal without a running reference', async () => { + mockBuildInfo({ channel: 'unknown', imageRef: null, revision: null }); + const res = await request(app).get('/api/build-info').set('Authorization', viewerAuth); + expect(res.status).toBe(200); + // No reference means no classification, and no redaction can apply. + expect(res.body.channel).toBe('unknown'); + expect(res.body.imageChannel).toBe('unknown'); + expect(res.body.imageRef).toBeNull(); + expect(res.body.revision).toBeNull(); + expect(res.body.restricted).toBe(false); + }); +}); + +describe('GET /api/build-info awaits revision enrichment', () => { + it('blocks the response until the enrichment settle promise resolves', async () => { + mockBuildInfo(); + const svc = SelfIdentityService.getInstance(); + let release!: () => void; + vi.spyOn(svc, 'whenRevisionResolved').mockImplementation( + () => new Promise((res) => { release = res; }), + ); + + let settled = false; + const pending = request(app) + .get('/api/build-info') + .set('Authorization', adminAuth) + .then((res) => { settled = true; return res; }); + + // Give the route a tick to reach the await. It must not have responded yet, + // proving a transient null is never the settled value of a success. + await new Promise((r) => setTimeout(r, 10)); + expect(settled).toBe(false); + + release(); + const res = await pending; + expect(res.status).toBe(200); + expect(res.body.revision).toBe('dev-abc1234'); + }); +}); + +describe('GET /api/meta buildChannel', () => { + it('exposes the bounded build channel on the public endpoint', async () => { + mockBuildInfo(); + const res = await request(app).get('/api/meta'); + expect(res.status).toBe(200); + expect(res.body.buildChannel).toBe('dev'); + }); + + it('never leaks the running image reference on the public endpoint', async () => { + mockBuildInfo(); + const res = await request(app).get('/api/meta'); + const body = JSON.stringify(res.body); + expect(body).not.toContain('ghcr.io/studio-saelix/sencho-dev'); + expect(body).not.toContain('dev-abc1234'); + }); + + it('omits buildChannel when the running image reference is unknown', async () => { + mockBuildInfo({ imageRef: null, channel: 'unknown', revision: null }); + const res = await request(app).get('/api/meta'); + expect(res.status).toBe(200); + expect(res.body.buildChannel).toBeUndefined(); + }); +}); \ No newline at end of file diff --git a/backend/src/__tests__/classify-build-channel.test.ts b/backend/src/__tests__/classify-build-channel.test.ts new file mode 100644 index 00000000..825fa1d4 --- /dev/null +++ b/backend/src/__tests__/classify-build-channel.test.ts @@ -0,0 +1,42 @@ +/** + * Truth table for classifyBuildChannel, the canonical build-identity classifier. + * Answers "is this a dev, preview, or stable build" from the image reference + * alone, independent of the packaged semver. + */ +import { describe, it, expect } from 'vitest'; +import { classifyBuildChannel } from '../helpers/selfUpdateCompose'; + +describe('classifyBuildChannel', () => { + it('classifies the dev repository as dev regardless of tag', () => { + expect(classifyBuildChannel('ghcr.io/studio-saelix/sencho-dev:dev')).toBe('dev'); + expect(classifyBuildChannel('ghcr.io/studio-saelix/sencho-dev:dev-abc1234')).toBe('dev'); + expect(classifyBuildChannel('ghcr.io/studio-saelix/sencho-dev:latest')).toBe('dev'); + expect(classifyBuildChannel('ghcr.io/studio-saelix/sencho-dev@sha256:abc')).toBe('dev'); + }); + + it('classifies stable-repo preview tags as preview', () => { + expect(classifyBuildChannel('saelix/sencho:pr-42')).toBe('preview'); + expect(classifyBuildChannel('saelix/sencho:preview-abc1234')).toBe('preview'); + expect(classifyBuildChannel('ghcr.io/studio-saelix/sencho:pr-7')).toBe('preview'); + }); + + it('classifies stable-repo release/floating tags as stable', () => { + expect(classifyBuildChannel('saelix/sencho:0.97.1')).toBe('stable'); + expect(classifyBuildChannel('saelix/sencho:latest')).toBe('stable'); + expect(classifyBuildChannel('ghcr.io/studio-saelix/sencho:v1.2.3')).toBe('stable'); + expect(classifyBuildChannel('ghcr.io/studio-saelix/sencho-hardened:1.2.3')).toBe('stable'); + expect(classifyBuildChannel('ghcr.io/studio-saelix/sencho-hardened:latest')).toBe('stable'); + }); + + it('treats a dev- tag as preview-matching-tolerant (still dev repo wins)', () => { + // dev repo takes precedence over the preview/stable tag classification + expect(classifyBuildChannel('ghcr.io/studio-saelix/sencho-dev:pr-42')).toBe('dev'); + }); + + it('classifies unknown repositories as unknown', () => { + expect(classifyBuildChannel('ubuntu:22.04')).toBe('unknown'); + expect(classifyBuildChannel('registry.example.com/private/app:1.0')).toBe('unknown'); + expect(classifyBuildChannel('')).toBe('unknown'); + expect(classifyBuildChannel(' ')).toBe('unknown'); + }); +}); \ No newline at end of file diff --git a/backend/src/__tests__/git-project-manifest.test.ts b/backend/src/__tests__/git-project-manifest.test.ts index e8790b4d..464e3012 100644 --- a/backend/src/__tests__/git-project-manifest.test.ts +++ b/backend/src/__tests__/git-project-manifest.test.ts @@ -92,6 +92,7 @@ function makeClone(files: Record): string { } const REPO = { repo_url: 'https://github.com/example/repo.git', branch: 'main' }; +const NO_CANDIDATE_CLAIMS = { complete: true as const, dirs: new Set() }; function seedGitSource(stackName: string): void { DatabaseService.getInstance().upsertGitSource({ @@ -731,6 +732,59 @@ describe('promoteGeneration', () => { }); describe('sweepManagedArea (crash recovery)', () => { + it('does not delete a candidate when its completion marker cannot be inspected', async () => { + const svc = GitProjectManifestService.getInstance(); + const stackName = 'sweep-candidate-marker-io'; + const candidateAbs = path.join(tmpDir, 'git-managed', '1', stackName, 'generations', 'candidate-marker-io'); + const markerPath = path.join(candidateAbs, CANDIDATE_COMPLETE_MARKER); + fs.mkdirSync(candidateAbs, { recursive: true }); + fs.writeFileSync(markerPath, 'complete'); + const originalAccess = fs.promises.access.bind(fs.promises); + const accessSpy = vi.spyOn(fs.promises, 'access').mockImplementation(async (...args: Parameters) => { + if (String(args[0]) === markerPath) { + throw Object.assign(new Error('candidate marker permission denied'), { code: 'EACCES' }); + } + return originalAccess(...args); + }); + + try { + await expect(svc.sweepManagedArea(stackName, { + repoUrl: REPO.repo_url, + branch: REPO.branch, + stackExists: true, + candidateClaims: NO_CANDIDATE_CLAIMS, + })).rejects.toThrow(/candidate marker permission denied/); + expect(fs.existsSync(candidateAbs)).toBe(true); + } finally { + accessSpy.mockRestore(); + } + }); + + it('surfaces a generations-directory read failure', async () => { + const svc = GitProjectManifestService.getInstance(); + const stackName = 'sweep-generations-read-io'; + const generationsDir = path.join(tmpDir, 'git-managed', '1', stackName, 'generations'); + fs.mkdirSync(generationsDir, { recursive: true }); + const originalReaddir = fs.promises.readdir.bind(fs.promises); + const readdirSpy = vi.spyOn(fs.promises, 'readdir').mockImplementation(async (...args: Parameters) => { + if (String(args[0]) === generationsDir) { + throw Object.assign(new Error('generations directory unavailable'), { code: 'EIO' }); + } + return originalReaddir(...args); + }); + + try { + await expect(svc.sweepManagedArea(stackName, { + repoUrl: REPO.repo_url, + branch: REPO.branch, + stackExists: true, + candidateClaims: NO_CANDIDATE_CLAIMS, + })).rejects.toThrow(/generations directory unavailable/); + } finally { + readdirSpy.mockRestore(); + } + }); + it('restores the previous applied generation when the marker matches the stack dir', async () => { const svc = GitProjectManifestService.getInstance(); const stackName = 'sweep-restore'; @@ -763,7 +817,7 @@ describe('sweepManagedArea (crash recovery)', () => { affected: ['app.env', 'compose.yaml'], }); - await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true }); + await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true, candidateClaims: NO_CANDIDATE_CLAIMS }); expect(readStackFile(stackName, 'compose.yaml')).toBe('PRIOR\n'); expect(fs.existsSync(path.join(tmpDir, 'git-managed', '1', stackName, PROMOTION_MARKER))).toBe(false); @@ -817,7 +871,7 @@ describe('sweepManagedArea (crash recovery)', () => { affected: ['compose.yaml'], }); - await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true }); + await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true, candidateClaims: NO_CANDIDATE_CLAIMS }); expect(readStackFile(stackName, 'compose.yaml')).toBe('OPERATOR FIXED ME\n'); expect(fs.existsSync(path.join(tmpDir, 'git-managed', '1', stackName, PROMOTION_MARKER))).toBe(false); @@ -852,7 +906,7 @@ describe('sweepManagedArea (crash recovery)', () => { affected: ['compose.yaml'], }); - await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true }); + await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true, candidateClaims: NO_CANDIDATE_CLAIMS }); expect(readStackFile(stackName, 'compose.yaml')).toBe('PRIOR\n'); expect(fs.existsSync(path.join(tmpDir, 'git-managed', '1', stackName, PROMOTION_MARKER))).toBe(false); @@ -887,7 +941,7 @@ describe('sweepManagedArea (crash recovery)', () => { affected: ['compose.yaml'], }); - await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true }); + await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true, candidateClaims: NO_CANDIDATE_CLAIMS }); expect(readStackFile(stackName, 'compose.yaml')).toBe('NEW\n'); expect(fs.existsSync(path.join(tmpDir, 'git-managed', '1', stackName, PROMOTION_MARKER))).toBe(false); @@ -927,7 +981,7 @@ describe('sweepManagedArea (crash recovery)', () => { affected: ['compose.yaml'], }); - await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true }); + await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true, candidateClaims: NO_CANDIDATE_CLAIMS }); expect(readStackFile(stackName, 'compose.yaml')).toBe('PRIOR\n'); const row = DatabaseService.getInstance().getGitSource(stackName); @@ -969,7 +1023,7 @@ describe('sweepManagedArea (crash recovery)', () => { affected: ['app.env', 'compose.yaml'], }); - await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true }); + await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true, candidateClaims: NO_CANDIDATE_CLAIMS }); expect(readStackFile(stackName, 'compose.yaml')).toBe('NEW\n'); expect(readStackFile(stackName, 'app.env')).toBe('OPERATOR\n'); @@ -981,7 +1035,7 @@ describe('sweepManagedArea (crash recovery)', () => { const svc = GitProjectManifestService.getInstance(); const stackName = 'sweep-orphan'; await svc.writeManifest(stackName, buildManifest(stackName, [managedEntry({ materializedPath: 'compose.yaml' })])); - await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: false }); + await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: false, candidateClaims: NO_CANDIDATE_CLAIMS }); expect(await svc.readManifest(stackName, REPO.repo_url, REPO.branch)).toBeNull(); }); }); @@ -1059,7 +1113,7 @@ describe('detach crash recovery', () => { writeStackFile(stackName, 'compose.yaml', 'services:\n web:\n image: nginx:new\n'); expect(await svc.stageManagedAreaForDetach(stackName)).toBe(true); - await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true }); + await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true, candidateClaims: NO_CANDIDATE_CLAIMS }); expect(readStackFile(stackName, 'compose.yaml')).toBe(original.toString('utf8')); const restored = await svc.readManifest(stackName, REPO.repo_url, REPO.branch); @@ -1303,7 +1357,7 @@ describe('promoteGeneration mid-write failure recovery', () => { }); fs.mkdirSync(path.join(tmpDir, 'git-managed', '1', stackName), { recursive: true }); fs.writeFileSync(path.join(tmpDir, 'git-managed', '1', stackName, PROMOTION_MARKER), '{"v":3 torn', 'utf8'); - await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true }); + await svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true, candidateClaims: NO_CANDIDATE_CLAIMS }); expect(fs.existsSync(path.join(tmpDir, 'git-managed', '1', stackName, PROMOTION_MARKER))).toBe(false); expect(DatabaseService.getInstance().getGitSource(stackName)?.manifest_state).toBe('migration_required'); }); @@ -1325,7 +1379,7 @@ describe('promoteGeneration mid-write failure recovery', () => { }), 'utf8'); const stateSpy = vi.spyOn(DatabaseService.getInstance(), 'setGitSourceManifestState'); try { - await expect(svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true })).resolves.toBeUndefined(); + await expect(svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true, candidateClaims: NO_CANDIDATE_CLAIMS })).resolves.toBeUndefined(); expect(stateSpy).toHaveBeenCalledWith(stackName, null, 'migration_required', null); } finally { stateSpy.mockRestore(); @@ -1343,7 +1397,7 @@ describe('promoteGeneration mid-write failure recovery', () => { throw new Error('database unavailable'); }); try { - await expect(svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true })).rejects.toThrow(/database unavailable/); + await expect(svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true, candidateClaims: NO_CANDIDATE_CLAIMS })).rejects.toThrow(/database unavailable/); expect(fs.existsSync(markerPath)).toBe(true); } finally { stateSpy.mockRestore(); @@ -1371,7 +1425,7 @@ describe('promoteGeneration mid-write failure recovery', () => { return originalAccess(...args); }); try { - await expect(svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true })).rejects.toThrow(/permission denied/); + await expect(svc.sweepManagedArea(stackName, { repoUrl: REPO.repo_url, branch: REPO.branch, stackExists: true, candidateClaims: NO_CANDIDATE_CLAIMS })).rejects.toThrow(/permission denied/); expect(fs.existsSync(markerPath)).toBe(true); } finally { accessSpy.mockRestore(); diff --git a/backend/src/__tests__/git-source-apply-recovery.test.ts b/backend/src/__tests__/git-source-apply-recovery.test.ts index 3f080b7b..706339d0 100644 --- a/backend/src/__tests__/git-source-apply-recovery.test.ts +++ b/backend/src/__tests__/git-source-apply-recovery.test.ts @@ -13,6 +13,38 @@ const mockMarkReconciling = vi.fn().mockReturnValue(true); const mockMarkImmediateVerified = vi.fn().mockReturnValue(true); const mockGet = vi.fn(); const mockCompensate = vi.fn(); +const mockGitOpsApplication = { + id: 'gitops-app', + lifecycle_status: 'active', + stack_name: 'app', + candidate_generation_id: null, +}; +const mockGitOpsStore = { + getLiveDirectApplication: vi.fn().mockReturnValue(mockGitOpsApplication), + getApplication: vi.fn().mockReturnValue(mockGitOpsApplication), + getGeneration: vi.fn().mockReturnValue(undefined), + getSettledAttempt: vi.fn().mockReturnValue(undefined), +}; +const mockGitOpsTransitions = { + allocateReconcileAttempt: vi.fn().mockReturnValue({ operationId: 'gitops-app:attempt:1', reserved: true }), + settleReconcileAttempt: vi.fn().mockReturnValue({ settled: true }), +}; + +vi.mock('../services/gitops/store', () => ({ + GitOpsStore: { + getInstance: () => mockGitOpsStore, + }, +})); + +vi.mock('../services/gitops/transitions', async () => { + const actual = await vi.importActual('../services/gitops/transitions'); + return { + ...actual, + GitOpsTransitions: { + getInstance: () => mockGitOpsTransitions, + }, + }; +}); vi.mock('../services/StackUpdateRecoveryService', () => ({ StackUpdateRecoveryService: { @@ -129,10 +161,6 @@ vi.mock('../services/DatabaseService', () => ({ setGitSourceLastPlan: mockSetGitSourceLastPlan, addNotificationHistory: mockAddNotificationHistory, getStackProjectEnvFiles: vi.fn().mockReturnValue([]), - // The apply path now asks whether this stack has a GitOps application. - // These fixtures predate the revision-state model, so the lookup finds - // nothing and every GitOps producer stays a no-op, which is exactly the - // behavior an install with pre-existing Git stacks gets. getDb: () => ({ prepare: () => ({ get: () => undefined, all: () => [], run: () => ({ changes: 0 }) }), transaction: (fn: () => unknown) => () => fn(), @@ -197,7 +225,7 @@ describe('git-source apply recovery (R1)', () => { v: 4, files: { 'compose.yaml': 'services:\n web:\n image: nginx\n' }, contextDir: null, - candidateRelPath: 'generations/cand', + candidateRelPath: 'generations/candidate-abc1234deadbeef', inventory: { inputs: [], refusals: [], @@ -246,7 +274,7 @@ describe('git-source apply recovery (R1)', () => { version: 4, files: [{ path: 'compose.yaml', content: 'services:\n web:\n image: nginx\n' }], contextDir: null, - candidateRelPath: 'generations/cand', + candidateRelPath: 'generations/candidate-abc1234deadbeef', inventory: { inputs: [], refusals: [], buildContexts: [] }, planFingerprint: 'fp-test', planSchemaVersion: GIT_CHANGE_PLAN_SCHEMA_VERSION, diff --git a/backend/src/__tests__/git-source-routes.test.ts b/backend/src/__tests__/git-source-routes.test.ts index 52a996ad..5de0ccc8 100644 --- a/backend/src/__tests__/git-source-routes.test.ts +++ b/backend/src/__tests__/git-source-routes.test.ts @@ -26,11 +26,13 @@ import { ComposeService } from '../services/ComposeService'; import { GitSourceService, GitSourceError } from '../services/GitSourceService'; import { GitOpsStore } from '../services/gitops/store'; import { GitOpsTransitions } from '../services/gitops/transitions'; +import { deliveryKey } from '../services/gitops/triggers'; import { insertHistory } from '../services/gitops/history'; import type { GitOpsApplicationRow } from '../services/gitops/types'; import { PROXY_DEPLOY_ACTOR_HEADER, PROXY_DEPLOY_SOURCE_HEADER } from '../services/license-headers'; import { withLoopbackTargetProtection } from './helpers/allowLoopbackTargets'; import { directApplicationFixture } from './helpers/gitopsFixtures'; +import { ROLE_PERMISSIONS } from '../middleware/permissions'; // ── Hoisted mocks (must come before importing the app) ───────────────── @@ -75,8 +77,17 @@ function adminToken(): string { return jwt.sign({ username: TEST_USERNAME, role: 'admin' }, TEST_JWT_SECRET, { expiresIn: '1m' }); } +function viewerToken(): string { + return jwt.sign({ username: 'viewer', role: 'viewer' }, TEST_JWT_SECRET, { expiresIn: '1m' }); +} + +function nodeAdminToken(): string { + return jwt.sign({ username: 'node-admin', role: 'node-admin' }, TEST_JWT_SECRET, { expiresIn: '1m' }); +} + beforeAll(async () => { tmpDir = await setupTestDb(); + DatabaseService.getInstance().addUser({ username: 'node-admin', password_hash: 'test', role: 'node-admin' }); ({ app } = await import('../index')); // Seed a real stack directory so the PUT handler's existence guard is satisfied @@ -646,6 +657,253 @@ describe('POST /api/stacks/:stackName/git-source/webhook-pull status codes', () expect(res.status).toBe(200); pullSpy.mockRestore(); }); + + it('passes a remote webhook delivery id into the durable pull path', async () => { + seedGitSource('webhook-delivery-id'); + const pullSpy = vi.spyOn(GitSourceService.getInstance(), 'handleWebhookPull') + .mockResolvedValue({ status: 'success', message: 'Pending update ready at abc1234.' }); + const res = await request(app) + .post('/api/stacks/webhook-delivery-id/git-source/webhook-pull') + .set('Authorization', `Bearer ${adminToken()}`) + .send({ deliveryId: 'webhook:42:provider-delivery-1' }); + + expect(res.status).toBe(200); + expect(pullSpy).toHaveBeenCalledWith('webhook-delivery-id', true, 'webhook:42:provider-delivery-1'); + pullSpy.mockRestore(); + }); + + it('requires stack:deploy when a redelivery carries a persisted deploy intent', async () => { + const stackName = 'webhook-delivery-deploy-auth'; + const applicationId = 'webhook-delivery-deploy-auth-app'; + const deliveryId = 'webhook:control:42:provider-delivery-deploy'; + seedGitSource(stackName); + GitOpsStore.getInstance().insertApplication(directApplicationFixture(applicationId, stackName)); + GitOpsTransitions.getInstance().reserveReconcileAttempt( + applicationId, + { + operationId: deliveryKey('webhook', 'fetch', deliveryId), + actor: 'system:webhook', + trigger: 'webhook', + at: Date.now(), + }, + undefined, + { autoApply: true, deploy: true }, + ); + const pullSpy = vi.spyOn(GitSourceService.getInstance(), 'handleWebhookPull'); + const originalPermissions = ROLE_PERMISSIONS['node-admin']; + ROLE_PERMISSIONS['node-admin'] = originalPermissions.filter((permission) => permission !== 'stack:deploy'); + + try { + const res = await request(app) + .post(`/api/stacks/${stackName}/git-source/webhook-pull`) + .set('Authorization', `Bearer ${nodeAdminToken()}`) + .send({ deliveryId }); + + expect(res.status).toBe(403); + expect(res.body.code).toBe('PERMISSION_DENIED'); + expect(pullSpy).not.toHaveBeenCalled(); + } finally { + ROLE_PERMISSIONS['node-admin'] = originalPermissions; + pullSpy.mockRestore(); + } + }); + + it('requires stack:deploy when a first delivery is configured to auto-apply and deploy', async () => { + const stackName = 'webhook-first-delivery-deploy-auth'; + seedGitSource(stackName); + DatabaseService.getInstance().getDb() + .prepare('UPDATE stack_git_sources SET auto_apply_on_webhook = 1, auto_deploy_on_apply = 1 WHERE stack_name = ?') + .run(stackName); + const pullSpy = vi.spyOn(GitSourceService.getInstance(), 'handleWebhookPull'); + const originalPermissions = ROLE_PERMISSIONS['node-admin']; + ROLE_PERMISSIONS['node-admin'] = originalPermissions.filter((permission) => permission !== 'stack:deploy'); + + try { + const res = await request(app) + .post(`/api/stacks/${stackName}/git-source/webhook-pull`) + .set('Authorization', `Bearer ${nodeAdminToken()}`); + + expect(res.status).toBe(403); + expect(res.body.code).toBe('PERMISSION_DENIED'); + expect(pullSpy).not.toHaveBeenCalled(); + } finally { + ROLE_PERMISSIONS['node-admin'] = originalPermissions; + pullSpy.mockRestore(); + } + }); + + it.each([ + ['an object', { nested: true }], + ['a blank string', ' '], + ['a string over 512 characters', 'x'.repeat(513)], + ])('rejects %s as a remote webhook delivery id', async (_caseName, deliveryId) => { + seedGitSource('webhook-delivery-id-invalid'); + const pullSpy = vi.spyOn(GitSourceService.getInstance(), 'handleWebhookPull'); + + try { + const res = await request(app) + .post('/api/stacks/webhook-delivery-id-invalid/git-source/webhook-pull') + .set('Authorization', `Bearer ${adminToken()}`) + .send({ deliveryId }); + + expect(res.status).toBe(400); + expect(pullSpy).not.toHaveBeenCalled(); + } finally { + pullSpy.mockRestore(); + } + }); +}); + +describe('POST /api/stacks/:stackName/git-source/suspend', () => { + it('returns 401 without auth', async () => { + const res = await request(app).post('/api/stacks/existing-stack/git-source/suspend'); + expect(res.status).toBe(401); + }); + + it('returns 400 for an invalid stack name', async () => { + const res = await request(app) + .post('/api/stacks/..%2fescape/git-source/suspend') + .set('Authorization', `Bearer ${adminToken()}`) + .send({}); + expect([400, 404]).toContain(res.status); + }); + + it('passes the reason through and returns the normalized result', async () => { + const suspendSpy = vi.spyOn(GitSourceService.getInstance(), 'suspend') + .mockResolvedValue({ outcome: 'suspended', reason: 'Reconciliation is suspended: maintenance', nextAction: 'resume' }); + const res = await request(app) + .post('/api/stacks/existing-stack/git-source/suspend') + .set('Authorization', `Bearer ${adminToken()}`) + .send({ reason: 'maintenance' }); + expect(res.status).toBe(200); + expect(res.body.outcome).toBe('suspended'); + expect(suspendSpy).toHaveBeenCalledWith('existing-stack', expect.objectContaining({ reason: 'maintenance' })); + suspendSpy.mockRestore(); + }); + + it('omits reason when none is given in the body', async () => { + const suspendSpy = vi.spyOn(GitSourceService.getInstance(), 'suspend') + .mockResolvedValue({ outcome: 'suspended', reason: 'Reconciliation is suspended.', nextAction: 'resume' }); + await request(app) + .post('/api/stacks/existing-stack/git-source/suspend') + .set('Authorization', `Bearer ${adminToken()}`) + .send({}); + expect(suspendSpy).toHaveBeenCalledWith('existing-stack', expect.objectContaining({ reason: undefined })); + suspendSpy.mockRestore(); + }); + + it('maps a refused suspend to 409', async () => { + const suspendSpy = vi.spyOn(GitSourceService.getInstance(), 'suspend') + .mockRejectedValue(new GitSourceError('OPERATION_IN_FLIGHT', 'Cannot suspend existing-stack: source is not live')); + const res = await request(app) + .post('/api/stacks/existing-stack/git-source/suspend') + .set('Authorization', `Bearer ${adminToken()}`) + .send({}); + expect(res.status).toBe(409); + suspendSpy.mockRestore(); + }); + + it('denies without the stack:edit permission', async () => { + const res = await request(app) + .post('/api/stacks/existing-stack/git-source/suspend') + .set('Authorization', `Bearer ${viewerToken()}`) + .send({}); + expect([401, 403]).toContain(res.status); + }); + + it('rejects an oversized reason with 400', async () => { + const suspendSpy = vi.spyOn(GitSourceService.getInstance(), 'suspend'); + const res = await request(app) + .post('/api/stacks/existing-stack/git-source/suspend') + .set('Authorization', `Bearer ${adminToken()}`) + .send({ reason: 'x'.repeat(513) }); + expect(res.status).toBe(400); + expect(suspendSpy).not.toHaveBeenCalled(); + suspendSpy.mockRestore(); + }); +}); + +describe('POST /api/stacks/:stackName/git-source/resume', () => { + it('returns 401 without auth', async () => { + const res = await request(app).post('/api/stacks/existing-stack/git-source/resume'); + expect(res.status).toBe(401); + }); + + it('returns 400 for an invalid stack name', async () => { + const res = await request(app) + .post('/api/stacks/..%2fescape/git-source/resume') + .set('Authorization', `Bearer ${adminToken()}`) + .send({}); + expect([400, 404]).toContain(res.status); + }); + + it('returns the normalized result', async () => { + const resumeSpy = vi.spyOn(GitSourceService.getInstance(), 'resume') + .mockResolvedValue({ outcome: 'no_source_change', reason: 'ok', nextAction: 'none' }); + const res = await request(app) + .post('/api/stacks/existing-stack/git-source/resume') + .set('Authorization', `Bearer ${adminToken()}`) + .send({}); + expect(res.status).toBe(200); + expect(res.body.outcome).toBe('no_source_change'); + expect(resumeSpy).toHaveBeenCalledWith('existing-stack', expect.objectContaining({ actor: expect.any(String) })); + resumeSpy.mockRestore(); + }); + + it('denies without the stack:edit permission', async () => { + const res = await request(app) + .post('/api/stacks/existing-stack/git-source/resume') + .set('Authorization', `Bearer ${viewerToken()}`) + .send({}); + expect([401, 403]).toContain(res.status); + }); +}); + +describe('POST /api/stacks/:stackName/git-source/retry', () => { + it('returns 401 without auth', async () => { + const res = await request(app).post('/api/stacks/existing-stack/git-source/retry'); + expect(res.status).toBe(401); + }); + + it('returns 400 for an invalid stack name', async () => { + const res = await request(app) + .post('/api/stacks/..%2fescape/git-source/retry') + .set('Authorization', `Bearer ${adminToken()}`) + .send({}); + expect([400, 404]).toContain(res.status); + }); + + it('returns the normalized result', async () => { + const retrySpy = vi.spyOn(GitSourceService.getInstance(), 'retry') + .mockResolvedValue({ outcome: 'candidate_already_fetched', reason: 'ok', nextAction: 'none' }); + const res = await request(app) + .post('/api/stacks/existing-stack/git-source/retry') + .set('Authorization', `Bearer ${adminToken()}`) + .send({}); + expect(res.status).toBe(200); + expect(res.body.outcome).toBe('candidate_already_fetched'); + expect(retrySpy).toHaveBeenCalledWith('existing-stack', expect.objectContaining({ actor: expect.any(String) })); + retrySpy.mockRestore(); + }); + + it('maps an unexpected failure to 500', async () => { + const retrySpy = vi.spyOn(GitSourceService.getInstance(), 'retry') + .mockRejectedValue(new Error('unexpected')); + const res = await request(app) + .post('/api/stacks/existing-stack/git-source/retry') + .set('Authorization', `Bearer ${adminToken()}`) + .send({}); + expect(res.status).toBe(500); + retrySpy.mockRestore(); + }); + + it('denies without the stack:edit permission', async () => { + const res = await request(app) + .post('/api/stacks/existing-stack/git-source/retry') + .set('Authorization', `Bearer ${viewerToken()}`) + .send({}); + expect([401, 403]).toContain(res.status); + }); }); describe('DELETE /api/stacks/:stackName/git-source, detach/export contract', () => { diff --git a/backend/src/__tests__/git-source-service.test.ts b/backend/src/__tests__/git-source-service.test.ts index 24f422e5..5e91609d 100644 --- a/backend/src/__tests__/git-source-service.test.ts +++ b/backend/src/__tests__/git-source-service.test.ts @@ -21,10 +21,13 @@ import type { TransportFailure } from '../services/git/errors'; import { GitOpsStore } from '../services/gitops/store'; import { GitOpsTransitions } from '../services/gitops/transitions'; import { StackOpLockService } from '../services/StackOpLockService'; +import { coalesceKey, deliveryKey, type ReconcileRequest, type ReconcileTrigger } from '../services/gitops/triggers'; import { + buildDirectApplicationRow, buildGenerationRow, directSourceIdentity, newGitOpsId, + stackManagedRoot, type DirectSourceConfig, } from '../services/gitops/directApplication'; @@ -239,6 +242,41 @@ function mockSuccessfulClone(options: { return sha; } +/** + * Configure a plain single-file Git source for a stack, without creating the + * stack itself. The caller stages the clone mock first: upsert runs a + * reachability fetch. + */ +async function configureGitSource(stackName: string): Promise { + await GitSourceService.getInstance().upsert({ + stackName, + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); +} + +/** Operation ids of every gitops_history row an application recorded at one stage. */ +function historyOperationIds(applicationId: string, stage: string): string[] { + const rows = DatabaseService.getInstance().getDb() + .prepare('SELECT operation_id FROM gitops_history WHERE application_id = ? AND stage = ?') + .all(applicationId, stage) as { operation_id: string }[]; + return rows.map((r) => r.operation_id); +} + +/** Settled attempt rows for one application, used to compare follower results. */ +function settledAttemptsForApplication(applicationId: string): { operation_id: string; after_json: string }[] { + return DatabaseService.getInstance().getDb() + .prepare("SELECT operation_id, after_json FROM gitops_history WHERE application_id = ? AND stage = 'source_reconcile_settled'") + .all(applicationId) as { operation_id: string; after_json: string }[]; +} + /** Wrap a single compose string in the ComposeFile[] shape the new APIs take. */ function asFiles(content: string): import('../services/GitSourceService').ComposeFile[] { return [{ path: 'compose.yaml', content }]; @@ -258,6 +296,27 @@ async function cleanupStackDir(name: string) { const SKIP_PLAN_FINGERPRINT = { requirePlanFingerprint: false as const }; +describe('GitSourceService.shortOperationId', () => { + function shortOperationId(operationId: string): string { + return (GitSourceService as unknown as { shortOperationId: (id: string) => string }).shortOperationId(operationId); + } + + it('discriminates between reserved attempts on the same application, unlike a fixed-width prefix', () => { + const appId = '4457ddc3-3eb0-444e-902c-7e65d355b36b'; + expect(shortOperationId(`${appId}:attempt:1`)).toBe('1'); + expect(shortOperationId(`${appId}:attempt:2`)).toBe('2'); + }); + + it('uses the delivery-key suffix for a webhook-triggered reservation', () => { + expect(shortOperationId('webhook:fetch:delivery-abc')).toBe('delivery-abc'); + }); + + it('falls back to a prefix for a plain UUID with no colon', () => { + const uuid = '29ec01cd-4129-4c4c-a5f2-4e2368f44490'; + expect(shortOperationId(uuid)).toBe(uuid.slice(0, 8)); + }); +}); + describe('GitSourceService.hashContent', () => { it('produces stable hashes for identical inputs', () => { const svc = GitSourceService.getInstance(); @@ -1488,23 +1547,379 @@ describe('GitSourceService.handleWebhookPull debounce', () => { // Stamp a recent debounce timestamp directly DatabaseService.getInstance().touchGitSourceDebounce('debounce-stack'); - const result = await svc.handleWebhookPull('debounce-stack'); + const result = await svc.handleWebhookPull('debounce-stack', true); expect(result.status).toBe('skipped'); expect(result.message).toMatch(/rate limited/i); }); it('returns error when stack has no Git source configured', async () => { const svc = GitSourceService.getInstance(); - const result = await svc.handleWebhookPull('does-not-exist'); + const result = await svc.handleWebhookPull('does-not-exist', true); expect(result.status).toBe('error'); expect(result.message).toMatch(/no git source/i); }); + it('fails closed and does not clone when reservation itself fails', async () => { + mockSuccessfulClone({ sha: '2'.repeat(40) }); + const svc = GitSourceService.getInstance(); + await configureGitSource('webhook-reservation-fails-closed'); + mockGitClone.mockClear(); + const reserveSpy = vi.spyOn(GitOpsTransitions.prototype, 'allocateReconcileAttempt') + .mockImplementationOnce(() => { throw new Error('simulated reservation failure'); }); + + try { + const result = await svc.handleWebhookPull('webhook-reservation-fails-closed', true); + expect(result.status).toBe('error'); + expect(mockGitClone).not.toHaveBeenCalled(); + } finally { + reserveSpy.mockRestore(); + } + }); + + it('fails closed and does not clone when the application was detached but the source config survives', async () => { + mockSuccessfulClone({ sha: '3'.repeat(40) }); + const svc = GitSourceService.getInstance(); + await configureGitSource('webhook-tombstoned-app'); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('webhook-tombstoned-app')!.id; + GitOpsTransitions.getInstance().applicationTombstoned(applicationId, 'detached', { + operationId: 'op-detach-3', actor: 'tester', trigger: 'test', at: Date.now(), + }); + mockGitClone.mockClear(); + + const result = await svc.handleWebhookPull('webhook-tombstoned-app', true); + + // Skipped, not error: this is a permanent state, and reporting it + // as a delivery failure on every future push risks the Git host + // disabling the webhook for a condition retrying can never fix. + expect(result.status).toBe('skipped'); + expect(result.message).toMatch(/GitOps tracking was removed/); + expect(mockGitClone).not.toHaveBeenCalled(); + }); + + it('fails closed and does not clone when the application was deleted but the source config survives', async () => { + mockSuccessfulClone({ sha: '4'.repeat(40) }); + const svc = GitSourceService.getInstance(); + await configureGitSource('webhook-deleted-app'); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('webhook-deleted-app')!.id; + GitOpsTransitions.getInstance().applicationTombstoned(applicationId, 'deleted', { + operationId: 'op-delete-webhook', actor: 'tester', trigger: 'test', at: Date.now(), + }); + mockGitClone.mockClear(); + + const result = await svc.handleWebhookPull('webhook-deleted-app', true); + + expect(result.status).toBe('skipped'); + expect(result.message).toMatch(/GitOps tracking is unavailable/); + expect(mockGitClone).not.toHaveBeenCalled(); + }); + + it('does not execute a persisted deploy intent when the caller lacks deploy authorization', async () => { + const stackName = 'webhook-persisted-deploy-auth'; + const deliveryId = 'webhook:control:7:deploy-auth'; + mockSuccessfulClone({ sha: '5'.repeat(40) }); + const svc = GitSourceService.getInstance(); + await configureGitSource(stackName); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication(stackName)!.id; + GitOpsTransitions.getInstance().reserveReconcileAttempt( + applicationId, + { + operationId: deliveryKey('webhook', 'fetch', deliveryId), + actor: 'system:webhook', + trigger: 'webhook', + at: Date.now(), + }, + undefined, + { autoApply: true, deploy: true }, + ); + mockGitClone.mockClear(); + + const result = await svc.handleWebhookPull(stackName, false, deliveryId); + + expect(result.status).toBe('error'); + expect(result.message).toMatch(/deploy permission/i); + expect(mockGitClone).not.toHaveBeenCalled(); + }); + + it('does not require deploy authorization when auto-apply is disabled', async () => { + const stackName = 'webhook-fetch-only-deploy-setting'; + const deliveryId = 'delivery-fetch-only-deploy-setting'; + mockSuccessfulClone({ sha: '7'.repeat(40) }); + const svc = GitSourceService.getInstance(); + await svc.upsert({ + stackName, + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + DatabaseService.getInstance().getDb() + .prepare('UPDATE stack_git_sources SET auto_deploy_on_apply = 1 WHERE stack_name = ?') + .run(stackName); + mockGitClone.mockClear(); + + expect(svc.webhookDeliveryRequiresDeploy(stackName, deliveryId)).toBe(false); + const first = await svc.handleWebhookPull(stackName, false, deliveryId); + expect(first.status).toBe('success'); + expect(svc.webhookDeliveryRequiresDeploy(stackName, deliveryId)).toBe(false); + + DatabaseService.getInstance().getDb() + .prepare('UPDATE stack_git_sources SET last_debounce_at = ? WHERE stack_name = ?') + .run(Date.now() - 999_999, stackName); + mockGitClone.mockClear(); + const redelivery = await svc.handleWebhookPull(stackName, false, deliveryId); + + expect(redelivery.status).toBe('success'); + expect(mockGitClone).not.toHaveBeenCalled(); + }); + + it('fails closed on the auto-apply step alone: the fetch settles, the apply reservation fails, and no apply proceeds', async () => { + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: '6'.repeat(40) }); + const svc = GitSourceService.getInstance(); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + try { + await svc.upsert({ + stackName: 'webhook-apply-reservation-fails-closed', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: true, + autoDeployOnApply: false, + }); + } finally { + validateSpy.mockRestore(); + } + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('webhook-apply-reservation-fails-closed')!.id; + + // First call (the fetch) reserves normally; the second (the + // auto-apply) fails, isolating the apply-stage reservation path. + const originalReserve = GitOpsTransitions.prototype.reserveReconcileAttempt; + const reserveSpy = vi.spyOn(GitOpsTransitions.prototype, 'reserveReconcileAttempt') + .mockImplementationOnce(function (this: GitOpsTransitions, ...args: Parameters) { + return originalReserve.apply(this, args); + }) + .mockImplementationOnce(() => { throw new Error('simulated apply reservation failure'); }); + + try { + const result = await svc.handleWebhookPull( + 'webhook-apply-reservation-fails-closed', + true, + 'delivery-apply-reservation-failure', + ); + expect(result.status).toBe('error'); + expect(saveSpy).not.toHaveBeenCalled(); + // The fetch attempt settled normally; only the apply attempt + // never got as far as being reserved at all. + const unsettled = GitOpsStore.getInstance().listUnsettledReconcileAttempts() + .filter((r) => r.application_id === applicationId); + expect(unsettled).toHaveLength(0); + } finally { + reserveSpy.mockRestore(); + } + + DatabaseService.getInstance().getDb() + .prepare('UPDATE stack_git_sources SET auto_apply_on_webhook = 0, last_debounce_at = ? WHERE stack_name = ?') + .run(Date.now() - 999_999, 'webhook-apply-reservation-fails-closed'); + try { + const redelivery = await svc.handleWebhookPull( + 'webhook-apply-reservation-fails-closed', + true, + 'delivery-apply-reservation-failure', + ); + expect(redelivery.status).toBe('success'); + expect(saveSpy).toHaveBeenCalledTimes(1); + } finally { + saveSpy.mockRestore(); + } + }); + + it('reserves and durably settles an attempt for a successful webhook fetch', async () => { + mockSuccessfulClone({ sha: '8'.repeat(40) }); + const svc = GitSourceService.getInstance(); + await configureGitSource('webhook-reserves'); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('webhook-reserves')!.id; + mockGitClone.mockClear(); + mockSuccessfulClone({ sha: '9'.repeat(40) }); + + const result = await svc.handleWebhookPull('webhook-reserves', true); + + expect(result.status).toBe('success'); + expect(historyOperationIds(applicationId, 'source_reconcile_settled').length).toBeGreaterThanOrEqual(1); + expect(GitOpsStore.getInstance().listUnsettledReconcileAttempts().some((r) => r.application_id === applicationId)).toBe(false); + }); + + it('deduplicates a webhook redelivery by its stable delivery id after the debounce window expires', async () => { + const sha = 'a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1'; + mockSuccessfulClone({ sha }); + const svc = GitSourceService.getInstance(); + await configureGitSource('webhook-delivery-recorded'); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('webhook-delivery-recorded')!.id; + + const first = await svc.handleWebhookPull('webhook-delivery-recorded', true, 'delivery-xyz'); + expect(first.status).toBe('success'); + + DatabaseService.getInstance().getDb() + .prepare('UPDATE stack_git_sources SET last_debounce_at = ? WHERE stack_name = ?') + .run(Date.now() - 999_999, 'webhook-delivery-recorded'); + mockGitClone.mockClear(); + + const redelivery = await svc.handleWebhookPull('webhook-delivery-recorded', true, 'delivery-xyz'); + + expect(mockGitClone).not.toHaveBeenCalled(); + expect(redelivery.status).toBe('success'); + expect(historyOperationIds(applicationId, 'source_reconcile_started')).toHaveLength(1); + expect(historyOperationIds(applicationId, 'source_reconcile_settled')).toHaveLength(1); + }); + + it('joins a concurrent redelivery to the whole webhook fetch-and-apply execution', async () => { + const sha = 'a4'.repeat(20); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha }); + const svc = GitSourceService.getInstance(); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + let releaseSave!: () => void; + const saveGate = new Promise((resolve) => { releaseSave = resolve; }); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockImplementation(async () => { await saveGate; }); + + try { + await svc.upsert({ + stackName: 'webhook-whole-delivery-coalesce', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: true, + autoDeployOnApply: false, + }); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('webhook-whole-delivery-coalesce')!.id; + mockGitClone.mockClear(); + + const first = svc.handleWebhookPull('webhook-whole-delivery-coalesce', true, 'delivery-whole-execution'); + await vi.waitFor(() => expect(saveSpy).toHaveBeenCalledTimes(1)); + const redelivery = svc.handleWebhookPull('webhook-whole-delivery-coalesce', true, 'delivery-whole-execution'); + releaseSave(); + const [firstResult, redeliveryResult] = await Promise.all([first, redelivery]); + + expect(redeliveryResult).toEqual(firstResult); + expect(mockGitClone).toHaveBeenCalledTimes(1); + expect(saveSpy).toHaveBeenCalledTimes(1); + expect(historyOperationIds(applicationId, 'source_reconcile_started')).toHaveLength(2); + expect(historyOperationIds(applicationId, 'source_reconcile_settled')).toHaveLength(2); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + } + }); + + it('does not add an apply step to a fetch-only delivery when settings change before redelivery', async () => { + const sha = 'a2'.repeat(20); + mockSuccessfulClone({ sha }); + const svc = GitSourceService.getInstance(); + await configureGitSource('webhook-redelivery-settings-change'); + + const first = await svc.handleWebhookPull('webhook-redelivery-settings-change', true, 'delivery-settings-change'); + expect(first.status).toBe('success'); + + const db = DatabaseService.getInstance().getDb(); + db.prepare('UPDATE stack_git_sources SET auto_apply_on_webhook = 1, last_debounce_at = ? WHERE stack_name = ?') + .run(Date.now() - 999_999, 'webhook-redelivery-settings-change'); + const { FileSystemService } = await import('../services/FileSystemService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + mockGitClone.mockClear(); + + try { + const redelivery = await svc.handleWebhookPull('webhook-redelivery-settings-change', true, 'delivery-settings-change'); + expect(redelivery.status).toBe('success'); + expect(mockGitClone).not.toHaveBeenCalled(); + expect(saveSpy).not.toHaveBeenCalled(); + } finally { + saveSpy.mockRestore(); + } + }); + + it('returns the stored apply failure when auto-apply is disabled before redelivery', async () => { + const sha = 'a3'.repeat(20); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha }); + const svc = GitSourceService.getInstance(); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const { ComposeService } = await import('../services/ComposeService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockRejectedValue(new Error('simulated webhook deploy failure')); + + try { + await svc.upsert({ + stackName: 'webhook-redelivery-apply-failure', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: true, + autoDeployOnApply: true, + }); + + const first = await svc.handleWebhookPull('webhook-redelivery-apply-failure', true, 'delivery-apply-failure'); + expect(first.status).toBe('error'); + expect(first.message).toContain('simulated webhook deploy failure'); + + DatabaseService.getInstance().getDb() + .prepare('UPDATE stack_git_sources SET auto_apply_on_webhook = 0, auto_deploy_on_apply = 0, last_debounce_at = ? WHERE stack_name = ?') + .run(Date.now() - 999_999, 'webhook-redelivery-apply-failure'); + mockGitClone.mockClear(); + saveSpy.mockClear(); + deploySpy.mockClear(); + + const redelivery = await svc.handleWebhookPull('webhook-redelivery-apply-failure', true, 'delivery-apply-failure'); + + expect(redelivery.status).toBe('error'); + expect(redelivery.message).toContain('simulated webhook deploy failure'); + expect(mockGitClone).not.toHaveBeenCalled(); + expect(saveSpy).not.toHaveBeenCalled(); + expect(deploySpy).not.toHaveBeenCalled(); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('logs the recognized delivery id as a traceability breadcrumb when a webhook pull fails', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ sha: '6'.repeat(40) }); + await configureGitSource('webhook-delivery-breadcrumb'); + mockGitClone.mockClear(); + mockGitClone.mockRejectedValueOnce(new Error('simulated network failure')); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + + try { + const result = await svc.handleWebhookPull('webhook-delivery-breadcrumb', true, 'delivery-log-1'); + expect(result.status).toBe('error'); + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('(delivery delivery-log-1)')); + } finally { + errorSpy.mockRestore(); + } + }); + it('runs a single clone for a concurrent webhook fan-out', async () => { - // The original failure: N webhooks for one push each ran a full clone - // because the debounce gate was read before the per-stack lock. The - // gate now lives inside the lock, so the first request stamps the - // window and the rest skip. + // Concurrent deliveries all reserve and join one shared fetch. Every + // caller receives the leader's normalized result, while only the + // leader performs the clone. const sha = 'eeee555eeee555eeee555eeee555eeee555eeee5'; mockSuccessfulClone({ sha }); const svc = GitSourceService.getInstance(); @@ -1526,15 +1941,85 @@ describe('GitSourceService.handleWebhookPull debounce', () => { mockGitClone.mockClear(); const results = await Promise.all( - Array.from({ length: 5 }, () => svc.handleWebhookPull('fanout-stack')), + Array.from({ length: 5 }, () => svc.handleWebhookPull('fanout-stack', true)), ); expect(mockGitClone.mock.calls.length).toBe(1); - expect(results.filter(r => r.status === 'success')).toHaveLength(1); - expect(results.filter(r => r.status === 'skipped')).toHaveLength(4); + expect(results.filter(r => r.status === 'success')).toHaveLength(5); + expect(results.filter(r => r.status === 'skipped')).toHaveLength(0); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('fanout-stack')!.id; + const settled = settledAttemptsForApplication(applicationId); + expect(settled).toHaveLength(5); + expect(new Set(settled.map((row) => row.after_json)).size).toBe(1); validateSpy.mockRestore(); }); + it('coalesces a webhook fetch with a concurrent manual pull', async () => { + const sha = 'ef'.repeat(20); + mockSuccessfulClone({ sha }); + const svc = GitSourceService.getInstance(); + await configureGitSource('webhook-manual-fetch-coalesce'); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('webhook-manual-fetch-coalesce')!.id; + + let releaseClone!: () => void; + const cloneGate = new Promise((resolve) => { releaseClone = resolve; }); + mockGitClone.mockClear(); + mockGitClone.mockImplementation(async (args: { dir: string }) => { + await cloneGate; + const { promises: fsp } = await import('fs'); + const path = await import('path'); + await fsp.writeFile(path.join(args.dir, 'compose.yaml'), 'services:\n x:\n image: alpine\n', 'utf-8'); + }); + mockGitLog.mockResolvedValue([{ oid: sha }]); + + const webhook = svc.handleWebhookPull('webhook-manual-fetch-coalesce', true, 'delivery-cross-producer'); + await vi.waitFor(() => expect(mockGitClone).toHaveBeenCalledTimes(1)); + const manual = svc.pull('webhook-manual-fetch-coalesce'); + releaseClone(); + const [webhookResult, manualResult] = await Promise.all([webhook, manual]); + + expect(webhookResult.status).toBe('success'); + expect(manualResult.commitSha).toBe(sha); + expect(mockGitClone).toHaveBeenCalledTimes(1); + const settled = DatabaseService.getInstance().getDb() + .prepare("SELECT after_json FROM gitops_history WHERE application_id = ? AND stage = 'source_reconcile_settled'") + .all(applicationId) as { after_json: string }[]; + expect(settled).toHaveLength(2); + expect(new Set(settled.map((row) => row.after_json)).size).toBe(1); + }); + + it('coalesces a manual pull with a concurrent webhook fetch', async () => { + const sha = 'f0'.repeat(20); + mockSuccessfulClone({ sha }); + const svc = GitSourceService.getInstance(); + await configureGitSource('manual-webhook-fetch-coalesce'); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('manual-webhook-fetch-coalesce')!.id; + + let releaseClone!: () => void; + const cloneGate = new Promise((resolve) => { releaseClone = resolve; }); + mockGitClone.mockClear(); + mockGitClone.mockImplementation(async (args: { dir: string }) => { + await cloneGate; + const { promises: fsp } = await import('fs'); + const path = await import('path'); + await fsp.writeFile(path.join(args.dir, 'compose.yaml'), 'services:\n x:\n image: alpine\n', 'utf-8'); + }); + mockGitLog.mockResolvedValue([{ oid: sha }]); + + const manual = svc.pull('manual-webhook-fetch-coalesce'); + await vi.waitFor(() => expect(mockGitClone).toHaveBeenCalledTimes(1)); + const webhook = svc.handleWebhookPull('manual-webhook-fetch-coalesce', true, 'delivery-manual-leader'); + releaseClone(); + const [manualResult, webhookResult] = await Promise.all([manual, webhook]); + + expect(manualResult.commitSha).toBe(sha); + expect(webhookResult.status).toBe('success'); + expect(mockGitClone).toHaveBeenCalledTimes(1); + const settled = settledAttemptsForApplication(applicationId); + expect(settled).toHaveLength(2); + expect(new Set(settled.map((row) => row.after_json)).size).toBe(1); + }); + it('returns error when the pulled compose fails validation', async () => { mockSuccessfulClone(); const svc = GitSourceService.getInstance(); @@ -1556,7 +2041,7 @@ describe('GitSourceService.handleWebhookPull debounce', () => { .spyOn(svc as unknown as { runDockerCompose: (a: string[], c: string, t: number) => Promise<{ code: number; stdout: string; stderr: string }> }, 'runDockerCompose') .mockResolvedValue({ code: 1, stdout: '', stderr: 'bad compose' }); - const result = await svc.handleWebhookPull('webhook-validate-fail'); + const result = await svc.handleWebhookPull('webhook-validate-fail', true); expect(result.status).toBe('error'); expect(result.message).toMatch(/validation failed/i); runSpy.mockRestore(); @@ -1588,7 +2073,7 @@ describe('GitSourceService.handleWebhookPull debounce', () => { existing: { action: 'update', actor: 'user:admin', startedAt: Date.now() }, } as never); - const result = await svc.handleWebhookPull('webhook-shared-lock'); + const result = await svc.handleWebhookPull('webhook-shared-lock', true); expect(result.status).toBe('error'); expect(result.message).toMatch(/already in progress/i); expect(runExclusive).toHaveBeenCalledWith( @@ -1789,6 +2274,224 @@ describe('GitSourceService.pull', () => { await expect(svc.pull('does-not-exist')).rejects.toMatchObject({ code: 'GIT_ERROR' }); }); + it('reserves and durably settles an attempt for a successful pull', async () => { + await createFromGit('pull-reserves', '2'.repeat(40)); + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n web:\n image: nginx:2\n', sha: '3'.repeat(40) }); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('pull-reserves')!.id; + + await svc.pull('pull-reserves'); + + expect(historyOperationIds(applicationId, 'source_reconcile_settled').length).toBeGreaterThanOrEqual(1); + }); + + it('stamps the pending fetch record with the same operation id the reserved attempt used, not an independent one', async () => { + await createFromGit('pull-pending-lineage', '2'.repeat(40)); + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n web:\n image: nginx:2\n', sha: '3'.repeat(40) }); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('pull-pending-lineage')!.id; + + await svc.pull('pull-pending-lineage'); + + const [reservedOperationId] = historyOperationIds(applicationId, 'source_reconcile_started'); + expect(reservedOperationId).toBeTruthy(); + const row = DatabaseService.getInstance().getGitSource('pull-pending-lineage'); + const decoded = (svc as unknown as { + decodePendingCompose: (raw: string) => { operationId: string | null }; + }).decodePendingCompose(row!.pending_compose_content!); + expect(decoded.operationId).toBe(reservedOperationId); + }); + + it('coalesces two concurrent pulls for the same stack into one clone', async () => { + const svc = GitSourceService.getInstance(); + await createFromGit('pull-coalesce', '4'.repeat(40)); + + let releaseClone!: () => void; + const gate = new Promise((resolve) => { releaseClone = resolve; }); + mockGitClone.mockClear(); + mockGitClone.mockImplementation(async (args: { dir: string }) => { + await gate; + const { promises: fsp } = await import('fs'); + const path = await import('path'); + const composeAbs = path.join(args.dir, 'compose.yaml'); + await fsp.mkdir(path.dirname(composeAbs), { recursive: true }); + await fsp.writeFile(composeAbs, 'services:\n web:\n image: nginx:3\n', 'utf-8'); + }); + mockGitLog.mockResolvedValue([{ oid: '5'.repeat(40) }]); + + const first = svc.pull('pull-coalesce'); + const second = svc.pull('pull-coalesce'); + releaseClone(); + const [firstResult, secondResult] = await Promise.all([first, second]); + + expect(mockGitClone).toHaveBeenCalledTimes(1); + expect(secondResult).toEqual(firstResult); + }); + + it('fails closed and does not clone when reservation itself fails', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ sha: '1'.repeat(40) }); + await configureGitSource('pull-reservation-fails-closed'); + mockGitClone.mockClear(); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('pull-reservation-fails-closed')!.id; + const reserveSpy = vi.spyOn(GitOpsTransitions.prototype, 'allocateReconcileAttempt') + .mockImplementationOnce(() => { throw new Error('simulated reservation failure'); }); + + try { + await expect(svc.pull('pull-reservation-fails-closed')).rejects.toMatchObject({ code: 'GIT_ERROR' }); + expect(mockGitClone).not.toHaveBeenCalled(); + expect(historyOperationIds(applicationId, 'source_reconcile_started')).toHaveLength(0); + } finally { + reserveSpy.mockRestore(); + } + }); + + it('fails closed and does not clone when the application was detached but the source config survives', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ sha: '1'.repeat(40) }); + await configureGitSource('pull-tombstoned-app'); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('pull-tombstoned-app')!.id; + GitOpsTransitions.getInstance().applicationTombstoned(applicationId, 'detached', { + operationId: 'op-detach-1', actor: 'tester', trigger: 'test', at: Date.now(), + }); + expect(DatabaseService.getInstance().getGitSource('pull-tombstoned-app')).toBeDefined(); + mockGitClone.mockClear(); + const activitySpy = vi.spyOn(DatabaseService.getInstance(), 'addNotificationHistory'); + + try { + await expect(svc.pull('pull-tombstoned-app')).rejects.toMatchObject({ + code: 'GIT_ERROR', + message: expect.stringContaining('GitOps tracking was removed'), + }); + expect(mockGitClone).not.toHaveBeenCalled(); + expect(historyOperationIds(applicationId, 'source_reconcile_started')).toHaveLength(0); + expect(activitySpy).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ + category: 'git_pull_failed', + stack_name: 'pull-tombstoned-app', + })); + } finally { + activitySpy.mockRestore(); + } + }); + + it('fails closed when the application was deleted, then restores tracked pulls after reconfiguration', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ sha: '4'.repeat(40) }); + await configureGitSource('pull-deleted-app-refused'); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('pull-deleted-app-refused')!.id; + GitOpsTransitions.getInstance().applicationTombstoned(applicationId, 'deleted', { + operationId: 'op-delete-1', actor: 'tester', trigger: 'test', at: Date.now(), + }); + mockGitClone.mockClear(); + + await expect(svc.pull('pull-deleted-app-refused')).rejects.toMatchObject({ + code: 'GIT_ERROR', + message: expect.stringContaining('GitOps tracking is unavailable'), + }); + expect(mockGitClone).not.toHaveBeenCalled(); + + await configureGitSource('pull-deleted-app-refused'); + expect(GitOpsStore.getInstance().getLiveDirectApplication('pull-deleted-app-refused')?.id).not.toBe(applicationId); + mockGitClone.mockClear(); + await svc.pull('pull-deleted-app-refused'); + expect(mockGitClone).toHaveBeenCalledTimes(1); + }); + + it('falls through to the ordinary no-source error for a completed detach, not the detach-in-progress message', async () => { + // detach() commits applicationTombstoned('detached') and + // deleteGitSource in one transaction, so a routine, fully + // successful detach leaves exactly this state: a detached + // tombstone with NO surviving source row. The detach-in-progress + // refusal must not fire here, or every previously-detached stack + // name would get a false, unactionable message instead of the + // real, correct "no source configured" error. + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ sha: '5'.repeat(40) }); + await configureGitSource('pull-completed-detach'); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('pull-completed-detach')!.id; + GitOpsTransitions.getInstance().applicationTombstoned(applicationId, 'detached', { + operationId: 'op-detach-4', actor: 'tester', trigger: 'test', at: Date.now(), + }); + DatabaseService.getInstance().deleteGitSource('pull-completed-detach'); + mockGitClone.mockClear(); + + await expect(svc.pull('pull-completed-detach')).rejects.toMatchObject({ + code: 'GIT_ERROR', + message: 'No Git source configured for this stack.', + }); + expect(mockGitClone).not.toHaveBeenCalled(); + }); + + it('does not mask the real pull failure when deriving the settlement result afterward also throws', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ sha: '1'.repeat(40) }); + await configureGitSource('pull-mask-error'); + // A real gitops application exists (so this call reserves an + // attempt), but the source config row is now gone, so pullLocked + // itself throws a specific, truthful error. + DatabaseService.getInstance().deleteGitSource('pull-mask-error'); + const deriveSpy = vi.spyOn(svc as unknown as { deriveReconcileResult: (s: string) => unknown }, 'deriveReconcileResult') + .mockImplementationOnce(() => { throw new Error('derivation boom'); }); + + try { + await expect(svc.pull('pull-mask-error')).rejects.toMatchObject({ + code: 'GIT_ERROR', + message: expect.stringContaining('No Git source configured'), + }); + } finally { + deriveSpy.mockRestore(); + } + }); + + it('fails closed when a reservation collision is forced with no in-process leader', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ sha: '1'.repeat(40) }); + await configureGitSource('pull-forced-collision'); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('pull-forced-collision')!.id; + const nextSeq = GitOpsStore.getInstance().getApplication(applicationId)!.attempt_seq + 1; + const predictedOperationId = `${applicationId}:attempt:${nextSeq}`; + // Force the exact operation id pull() is about to allocate to + // already be reserved, simulating a collision with no in-process + // leader for it. The call must not run untracked work under an + // operation id already owned by another submission. + GitOpsTransitions.getInstance().reserveReconcileAttempt(applicationId, { + operationId: predictedOperationId, actor: 'someone-else', trigger: 'poll', at: Date.now(), + }); + mockSuccessfulClone({ sha: '2'.repeat(40) }); + mockGitClone.mockClear(); + await expect(svc.pull('pull-forced-collision')).rejects.toMatchObject({ + code: 'GIT_ERROR', + message: expect.stringContaining('already recorded'), + }); + expect(mockGitClone).not.toHaveBeenCalled(); + }); + + it('settles a coalesced follower\'s own attempt even when the leader\'s fetch rejects', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ sha: '1'.repeat(40) }); + await configureGitSource('pull-follower-settles-on-reject'); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('pull-follower-settles-on-reject')!.id; + + let releaseClone!: () => void; + const gate = new Promise((resolve) => { releaseClone = resolve; }); + mockGitClone.mockClear(); + mockGitClone.mockImplementation(async () => { + await gate; + throw new Error('simulated clone failure'); + }); + + const first = svc.pull('pull-follower-settles-on-reject'); + const second = svc.pull('pull-follower-settles-on-reject'); + releaseClone(); + await expect(first).rejects.toThrow('simulated clone failure'); + await expect(second).rejects.toThrow('simulated clone failure'); + + // Both the leader's and the follower's own reservations must be + // durably settled; neither may be left open waiting for a crash + // that never happened. + expect(GitOpsStore.getInstance().listUnsettledReconcileAttempts().some((r) => r.application_id === applicationId)).toBe(false); + }); + function generationCount(stackName: string): number { const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName)!; return (DatabaseService.getInstance().getDb() @@ -2236,6 +2939,1701 @@ describe('GitSourceService.apply', () => { return svc; } + function liveApp(stackName: string) { + return GitOpsStore.getInstance().getLiveDirectApplication(stackName); + } + + it('fails closed and does not write or deploy when reservation itself fails, even for a deploying apply', async () => { + const sha = 'df'.repeat(20); + const svc = await seedPending('apply-reservation-fails-closed', 'services:\n x:\n image: alpine\n', sha); + const applicationId = liveApp('apply-reservation-fails-closed')!.id; + const { FileSystemService } = await import('../services/FileSystemService'); + const { ComposeService } = await import('../services/ComposeService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack'); + const startedBefore = historyOperationIds(applicationId, 'source_reconcile_started').length; + const reserveSpy = vi.spyOn(GitOpsTransitions.prototype, 'allocateReconcileAttempt') + .mockImplementationOnce(() => { throw new Error('simulated reservation failure'); }); + + try { + await expect(svc.apply('apply-reservation-fails-closed', sha, { ...SKIP_PLAN_FINGERPRINT, deploy: true })) + .rejects.toMatchObject({ code: 'GIT_ERROR' }); + expect(saveSpy).not.toHaveBeenCalled(); + expect(deploySpy).not.toHaveBeenCalled(); + // No new attempt at all, settled or unsettled: reservation + // itself never landed, so there is nothing new to track. + expect(historyOperationIds(applicationId, 'source_reconcile_started')).toHaveLength(startedBefore); + } finally { + saveSpy.mockRestore(); + deploySpy.mockRestore(); + reserveSpy.mockRestore(); + } + }); + + it('fails closed and does not write when the application was detached but the pending commit survives', async () => { + const sha = 'db'.repeat(20); + const svc = await seedPending('apply-tombstoned-app', 'services:\n x:\n image: alpine\n', sha); + const applicationId = liveApp('apply-tombstoned-app')!.id; + GitOpsTransitions.getInstance().applicationTombstoned(applicationId, 'detached', { + operationId: 'op-detach-2', actor: 'tester', trigger: 'test', at: Date.now(), + }); + expect(DatabaseService.getInstance().getGitSource('apply-tombstoned-app')?.pending_commit_sha).toBe(sha); + const { FileSystemService } = await import('../services/FileSystemService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + const activitySpy = vi.spyOn(DatabaseService.getInstance(), 'addNotificationHistory'); + + try { + await expect(svc.apply('apply-tombstoned-app', sha, SKIP_PLAN_FINGERPRINT)).rejects.toMatchObject({ + code: 'GIT_ERROR', + message: expect.stringContaining('GitOps tracking was removed'), + }); + expect(saveSpy).not.toHaveBeenCalled(); + expect(activitySpy).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ + category: 'git_apply_failed', + stack_name: 'apply-tombstoned-app', + })); + } finally { + saveSpy.mockRestore(); + activitySpy.mockRestore(); + } + }); + + it('fails closed and does not write or deploy when the application was deleted but the pending commit survives', async () => { + const sha = 'dc'.repeat(20); + const svc = await seedPending('apply-deleted-app', 'services:\n x:\n image: alpine\n', sha); + const applicationId = liveApp('apply-deleted-app')!.id; + GitOpsTransitions.getInstance().applicationTombstoned(applicationId, 'deleted', { + operationId: 'op-delete-apply', actor: 'tester', trigger: 'test', at: Date.now(), + }); + const { FileSystemService } = await import('../services/FileSystemService'); + const { ComposeService } = await import('../services/ComposeService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack'); + + try { + await expect(svc.apply('apply-deleted-app', sha, { ...SKIP_PLAN_FINGERPRINT, deploy: true })) + .rejects.toMatchObject({ + code: 'GIT_ERROR', + message: expect.stringContaining('GitOps tracking is unavailable'), + }); + expect(saveSpy).not.toHaveBeenCalled(); + expect(deploySpy).not.toHaveBeenCalled(); + } finally { + saveSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('reserves and durably settles an attempt for a successful apply', async () => { + const sha = '6'.repeat(40); + const svc = await seedPending('apply-reserves', 'services:\n x:\n image: alpine\n', sha); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + + try { + const applicationId = liveApp('apply-reserves')!.id; + await svc.apply('apply-reserves', sha, SKIP_PLAN_FINGERPRINT); + + const settled = historyOperationIds(applicationId, 'source_reconcile_settled'); + const applied = historyOperationIds(applicationId, 'applied'); + expect(settled.length).toBeGreaterThanOrEqual(1); + expect(applied).toHaveLength(1); + expect(settled).toContain(applied[0]); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + } + }); + + it('coalesces two concurrent applies for the same commit into one execution', async () => { + const sha = '7'.repeat(40); + const svc = await seedPending('apply-coalesce', 'services:\n x:\n image: alpine\n', sha); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + let releaseSave!: () => void; + const gate = new Promise((resolve) => { releaseSave = resolve; }); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockImplementation(async () => { await gate; }); + + try { + const first = svc.apply('apply-coalesce', sha, SKIP_PLAN_FINGERPRINT); + const second = svc.apply('apply-coalesce', sha, SKIP_PLAN_FINGERPRINT); + releaseSave(); + const [firstResult, secondResult] = await Promise.all([first, second]); + + expect(saveSpy).toHaveBeenCalledTimes(1); + expect(secondResult).toEqual(firstResult); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + } + }); + + it('does not coalesce two concurrent applies that resolve to different deploy behavior', async () => { + const sha = 'ba'.repeat(20); + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha }); + const seedValidateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + try { + await svc.upsert({ + stackName: 'apply-deploy-mismatch', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: true, + autoDeployOnApply: true, + }); + await svc.pull('apply-deploy-mismatch'); + } finally { + seedValidateSpy.mockRestore(); + } + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + let releaseSave!: () => void; + const gate = new Promise((resolve) => { releaseSave = resolve; }); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockImplementation(async () => { await gate; }); + const { ComposeService } = await import('../services/ComposeService'); + const { HealthGateService } = await import('../services/HealthGateService'); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); + const beginSpy = vi.spyOn(HealthGateService.getInstance(), 'beginStack').mockReturnValue('gate-git'); + + try { + // The stack has auto_deploy_on_apply: true. The first call + // leaves deploy unresolved (so it resolves to true); the + // second explicitly asks not to deploy. These must never + // share a coalesce key: if they wrongly joined, the second + // call would resolve successfully with the first's deployed + // result instead of running (or failing) on its own terms. + // Since they do not join, and applying clears the pending + // commit, the second genuinely has nothing left to apply + // once the first (which the per-stack lock serializes first) + // completes -- a real, honest failure, not a borrowed result. + const first = svc.apply('apply-deploy-mismatch', sha, SKIP_PLAN_FINGERPRINT); + const second = svc.apply('apply-deploy-mismatch', sha, { ...SKIP_PLAN_FINGERPRINT, deploy: false }); + releaseSave(); + const firstResult = await first; + expect(firstResult.deployed).toBe(true); + await expect(second).rejects.toMatchObject({ code: 'GIT_ERROR' }); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + deploySpy.mockRestore(); + beginSpy.mockRestore(); + } + }); + + describe('reconcile', () => { + it('reports candidate_already_fetched after a fetch-intent reconcile stages a new candidate', async () => { + const sha = 'e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1'; + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha }); + const svc = GitSourceService.getInstance(); + await svc.upsert({ + stackName: 'reconcile-fetch', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + try { + const applicationId = liveApp('reconcile-fetch')!.id; + const result = await svc.reconcile({ + intent: 'fetch', + applicationId, + stackName: 'reconcile-fetch', + trigger: 'manual', + actor: 'tester', + }); + expect(result.outcome).toBe('candidate_already_fetched'); + } finally { + validateSpy.mockRestore(); + } + }); + + it('reports no_source_change after an apply-intent reconcile accepts the candidate', async () => { + const sha = 'e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2e2'; + const svc = await seedPending('reconcile-apply', 'services:\n x:\n image: alpine\n', sha); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + + try { + const applicationId = liveApp('reconcile-apply')!.id; + const result = await svc.reconcile({ + intent: 'apply', + applicationId, + stackName: 'reconcile-apply', + trigger: 'manual', + actor: 'tester', + commitSha: sha, + planFingerprint: '', + deploy: false, + }); + expect(result.outcome).toBe('no_source_change'); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + } + }); + + it('does not report success when the source applied but the deploy failed', async () => { + const sha = 'e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7'; + const svc = await seedPending('reconcile-apply-deploy-fail', 'services:\n x:\n image: alpine\n', sha); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const { ComposeService } = await import('../services/ComposeService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockRejectedValue( + new Error('compose up failed: docker unavailable'), + ); + + try { + const applicationId = liveApp('reconcile-apply-deploy-fail')!.id; + const result = await svc.reconcile({ + intent: 'apply', + applicationId, + stackName: 'reconcile-apply-deploy-fail', + trigger: 'manual', + actor: 'tester', + commitSha: sha, + planFingerprint: '', + deploy: true, + }); + // The promotion itself succeeded (files landed, generation + // accepted), so the source facet alone reads as converged. + // reconcile must not let that mask the deploy failure, and must + // not claim the previous generation is unchanged either: it isn't. + expect(result.outcome).toBe('recovery_required'); + expect(result.nextAction).toBe('view_target_results'); + expect(result.reason).toMatch(/deploy/i); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('reports a truthful failure, not the stale staged-candidate outcome, when fetch throws before touching the application row', async () => { + const sha = 'e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3'; + const svc = await seedPending('reconcile-fetch-fail', 'services:\n x:\n image: alpine\n', sha); + const applicationId = liveApp('reconcile-fetch-fail')!.id; + // Deleting the config row makes pullLocked throw its `!src` guard, + // which fires before fetchStarted opens any transition: the + // application row is untouched by this failure. + DatabaseService.getInstance().deleteGitSource('reconcile-fetch-fail'); + + const result = await svc.reconcile({ + intent: 'fetch', + applicationId, + stackName: 'reconcile-fetch-fail', + trigger: 'manual', + actor: 'tester', + }); + + expect(result.outcome).not.toBe('candidate_already_fetched'); + expect(result.nextAction).not.toBe('none'); + }); + + it('reports a truthful failure, not the stale staged-candidate outcome, when apply throws on a stale commitSha', async () => { + const sha = 'e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4e4'; + const svc = await seedPending('reconcile-apply-stale-sha', 'services:\n x:\n image: alpine\n', sha); + const applicationId = liveApp('reconcile-apply-stale-sha')!.id; + + const result = await svc.reconcile({ + intent: 'apply', + applicationId, + stackName: 'reconcile-apply-stale-sha', + trigger: 'manual', + actor: 'tester', + commitSha: 'ffffffffffffffffffffffffffffffffffffffff', + planFingerprint: '', + deploy: false, + }); + + expect(result.outcome).not.toBe('candidate_already_fetched'); + expect(result.nextAction).not.toBe('none'); + }); + + it('fails closed instead of silently reconciling the wrong application when the requested applicationId is stale', async () => { + const sha = 'e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5'; + const svc = await seedPending('reconcile-stale-app-id', 'services:\n x:\n image: alpine\n', sha); + + const result = await svc.reconcile({ + intent: 'fetch', + applicationId: 'no-longer-the-live-application', + stackName: 'reconcile-stale-app-id', + trigger: 'manual', + actor: 'tester', + }); + + expect(result.outcome).toBe('unknown'); + expect(result.nextAction).toBe('none'); + // Fails closed before doing anything: the candidate this stack had + // staged before the call is still exactly as it was. + const stillStaged = liveApp('reconcile-stale-app-id'); + expect(stillStaged?.candidate_generation_id).toBeTruthy(); + }); + + it('fails closed on a stale applicationId even when the live application is stuck in creating, not active', async () => { + const sha = 'e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6e6'; + const svc = await seedPending('reconcile-creating-app', 'services:\n x:\n image: alpine\n', sha); + const applicationId = liveApp('reconcile-creating-app')!.id; + // gitopsApplicationFor() (used elsewhere to gate transitions) only + // recognizes 'active' rows, but getLiveDirectApplication() (used + // by deriveReconcileResult) recognizes 'active' and 'creating' + // alike. reconcile's identity guard must use the same broad + // definition, or a 'creating' row slips past it entirely. + DatabaseService.getInstance().getDb() + .prepare("UPDATE gitops_applications SET lifecycle_status = 'creating' WHERE id = ?") + .run(applicationId); + + const result = await svc.reconcile({ + intent: 'apply', + applicationId: 'deliberately-mismatched-id', + stackName: 'reconcile-creating-app', + trigger: 'manual', + actor: 'tester', + commitSha: 'ffffffffffffffffffffffffffffffffffffffff', + planFingerprint: '', + deploy: false, + }); + + expect(result.outcome).toBe('unknown'); + expect(result.nextAction).toBe('none'); + // Fails closed before doing anything: the candidate this stack + // had staged before the call is still exactly as it was. + const stillStaged = liveApp('reconcile-creating-app'); + expect(stillStaged?.candidate_generation_id).toBeTruthy(); + }); + + it('fails closed instead of silently applying under the current live application when the requested applicationId still exists but was superseded', async () => { + const sha = 'e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7e7'; + const svc = await seedPending('reconcile-apply-superseded-id', 'services:\n x:\n image: alpine\n', sha); + const staleApplicationId = liveApp('reconcile-apply-superseded-id')!.id; + // A real row that used to be live for this stack, not a + // fabricated id: this is the exact gap the existing stale-id + // tests (using ids that never existed as any row) do not cover, + // since GitOpsStore.getApplication finds this row just fine. + GitOpsTransitions.getInstance().applicationTombstoned(staleApplicationId, 'detached', { + operationId: 'op-supersede-1', actor: 'tester', trigger: 'test', at: Date.now(), + }); + const config: DirectSourceConfig = { + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + }; + GitOpsStore.getInstance().insertApplication(buildDirectApplicationRow({ + id: newGitOpsId(), + stackName: 'reconcile-apply-superseded-id', + config, + identity: directSourceIdentity(config), + lifecycleStatus: 'active', + at: Date.now(), + })); + const newLiveId = liveApp('reconcile-apply-superseded-id')!.id; + expect(newLiveId).not.toBe(staleApplicationId); + + const applySpy = vi.spyOn( + svc as unknown as { applyWithSharedLock: (...args: unknown[]) => Promise }, + 'applyWithSharedLock', + ); + + try { + const result = await svc.reconcile({ + intent: 'apply', + applicationId: staleApplicationId, + stackName: 'reconcile-apply-superseded-id', + trigger: 'manual', + actor: 'tester', + commitSha: 'ffffffffffffffffffffffffffffffffffffffff', + planFingerprint: '', + deploy: false, + }); + + expect(result.outcome).toBe('unknown'); + expect(result.nextAction).toBe('none'); + expect(applySpy).not.toHaveBeenCalled(); + } finally { + applySpy.mockRestore(); + } + }); + + it('revalidates the application identity after acquiring the fetch lock', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ sha: 'fa'.repeat(20) }); + await configureGitSource('reconcile-fetch-replaced-while-queued'); + const staleApplicationId = liveApp('reconcile-fetch-replaced-while-queued')!.id; + mockGitClone.mockClear(); + + let releaseLock!: () => void; + const lockGate = new Promise((resolve) => { releaseLock = resolve; }); + const lockHolder = (svc as unknown as { + withStackLock: (stackName: string, fn: () => Promise) => Promise; + }).withStackLock('reconcile-fetch-replaced-while-queued', () => lockGate); + + const reconcile = svc.reconcile({ + intent: 'fetch', + applicationId: staleApplicationId, + stackName: 'reconcile-fetch-replaced-while-queued', + trigger: 'poll', + actor: 'system:source-controller', + }); + await vi.waitFor(() => { + expect(historyOperationIds(staleApplicationId, 'source_reconcile_started')).toHaveLength(1); + }); + + GitOpsTransitions.getInstance().applicationTombstoned(staleApplicationId, 'detached', { + operationId: 'op-replace-queued-fetch', actor: 'tester', trigger: 'test', at: Date.now(), + }); + const config: DirectSourceConfig = { + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + }; + GitOpsStore.getInstance().insertApplication(buildDirectApplicationRow({ + id: newGitOpsId(), + stackName: 'reconcile-fetch-replaced-while-queued', + config, + identity: directSourceIdentity(config), + lifecycleStatus: 'active', + at: Date.now(), + })); + + releaseLock(); + await lockHolder; + const result = await reconcile; + + expect(result.outcome).toBe('unknown'); + expect(result.nextAction).toBe('none'); + expect(mockGitClone).not.toHaveBeenCalled(); + expect(settledAttempts(staleApplicationId)).toHaveLength(1); + expect(historyOperationIds(liveApp('reconcile-fetch-replaced-while-queued')!.id, 'fetch_started')).toHaveLength(0); + }); + + it('captures the fetch result before a queued source mutation can change row state', async () => { + const stackName = 'reconcile-settlement-before-queued-suspend'; + const sha = 'f1'.repeat(20); + mockSuccessfulClone({ sha }); + const svc = GitSourceService.getInstance(); + await configureGitSource(stackName); + const applicationId = liveApp(stackName)!.id; + const releaseClone = gatedClone(); + mockGitLog.mockResolvedValue([{ oid: sha }]); + + const fetch = svc.reconcile({ + intent: 'fetch', + applicationId, + stackName, + trigger: 'poll', + actor: 'system:source-controller', + }); + await vi.waitFor(() => expect(mockGitClone).toHaveBeenCalledTimes(1)); + const suspend = svc.suspend(stackName, { actor: 'tester', reason: 'queue behind fetch' }); + releaseClone(); + + const [fetchResult, suspendResult] = await Promise.all([fetch, suspend]); + + expect(fetchResult.outcome).toBe('candidate_already_fetched'); + expect(suspendResult.outcome).toBe('suspended'); + const settled = settledAttempts(applicationId); + expect(settled).toHaveLength(1); + expect(JSON.parse(settled[0].after_json).outcome).toBe('candidate_already_fetched'); + }); + + it('reports unknown for a stack with no GitOps application', async () => { + const svc = GitSourceService.getInstance(); + const result = await svc.reconcile({ + intent: 'fetch', + applicationId: 'unused', + stackName: 'reconcile-no-app', + trigger: 'manual', + actor: 'tester', + }); + expect(result.outcome).toBe('unknown'); + }); + + function settledAttempts(applicationId: string): { operation_id: string; after_json: string }[] { + return DatabaseService.getInstance().getDb() + .prepare("SELECT operation_id, after_json FROM gitops_history WHERE application_id = ? AND stage = 'source_reconcile_settled'") + .all(applicationId) as { operation_id: string; after_json: string }[]; + } + + function unsettledAttempts(applicationId: string): { operation_id: string }[] { + return DatabaseService.getInstance().getDb() + .prepare("SELECT operation_id FROM gitops_history started WHERE started.application_id = ? AND started.stage = 'source_reconcile_started' AND NOT EXISTS (SELECT 1 FROM gitops_history settled WHERE settled.application_id = started.application_id AND settled.operation_id = started.operation_id AND settled.stage = 'source_reconcile_settled')") + .all(applicationId) as { operation_id: string }[]; + } + + /** + * Hold the clone open so a second reconcile submission is guaranteed + * to arrive while the first is still executing. Returns the release + * function; calling it lets the clone finish and write its compose + * file. + */ + function gatedClone(): () => void { + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + mockGitClone.mockClear(); + mockGitClone.mockImplementation(async (args: { dir: string }) => { + await gate; + const { promises: fsp } = await import('fs'); + const path = await import('path'); + const composeAbs = path.join(args.dir, 'compose.yaml'); + await fsp.mkdir(path.dirname(composeAbs), { recursive: true }); + await fsp.writeFile(composeAbs, 'services:\n x:\n image: alpine\n', 'utf-8'); + }); + return release; + } + + it('durably settles a reconcile attempt for a successful fetch, leaving nothing unsettled', async () => { + const sha = 'e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8e8'; + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha }); + const svc = GitSourceService.getInstance(); + await svc.upsert({ + stackName: 'reconcile-durable', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + try { + const applicationId = liveApp('reconcile-durable')!.id; + const result = await svc.reconcile({ + intent: 'fetch', + applicationId, + stackName: 'reconcile-durable', + trigger: 'manual', + actor: 'tester', + }); + + const settled = settledAttempts(applicationId); + expect(settled).toHaveLength(1); + expect(JSON.parse(settled[0].after_json)).toMatchObject({ outcome: result.outcome, reason: result.reason }); + expect(unsettledAttempts(applicationId)).toHaveLength(0); + } finally { + validateSpy.mockRestore(); + } + }); + + it('reserves and settles an attempt for every normalized trigger kind', async () => { + const stackName = 'reconcile-trigger-matrix'; + const sha = 'e9'.repeat(20); + const triggers: ReconcileTrigger[] = [ + 'manual', + 'api', + 'webhook', + 'poll', + 'retry', + 'config_change', + 'startup', + 'resume', + 'provider_event', + 'schedule', + 'binding_change', + ]; + mockSuccessfulClone({ sha }); + const svc = GitSourceService.getInstance(); + await configureGitSource(stackName); + const applicationId = liveApp(stackName)!.id; + + for (const trigger of triggers) { + await svc.reconcile({ + intent: 'fetch', + applicationId, + stackName, + trigger, + actor: `system:${trigger}`, + ...(trigger === 'webhook' ? { deliveryId: 'trigger-matrix-webhook' } : {}), + }); + } + + const started = DatabaseService.getInstance().getDb() + .prepare("SELECT operation_id, trigger FROM gitops_history WHERE application_id = ? AND stage = 'source_reconcile_started' ORDER BY rowid") + .all(applicationId) as { operation_id: string; trigger: string }[]; + expect(started.map((row) => row.trigger)).toEqual(triggers); + expect(new Set(started.map((row) => row.operation_id)).size).toBe(triggers.length); + expect(started.find((row) => row.trigger === 'webhook')?.operation_id) + .toBe(deliveryKey('webhook', 'fetch', 'trigger-matrix-webhook')); + expect(unsettledAttempts(applicationId)).toHaveLength(0); + }); + + it('threads the reserved attempt\'s operation id into the generation the fetch produces', async () => { + const sha = 'ed'.repeat(20); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha }); + const svc = GitSourceService.getInstance(); + await svc.upsert({ + stackName: 'reconcile-operation-id-threading', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + try { + const applicationId = liveApp('reconcile-operation-id-threading')!.id; + await svc.reconcile({ + intent: 'fetch', + applicationId, + stackName: 'reconcile-operation-id-threading', + trigger: 'manual', + actor: 'tester', + }); + + const settled = settledAttempts(applicationId); + expect(settled).toHaveLength(1); + const candidateGenerationId = liveApp('reconcile-operation-id-threading')!.candidate_generation_id; + expect(candidateGenerationId).toBeTruthy(); + const generation = GitOpsStore.getInstance().getGeneration(candidateGenerationId!); + expect(generation?.operation_id).toBe(settled[0].operation_id); + } finally { + validateSpy.mockRestore(); + } + }); + + it('threads the reserved attempt\'s operation id into the apply-side transition an apply-intent reconcile produces', async () => { + const sha = 'ee'.repeat(20); + const svc = await seedPending('reconcile-apply-operation-id-threading', 'services:\n x:\n image: alpine\n', sha); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + + try { + const applicationId = liveApp('reconcile-apply-operation-id-threading')!.id; + await svc.reconcile({ + intent: 'apply', + applicationId, + stackName: 'reconcile-apply-operation-id-threading', + trigger: 'manual', + actor: 'tester', + commitSha: sha, + planFingerprint: '', + deploy: false, + }); + + // seedPending's own pull() reserves and settles its own fetch + // attempt now that pull() is wired through reservation too, so + // more than one settled row is expected here; only the + // apply-intent one needs to match the applied-stage transition. + const settled = settledAttempts(applicationId).map((r) => r.operation_id); + const applied = historyOperationIds(applicationId, 'applied'); + expect(applied).toHaveLength(1); + expect(settled).toContain(applied[0]); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + } + }); + + it('coalesces two concurrent fetch-intent reconciles into one execution, each settling its own durable attempt', async () => { + const newSha = 'e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9e9'; + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0' }); + const svc = GitSourceService.getInstance(); + await svc.upsert({ + stackName: 'reconcile-coalesce', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + const releaseClone = gatedClone(); + mockGitLog.mockResolvedValue([{ oid: newSha }]); + + try { + const applicationId = liveApp('reconcile-coalesce')!.id; + const first = svc.reconcile({ + intent: 'fetch', applicationId, stackName: 'reconcile-coalesce', trigger: 'manual', actor: 'tester-a', + }); + const second = svc.reconcile({ + intent: 'fetch', applicationId, stackName: 'reconcile-coalesce', trigger: 'manual', actor: 'tester-b', + }); + releaseClone(); + const [firstResult, secondResult] = await Promise.all([first, second]); + + expect(mockGitClone).toHaveBeenCalledTimes(1); + expect(firstResult).toEqual(secondResult); + + const settled = settledAttempts(applicationId); + expect(settled).toHaveLength(2); + expect(new Set(settled.map((r) => r.operation_id)).size).toBe(2); + expect(unsettledAttempts(applicationId)).toHaveLength(0); + } finally { + validateSpy.mockRestore(); + } + }); + + it('coalesces a concurrent manual pull and a controller-triggered reconcile into one clone, each with its own complete settled history and a follower-to-leader link', async () => { + const newSha = 'f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1'; + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0' }); + const svc = GitSourceService.getInstance(); + await configureGitSource('pull-reconcile-coalesce'); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + const releaseClone = gatedClone(); + mockGitLog.mockResolvedValue([{ oid: newSha }]); + + try { + const applicationId = liveApp('pull-reconcile-coalesce')!.id; + const generationCountBefore = (DatabaseService.getInstance().getDb() + .prepare('SELECT COUNT(*) AS count FROM gitops_generations WHERE application_id = ?') + .get(applicationId) as { count: number }).count; + const candidateReadyCountBefore = historyOperationIds(applicationId, 'candidate_ready').length; + // Two different producers, one a manual pull() and the other + // a controller poll driving reconcile(), submitting for the + // exact same live application: coalesceKey() does not vary + // by trigger or producer, so this must join into one clone + // rather than each running its own. + const manualPull = svc.pull('pull-reconcile-coalesce'); + const controllerReconcile = svc.reconcile({ + intent: 'fetch', + applicationId, + stackName: 'pull-reconcile-coalesce', + trigger: 'poll', + actor: 'system:source-controller', + }); + releaseClone(); + const [pullResult, reconcileResult] = await Promise.all([manualPull, controllerReconcile]); + + expect(mockGitClone).toHaveBeenCalledTimes(1); + expect(pullResult.commitSha).toBe(newSha); + expect(pullResult.candidateReady).toBe(true); + expect(reconcileResult.outcome).toBe('candidate_already_fetched'); + const generationCountAfter = (DatabaseService.getInstance().getDb() + .prepare('SELECT COUNT(*) AS count FROM gitops_generations WHERE application_id = ?') + .get(applicationId) as { count: number }).count; + expect(generationCountAfter).toBe(generationCountBefore + 1); + expect(historyOperationIds(applicationId, 'candidate_ready')).toHaveLength(candidateReadyCountBefore + 1); + + // Two complete histories: each caller reserved and settled + // its own durable attempt, neither left dangling. + const settled = settledAttempts(applicationId); + expect(settled).toHaveLength(2); + const settledOperationIds = settled.map((r) => r.operation_id); + expect(new Set(settledOperationIds).size).toBe(2); + expect(unsettledAttempts(applicationId)).toHaveLength(0); + + // A follower-to-leader link: exactly one of the two + // reservations recorded that it was made on behalf of the + // other. + const started = DatabaseService.getInstance().getDb() + .prepare("SELECT operation_id, after_json FROM gitops_history WHERE application_id = ? AND stage = 'source_reconcile_started'") + .all(applicationId) as { operation_id: string; after_json: string }[]; + const followerLinks = started + .map((r) => (JSON.parse(r.after_json) as { followerOf?: string }).followerOf) + .filter((followerOf): followerOf is string => followerOf !== undefined); + expect(followerLinks).toHaveLength(1); + expect(settledOperationIds).toContain(followerLinks[0]); + } finally { + validateSpy.mockRestore(); + } + }); + + it('leaves every coalesced attempt unsettled when the leader result is not durable, so recovery gives them one result', async () => { + const stackName = 'reconcile-shared-settlement-failure'; + const sha = 'c7'.repeat(20); + mockSuccessfulClone({ sha }); + const svc = GitSourceService.getInstance(); + await configureGitSource(stackName); + const applicationId = liveApp(stackName)!.id; + const priorStarted = new Set(historyOperationIds(applicationId, 'source_reconcile_started')); + const releaseClone = gatedClone(); + mockGitLog.mockResolvedValue([{ oid: sha }]); + const originalSettle = GitOpsTransitions.prototype.settleReconcileAttempt; + const settleSpy = vi.spyOn(GitOpsTransitions.prototype, 'settleReconcileAttempt') + .mockImplementationOnce(() => { throw new Error('simulated leader settlement failure'); }) + .mockImplementation(function (this: GitOpsTransitions, ...args: Parameters) { + return originalSettle.apply(this, args); + }); + + try { + const manual = svc.pull(stackName); + const controller = svc.reconcile({ + intent: 'fetch', + applicationId, + stackName, + trigger: 'poll', + actor: 'system:source-controller', + }); + releaseClone(); + await Promise.all([manual, controller]); + + const operationIds = historyOperationIds(applicationId, 'source_reconcile_started') + .filter((operationId) => !priorStarted.has(operationId)); + expect(operationIds).toHaveLength(2); + expect(unsettledAttempts(applicationId).map((row) => row.operation_id).sort()) + .toEqual([...operationIds].sort()); + + await svc.suspend(stackName, { actor: 'tester', reason: 'settle through recovery' }); + await svc.recoverUnsettledReconcileAttempts(); + + const recovered = settledAttempts(applicationId) + .filter((row) => operationIds.includes(row.operation_id)); + expect(recovered).toHaveLength(2); + expect(new Set(recovered.map((row) => row.after_json)).size).toBe(1); + expect(JSON.parse(recovered[0].after_json).outcome).toBe('suspended'); + } finally { + settleSpy.mockRestore(); + } + }); + + it.each([ + ['manual pull', true], + ['controller reconcile', false], + ] as const)('settles both callers from one failed shared fetch when %s leads', async (_leader, manualLeads) => { + const stackName = manualLeads + ? 'pull-reconcile-failure-manual-leads' + : 'pull-reconcile-failure-controller-leads'; + mockSuccessfulClone({ sha: 'f2'.repeat(20) }); + const svc = GitSourceService.getInstance(); + await configureGitSource(stackName); + const applicationId = liveApp(stackName)!.id; + + let releaseClone!: () => void; + const cloneGate = new Promise((resolve) => { releaseClone = resolve; }); + mockGitClone.mockClear(); + mockGitClone.mockImplementation(async () => { + await cloneGate; + throw new Error('shared fetch failed'); + }); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const request = { + intent: 'fetch' as const, + applicationId, + stackName, + trigger: 'poll' as const, + actor: 'system:source-controller', + }; + + try { + const first = manualLeads ? svc.pull(stackName) : svc.reconcile(request); + const second = manualLeads ? svc.reconcile(request) : svc.pull(stackName); + releaseClone(); + const [firstResult, secondResult] = await Promise.allSettled([first, second]); + + const reconcileResult = manualLeads ? secondResult : firstResult; + expect(reconcileResult.status).toBe('fulfilled'); + if (reconcileResult.status !== 'fulfilled') throw reconcileResult.reason; + const settled = settledAttempts(applicationId).map((row) => JSON.parse(row.after_json)); + expect(settled).toHaveLength(2); + expect(settled).toEqual([reconcileResult.value, reconcileResult.value]); + expect(mockGitClone).toHaveBeenCalledTimes(1); + expect(unsettledAttempts(applicationId)).toHaveLength(0); + } finally { + errorSpy.mockRestore(); + } + }); + + it('settles a fetch-intent reconcile durably with the same classified result it returns, even for a pre-transition failure the row does not yet reflect', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ sha: '3'.repeat(40) }); + await configureGitSource('reconcile-pretransition-failure'); + const applicationId = liveApp('reconcile-pretransition-failure')!.id; + // Deletes the config row pullLocked itself checks for, before + // any transition table write, so the row state alone (a + // generic row-derivation, with no notion of this failure) would + // misreport the outcome as an unremarkable "never reconciled" + // rather than the real, classified failure the caller receives. + DatabaseService.getInstance().deleteGitSource('reconcile-pretransition-failure'); + + const result = await svc.reconcile({ + intent: 'fetch', + applicationId, + stackName: 'reconcile-pretransition-failure', + trigger: 'poll', + actor: 'system:source-controller', + }); + + expect(result.outcome).not.toBe('unknown'); + const settled = settledAttempts(applicationId); + expect(settled).toHaveLength(1); + expect(JSON.parse(settled[0].after_json)).toEqual(result); + }); + + it('does not re-execute a redelivered request whose original attempt was reserved but never settled', async () => { + const sha = 'dededededededededededededededededededede'; + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha }); + const svc = GitSourceService.getInstance(); + await configureGitSource('reconcile-orphaned-redelivery'); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + try { + const applicationId = liveApp('reconcile-orphaned-redelivery')!.id; + const request = { + intent: 'fetch' as const, + applicationId, + stackName: 'reconcile-orphaned-redelivery', + trigger: 'webhook' as const, + actor: 'tester', + deliveryId: 'delivery-orphaned', + }; + await svc.reconcile(request); + // Simulate a crash between reservation and settlement: the + // original attempt's reservation survives, but its + // settlement row never got written, and no in-process + // leader remains for it in this fresh call. + DatabaseService.getInstance().getDb() + .prepare("DELETE FROM gitops_history WHERE application_id = ? AND stage = 'source_reconcile_settled'") + .run(applicationId); + mockGitClone.mockClear(); + + const redeliveryResult = await svc.reconcile(request); + + expect(mockGitClone).not.toHaveBeenCalled(); + expect(redeliveryResult.outcome).not.toBe('unknown'); + } finally { + validateSpy.mockRestore(); + } + }); + + it('reports an unknown redelivery result when durable settlement fails', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ sha: 'd1'.repeat(20) }); + await configureGitSource('reconcile-redelivery-settlement-failure'); + const applicationId = liveApp('reconcile-redelivery-settlement-failure')!.id; + const request: ReconcileRequest & { intent: 'fetch'; deliveryId: string } = { + intent: 'fetch', + applicationId, + stackName: 'reconcile-redelivery-settlement-failure', + trigger: 'webhook', + actor: 'tester', + deliveryId: 'delivery-settlement-failure', + }; + GitOpsTransitions.getInstance().reserveReconcileAttempt(applicationId, { + operationId: deliveryKey('webhook', 'fetch', request.deliveryId), + actor: request.actor, + trigger: request.trigger, + at: Date.now(), + }); + mockGitClone.mockClear(); + const settleSpy = vi.spyOn(GitOpsTransitions.prototype, 'settleReconcileAttempt') + .mockImplementationOnce(() => { throw new Error('simulated redelivery settlement failure'); }); + + try { + const result = await svc.reconcile(request); + + expect(mockGitClone).not.toHaveBeenCalled(); + expect(result).toEqual({ + outcome: 'unknown', + reason: 'This attempt could not be durably resolved.', + nextAction: 'none', + }); + expect(unsettledAttempts(applicationId)).toHaveLength(1); + } finally { + settleSpy.mockRestore(); + } + }); + + it('resolves a redelivery from its own settled history rather than an unrelated leader that happens to be running under the shared coalesce key', async () => { + const sha = 'cececececececececececececececececececece'; + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha }); + const svc = GitSourceService.getInstance(); + await configureGitSource('reconcile-redelivery-vs-unrelated-leader'); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + try { + const applicationId = liveApp('reconcile-redelivery-vs-unrelated-leader')!.id; + const request = { + intent: 'fetch' as const, + applicationId, + stackName: 'reconcile-redelivery-vs-unrelated-leader', + trigger: 'webhook' as const, + actor: 'tester', + deliveryId: 'delivery-vs-unrelated-leader', + }; + const first = await svc.reconcile(request); + + // A completely unrelated fetch (a plain manual pull, no + // deliveryId) becomes the in-process leader registered + // under this application's shared fetch coalesce key, + // which does not vary by deliveryId or trigger. + const releaseClone = gatedClone(); + mockGitLog.mockResolvedValue([{ oid: 'dfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdfdf' }]); + const unrelatedPull = svc.pull('reconcile-redelivery-vs-unrelated-leader'); + + // A redelivery of the original event arrives while that + // unrelated pull is still running: it must resolve from its + // own settled history, not from the unrelated in-flight + // leader it happens to find under the shared key. + const redelivery = await svc.reconcile(request); + releaseClone(); + await unrelatedPull; + + expect(redelivery).toEqual(first); + } finally { + validateSpy.mockRestore(); + } + }); + + it('does not re-execute a redelivered request carrying the same deliveryId, and returns the original settled result', async () => { + const sha = 'eaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaeaea'; + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha }); + const svc = GitSourceService.getInstance(); + await svc.upsert({ + stackName: 'reconcile-dedupe', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + try { + const applicationId = liveApp('reconcile-dedupe')!.id; + const request = { + intent: 'fetch' as const, + applicationId, + stackName: 'reconcile-dedupe', + trigger: 'webhook' as const, + actor: 'tester', + deliveryId: 'delivery-1', + }; + const first = await svc.reconcile(request); + mockGitClone.mockClear(); + const second = await svc.reconcile(request); + + expect(mockGitClone).not.toHaveBeenCalled(); + expect(second).toEqual(first); + expect(settledAttempts(applicationId)).toHaveLength(1); + } finally { + validateSpy.mockRestore(); + } + }); + + it('joins a concurrent redelivery of the same deliveryId to the in-flight leader, returning the leader\'s real result rather than a stale snapshot', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'e0'.repeat(20) }); + await svc.upsert({ + stackName: 'reconcile-concurrent-redelivery', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + const releaseClone = gatedClone(); + mockGitLog.mockResolvedValue([{ oid: 'e1'.repeat(20) }]); + + try { + const applicationId = liveApp('reconcile-concurrent-redelivery')!.id; + const request = { + intent: 'fetch' as const, + applicationId, + stackName: 'reconcile-concurrent-redelivery', + trigger: 'webhook' as const, + actor: 'tester', + deliveryId: 'delivery-race', + }; + const first = svc.reconcile(request); + const redelivery = svc.reconcile(request); + releaseClone(); + const [firstResult, redeliveryResult] = await Promise.all([first, redelivery]); + + expect(mockGitClone).toHaveBeenCalledTimes(1); + // The redelivery must report the leader's real post-fetch + // outcome, not a snapshot of the row from before the fetch + // ran (which would still show no candidate staged). + expect(redeliveryResult).toEqual(firstResult); + expect(firstResult.outcome).toBe('candidate_already_fetched'); + } finally { + validateSpy.mockRestore(); + } + }); + + it('logs and reports unknown rather than throwing when a settled attempt\'s stored result is corrupted', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'ef'.repeat(20) }); + await svc.upsert({ + stackName: 'reconcile-corrupt-settled', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + + try { + const applicationId = liveApp('reconcile-corrupt-settled')!.id; + const request = { + intent: 'fetch' as const, + applicationId, + stackName: 'reconcile-corrupt-settled', + trigger: 'webhook' as const, + actor: 'tester', + deliveryId: 'delivery-corrupt', + }; + await svc.reconcile(request); + DatabaseService.getInstance().getDb() + .prepare("UPDATE gitops_history SET after_json = ? WHERE application_id = ? AND stage = 'source_reconcile_settled'") + .run('not valid json{{{', applicationId); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + + const result = await svc.reconcile(request); + + expect(result.outcome).toBe('unknown'); + expect(errorSpy).toHaveBeenCalled(); + errorSpy.mockRestore(); + } finally { + validateSpy.mockRestore(); + } + }); + + it('joins a same-delivery apply under a different coalesce key to the real in-flight leader rather than settling a stale snapshot', async () => { + const svc = GitSourceService.getInstance(); + const sha = 'a6'.repeat(20); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha }); + await svc.upsert({ + stackName: 'reconcile-apply-delivery-race', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + await svc.pull('reconcile-apply-delivery-race'); + const applicationId = liveApp('reconcile-apply-delivery-race')!.id; + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + let releaseSave!: () => void; + const gate = new Promise((resolve) => { releaseSave = resolve; }); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockImplementation(async () => { await gate; }); + + try { + // Two apply requests sharing a delivery id (so their + // operation ids collide) but differing in commitSha, which + // coalesceKey includes for an apply intent -- so they run + // under different coalesce keys despite the shared + // operation id. + const first = svc.reconcile({ + intent: 'apply', + applicationId, + stackName: 'reconcile-apply-delivery-race', + trigger: 'webhook', + actor: 'tester', + commitSha: sha, + planFingerprint: '', + deploy: false, + deliveryId: 'shared-delivery', + }); + const second = svc.reconcile({ + intent: 'apply', + applicationId, + stackName: 'reconcile-apply-delivery-race', + trigger: 'webhook', + actor: 'tester', + commitSha: 'ff'.repeat(20), + planFingerprint: 'different-fingerprint', + deploy: true, + deliveryId: 'shared-delivery', + }); + releaseSave(); + const [firstResult, secondResult] = await Promise.all([first, second]); + + // The second request must never have run its own apply + // (a different, unstaged commitSha would fail on its own + // terms); it must instead have joined the first's real + // execution and returned its actual result. + expect(secondResult).toEqual(firstResult); + expect(saveSpy).toHaveBeenCalledTimes(1); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + } + }); + + it('coalesces a concurrent manual apply and webhook apply into one promotion and one normalized result', async () => { + const sha = 'a7'.repeat(20); + const svc = await seedPending('apply-reconcile-coalesce', 'services:\n x:\n image: alpine\n', sha); + const applicationId = liveApp('apply-reconcile-coalesce')!.id; + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + let releaseSave!: () => void; + const saveGate = new Promise((resolve) => { releaseSave = resolve; }); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockImplementation(async () => { await saveGate; }); + const priorSettlements = new Set(settledAttempts(applicationId).map((row) => row.operation_id)); + + try { + const manualApply = svc.apply('apply-reconcile-coalesce', sha, SKIP_PLAN_FINGERPRINT); + const controllerApply = svc.reconcile({ + intent: 'apply', + applicationId, + stackName: 'apply-reconcile-coalesce', + trigger: 'webhook', + actor: 'system:webhook', + commitSha: sha, + planFingerprint: '', + deploy: false, + deliveryId: 'apply-cross-producer', + }); + releaseSave(); + const [manualResult, reconcileResult] = await Promise.all([manualApply, controllerApply]); + + expect(manualResult.applied).toBe(true); + expect(saveSpy).toHaveBeenCalledTimes(1); + const settled = settledAttempts(applicationId) + .filter((row) => !priorSettlements.has(row.operation_id)) + .map((row) => JSON.parse(row.after_json)); + expect(settled).toHaveLength(2); + expect(settled).toEqual([reconcileResult, reconcileResult]); + expect(unsettledAttempts(applicationId)).toHaveLength(0); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + } + }); + + it('does not let a fingerprint-enforcing manual apply borrow an internal apply that bypasses the fingerprint check', async () => { + const sha = 'b7'.repeat(20); + const svc = await seedPending('apply-fingerprint-mode-isolation', 'services:\n x:\n image: alpine\n', sha); + const applicationId = liveApp('apply-fingerprint-mode-isolation')!.id; + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + let releaseSave!: () => void; + const saveGate = new Promise((resolve) => { releaseSave = resolve; }); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockImplementation(async () => { await saveGate; }); + + try { + const internalApply = svc.reconcile({ + intent: 'apply', + applicationId, + stackName: 'apply-fingerprint-mode-isolation', + trigger: 'webhook', + actor: 'system:webhook', + commitSha: sha, + planFingerprint: 'stale-fingerprint', + deploy: false, + deliveryId: 'apply-fingerprint-mode-isolation', + }); + await vi.waitFor(() => expect(saveSpy).toHaveBeenCalledTimes(1)); + const manualApply = svc.apply('apply-fingerprint-mode-isolation', sha, { + planFingerprint: 'stale-fingerprint', + deploy: false, + }); + releaseSave(); + + await expect(internalApply).resolves.toMatchObject({ outcome: expect.any(String) }); + await expect(manualApply).rejects.toMatchObject({ code: expect.any(String) }); + expect(saveSpy).toHaveBeenCalledTimes(1); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + } + }); + + it('resolves an apply redelivery from its own history while an unrelated matching apply is running', async () => { + const sha = 'a8'.repeat(20); + const svc = await seedPending('apply-redelivery-vs-unrelated-leader', 'services:\n x:\n image: alpine\n', sha); + const applicationId = liveApp('apply-redelivery-vs-unrelated-leader')!.id; + const originalRequest: ReconcileRequest & { intent: 'apply' } = { + intent: 'apply' as const, + applicationId, + stackName: 'apply-redelivery-vs-unrelated-leader', + trigger: 'webhook' as const, + actor: 'tester', + commitSha: sha, + planFingerprint: '', + deploy: true, + deliveryId: 'original-delivery', + }; + const originalOperationId = deliveryKey('webhook', 'apply', 'original-delivery'); + const originalResult = { + outcome: 'recovery_required' as const, + reason: 'The source applied, but the deploy failed: first delivery deploy failed', + nextAction: 'view_target_results' as const, + }; + const tx = GitOpsTransitions.getInstance(); + const envelope = { operationId: originalOperationId, actor: 'tester', trigger: 'webhook', at: Date.now() }; + tx.reserveReconcileAttempt(applicationId, envelope); + tx.settleReconcileAttempt(applicationId, envelope, originalResult); + + type ApplyExecution = { + status: 'fulfilled'; + value: { applied: boolean; deployed: boolean }; + result: { outcome: 'no_source_change'; reason: string; nextAction: 'none' }; + }; + type ApplyCompletion = { execution: ApplyExecution; settled: boolean }; + let releaseLeader!: (completion: ApplyCompletion) => void; + const leaderPromise = new Promise((resolve) => { releaseLeader = resolve; }); + const inFlightApplies = (svc as unknown as { + inFlightApplies: Map }>; + }).inFlightApplies; + const executionKey = `${coalesceKey(originalRequest)}:fingerprint-optional`; + inFlightApplies.set(executionKey, { + operationId: 'unrelated-operation', + promise: leaderPromise, + }); + + try { + const redeliveryResult = await svc.reconcile(originalRequest); + expect(redeliveryResult).toEqual(originalResult); + } finally { + inFlightApplies.delete(executionKey); + releaseLeader({ + settled: true, + execution: { + status: 'fulfilled', + value: { applied: true, deployed: true }, + result: { outcome: 'no_source_change', reason: 'Unrelated leader finished.', nextAction: 'none' }, + }, + }); + } + }); + }); + + describe('dispatchAcceptedGeneration', () => { + const directContext = { targetMode: 'direct', nodeId: null, bindingRevision: null } as const; + const manualDispatch = { trigger: 'manual', actor: 'tester' } as const; + + async function acceptedGenerationFor(stackName: string) { + const { buildAcceptedGeneration } = await import('../services/gitops/handoff'); + const app = liveApp(stackName)!; + const row = GitOpsStore.getInstance().getGeneration(app.candidate_generation_id!)!; + return buildAcceptedGeneration(row); + } + + it('dispatches a direct-mode generation by driving the existing apply path', async () => { + const sha = 'd1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1d1'; + const svc = await seedPending('dispatch-direct', 'services:\n x:\n image: alpine\n', sha); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + + try { + const generation = await acceptedGenerationFor('dispatch-direct'); + const result = await svc.dispatchAcceptedGeneration(generation, directContext, manualDispatch); + expect(result).toEqual({ status: 'dispatched' }); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + } + }); + + it('blocks a blueprint-mode generation by delegating to BlueprintTargetAdapter', async () => { + const sha = 'd2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2'; + const svc = await seedPending('dispatch-blueprint', 'services:\n x:\n image: alpine\n', sha); + const generation = await acceptedGenerationFor('dispatch-blueprint'); + + const result = await svc.dispatchAcceptedGeneration( + generation, + { targetMode: 'blueprint', nodeId: 1, bindingRevision: 'rev-1' }, + manualDispatch, + ); + + expect(result).toEqual({ + status: 'blocked', + reason: 'Blueprint rollout orchestration is not yet implemented.', + }); + }); + + it('blocks and forwards the reason when the underlying apply fails', async () => { + const sha = 'd3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3'; + const svc = await seedPending('dispatch-apply-fails', 'services:\n x:\n image: alpine\n', sha); + const generation = await acceptedGenerationFor('dispatch-apply-fails'); + const staleGeneration = { ...generation, commitSha: 'ffffffffffffffffffffffffffffffffffffffff' }; + + const result = await svc.dispatchAcceptedGeneration(staleGeneration, directContext, manualDispatch); + + expect(result).toEqual({ + status: 'blocked', + reason: expect.stringMatching(/pending commit has changed/i), + }); + }); + + it('blocks a direct-mode dispatch when the generation names an application that no longer exists', async () => { + const sha = 'd4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4'; + const svc = await seedPending('dispatch-no-stack', 'services:\n x:\n image: alpine\n', sha); + const generation = await acceptedGenerationFor('dispatch-no-stack'); + const orphanGeneration = { ...generation, applicationId: 'no-such-application' }; + + const result = await svc.dispatchAcceptedGeneration(orphanGeneration, directContext, manualDispatch); + + expect(result).toEqual({ + status: 'blocked', + reason: expect.stringMatching(/no direct stack is bound/i), + }); + }); + + it('honors an auto_deploy_on_apply source setting by requesting a deploy on dispatch', async () => { + const sha = 'd5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5d5'; + const svc = await seedPending('dispatch-auto-deploy', 'services:\n x:\n image: alpine\n', sha); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const { ComposeService } = await import('../services/ComposeService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue({ recoveryId: null, deployedGenerationId: null }); + DatabaseService.getInstance().getDb() + .prepare('UPDATE stack_git_sources SET auto_deploy_on_apply = 1 WHERE stack_name = ?') + .run('dispatch-auto-deploy'); + + try { + const generation = await acceptedGenerationFor('dispatch-auto-deploy'); + const result = await svc.dispatchAcceptedGeneration(generation, directContext, manualDispatch); + expect(result).toEqual({ status: 'dispatched' }); + expect(deploySpy).toHaveBeenCalled(); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + + it('blocks, rather than reporting dispatched, when an auto_deploy_on_apply dispatch applies but the deploy fails', async () => { + const sha = 'd6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6d6'; + const svc = await seedPending('dispatch-auto-deploy-fails', 'services:\n x:\n image: alpine\n', sha); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const { ComposeService } = await import('../services/ComposeService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockRejectedValue( + new Error('compose up failed: docker unavailable'), + ); + DatabaseService.getInstance().getDb() + .prepare('UPDATE stack_git_sources SET auto_deploy_on_apply = 1 WHERE stack_name = ?') + .run('dispatch-auto-deploy-fails'); + + try { + const generation = await acceptedGenerationFor('dispatch-auto-deploy-fails'); + const result = await svc.dispatchAcceptedGeneration(generation, directContext, manualDispatch); + expect(result).toEqual({ status: 'blocked', reason: expect.stringMatching(/deploy/i) }); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + deploySpy.mockRestore(); + } + }); + }); + + describe('suspend / resume / retry', () => { + it('suspends an active source and reflects it in the reconcile result', async () => { + const svc = await seedPending('suspend-basic', 'services:\n x:\n image: alpine\n', 's1s1s1s1s1s1s1s1s1s1s1s1s1s1s1s1s1s1s1s1'); + + const result = await svc.suspend('suspend-basic', { actor: 'tester', reason: 'maintenance window' }); + + expect(result.outcome).toBe('suspended'); + expect(result.reason).toMatch(/maintenance window/i); + const app = liveApp('suspend-basic'); + expect(app?.suspended_at).toBeTruthy(); + expect(app?.source_suspended_reason).toBe('maintenance window'); + }); + + it('suspends with a default reason when none is given', async () => { + const svc = await seedPending('suspend-default-reason', 'services:\n x:\n image: alpine\n', 's2s2s2s2s2s2s2s2s2s2s2s2s2s2s2s2s2s2s2s2'); + + const result = await svc.suspend('suspend-default-reason', { actor: 'tester' }); + + expect(result.outcome).toBe('suspended'); + expect(liveApp('suspend-default-reason')?.source_suspended_reason).toBe('Suspended by operator.'); + }); + + it('falls back to the default reason when only whitespace is given', async () => { + const svc = await seedPending('suspend-whitespace-reason', 'services:\n x:\n image: alpine\n', 's9s9s9s9s9s9s9s9s9s9s9s9s9s9s9s9s9s9s9s9'); + + await svc.suspend('suspend-whitespace-reason', { actor: 'tester', reason: ' ' }); + + expect(liveApp('suspend-whitespace-reason')?.source_suspended_reason).toBe('Suspended by operator.'); + }); + + it('reports unknown when suspending a stack with no GitOps application', async () => { + const result = await GitSourceService.getInstance().suspend('suspend-no-app', { actor: 'tester' }); + + expect(result.outcome).toBe('unknown'); + }); + + it('surfaces a real error, rather than a silent no-op, when suspending an application that is not live', async () => { + const config: DirectSourceConfig = { + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + }; + GitOpsStore.getInstance().insertApplication(buildDirectApplicationRow({ + id: newGitOpsId(), + stackName: 'suspend-creating-app', + config, + identity: directSourceIdentity(config), + lifecycleStatus: 'creating', + at: Date.now(), + })); + + await expect(GitSourceService.getInstance().suspend('suspend-creating-app', { actor: 'tester' })) + .rejects.toMatchObject({ code: 'OPERATION_IN_FLIGHT' }); + }); + + it('actually stops a fetch, not just the projected outcome, once suspended', async () => { + const svc = await seedPending('suspend-blocks-fetch', 'services:\n x:\n image: alpine\n', 's6s6s6s6s6s6s6s6s6s6s6s6s6s6s6s6s6s6s6s6'); + await svc.suspend('suspend-blocks-fetch', { actor: 'tester', reason: 'pausing' }); + mockGitClone.mockClear(); + + await expect(svc.pull('suspend-blocks-fetch')).rejects.toMatchObject({ code: 'OPERATION_IN_FLIGHT' }); + + expect(mockGitClone).not.toHaveBeenCalled(); + }); + + it('actually stops an apply once suspended, even for a pending commit fetched before suspension', async () => { + const sha = 's7s7s7s7s7s7s7s7s7s7s7s7s7s7s7s7s7s7s7s7'; + const svc = await seedPending('suspend-blocks-apply', 'services:\n x:\n image: alpine\n', sha); + await svc.suspend('suspend-blocks-apply', { actor: 'tester', reason: 'pausing' }); + const { FileSystemService } = await import('../services/FileSystemService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + + try { + await expect(svc.apply('suspend-blocks-apply', sha, SKIP_PLAN_FINGERPRINT)) + .rejects.toMatchObject({ code: 'OPERATION_IN_FLIGHT' }); + expect(saveSpy).not.toHaveBeenCalled(); + } finally { + saveSpy.mockRestore(); + } + }); + + it('a webhook delivery to a suspended source is skipped, not reported as a failed pull', async () => { + const svc = await seedPending('suspend-webhook', 'services:\n x:\n image: alpine\n', 'scscscscscscscscscscscscscscscscscscscsc'); + await svc.suspend('suspend-webhook', { actor: 'tester', reason: 'pausing' }); + mockGitClone.mockClear(); + const activitySpy = vi.spyOn(DatabaseService.getInstance(), 'addNotificationHistory'); + + try { + const result = await svc.handleWebhookPull('suspend-webhook', true); + + expect(result.status).toBe('skipped'); + expect(mockGitClone).not.toHaveBeenCalled(); + expect(activitySpy).not.toHaveBeenCalled(); + } finally { + activitySpy.mockRestore(); + } + }); + + it('resumes a suspended source', async () => { + const svc = await seedPending('resume-basic', 'services:\n x:\n image: alpine\n', 's3s3s3s3s3s3s3s3s3s3s3s3s3s3s3s3s3s3s3s3'); + await svc.suspend('resume-basic', { actor: 'tester', reason: 'pausing' }); + + const result = await svc.resume('resume-basic', { actor: 'tester' }); + + expect(result.outcome).not.toBe('suspended'); + const app = liveApp('resume-basic'); + expect(app?.suspended_at).toBeNull(); + expect(app?.source_suspended_reason).toBeNull(); + }); + + it('resuming a source that is not suspended is a harmless no-op, not an error', async () => { + const svc = await seedPending('resume-noop', 'services:\n x:\n image: alpine\n', 's8s8s8s8s8s8s8s8s8s8s8s8s8s8s8s8s8s8s8s8'); + + const result = await svc.resume('resume-noop', { actor: 'tester' }); + + expect(result.outcome).toBe('candidate_already_fetched'); + }); + + it('pulling and applying again succeeds once a suspended source is resumed', async () => { + const sha = 'sbsbsbsbsbsbsbsbsbsbsbsbsbsbsbsbsbsbsbsb'; + const svc = await seedPending('suspend-resume-roundtrip', 'services:\n x:\n image: alpine\n', sha); + await svc.suspend('suspend-resume-roundtrip', { actor: 'tester', reason: 'pausing' }); + await expect(svc.pull('suspend-resume-roundtrip')).rejects.toMatchObject({ code: 'OPERATION_IN_FLIGHT' }); + await expect(svc.apply('suspend-resume-roundtrip', sha, SKIP_PLAN_FINGERPRINT)) + .rejects.toMatchObject({ code: 'OPERATION_IN_FLIGHT' }); + + await svc.resume('suspend-resume-roundtrip', { actor: 'tester' }); + const pullResult = await svc.pull('suspend-resume-roundtrip'); + expect(pullResult.candidateReady).toBe(true); + + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const { FileSystemService } = await import('../services/FileSystemService'); + const saveSpy = vi.spyOn(FileSystemService.prototype, 'saveStackContent').mockResolvedValue(); + try { + const applyResult = await svc.apply('suspend-resume-roundtrip', sha, SKIP_PLAN_FINGERPRINT); + expect(applyResult.applied).toBe(true); + } finally { + validateSpy.mockRestore(); + saveSpy.mockRestore(); + } + }); + + it('retries by driving a fresh fetch-intent reconcile', async () => { + const svc = await seedPending('retry-basic', 'services:\n x:\n image: alpine\n', 's5s5s5s5s5s5s5s5s5s5s5s5s5s5s5s5s5s5s5s5'); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + const reconcileSpy = vi.spyOn(svc, 'reconcile'); + + try { + const result = await svc.retry('retry-basic', { actor: 'tester' }); + expect(result.outcome).toBe('candidate_already_fetched'); + expect(reconcileSpy).toHaveBeenCalledWith(expect.objectContaining({ trigger: 'retry', intent: 'fetch' })); + } finally { + validateSpy.mockRestore(); + reconcileSpy.mockRestore(); + } + }); + + it('reports unknown when retrying a stack with no GitOps application', async () => { + const result = await GitSourceService.getInstance().retry('retry-no-app', { actor: 'tester' }); + + expect(result.outcome).toBe('unknown'); + }); + + it('retrying a suspended source reports suspended, not a generic unknown', async () => { + const svc = await seedPending('retry-while-suspended', 'services:\n x:\n image: alpine\n', 'sasasasasasasasasasasasasasasasasasasasa'); + await svc.suspend('retry-while-suspended', { actor: 'tester', reason: 'pausing' }); + + const result = await svc.retry('retry-while-suspended', { actor: 'tester' }); + + expect(result.outcome).toBe('suspended'); + expect(result.nextAction).toBe('resume'); + }); + }); + it('throws when pending has been cleared between pull and apply', async () => { const svc = await seedPending('apply-cleared', 'services:\n x:\n image: alpine\n', 'aaaa111aaaa111aaaa111aaaa111aaaa111aaaa1'); DatabaseService.getInstance().clearGitSourcePending('apply-cleared'); @@ -2292,6 +4690,13 @@ describe('GitSourceService.apply', () => { const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockRejectedValue( new Error('compose up failed: docker unavailable'), ); + const applicationId = liveApp('apply-deploy-fail')!.id; + const priorSettlements = new Set( + DatabaseService.getInstance().getDb() + .prepare("SELECT operation_id FROM gitops_history WHERE application_id = ? AND stage = 'source_reconcile_settled'") + .all(applicationId) + .map((row) => (row as { operation_id: string }).operation_id), + ); try { // Assert the return SHAPE: apply must not throw, deployError must @@ -2306,6 +4711,16 @@ describe('GitSourceService.apply', () => { const row = DatabaseService.getInstance().getGitSource('apply-deploy-fail'); expect(row?.last_applied_commit_sha).toBe(sha); expect(row?.pending_commit_sha).toBeNull(); + + const settled = DatabaseService.getInstance().getDb() + .prepare("SELECT operation_id, after_json FROM gitops_history WHERE application_id = ? AND stage = 'source_reconcile_settled'") + .all(applicationId) + .filter((item) => !priorSettlements.has((item as { operation_id: string }).operation_id)) as { after_json: string }[]; + expect(settled).toHaveLength(1); + expect(JSON.parse(settled[0].after_json)).toMatchObject({ + outcome: 'recovery_required', + nextAction: 'view_target_results', + }); } finally { validateSpy.mockRestore(); saveSpy.mockRestore(); @@ -2502,6 +4917,306 @@ describe('GitSourceService.apply', () => { }); }); +describe('GitSourceService.recoverUnsettledReconcileAttempts', () => { + it('settles a follower from its leader\'s stored result rather than deriving independently, when only the follower is unsettled', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'a1'.repeat(20) }); + await svc.upsert({ + stackName: 'recover-leader-follower', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('recover-leader-follower')!.id; + const tx = GitOpsTransitions.getInstance(); + + tx.reserveReconcileAttempt(applicationId, { operationId: 'leader-op', actor: 'tester', trigger: 'manual', at: Date.now() }); + tx.reserveReconcileAttempt( + applicationId, + { operationId: 'follower-op', actor: 'tester', trigger: 'manual', at: Date.now() + 1 }, + 'leader-op', + ); + // The leader settled with a specific result before the crash that + // orphaned only the follower. This must not match whatever + // independent derivation from current row state would produce, or + // the test cannot tell a real leader-link recovery from a + // coincidence. + tx.settleReconcileAttempt(applicationId, { operationId: 'leader-op', actor: 'tester', trigger: 'manual', at: Date.now() }, { + outcome: 'blocked', + reason: 'a specific reason only the leader would know', + nextAction: 'resolve_conflict', + }); + + await svc.recoverUnsettledReconcileAttempts(); + + const followerSettled = GitOpsStore.getInstance().getSettledAttempt(applicationId, 'follower-op'); + expect(followerSettled).toBeDefined(); + expect(JSON.parse(followerSettled!.after_json)).toMatchObject({ + outcome: 'blocked', + reason: 'a specific reason only the leader would know', + nextAction: 'resolve_conflict', + }); + }); + + it('leaves no unsettled follower after restart when both leader and follower crashed before settling', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'a2'.repeat(20) }); + await svc.upsert({ + stackName: 'recover-both-unsettled', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('recover-both-unsettled')!.id; + const tx = GitOpsTransitions.getInstance(); + tx.reserveReconcileAttempt(applicationId, { operationId: 'leader-op-2', actor: 'tester', trigger: 'manual', at: Date.now() }); + tx.reserveReconcileAttempt( + applicationId, + { operationId: 'follower-op-2', actor: 'tester', trigger: 'manual', at: Date.now() + 1 }, + 'leader-op-2', + ); + + await svc.recoverUnsettledReconcileAttempts(); + + expect(GitOpsStore.getInstance().getSettledAttempt(applicationId, 'leader-op-2')).toBeDefined(); + expect(GitOpsStore.getInstance().getSettledAttempt(applicationId, 'follower-op-2')).toBeDefined(); + expect(GitOpsStore.getInstance().listUnsettledReconcileAttempts().some((r) => r.application_id === applicationId)).toBe(false); + }); + + it('does not settle a follower independently when its leader also fails to settle in this same recovery pass', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'a5'.repeat(20) }); + await svc.upsert({ + stackName: 'recover-leader-also-fails', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('recover-leader-also-fails')!.id; + const tx = GitOpsTransitions.getInstance(); + tx.reserveReconcileAttempt(applicationId, { operationId: 'leader-op-3', actor: 'tester', trigger: 'manual', at: Date.now() }); + tx.reserveReconcileAttempt( + applicationId, + { operationId: 'follower-op-3', actor: 'tester', trigger: 'manual', at: Date.now() + 1 }, + 'leader-op-3', + ); + // The application vanishes before recovery runs, so the leader's + // own settlement (pass 1, independent branch) will itself throw + // and fail, not just "not yet have happened." + DatabaseService.getInstance().getDb().prepare('DELETE FROM gitops_applications WHERE id = ?').run(applicationId); + + await svc.recoverUnsettledReconcileAttempts(); + + // Neither settles: the follower must not be given an independently + // guessed result while its leader's own fate is still unresolved, + // even though the leader failed rather than merely being deferred. + expect(GitOpsStore.getInstance().getSettledAttempt(applicationId, 'leader-op-3')).toBeUndefined(); + expect(GitOpsStore.getInstance().getSettledAttempt(applicationId, 'follower-op-3')).toBeUndefined(); + }); + + it.each([ + ['malformed JSON', '{invalid'], + ['a non-string follower link', JSON.stringify({ followerOf: 123 })], + ])('leaves a follower unsettled when its reservation contains %s', async (_caseName, corruptAfterJson) => { + const svc = GitSourceService.getInstance(); + const stackName = `recover-corrupt-follower-${crypto.randomUUID()}`; + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'a6'.repeat(20) }); + await svc.upsert({ + stackName, + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication(stackName)!.id; + const tx = GitOpsTransitions.getInstance(); + tx.reserveReconcileAttempt(applicationId, { operationId: 'corrupt-leader', actor: 'tester', trigger: 'manual', at: Date.now() }); + tx.reserveReconcileAttempt( + applicationId, + { operationId: 'corrupt-follower', actor: 'tester', trigger: 'manual', at: Date.now() + 1 }, + 'corrupt-leader', + ); + DatabaseService.getInstance().getDb() + .prepare("UPDATE gitops_history SET after_json = ? WHERE application_id = ? AND operation_id = ? AND stage = 'source_reconcile_started'") + .run(corruptAfterJson, applicationId, 'corrupt-follower'); + const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + try { + await svc.recoverUnsettledReconcileAttempts(); + + expect(GitOpsStore.getInstance().getSettledAttempt(applicationId, 'corrupt-leader')).toBeDefined(); + expect(GitOpsStore.getInstance().getSettledAttempt(applicationId, 'corrupt-follower')).toBeUndefined(); + const unsettled = GitOpsStore.getInstance().listUnsettledReconcileAttempts(); + expect(unsettled.some((row) => row.application_id === applicationId && row.operation_id === 'corrupt-follower')).toBe(true); + } finally { + consoleSpy.mockRestore(); + } + }); + + it('drains the full backlog across multiple pages even when an earlier row can never be recovered', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'a3'.repeat(20) }); + await svc.upsert({ + stackName: 'recover-paginated-unrecoverable', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const unrecoverableAppId = GitOpsStore.getInstance().getLiveDirectApplication('recover-paginated-unrecoverable')!.id; + const tx = GitOpsTransitions.getInstance(); + // The oldest unsettled row's application is gone, so it can never + // settle. With a page size of 1, a query that keeps returning "the + // oldest still-unsettled row" would return only this one forever. + tx.reserveReconcileAttempt(unrecoverableAppId, { operationId: 'op-unrecoverable', actor: 'tester', trigger: 'manual', at: Date.now() }); + DatabaseService.getInstance().getDb().prepare('DELETE FROM gitops_applications WHERE id = ?').run(unrecoverableAppId); + + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'a4'.repeat(20) }); + await svc.upsert({ + stackName: 'recover-paginated-recoverable', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const recoverableAppId = GitOpsStore.getInstance().getLiveDirectApplication('recover-paginated-recoverable')!.id; + tx.reserveReconcileAttempt(recoverableAppId, { operationId: 'op-recoverable', actor: 'tester', trigger: 'manual', at: Date.now() + 1 }); + + await svc.recoverUnsettledReconcileAttempts(1); + + expect(GitOpsStore.getInstance().getSettledAttempt(recoverableAppId, 'op-recoverable')).toBeDefined(); + }); + + it('settles an attempt left unsettled by a crash, without re-executing a fetch', async () => { + const sha = 'eb'.repeat(20); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha }); + const svc = GitSourceService.getInstance(); + await svc.upsert({ + stackName: 'recover-unsettled', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('recover-unsettled')!.id; + GitOpsTransitions.getInstance().reserveReconcileAttempt(applicationId, { + operationId: 'orphaned-op-1', actor: 'tester', trigger: 'poll', at: Date.now(), + }); + mockGitClone.mockClear(); + + await svc.recoverUnsettledReconcileAttempts(); + + expect(mockGitClone).not.toHaveBeenCalled(); + const settled = GitOpsStore.getInstance().getSettledAttempt(applicationId, 'orphaned-op-1'); + expect(settled).toBeDefined(); + }); + + it('leaves a different application unaffected when only one has an unsettled attempt', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'ec'.repeat(20) }); + await svc.upsert({ + stackName: 'recover-clean', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const applicationId = GitOpsStore.getInstance().getLiveDirectApplication('recover-clean')!.id; + + await expect(svc.recoverUnsettledReconcileAttempts()).resolves.toBeUndefined(); + + expect(GitOpsStore.getInstance().listUnsettledReconcileAttempts().some((r) => r.application_id === applicationId)).toBe(false); + }); + + it('does not let one attempt whose application vanished block recovery of another, older, still-real attempt', async () => { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'ed'.repeat(20) }); + await svc.upsert({ + stackName: 'recover-poisoned', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const poisonedAppId = GitOpsStore.getInstance().getLiveDirectApplication('recover-poisoned')!.id; + GitOpsTransitions.getInstance().reserveReconcileAttempt(poisonedAppId, { + operationId: 'poisoned-op-1', actor: 'tester', trigger: 'poll', at: Date.now(), + }); + // Simulate the application row vanishing between listing and processing. + DatabaseService.getInstance().getDb().prepare('DELETE FROM gitops_applications WHERE id = ?').run(poisonedAppId); + + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: 'ee'.repeat(20) }); + await svc.upsert({ + stackName: 'recover-healthy', + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + const healthyAppId = GitOpsStore.getInstance().getLiveDirectApplication('recover-healthy')!.id; + GitOpsTransitions.getInstance().reserveReconcileAttempt(healthyAppId, { + operationId: 'healthy-op-1', actor: 'tester', trigger: 'poll', at: Date.now(), + }); + + await expect(svc.recoverUnsettledReconcileAttempts()).resolves.toBeUndefined(); + + expect(GitOpsStore.getInstance().getSettledAttempt(healthyAppId, 'healthy-op-1')).toBeDefined(); + }); +}); + describe('GitSourceService DB normalization (compose_paths back-compat)', () => { it('reads back [compose_path] when a row stores compose_paths as null (legacy)', async () => { mockSuccessfulClone({ composePath: 'stacks/web/compose.yaml' }); @@ -3123,28 +5838,8 @@ describe('GitSourceService legacy pending apply (migration path)', () => { const { FileSystemService } = await import('../services/FileSystemService'); const fsSvc = FileSystemService.getInstance(); await fsSvc.createStack('legacy-apply'); - db.upsertGitSource({ - stack_name: 'legacy-apply', - repo_url: 'https://github.com/example/repo.git', - branch: 'main', - compose_path: 'compose.yaml', - compose_paths: ['compose.yaml'], - context_dir: null, - sync_env: false, - env_path: null, - auth_type: 'none', - encrypted_token: null, encrypted_deploy_key: null, ssh_known_hosts_entry: null, ssh_host_key_fingerprint: null, - encrypted_ca_bundle: null, - auto_apply_on_webhook: false, - auto_deploy_on_apply: false, - last_applied_commit_sha: null, - last_applied_content_hash: null, - pending_commit_sha: sha, - pending_compose_content: null, - pending_env_content: null, - pending_fetched_at: null, - last_debounce_at: null, - }); + mockSuccessfulClone({ sha }); + await configureGitSource('legacy-apply'); // Seed the v2 blob directly, as a pre-upgrade row would carry it. const svcPriv = svc as unknown as { crypto: { encrypt(s: string): string } }; db.setGitSourcePending('legacy-apply', sha, svcPriv.crypto.encrypt(JSON.stringify({ v: 2, files: [{ path: 'compose.yaml', content: 'services:\n web:\n image: nginx\n' }], contextDir: null })), null); @@ -3402,28 +6097,8 @@ describe('GitSourceService classified plan fingerprint', () => { const db = DatabaseService.getInstance(); const { FileSystemService } = await import('../services/FileSystemService'); await FileSystemService.getInstance().createStack('plan-unavail'); - db.upsertGitSource({ - stack_name: 'plan-unavail', - repo_url: 'https://github.com/example/repo.git', - branch: 'main', - compose_path: 'compose.yaml', - compose_paths: ['compose.yaml'], - context_dir: null, - sync_env: false, - env_path: null, - auth_type: 'none', - encrypted_token: null, encrypted_deploy_key: null, ssh_known_hosts_entry: null, ssh_host_key_fingerprint: null, - encrypted_ca_bundle: null, - auto_apply_on_webhook: false, - auto_deploy_on_apply: false, - last_applied_commit_sha: null, - last_applied_content_hash: null, - pending_commit_sha: sha, - pending_compose_content: null, - pending_env_content: null, - pending_fetched_at: null, - last_debounce_at: null, - }); + mockSuccessfulClone({ sha }); + await configureGitSource('plan-unavail'); const svcPriv = svc as unknown as { crypto: { encrypt(s: string): string } }; db.setGitSourcePending( 'plan-unavail', @@ -3494,3 +6169,180 @@ function seedDirectCandidate(stackName: string): { appId: string; generationId: GitOpsTransitions.getInstance().candidateReady(appId, generationId, false, testEnvelope()); return { appId, generationId }; } + +// ── sweepOrphans claimant fixtures ───────────────────────────────────── + +/** A deterministic 40-char hex sha derived from a short seed. */ +function shaFromSeed(seed: string): string { + return seed.repeat(40).slice(0, 40); +} + +/** + * Create a Git-backed stack, pull one update into it, and backdate the + * resulting candidate directory past the orphan-candidate age threshold so a + * claimant-blind sweep would reap it as stale. Each test then arranges only + * the claimant pointers it exercises before running the sweep. + */ +async function stageStaleCandidate( + stackName: string, + shaSeed: string, +): Promise<{ appId: string; generationId: string; candidateAbs: string }> { + const svc = GitSourceService.getInstance(); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine\n', sha: shaFromSeed(shaSeed) }); + const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true }); + try { + await svc.createStackFromGit({ + stackName, + repoUrl: 'https://github.com/example/repo.git', + branch: 'main', + composePaths: ['compose.yaml'], + contextDir: null, + syncEnv: false, + envPath: null, + authType: 'none', + token: null, + autoApplyOnWebhook: false, + autoDeployOnApply: false, + }); + mockSuccessfulClone({ compose: 'services:\n x:\n image: alpine:2\n', sha: shaFromSeed(`${shaSeed}a`) }); + await svc.pull(stackName); + } finally { + validateSpy.mockRestore(); + } + const store = GitOpsStore.getInstance(); + const app = store.getLiveDirectApplication(stackName)!; + expect(app.candidate_generation_id).toBeTruthy(); + const generation = store.getGeneration(app.candidate_generation_id!)!; + const candidateAbs = path.join(stackManagedRoot(stackName), generation.candidate_dir); + expect(fs.existsSync(candidateAbs)).toBe(true); + const staleMtime = (Date.now() - 25 * 60 * 60 * 1000) / 1000; + fs.utimesSync(candidateAbs, staleMtime, staleMtime); + return { appId: app.id, generationId: generation.id, candidateAbs }; +} + +/** Drop the application row's own pointer at its staged candidate. */ +function clearCandidatePointer(appId: string): void { + DatabaseService.getInstance().getDb() + .prepare('UPDATE gitops_applications SET candidate_generation_id = NULL WHERE id = ?') + .run(appId); +} + +/** Drop the pending fetch record, the claimant that is independent of the application row. */ +function clearPendingFetch(stackName: string): void { + DatabaseService.getInstance().getDb() + .prepare('UPDATE stack_git_sources SET pending_commit_sha = NULL, pending_compose_content = NULL WHERE stack_name = ?') + .run(stackName); +} + +describe('GitSourceService.sweepOrphans candidate claimant preservation', () => { + it('preserves a stale, complete candidate directory still referenced by the live application\'s candidate_generation_id', async () => { + const { candidateAbs } = await stageStaleCandidate('sweep-claims-candidate', 'c1'); + + await GitSourceService.getInstance().sweepOrphans(); + + expect(fs.existsSync(candidateAbs)).toBe(true); + }); + + it('still reaps a stale, complete candidate directory nothing on the application row references', async () => { + const { appId, candidateAbs } = await stageStaleCandidate('sweep-reaps-unclaimed', 'c2'); + // Nothing on the row, and nothing pending, points at this generation + // any more: the exact "leftover from an earlier attempt" case the + // sweep exists to clean up. + clearCandidatePointer(appId); + clearPendingFetch('sweep-reaps-unclaimed'); + + await GitSourceService.getInstance().sweepOrphans(); + + expect(fs.existsSync(candidateAbs)).toBe(false); + }); + + it('preserves a stale, complete candidate directory referenced only by the pending fetch record, with no generation row yet', async () => { + const { appId, candidateAbs } = await stageStaleCandidate('sweep-claims-pending-only', 'c3'); + // Simulate the row-level pointer being gone (the exact + // fetchedInvalid/no-live-application gap where a generation may never + // have existed at all) while the pending fetch record, written + // independently, still names this candidate. + clearCandidatePointer(appId); + + await GitSourceService.getInstance().sweepOrphans(); + + expect(fs.existsSync(candidateAbs)).toBe(true); + }); + + it('preserves a stale, complete candidate directory referenced only by accepted_generation_id', async () => { + const { appId, generationId, candidateAbs } = await stageStaleCandidate('sweep-claims-accepted-only', 'c4'); + // Simulate the sourceAccepted-committed-but-not-yet-promoted window: + // accepted_generation_id names this candidate, the row's own candidate + // pointer is gone, and the pending record no longer references it + // either, so this pointer alone must be what preserves the directory. + DatabaseService.getInstance().getDb() + .prepare('UPDATE gitops_applications SET accepted_generation_id = ? WHERE id = ?') + .run(generationId, appId); + clearCandidatePointer(appId); + clearPendingFetch('sweep-claims-accepted-only'); + + await GitSourceService.getInstance().sweepOrphans(); + + expect(fs.existsSync(candidateAbs)).toBe(true); + }); + + it('preserves a stale candidate referenced only by an unsettled attempt', async () => { + const { appId, generationId, candidateAbs } = await stageStaleCandidate('sweep-claims-unsettled', 'c5'); + const generation = GitOpsStore.getInstance().getGeneration(generationId)!; + clearCandidatePointer(appId); + clearPendingFetch('sweep-claims-unsettled'); + DatabaseService.getInstance().getDb() + .prepare("DELETE FROM gitops_history WHERE application_id = ? AND operation_id = ? AND stage = 'source_reconcile_settled'") + .run(appId, generation.operation_id); + + await GitSourceService.getInstance().sweepOrphans(); + + expect(fs.existsSync(candidateAbs)).toBe(true); + }); + + it('preserves stale candidates when pending claimant metadata is unreadable', async () => { + const stackName = 'sweep-preserves-unreadable-claims'; + const { appId, candidateAbs } = await stageStaleCandidate(stackName, 'c6'); + clearCandidatePointer(appId); + DatabaseService.getInstance().getDb() + .prepare('UPDATE stack_git_sources SET pending_compose_content = ? WHERE stack_name = ?') + .run('{"v":4 invalid', stackName); + + await GitSourceService.getInstance().sweepOrphans(); + + expect(fs.existsSync(candidateAbs)).toBe(true); + expect(DatabaseService.getInstance().getGitSource(stackName)?.manifest_state).toBe('migration_required'); + }); + + it('preserves stale candidates when a generation claimant pointer is dangling', async () => { + const stackName = 'sweep-preserves-dangling-claim'; + const { appId, candidateAbs } = await stageStaleCandidate(stackName, 'c7'); + clearPendingFetch(stackName); + DatabaseService.getInstance().getDb() + .prepare('UPDATE gitops_applications SET candidate_generation_id = ? WHERE id = ?') + .run('missing-generation', appId); + + await GitSourceService.getInstance().sweepOrphans(); + + expect(fs.existsSync(candidateAbs)).toBe(true); + expect(DatabaseService.getInstance().getGitSource(stackName)?.manifest_state).toBe('migration_required'); + }); + + it('preserves stale candidates when unsettled-attempt claimant lookup fails', async () => { + const stackName = 'sweep-preserves-claim-query-failure'; + const { appId, candidateAbs } = await stageStaleCandidate(stackName, 'c8'); + clearCandidatePointer(appId); + clearPendingFetch(stackName); + const claimantSpy = vi.spyOn(GitOpsStore.prototype, 'listGenerationsClaimedByUnsettledAttempts') + .mockImplementationOnce(() => { throw new Error('simulated claimant query failure'); }); + + try { + await GitSourceService.getInstance().sweepOrphans(); + + expect(fs.existsSync(candidateAbs)).toBe(true); + expect(DatabaseService.getInstance().getGitSource(stackName)?.manifest_state).toBe('migration_required'); + } finally { + claimantSpy.mockRestore(); + } + }); +}); diff --git a/backend/src/__tests__/gitops-approvals.test.ts b/backend/src/__tests__/gitops-approvals.test.ts index 75b0d132..1ab89bab 100644 --- a/backend/src/__tests__/gitops-approvals.test.ts +++ b/backend/src/__tests__/gitops-approvals.test.ts @@ -309,6 +309,10 @@ function directApp(id: string, stackName: string): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, @@ -351,6 +355,12 @@ function generation(id: string, applicationId: string): GitOpsGenerationRow { actor: 'tester', previous_generation_id: null, redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, created_at: 1, }; } diff --git a/backend/src/__tests__/gitops-backoff.test.ts b/backend/src/__tests__/gitops-backoff.test.ts new file mode 100644 index 00000000..79a84f82 --- /dev/null +++ b/backend/src/__tests__/gitops-backoff.test.ts @@ -0,0 +1,157 @@ +/** + * Failure classification and retry-delay coverage for the GitOps source + * controller. classifyFailure is a compile-enforced total map: adding a new + * TransportFailureReason or GitSourceErrorCode without updating the lookup + * tables here fails the build, not just these tests. + */ +import { describe, it, expect } from 'vitest'; +import { + classifyFailure, + nextRetryAt, + DEFAULT_TRANSIENT_CEILING, + LOW_TRANSIENT_CEILING, + type FailureEvidence, +} from '../services/gitops/backoff'; + +function gitSourceError(code: string, transportReason?: string): FailureEvidence { + return { kind: 'git_source_error', code: code as never, transportReason: transportReason as never }; +} + +describe('classifyFailure', () => { + it('classifies a tip-changed race as supersession, not backoff', () => { + expect(classifyFailure(gitSourceError('GIT_ERROR', 'tip-changed'))).toEqual({ class: 'supersession' }); + }); + + it('classifies a standalone timeout reason as transient', () => { + expect(classifyFailure(gitSourceError('NETWORK_TIMEOUT', 'timeout'))) + .toEqual({ class: 'transient', retryCeiling: DEFAULT_TRANSIENT_CEILING }); + }); + + it('classifies DNS resolution failure (target-unresolved) as transient', () => { + expect(classifyFailure(gitSourceError('NETWORK_TIMEOUT', 'target-unresolved'))) + .toEqual({ class: 'transient', retryCeiling: DEFAULT_TRANSIENT_CEILING }); + }); + + it('classifies an exit-coded network timeout as transient', () => { + expect(classifyFailure(gitSourceError('NETWORK_TIMEOUT', 'exit'))) + .toEqual({ class: 'transient', retryCeiling: DEFAULT_TRANSIENT_CEILING }); + }); + + it('classifies an exit-coded rate limit as transient', () => { + expect(classifyFailure(gitSourceError('RATE_LIMITED', 'exit'))) + .toEqual({ class: 'transient', retryCeiling: DEFAULT_TRANSIENT_CEILING }); + }); + + it('classifies an exit-coded unrecognized git error with a low retry ceiling', () => { + expect(classifyFailure(gitSourceError('GIT_ERROR', 'exit'))) + .toEqual({ class: 'transient', retryCeiling: LOW_TRANSIENT_CEILING }); + }); + + it.each(['invalid-url', 'unsafe-target', 'invalid-ref', 'redirect-scope'])( + 'classifies %s as permanent configuration', + (reason) => { + expect(classifyFailure(gitSourceError('GIT_ERROR', reason))).toEqual({ class: 'permanent' }); + }, + ); + + it.each(['git-missing', 'git-old'])('classifies %s as permanent environment', (reason) => { + expect(classifyFailure(gitSourceError('GIT_ERROR', reason))).toEqual({ class: 'permanent' }); + }); + + it('classifies a repository over the size cap as permanent', () => { + expect(classifyFailure(gitSourceError('GIT_ERROR', 'size'))).toEqual({ class: 'permanent' }); + }); + + it.each(['ssh-auth-required'])('classifies %s as permanent authorization', (reason) => { + expect(classifyFailure(gitSourceError('GIT_ERROR', reason))).toEqual({ class: 'permanent' }); + }); + + it.each(['AUTH_FAILED', 'SSH_HOST_KEY_FAILED'])('classifies %s (no transport reason) as permanent', (code) => { + expect(classifyFailure(gitSourceError(code))).toEqual({ class: 'permanent' }); + }); + + it.each(['ref-not-found', 'unsupported-ref'])('classifies %s as permanent configuration', (reason) => { + expect(classifyFailure(gitSourceError('GIT_ERROR', reason))).toEqual({ class: 'permanent' }); + }); + + it.each(['REPO_NOT_FOUND', 'REF_NOT_FOUND', 'REF_DELETED', 'UNSUPPORTED_REF'])( + 'classifies %s (no transport reason) as permanent', + (code) => { + expect(classifyFailure(gitSourceError(code))).toEqual({ class: 'permanent' }); + }, + ); + + it.each(['STALE_PLAN', 'PLAN_BLOCKED', 'PLAN_FINGERPRINT_REQUIRED', 'LEGACY_PENDING', 'PLAN_UNAVAILABLE', 'FILE_NOT_FOUND'])( + 'classifies %s as requiring operator action', + (code) => { + expect(classifyFailure(gitSourceError(code))).toEqual({ class: 'operator_action_required' }); + }, + ); + + it('classifies a conflicting in-flight operation for reconciliation, not blind retry', () => { + expect(classifyFailure(gitSourceError('OPERATION_IN_FLIGHT'))).toEqual({ class: 'reconcile' }); + }); + + it('classifies an unavailable policy scanner as degraded', () => { + expect(classifyFailure({ kind: 'policy_unavailable' })).toEqual({ class: 'degraded' }); + }); + + it('classifies unavailable persistence as transient with no source-stage progress', () => { + expect(classifyFailure({ kind: 'persistence_unavailable' })) + .toEqual({ class: 'transient', retryCeiling: DEFAULT_TRANSIENT_CEILING }); + }); + + it('classifies an invalid target binding as permanent at the target level', () => { + expect(classifyFailure({ kind: 'target_binding_invalid' })).toEqual({ class: 'target_permanent' }); + }); + + it('classifies a temporarily unavailable target as transient at the target level', () => { + expect(classifyFailure({ kind: 'target_unavailable' })).toEqual({ class: 'target_transient' }); + }); + + it('classifies a deploy/health failure after a successful apply as its own class, never refetch or reapply', () => { + expect(classifyFailure({ kind: 'target_mutation_failed' })).toEqual({ class: 'target_mutation_failed' }); + }); + + it('classifies unavailable Blueprint evaluation as blocked, not retried', () => { + expect(classifyFailure({ kind: 'blueprint_unavailable' })).toEqual({ class: 'blocked' }); + }); + + it('classifies an interrupted or unknown-completion operation for reconciliation', () => { + expect(classifyFailure({ kind: 'interrupted' })).toEqual({ class: 'reconcile' }); + }); +}); + +describe('nextRetryAt', () => { + it('computes the base delay with up to +-10% jitter on the first attempt', () => { + const now = 1_000_000; + const at = nextRetryAt(now, 0); + expect(at).toBeGreaterThanOrEqual(now + 54_000); + expect(at).toBeLessThanOrEqual(now + 66_000); + }); + + it('doubles the delay per retry count', () => { + const now = 1_000_000; + const at = nextRetryAt(now, 3); // 60s * 2^3 = 480s + expect(at).toBeGreaterThanOrEqual(now + 432_000); + expect(at).toBeLessThanOrEqual(now + 528_000); + }); + + it('caps the delay at one hour regardless of retry count', () => { + const now = 1_000_000; + const at = nextRetryAt(now, 20); + expect(at).toBeLessThanOrEqual(now + 3_600_000 * 1.1); + }); + + it('honors a provider retry floor larger than the computed delay', () => { + const now = 1_000_000; + const at = nextRetryAt(now, 0, 10_000_000); + expect(at).toBe(now + 10_000_000); + }); + + it('ignores a provider retry floor smaller than the computed delay', () => { + const now = 1_000_000; + const at = nextRetryAt(now, 5, 1_000); + expect(at).toBeGreaterThan(now + 1_000); + }); +}); diff --git a/backend/src/__tests__/gitops-blueprint-transitions.test.ts b/backend/src/__tests__/gitops-blueprint-transitions.test.ts index 9c5b6632..ff13ab72 100644 --- a/backend/src/__tests__/gitops-blueprint-transitions.test.ts +++ b/backend/src/__tests__/gitops-blueprint-transitions.test.ts @@ -609,6 +609,10 @@ function inlineApp(id: string, blueprintId: number): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/__tests__/gitops-create-recovery.test.ts b/backend/src/__tests__/gitops-create-recovery.test.ts index 85f4c614..71dde879 100644 --- a/backend/src/__tests__/gitops-create-recovery.test.ts +++ b/backend/src/__tests__/gitops-create-recovery.test.ts @@ -441,6 +441,10 @@ function creatingApp(id: string, stackName: string): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, @@ -483,6 +487,12 @@ function gen(id: string, applicationId: string): GitOpsGenerationRow { actor: 'tester', previous_generation_id: null, redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, created_at: 1, }; } diff --git a/backend/src/__tests__/gitops-create.test.ts b/backend/src/__tests__/gitops-create.test.ts index 0fdee147..1eb9e824 100644 --- a/backend/src/__tests__/gitops-create.test.ts +++ b/backend/src/__tests__/gitops-create.test.ts @@ -707,6 +707,10 @@ function creatingApp(id: string, stackName: string): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, @@ -749,6 +753,12 @@ function gen(id: string, applicationId: string): GitOpsGenerationRow { actor: 'tester', previous_generation_id: null, redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, created_at: 1, }; } diff --git a/backend/src/__tests__/gitops-deferred.test.ts b/backend/src/__tests__/gitops-deferred.test.ts index b901e0e0..565712da 100644 --- a/backend/src/__tests__/gitops-deferred.test.ts +++ b/backend/src/__tests__/gitops-deferred.test.ts @@ -340,6 +340,10 @@ function app(id: string, stackName: string): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, @@ -382,6 +386,12 @@ function gen(id: string, applicationId: string): GitOpsGenerationRow { actor: 'tester', previous_generation_id: null, redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, created_at: 1, }; } diff --git a/backend/src/__tests__/gitops-derive.test.ts b/backend/src/__tests__/gitops-derive.test.ts index 7b03c86d..dd0f18b6 100644 --- a/backend/src/__tests__/gitops-derive.test.ts +++ b/backend/src/__tests__/gitops-derive.test.ts @@ -963,6 +963,10 @@ function app(id: string, stackName: string): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, @@ -1010,6 +1014,12 @@ function gen(id: string, applicationId: string): GitOpsGenerationRow { actor: 'tester', previous_generation_id: null, redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, created_at: 1, }; } diff --git a/backend/src/__tests__/gitops-direct-producers.test.ts b/backend/src/__tests__/gitops-direct-producers.test.ts index 42690eff..936a30a1 100644 --- a/backend/src/__tests__/gitops-direct-producers.test.ts +++ b/backend/src/__tests__/gitops-direct-producers.test.ts @@ -644,7 +644,7 @@ describe('Direct Git producers drive the revision state', () => { expect(recovered.active_operation_stage).toBeNull(); }); - it('leaves a stack with no GitOps application untouched', async () => { + it('refuses to fetch when a configured stack has no GitOps application', async () => { const svc = GitSourceService.getInstance(); const store = GitOpsStore.getInstance(); const stackName = 'producers-legacy'; @@ -681,9 +681,12 @@ describe('Direct Git producers drive the revision state', () => { expect(store.getLiveDirectApplication(stackName)).toBeUndefined(); stageRepo(COMPOSE_V2, 'fffffff6'); - await svc.pull(stackName, { actor: 'tester' }); + await expect(svc.pull(stackName, { actor: 'tester' })).rejects.toMatchObject({ + code: 'GIT_ERROR', + message: expect.stringContaining('GitOps tracking is unavailable'), + }); - // The pull succeeded operationally and wrote no GitOps rows. + // No untracked fetch or GitOps history was written. expect(store.getLiveDirectApplication(stackName)).toBeUndefined(); const historyRows = (await import('../services/DatabaseService')).DatabaseService .getInstance().getDb() diff --git a/backend/src/__tests__/gitops-handoff.test.ts b/backend/src/__tests__/gitops-handoff.test.ts new file mode 100644 index 00000000..7956af58 --- /dev/null +++ b/backend/src/__tests__/gitops-handoff.test.ts @@ -0,0 +1,115 @@ +/** + * The accepted-generation contract: a portable, content-only projection of + * a gitops_generations row, plus the target-dispatch boundary. + */ +import { describe, it, expect } from 'vitest'; +import { + buildAcceptedGeneration, + BlueprintTargetAdapter, + type AcceptedGeneration, +} from '../services/gitops/handoff'; +import type { GitOpsGenerationRow } from '../services/gitops/types'; + +function baseRow(overrides: Partial = {}): GitOpsGenerationRow { + return { + id: 'gen-1', + application_id: 'app-1', + commit_sha: 'a'.repeat(40), + repo_url: 'https://github.com/example/repo.git', + configured_ref: 'main', + resolved_ref_kind: 'branch', + repo_identity_json: '{"host":"github.com","pathname":"/example/repo.git"}', + manifest_version: 4, + candidate_dir: 'generations/candidate-a', + applied_dir: 'generations/applied-a-0', + expected_invocation_json: '{}', + materialization_fingerprint: 'f'.repeat(64), + validation_ok: 1, + plan_blocked: 0, + change_plan_fingerprint: 'fp-1', + operation_id: 'op-1', + trigger: 'manual', + actor: 'tester', + previous_generation_id: null, + redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, + created_at: 1, + ...overrides, + }; +} + +describe('buildAcceptedGeneration', () => { + it('decodes identity and lineage fields directly from the row', () => { + const gen = buildAcceptedGeneration(baseRow()); + expect(gen.contractVersion).toBe(1); + expect(gen.generationId).toBe('gen-1'); + expect(gen.applicationId).toBe('app-1'); + expect(gen.repoIdentity).toEqual({ host: 'github.com', pathname: '/example/repo.git' }); + expect(gen.configuredRef).toBe('main'); + expect(gen.commitSha).toBe('a'.repeat(40)); + expect(gen.resolvedRefKind).toBe('branch'); + expect(gen.validationOk).toBe(true); + expect(gen.trigger).toBe('manual'); + expect(gen.operationId).toBe('op-1'); + }); + + it('records an explicit limitation for each missing portable field on a legacy row, never inventing evidence', () => { + const gen = buildAcceptedGeneration(baseRow()); + expect(gen.portableManifest).toBeNull(); + expect(gen.composeInputs).toBeNull(); + expect(gen.sourcePolicyEvidence).toBeNull(); + expect(gen.limitations).toEqual(expect.arrayContaining([ + 'portable_manifest_missing', + 'compose_inputs_missing', + 'source_policy_evidence_missing', + 'security_policy_evidence_missing', + 'support_requirements_missing', + 'compatibility_requirements_missing', + ])); + }); + + it('decodes real evidence when the row carries it, recording no limitation for that field', () => { + const gen = buildAcceptedGeneration(baseRow({ + portable_manifest_json: '{"files":[]}', + compose_inputs_json: '{"composeFileOrder":["compose.yaml"]}', + })); + expect(gen.portableManifest).toEqual({ files: [] }); + expect(gen.composeInputs).toEqual({ composeFileOrder: ['compose.yaml'] }); + expect(gen.limitations).not.toContain('portable_manifest_missing'); + expect(gen.limitations).not.toContain('compose_inputs_missing'); + }); + + it('refuses to build a contract from an unparseable repo identity', () => { + expect(() => buildAcceptedGeneration(baseRow({ repo_identity_json: 'not json' }))).toThrow(); + }); + + it('never populates secretCapability with a value, only its absence as capability metadata', () => { + const gen = buildAcceptedGeneration(baseRow()); + expect(gen.secretCapability).toBeNull(); + }); +}); + +// A future field on AcceptedGeneration named like a target-mode concept +// (selector, frozen target set, node id, rollout batch, target project +// name, local path, or secret value) must fail this compile, not merely +// this test run: the contract stays structurally content-only. +type AssertNoTargetModeFields = T extends Record< + 'selector' | 'targetSet' | 'nodeId' | 'nodeIds' | 'rolloutBatch' | 'projectName' | 'candidateDir' | 'secretValue', + unknown +> ? never : true; +const _structurallyContentOnly: AssertNoTargetModeFields = true; +void _structurallyContentOnly; + +describe('BlueprintTargetAdapter', () => { + it('always returns a durable blocked result, never inspecting selectors or placement', async () => { + const adapter = new BlueprintTargetAdapter(); + const gen = buildAcceptedGeneration(baseRow()); + const result = await adapter.dispatch(gen, { targetMode: 'blueprint', nodeId: null, bindingRevision: null }); + expect(result.status).toBe('blocked'); + }); +}); diff --git a/backend/src/__tests__/gitops-history-read.test.ts b/backend/src/__tests__/gitops-history-read.test.ts index 8417dbcc..d1f10aef 100644 --- a/backend/src/__tests__/gitops-history-read.test.ts +++ b/backend/src/__tests__/gitops-history-read.test.ts @@ -531,6 +531,10 @@ function application(): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/__tests__/gitops-managed-sweep.test.ts b/backend/src/__tests__/gitops-managed-sweep.test.ts index 87d62944..32f104ed 100644 --- a/backend/src/__tests__/gitops-managed-sweep.test.ts +++ b/backend/src/__tests__/gitops-managed-sweep.test.ts @@ -167,6 +167,10 @@ function creatingApp(id: string, stackName: string): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/__tests__/gitops-outcomes.test.ts b/backend/src/__tests__/gitops-outcomes.test.ts new file mode 100644 index 00000000..48eb975b --- /dev/null +++ b/backend/src/__tests__/gitops-outcomes.test.ts @@ -0,0 +1,152 @@ +/** + * Normalized reconcile-outcome coverage. outcomeFromSourceFacet derives from + * the existing SourceFacet projection rather than inventing a second status + * source, so "no source change" and "converged" cannot silently collapse + * into the same result. + */ +import { describe, it, expect } from 'vitest'; +import { outcomeFromSourceFacet } from '../services/gitops/outcomes'; +import type { SourceFacet } from '../services/gitops/types'; + +const identity = { + configuredRepoUrl: 'https://github.com/example/repo.git', + repoIdentity: { host: 'github.com', pathname: '/example/repo.git' }, + configuredRef: 'main', + desiredCommitSha: 'a'.repeat(40), + fetchedCommitSha: 'a'.repeat(40), + candidateGenerationId: null, + acceptedGenerationId: 'gen-1', +}; + +describe('outcomeFromSourceFacet', () => { + it('reports no_source_change for an accepted generation, never converged on SHA alone', () => { + const facet: SourceFacet = { ...identity, status: 'application_generation_accepted' }; + const result = outcomeFromSourceFacet(facet); + expect(result.outcome).toBe('no_source_change'); + expect(result.commitSha).toBe('a'.repeat(40)); + }); + + it('reports candidate_already_fetched for a ready candidate', () => { + const facet: SourceFacet = { ...identity, status: 'candidate_ready' }; + expect(outcomeFromSourceFacet(facet).outcome).toBe('candidate_already_fetched'); + }); + + it('reports pending_review with a review next action', () => { + const facet: SourceFacet = { ...identity, status: 'source_review_pending' }; + const result = outcomeFromSourceFacet(facet); + expect(result.outcome).toBe('pending_review'); + expect(result.nextAction).toBe('review'); + }); + + it('reports blocked with a resolve_conflict next action for a source conflict', () => { + const facet: SourceFacet = { ...identity, status: 'source_conflict_blocker' }; + const result = outcomeFromSourceFacet(facet); + expect(result.outcome).toBe('blocked'); + expect(result.nextAction).toBe('resolve_conflict'); + }); + + it('reports superseded for a candidate a newer revision replaced', () => { + const facet: SourceFacet = { ...identity, status: 'source_superseded', supersededGenerationId: 'gen-old' }; + expect(outcomeFromSourceFacet(facet).outcome).toBe('superseded'); + }); + + it('reports retry_scheduled with the retry time surfaced', () => { + const facet: SourceFacet = { ...identity, status: 'source_retry_scheduled', retryAt: 12345, retryCount: 2 }; + const result = outcomeFromSourceFacet(facet); + expect(result.outcome).toBe('retry_scheduled'); + expect(result.retryAt).toBe(12345); + expect(result.nextAction).toBe('none'); + }); + + it('reports suspended with a resume next action', () => { + const facet: SourceFacet = { ...identity, status: 'source_suspended', suspendedAt: 999, suspendedReason: 'operator paused sync' }; + const result = outcomeFromSourceFacet(facet); + expect(result.outcome).toBe('suspended'); + expect(result.nextAction).toBe('resume'); + expect(result.reason).toContain('operator paused sync'); + }); + + it('reports failed_previous_intact for a source failure, with a retry next action when a retry is scheduled', () => { + const facet: SourceFacet = { + ...identity, + status: 'source_failed', + failureStage: 'fetch', + failureClass: 'permanent', + failureAt: 100, + retryAt: 200, + retryCount: 1, + }; + const result = outcomeFromSourceFacet(facet); + expect(result.outcome).toBe('failed_previous_intact'); + expect(result.nextAction).toBe('retry'); + expect(result.retryAt).toBe(200); + }); + + it('reports failed_previous_intact with no retry next action when no retry is scheduled', () => { + const facet: SourceFacet = { + ...identity, + status: 'source_failed', + failureStage: 'fetch', + failureClass: 'permanent', + failureAt: 100, + retryAt: null, + retryCount: 0, + }; + const result = outcomeFromSourceFacet(facet); + expect(result.outcome).toBe('failed_previous_intact'); + expect(result.nextAction).toBe('configure_credentials'); + }); + + it('reports recovery_required for an interrupted operation', () => { + const facet: SourceFacet = { + ...identity, + status: 'source_unknown', + interruptedStage: 'fetch_started', + interruptedAt: 100, + interruptedOperationId: 'op-1', + interruptedGenerationId: null, + }; + expect(outcomeFromSourceFacet(facet).outcome).toBe('recovery_required'); + }); + + it('reports recovery_required with a view_target_results next action when recovery is outstanding', () => { + const facet: SourceFacet = { ...identity, status: 'recovery_required', recoveryRef: 'rec-1', recoveryGenerationId: 'gen-1' }; + const result = outcomeFromSourceFacet(facet); + expect(result.outcome).toBe('recovery_required'); + expect(result.nextAction).toBe('view_target_results'); + }); + + it('reports recovery_required when recovery itself failed, distinguishing that in the reason', () => { + const facet: SourceFacet = { + ...identity, + status: 'recovery_failed', + recoveryRef: 'rec-1', + recoveryGenerationId: 'gen-1', + failureClass: 'io_error', + failureAt: 100, + }; + const result = outcomeFromSourceFacet(facet); + expect(result.outcome).toBe('recovery_required'); + expect(result.reason).toMatch(/recovery/i); + }); + + it('reports unknown for an application that is no longer live', () => { + const facet: SourceFacet = { ...identity, status: 'not_live', lifecycleStatus: 'detached' }; + expect(outcomeFromSourceFacet(facet).outcome).toBe('unknown'); + }); + + it.each(['not_applicable', 'never_reconciled', 'checking_fetching', 'source_reconcile_required'] as const)( + 'reports unknown for %s, which has no settled outcome yet', + (status) => { + const facet = status === 'not_applicable' + ? ({ status } as SourceFacet) + : ({ ...identity, status } as SourceFacet); + expect(outcomeFromSourceFacet(facet).outcome).toBe('unknown'); + }, + ); + + it('reports unknown while an operation is in flight (applying)', () => { + const facet: SourceFacet = { ...identity, status: 'applying', activeOperationId: 'op-1', activeGenerationId: 'gen-1' }; + expect(outcomeFromSourceFacet(facet).outcome).toBe('unknown'); + }); +}); diff --git a/backend/src/__tests__/gitops-reconcile-attempts.test.ts b/backend/src/__tests__/gitops-reconcile-attempts.test.ts new file mode 100644 index 00000000..9f1699ed --- /dev/null +++ b/backend/src/__tests__/gitops-reconcile-attempts.test.ts @@ -0,0 +1,387 @@ +/** + * Durable reconcile-attempt reservation and settlement: a bare history + * insert in its own transaction, never through mutateApp, so a reservation + * writes no application-row state and can be safely repeated. + */ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { GitOpsStore } from '../services/gitops/store'; +import { GitOpsTransitions, type EventEnvelope } from '../services/gitops/transitions'; +import { DatabaseService } from '../services/DatabaseService'; +import type { GitOpsApplicationRow } from '../services/gitops/types'; + +let tmpDir: string; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + GitOpsStore.resetForTests(); + GitOpsTransitions.resetForTests(); +}); + +afterAll(() => { + cleanupTestDb(tmpDir); +}); + +describe('reconcile attempt reservation and settlement', () => { + it('reserves an attempt without touching application state', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-res', 'res-web'), nodeId: 1, envelope: env('op-act-res') }); + const before = store.getApplication('app-res')!; + + const result = tx.reserveReconcileAttempt('app-res', env('op-res-1')); + + expect(result.reserved).toBe(true); + const after = store.getApplication('app-res')!; + expect(after.updated_at).toBe(before.updated_at); + expect(after.desired_commit_sha).toBe(before.desired_commit_sha); + }); + + it('returns reserved: false on a repeated reservation for the same operation', () => { + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-res2', 'res2-web'), nodeId: 1, envelope: env('op-act-res2') }); + + const first = tx.reserveReconcileAttempt('app-res2', env('op-res2-1')); + const second = tx.reserveReconcileAttempt('app-res2', env('op-res2-1')); + + expect(first.reserved).toBe(true); + expect(second.reserved).toBe(false); + }); + + it('allows two different operations to each reserve their own attempt', () => { + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-res3', 'res3-web'), nodeId: 1, envelope: env('op-act-res3') }); + + const first = tx.reserveReconcileAttempt('app-res3', env('op-res3-a')); + const second = tx.reserveReconcileAttempt('app-res3', env('op-res3-b')); + + expect(first.reserved).toBe(true); + expect(second.reserved).toBe(true); + }); + + it('settles a reserved attempt and finds it by operation id afterward', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-settle', 'settle-web'), nodeId: 1, envelope: env('op-act-settle') }); + tx.reserveReconcileAttempt('app-settle', env('op-settle-1')); + + const settleResult = tx.settleReconcileAttempt('app-settle', env('op-settle-1'), { + outcome: 'no_source_change', + reason: 'Nothing new to fetch.', + nextAction: 'none', + }); + + expect(settleResult.settled).toBe(true); + const settled = store.getSettledAttempt('app-settle', 'op-settle-1'); + expect(settled).toBeDefined(); + }); + + it('settling twice for the same operation is a no-op the second time', () => { + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-settle2', 'settle2-web'), nodeId: 1, envelope: env('op-act-settle2') }); + tx.reserveReconcileAttempt('app-settle2', env('op-settle2-1')); + + const first = tx.settleReconcileAttempt('app-settle2', env('op-settle2-1'), { + outcome: 'no_source_change', + reason: 'first', + nextAction: 'none', + }); + const second = tx.settleReconcileAttempt('app-settle2', env('op-settle2-1'), { + outcome: 'no_source_change', + reason: 'second, must not overwrite', + nextAction: 'none', + }); + + expect(first.settled).toBe(true); + expect(second.settled).toBe(false); + }); + + it('has no settled attempt for a reservation that was never settled', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-orphan', 'orphan-web'), nodeId: 1, envelope: env('op-act-orphan') }); + tx.reserveReconcileAttempt('app-orphan', env('op-orphan-1')); + + expect(store.getSettledAttempt('app-orphan', 'op-orphan-1')).toBeUndefined(); + }); + + it('lists an unsettled reservation but not one that has settled', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-unsettled', 'unsettled-web'), nodeId: 1, envelope: env('op-act-unsettled') }); + tx.reserveReconcileAttempt('app-unsettled', env('op-unsettled-orphan')); + tx.reserveReconcileAttempt('app-unsettled', env('op-unsettled-done')); + tx.settleReconcileAttempt('app-unsettled', env('op-unsettled-done'), { + outcome: 'no_source_change', + reason: 'done', + nextAction: 'none', + }); + + const unsettled = store.listUnsettledReconcileAttempts(); + const operationIds = unsettled.filter((r) => r.application_id === 'app-unsettled').map((r) => r.operation_id); + expect(operationIds).toContain('op-unsettled-orphan'); + expect(operationIds).not.toContain('op-unsettled-done'); + }); + + it('gets the started row for one exact attempt, or undefined when it was never reserved', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-started', 'started-web'), nodeId: 1, envelope: env('op-act-started') }); + tx.reserveReconcileAttempt('app-started', env('op-started-1')); + + expect(store.getStartedAttempt('app-started', 'op-started-1')).toBeDefined(); + expect(store.getStartedAttempt('app-started', 'op-never-reserved')).toBeUndefined(); + }); + + it('pages past a permanently unsettled row instead of returning it forever on every call with the same cursor', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-cursor', 'cursor-web'), nodeId: 1, envelope: env('op-act-cursor') }); + tx.reserveReconcileAttempt('app-cursor', { operationId: 'op-cursor-stuck', actor: 'tester', trigger: 'manual', at: 1_000 }); + tx.reserveReconcileAttempt('app-cursor', { operationId: 'op-cursor-next', actor: 'tester', trigger: 'manual', at: 2_000 }); + + const firstPage = store.listUnsettledReconcileAttempts(1); + expect(firstPage.map((r) => r.operation_id)).toEqual(['op-cursor-stuck']); + const cursor = { createdAt: firstPage[0].created_at, id: firstPage[0].id }; + // Simulate op-cursor-stuck being permanently unrecoverable: it is never + // settled, so a caller must page past it using the cursor rather than + // seeing it again on the next call. + const secondPage = store.listUnsettledReconcileAttempts(1, cursor); + expect(secondPage.map((r) => r.operation_id)).toEqual(['op-cursor-next']); + }); + + it('allocates a fresh attemptSeq-derived operation id and reserves it in one transaction', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-alloc', 'alloc-web'), nodeId: 1, envelope: env('op-act-alloc') }); + const before = store.getApplication('app-alloc')!; + + const first = tx.allocateReconcileAttempt('app-alloc', 'tester', 'manual', Date.now()); + const second = tx.allocateReconcileAttempt('app-alloc', 'tester', 'manual', Date.now()); + + expect(first.reserved).toBe(true); + expect(second.reserved).toBe(true); + expect(first.operationId).not.toBe(second.operationId); + const after = store.getApplication('app-alloc')!; + expect(after.attempt_seq).toBe(before.attempt_seq + 2); + }); + + it('rolls back the allocated sequence when reservation insertion fails', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-alloc-rollback', 'alloc-rollback-web'), nodeId: 1, envelope: env('op-act-alloc-rollback') }); + const db = DatabaseService.getInstance().getDb(); + const before = store.getApplication('app-alloc-rollback')!; + db.exec(` + CREATE TRIGGER fail_reconcile_reservation + BEFORE INSERT ON gitops_history + WHEN NEW.stage = 'source_reconcile_started' + BEGIN + SELECT RAISE(ABORT, 'simulated reservation insert failure'); + END + `); + + try { + expect(() => tx.allocateReconcileAttempt('app-alloc-rollback', 'tester', 'manual', Date.now())) + .toThrow('simulated reservation insert failure'); + expect(store.getApplication('app-alloc-rollback')!.attempt_seq).toBe(before.attempt_seq); + expect(store.listUnsettledReconcileAttempts().some((row) => row.application_id === 'app-alloc-rollback')).toBe(false); + } finally { + db.exec('DROP TRIGGER fail_reconcile_reservation'); + } + }); + + it('records a follower link on a reservation made on behalf of a coalesced request', () => { + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-follower', 'follower-web'), nodeId: 1, envelope: env('op-act-follower') }); + + const leader = tx.allocateReconcileAttempt('app-follower', 'tester', 'manual', Date.now()); + const follower = tx.allocateReconcileAttempt('app-follower', 'tester', 'manual', Date.now(), leader.operationId); + + expect(follower.reserved).toBe(true); + const started = DatabaseService.getInstance().getDb() + .prepare("SELECT after_json FROM gitops_history WHERE application_id = ? AND operation_id = ? AND stage = 'source_reconcile_started'") + .get('app-follower', follower.operationId) as { after_json: string }; + expect(JSON.parse(started.after_json)).toEqual({ followerOf: leader.operationId }); + }); + + it('records the original webhook delivery intent on its stable reservation', () => { + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-delivery-intent', 'delivery-intent-web'), nodeId: 1, envelope: env('op-act-delivery-intent') }); + + tx.reserveReconcileAttempt( + 'app-delivery-intent', + env('webhook:fetch:delivery-intent'), + undefined, + { autoApply: true, deploy: false }, + ); + + const started = GitOpsStore.getInstance().getStartedAttempt('app-delivery-intent', 'webhook:fetch:delivery-intent')!; + expect(JSON.parse(started.after_json)).toEqual({ + deliveryIntent: { autoApply: true, deploy: false }, + }); + }); + + it('reports the most recently settled attempt even when both share the same millisecond timestamp', () => { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + tx.activateDirect({ application: app('app-latest', 'latest-web'), nodeId: 1, envelope: env('op-act-latest') }); + // Same `at` on purpose: created_at alone cannot tell these two apart, + // so the query must break the tie by insertion order (rowid), not the + // id column, which is a random UUID unrelated to recency. + const sameInstant = 5_000; + tx.reserveReconcileAttempt('app-latest', { operationId: 'op-latest-1', actor: 'tester', trigger: 'manual', at: sameInstant }); + tx.settleReconcileAttempt( + 'app-latest', + { operationId: 'op-latest-1', actor: 'tester', trigger: 'manual', at: sameInstant }, + { outcome: 'no_source_change', reason: 'first', nextAction: 'none' }, + ); + tx.reserveReconcileAttempt('app-latest', { operationId: 'op-latest-2', actor: 'tester', trigger: 'manual', at: sameInstant }); + tx.settleReconcileAttempt( + 'app-latest', + { operationId: 'op-latest-2', actor: 'tester', trigger: 'manual', at: sameInstant }, + { outcome: 'retry_scheduled', reason: 'second', nextAction: 'none' }, + ); + + const latest = store.latestSettledAttempt('app-latest'); + expect(latest?.operation_id).toBe('op-latest-2'); + }); +}); + +describe('poll and retry eligibility queries', () => { + it('lists a source whose next_poll_at has arrived', () => { + const store = GitOpsStore.getInstance(); + store.insertApplication({ ...app('app-poll-due', 'poll-due-web'), next_poll_at: 1_000 }); + const due = store.listSourcesDueForPoll(1_000); + expect(due.map((a) => a.id)).toContain('app-poll-due'); + }); + + it('excludes a source whose next_poll_at has not arrived yet', () => { + const store = GitOpsStore.getInstance(); + store.insertApplication({ ...app('app-poll-future', 'poll-future-web'), next_poll_at: 5_000 }); + const due = store.listSourcesDueForPoll(1_000); + expect(due.map((a) => a.id)).not.toContain('app-poll-future'); + }); + + it('excludes a suspended source even when its poll time has arrived', () => { + const store = GitOpsStore.getInstance(); + store.insertApplication({ ...app('app-poll-susp', 'poll-susp-web'), next_poll_at: 1_000, suspended_at: 500 }); + const due = store.listSourcesDueForPoll(1_000); + expect(due.map((a) => a.id)).not.toContain('app-poll-susp'); + }); + + it('excludes a source with an operation already in flight', () => { + const store = GitOpsStore.getInstance(); + store.insertApplication({ + ...app('app-poll-busy', 'poll-busy-web'), + next_poll_at: 1_000, + active_operation_stage: 'fetch_started', + }); + const due = store.listSourcesDueForPoll(1_000); + expect(due.map((a) => a.id)).not.toContain('app-poll-busy'); + }); + + it('excludes a Blueprint-mode application from polling', () => { + const store = GitOpsStore.getInstance(); + store.insertApplication({ + ...app('app-poll-bp', 'unused-bp'), + stack_name: null, + blueprint_id: 42, + target_mode: 'blueprint', + configured_repo_url: 'https://github.com/org/repo.git', + next_poll_at: 1_000, + }); + const due = store.listSourcesDueForPoll(1_000); + expect(due.map((a) => a.id)).not.toContain('app-poll-bp'); + }); + + it('lists an application whose retry_at has arrived', () => { + const store = GitOpsStore.getInstance(); + store.insertApplication({ ...app('app-retry-due', 'retry-due-web'), retry_at: 1_000 }); + const due = store.listApplicationsDueForRetry(1_000); + expect(due.map((a) => a.id)).toContain('app-retry-due'); + }); + + it('excludes an application with no retry scheduled', () => { + const store = GitOpsStore.getInstance(); + store.insertApplication(app('app-retry-none', 'retry-none-web')); + const due = store.listApplicationsDueForRetry(1_000); + expect(due.map((a) => a.id)).not.toContain('app-retry-none'); + }); + + it('excludes a suspended application even when its retry time has arrived', () => { + const store = GitOpsStore.getInstance(); + store.insertApplication({ ...app('app-retry-susp', 'retry-susp-web'), retry_at: 1_000, suspended_at: 500 }); + const due = store.listApplicationsDueForRetry(1_000); + expect(due.map((a) => a.id)).not.toContain('app-retry-susp'); + }); +}); + +function env(operationId: string): EventEnvelope { + return { operationId, actor: 'tester', trigger: 'manual', at: Date.now() }; +} + +function app(id: string, stackName: string): GitOpsApplicationRow { + return { + id, + lifecycle_key: `direct:${stackName}`, + lifecycle_status: 'active', + target_mode: 'direct', + stack_name: stackName, + blueprint_id: null, + configured_repo_url: 'https://github.com/org/repo.git', + repo_identity_json: '{"host":"github.com","pathname":"/org/repo.git"}', + configured_ref: 'main', + compose_paths_json: '["compose.yml"]', + context_dir: null, + sync_env: 0, + env_path: null, + materialization_fingerprint: 'a'.repeat(64), + desired_commit_sha: null, + fetched_commit_sha: null, + fetched_resolved_ref_kind: null, + candidate_generation_id: null, + accepted_generation_id: null, + candidate_plan_blocked: 0, + review_required: 0, + artifact_set_id: null, + latest_artifact_set_id: null, + intent_revision_id: null, + rollout_candidate_id: null, + rollout_generation_id: null, + source_acceptance_ref: null, + placement_approval_ref: null, + rollout_authorization_ref: null, + legacy_combined_approval_ref: null, + preflight_fingerprint: null, + latest_operation_id: null, + active_operation_id: null, + active_operation_stage: null, + active_operation_at: null, + active_generation_id: null, + pause_at: null, + pause_reason: null, + source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, + partial_json: null, + failure_stage: null, + failure_class: null, + failure_at: null, + retry_at: null, + retry_count: 0, + suspended_at: null, + recovery_ref: null, + recovery_phase: null, + interruption_stage: null, + interruption_at: null, + interruption_operation_id: null, + interruption_generation_id: null, + evidence_fresh_at: null, + evidence_limitations_json: null, + created_at: 1, + updated_at: 1, + }; +} diff --git a/backend/src/__tests__/gitops-recovery-capture.test.ts b/backend/src/__tests__/gitops-recovery-capture.test.ts index ee8b2957..ef76d704 100644 --- a/backend/src/__tests__/gitops-recovery-capture.test.ts +++ b/backend/src/__tests__/gitops-recovery-capture.test.ts @@ -147,6 +147,10 @@ function app(id: string, stackName: string): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, @@ -189,6 +193,12 @@ function gen(id: string, applicationId: string): GitOpsGenerationRow { actor: 'tester', previous_generation_id: null, redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, created_at: 1, }; } diff --git a/backend/src/__tests__/gitops-recovery.test.ts b/backend/src/__tests__/gitops-recovery.test.ts index eeb3a510..e754cf71 100644 --- a/backend/src/__tests__/gitops-recovery.test.ts +++ b/backend/src/__tests__/gitops-recovery.test.ts @@ -461,6 +461,10 @@ function app(id: string, stackName: string): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, @@ -503,6 +507,12 @@ function gen(id: string, applicationId: string): GitOpsGenerationRow { actor: 'tester', previous_generation_id: null, redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, created_at: 1, }; } diff --git a/backend/src/__tests__/gitops-schema.test.ts b/backend/src/__tests__/gitops-schema.test.ts index ca2fa446..11dbcd45 100644 --- a/backend/src/__tests__/gitops-schema.test.ts +++ b/backend/src/__tests__/gitops-schema.test.ts @@ -239,6 +239,61 @@ describe('gitops schema', () => { expect(row?.configured_ref).toBe('v1'); }); + it('round-trips the portable generation contract fields, defaulting to null for legacy rows', async () => { + const store = GitOpsStore.getInstance(); + store.insertApplication(directApp('app-portable', 'portable-web')); + store.insertGeneration({ + ...generation('gen-portable-legacy', 'app-portable'), + }); + const legacy = store.getGeneration('gen-portable-legacy'); + expect(legacy?.portable_manifest_json).toBeNull(); + expect(legacy?.compose_inputs_json).toBeNull(); + expect(legacy?.source_policy_evidence_json).toBeNull(); + expect(legacy?.security_policy_evidence_json).toBeNull(); + expect(legacy?.support_requirements_json).toBeNull(); + expect(legacy?.compatibility_requirements_json).toBeNull(); + + store.insertGeneration({ + ...generation('gen-portable-new', 'app-portable'), + portable_manifest_json: '{"files":[]}', + compose_inputs_json: '{"composeFileOrder":["compose.yaml"]}', + source_policy_evidence_json: '{"policy":"manual"}', + security_policy_evidence_json: '{"status":"allowed"}', + support_requirements_json: '{}', + compatibility_requirements_json: '{}', + }); + const populated = store.getGeneration('gen-portable-new'); + expect(populated?.portable_manifest_json).toBe('{"files":[]}'); + expect(populated?.compose_inputs_json).toBe('{"composeFileOrder":["compose.yaml"]}'); + expect(populated?.source_policy_evidence_json).toBe('{"policy":"manual"}'); + }); + + it('defaults controller-owned columns to manual, off, and zero on a fresh application', async () => { + const store = GitOpsStore.getInstance(); + store.insertApplication(directApp('app-ctrl', 'ctrl-web')); + const app = store.getApplication('app-ctrl'); + expect(app?.source_policy).toBe('manual'); + expect(app?.poll_interval_secs).toBeNull(); + expect(app?.next_poll_at).toBeNull(); + expect(app?.attempt_seq).toBe(0); + }); + + it('round-trips a configured poll interval and policy', async () => { + const store = GitOpsStore.getInstance(); + store.insertApplication({ + ...directApp('app-ctrl2', 'ctrl2-web'), + source_policy: 'automatic', + poll_interval_secs: 120, + next_poll_at: 5000, + attempt_seq: 3, + }); + const app = store.getApplication('app-ctrl2'); + expect(app?.source_policy).toBe('automatic'); + expect(app?.poll_interval_secs).toBe(120); + expect(app?.next_poll_at).toBe(5000); + expect(app?.attempt_seq).toBe(3); + }); + it('round-trips fetched_resolved_ref_kind on application fetch transitions', async () => { const store = GitOpsStore.getInstance(); store.insertApplication(directApp('app-fetch-kind', 'fetch-web')); @@ -294,6 +349,10 @@ function directApp(id: string, stackName: string): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, @@ -362,6 +421,12 @@ function generation(id: string, applicationId: string): GitOpsGenerationRow { actor: 'tester', previous_generation_id: null, redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, created_at: 1, }; } diff --git a/backend/src/__tests__/gitops-transitions.test.ts b/backend/src/__tests__/gitops-transitions.test.ts index 0d65150d..8cc883ae 100644 --- a/backend/src/__tests__/gitops-transitions.test.ts +++ b/backend/src/__tests__/gitops-transitions.test.ts @@ -813,6 +813,10 @@ function app(id: string, stackName: string): GitOpsApplicationRow { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, @@ -855,6 +859,12 @@ function gen(id: string, applicationId: string): GitOpsGenerationRow { actor: 'tester', previous_generation_id: null, redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, created_at: 1, }; } diff --git a/backend/src/__tests__/gitops-triggers.test.ts b/backend/src/__tests__/gitops-triggers.test.ts new file mode 100644 index 00000000..7577e5db --- /dev/null +++ b/backend/src/__tests__/gitops-triggers.test.ts @@ -0,0 +1,85 @@ +/** + * Pure trigger normalization and coalescing-key coverage for the GitOps + * source controller. No DB, no store: these are the identity/joining rules + * a controller submission goes through before anything durable happens. + */ +import { describe, it, expect } from 'vitest'; +import { coalesceKey, deliveryKey, type ReconcileRequest } from '../services/gitops/triggers'; + +function fetchRequest(overrides: Partial> = {}): ReconcileRequest { + return { + intent: 'fetch', + applicationId: 'app-1', + stackName: 'web', + trigger: 'manual', + actor: 'tester', + ...overrides, + }; +} + +function applyRequest(overrides: Partial> = {}): ReconcileRequest { + return { + intent: 'apply', + applicationId: 'app-1', + stackName: 'web', + trigger: 'manual', + actor: 'tester', + commitSha: 'a'.repeat(40), + planFingerprint: 'fp-1', + deploy: false, + ...overrides, + }; +} + +describe('coalesceKey', () => { + it('joins two fetch requests for the same application', () => { + expect(coalesceKey(fetchRequest())).toBe(coalesceKey(fetchRequest({ trigger: 'poll' }))); + }); + + it('does not join fetch requests for different applications', () => { + expect(coalesceKey(fetchRequest({ applicationId: 'app-1' }))) + .not.toBe(coalesceKey(fetchRequest({ applicationId: 'app-2' }))); + }); + + it('joins two apply requests with identical commit, fingerprint, and deploy flag', () => { + expect(coalesceKey(applyRequest())).toBe(coalesceKey(applyRequest({ trigger: 'webhook' }))); + }); + + it('does not join two applies with different plan fingerprints', () => { + expect(coalesceKey(applyRequest({ planFingerprint: 'fp-1' }))) + .not.toBe(coalesceKey(applyRequest({ planFingerprint: 'fp-2' }))); + }); + + it('does not join two applies with different commits', () => { + expect(coalesceKey(applyRequest({ commitSha: 'a'.repeat(40) }))) + .not.toBe(coalesceKey(applyRequest({ commitSha: 'b'.repeat(40) }))); + }); + + it('does not join two applies that differ only in deploy', () => { + expect(coalesceKey(applyRequest({ deploy: false }))) + .not.toBe(coalesceKey(applyRequest({ deploy: true }))); + }); + + it('never joins a fetch and an apply for the same application', () => { + expect(coalesceKey(fetchRequest())).not.toBe(coalesceKey(applyRequest())); + }); + + it('does not join two fetches for the same applicationId but different stack names', () => { + expect(coalesceKey(fetchRequest({ stackName: 'web' }))) + .not.toBe(coalesceKey(fetchRequest({ stackName: 'other-stack' }))); + }); +}); + +describe('deliveryKey', () => { + it('namespaces the same delivery id differently per trigger', () => { + expect(deliveryKey('webhook', 'fetch', 'delivery-1')).not.toBe(deliveryKey('api', 'fetch', 'delivery-1')); + }); + + it('namespaces the same delivery id differently per intent', () => { + expect(deliveryKey('webhook', 'fetch', 'delivery-1')).not.toBe(deliveryKey('webhook', 'apply', 'delivery-1')); + }); + + it('is stable for the same trigger, intent, and delivery id', () => { + expect(deliveryKey('webhook', 'fetch', 'delivery-1')).toBe(deliveryKey('webhook', 'fetch', 'delivery-1')); + }); +}); diff --git a/backend/src/__tests__/helpers/gitopsFixtures.ts b/backend/src/__tests__/helpers/gitopsFixtures.ts index a80878ed..71d08ebc 100644 --- a/backend/src/__tests__/helpers/gitopsFixtures.ts +++ b/backend/src/__tests__/helpers/gitopsFixtures.ts @@ -57,6 +57,10 @@ export function directApplicationFixture(id: string, stackName: string): GitOpsA pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/__tests__/monitor-service.test.ts b/backend/src/__tests__/monitor-service.test.ts index a35e387b..2a1bea18 100644 --- a/backend/src/__tests__/monitor-service.test.ts +++ b/backend/src/__tests__/monitor-service.test.ts @@ -1621,6 +1621,41 @@ describe('MonitorService - Sencho dev build check', () => { expect(mockDispatchAlert).not.toHaveBeenCalledWith('info', 'node_update_available', expect.anything()); }); + + // C1: update eligibility is compose-declared. A dev *running* identity must + // not make a stable-declared pin eligible for the dev-build detector, and an + // unknown running identity must still gate before any registry call. These + // two disagreement directions pin that the running identity surfaced by + // SelfIdentityService never changes update behavior. + it('does not treat a stable-declared pin as eligible even when the running identity is a dev build', async () => { + // beforeEach() already mints a dev running imageId; the declared pin is stable. + // Reset the version-update inputs: an earlier suppression test queues a + // once-value that a stable-pin run would otherwise drain into the version + // path, which is not what this test observes. + mockGetLatestVersionInfo.mockReset(); + mockGetPinInfo.mockResolvedValue(STABLE_PIN); + + await runEvaluate(); + + expect(mockDetectSelfDevBuildUpdate).not.toHaveBeenCalled(); + expect(devBuildCalls()).toHaveLength(0); + expect(mockSetSystemState).not.toHaveBeenCalledWith('sencho_dev_build_available_digest', expect.anything()); + }); + + it('still gates the registry comparison on a known running image id for an eligible dev-declared pin', async () => { + mockGetPinInfo.mockResolvedValue(DEV_PIN); + mockGetIdentity.mockReturnValue({ + containerId: null, containerName: null, composeProjectName: null, + imageId: null, networkNames: [], volumeNames: [], + }); + + await runEvaluate(); + + expect(mockDetectSelfDevBuildUpdate).not.toHaveBeenCalled(); + expect(devBuildCalls()).toHaveLength(0); + // Unknown running id takes the retry-sooner path, not the detector path. + expect((MonitorService.getInstance() as any).lastDevBuildCheckGateMs).toBe(5 * 60 * 1000); + }); }); // ── Per-container parallel fan-out ──────────────────────────────────── diff --git a/backend/src/__tests__/self-identity-buildinfo.test.ts b/backend/src/__tests__/self-identity-buildinfo.test.ts new file mode 100644 index 00000000..e5445e5b --- /dev/null +++ b/backend/src/__tests__/self-identity-buildinfo.test.ts @@ -0,0 +1,197 @@ +/** + * Unit tests for SelfIdentityService.getBuildInfo(): the canonical runtime + * build identity (version, channel, imageRef, imageId, revision), the detached + * bounded revision enrichment, and the failure-isolation guarantee. + */ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + +// ── Hoisted mocks ────────────────────────────────────────────────────── + +const { mockContainer, mockDocker, mockInspectImage, mockGetSenchoVersion } = vi.hoisted(() => { + const mockContainer = { inspect: vi.fn() }; + const mockDocker = { + getContainer: vi.fn(() => mockContainer), + getImage: vi.fn(), + listImages: vi.fn().mockResolvedValue([]), + listVolumes: vi.fn().mockResolvedValue({ Volumes: [] }), + listNetworks: vi.fn().mockResolvedValue([]), + listContainers: vi.fn().mockResolvedValue([]), + }; + return { + mockContainer, + mockDocker, + mockInspectImage: vi.fn(), + mockGetSenchoVersion: vi.fn(() => '0.97.1'), + }; +}); + +vi.mock('../services/NodeRegistry', () => ({ + NodeRegistry: { + getInstance: () => ({ + getDocker: () => mockDocker, + getDefaultNodeId: () => 1, + }), + }, +})); + +// Replace defaultInspectImage so enrichment is deterministic and isolated. +vi.mock('../services/selfDevBuildDetect', () => ({ + defaultInspectImage: (...args: unknown[]) => mockInspectImage(...args), +})); + +vi.mock('../services/CapabilityRegistry', () => ({ + getSenchoVersion: () => mockGetSenchoVersion(), +})); + +vi.mock('child_process', () => ({ exec: vi.fn(), execFile: vi.fn() })); +vi.mock('util', () => ({ promisify: () => vi.fn() })); + +import SelfIdentityService from '../services/SelfIdentityService'; + +const FULL_IMAGE_ID_HEX = 'b'.repeat(64); +const DIGEST = 'a'.repeat(64); + +const originalHostname = process.env.HOSTNAME; + +beforeEach(() => { + vi.clearAllMocks(); + mockContainer.inspect.mockReset(); + mockDocker.getImage.mockReset(); + mockInspectImage.mockReset(); + mockGetSenchoVersion.mockReturnValue('0.97.1'); + SelfIdentityService.getInstance().resetForTesting(); +}); + +afterEach(() => { + vi.restoreAllMocks(); + if (originalHostname === undefined) delete process.env.HOSTNAME; + else process.env.HOSTNAME = originalHostname; +}); + +async function initWith(configImage: string | undefined): Promise { + process.env.HOSTNAME = 'sencho-1'; + mockContainer.inspect.mockResolvedValue({ + Id: 'a'.repeat(64), + Name: '/sencho', + Image: 'sha256:' + FULL_IMAGE_ID_HEX, + ...(configImage !== undefined ? { Config: { Image: configImage } } : {}), + NetworkSettings: { Networks: {} }, + Mounts: [], + }); + const svc = SelfIdentityService.getInstance(); + await svc.initialize(); + return svc; +} + +/** Enrichment runs detached; poll until the condition holds so assertions are stable. */ +async function until(assert: () => void): Promise { + await vi.waitFor(assert, { timeout: 2000 }); +} + +describe('SelfIdentityService.getBuildInfo', () => { + it('identifies a dev image as DEV even when the packaged semver matches the previous stable', async () => { + // The regression case from the ticket: dev image, version still 0.97.1. + // The inspect mock must be set before initialize(): enrichment fires + // detached during initialize(), before the awaited call returns. + mockInspectImage.mockResolvedValue({ + RepoDigests: [`ghcr.io/studio-saelix/sencho-dev@sha256:${DIGEST}`], + Os: 'linux', + Architecture: 'amd64', + }); + const svc = await initWith('ghcr.io/studio-saelix/sencho-dev:dev'); + await until(() => expect(svc.getBuildInfo().revision).toBe(`sha256:${DIGEST}`)); + + const info = svc.getBuildInfo(); + expect(info.version).toBe('0.97.1'); + expect(info.channel).toBe('dev'); + expect(info.imageRef).toBe('ghcr.io/studio-saelix/sencho-dev:dev'); + expect(info.imageId).toBe(FULL_IMAGE_ID_HEX); + }); + + it('derives the revision from a pinned dev- tag without an image inspect', async () => { + const svc = await initWith('ghcr.io/studio-saelix/sencho-dev:dev-abc1234'); + await until(() => expect(svc.getBuildInfo().revision).toBe('dev-abc1234')); + + const info = svc.getBuildInfo(); + expect(info.channel).toBe('dev'); + expect(mockInspectImage).not.toHaveBeenCalled(); + }); + + it('classifies a stable image as stable', async () => { + mockInspectImage.mockResolvedValue({ + RepoDigests: [`ghcr.io/studio-saelix/sencho@sha256:${DIGEST}`], + Os: 'linux', + Architecture: 'amd64', + }); + const svc = await initWith('ghcr.io/studio-saelix/sencho:0.97.1'); + await until(() => expect(svc.getBuildInfo().revision).toBe(`sha256:${DIGEST}`)); + + expect(svc.getBuildInfo().channel).toBe('stable'); + }); + + it('reads unknown for partial metadata (no running image reference)', async () => { + const svc = await initWith(undefined); + + const info = svc.getBuildInfo(); + expect(info.imageRef).toBeNull(); + expect(info.channel).toBe('unknown'); + expect(info.revision).toBeNull(); + expect(info.imageId).toBe(FULL_IMAGE_ID_HEX); + }); + + it('keeps the dev channel but null revision when image inspection fails', async () => { + mockInspectImage.mockRejectedValue(new Error('docker daemon unreachable')); + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const svc = await initWith('ghcr.io/studio-saelix/sencho-dev:dev'); + await until(() => expect(warnSpy).toHaveBeenCalled()); + + const info = svc.getBuildInfo(); + expect(info.channel).toBe('dev'); + expect(info.revision).toBeNull(); + // C2: a failed enrichment must not corrupt the already-captured core identity. + expect(info.imageRef).toBe('ghcr.io/studio-saelix/sencho-dev:dev'); + expect(info.imageId).toBe(FULL_IMAGE_ID_HEX); + }); + + it('never inspects the image again on repeated reads', async () => { + mockInspectImage.mockResolvedValue({ + RepoDigests: [`ghcr.io/studio-saelix/sencho-dev@sha256:${DIGEST}`], + Os: 'linux', + Architecture: 'amd64', + }); + const svc = await initWith('ghcr.io/studio-saelix/sencho-dev:dev'); + await until(() => expect(mockInspectImage).toHaveBeenCalledTimes(1)); + + for (let i = 0; i < 5; i++) svc.getBuildInfo(); + expect(mockInspectImage).toHaveBeenCalledTimes(1); + }); + + it('exposes revision only after enrichment settles, and whenRevisionResolved awaits that', async () => { + let resolveInspect!: (v: { RepoDigests: string[]; Os: string; Architecture: string }) => void; + mockInspectImage.mockReturnValue(new Promise((res) => { resolveInspect = res; })); + + const svc = await initWith('ghcr.io/studio-saelix/sencho-dev:dev'); + // initialize() returned without awaiting the detached enrichment, so the + // revision is still transiently null and the settle promise is pending. + expect(svc.getBuildInfo().revision).toBeNull(); + + // A reader that awaits the settle promise (the build-info route) blocks + // until enrichment lands, then observes the resolved digest, so a single + // successful read never freezes a transient null. + const settled = svc.whenRevisionResolved(); + resolveInspect({ + RepoDigests: [`ghcr.io/studio-saelix/sencho-dev@sha256:${DIGEST}`], + Os: 'linux', + Architecture: 'amd64', + }); + await settled; + expect(svc.getBuildInfo().revision).toBe(`sha256:${DIGEST}`); + }); + + it('resolves whenRevisionResolved immediately when no enrichment ever started', async () => { + process.env.HOSTNAME = undefined; + const svc = SelfIdentityService.getInstance(); + await svc.whenRevisionResolved(); + expect(svc.getBuildInfo().revision).toBeNull(); + }); +}); diff --git a/backend/src/__tests__/source-controller.test.ts b/backend/src/__tests__/source-controller.test.ts new file mode 100644 index 00000000..1d9b204e --- /dev/null +++ b/backend/src/__tests__/source-controller.test.ts @@ -0,0 +1,207 @@ +/** + * SourceController: the background driver for unattended reconciliation. + * GitOpsStore's due-queries and GitSourceService.reconcile() are mocked so + * these tests exercise only the timer/coalescing behavior, not real fetch + * or apply mechanics (already covered by git-source-service.test.ts). + */ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; +import { directApplicationFixture } from './helpers/gitopsFixtures'; +import { GitOpsStore } from '../services/gitops/store'; +import { GitSourceService } from '../services/GitSourceService'; +import { SourceController } from '../services/gitops/SourceController'; +import type { GitOpsApplicationRow } from '../services/gitops/types'; +import type { ReconcileResult } from '../services/gitops/outcomes'; + +const TICK_MS = 60_000; +const okResult: ReconcileResult = { outcome: 'no_source_change', reason: 'ok', nextAction: 'none' }; + +let tmpDir: string; +let controller: SourceController; + +/** Point both due-queries at fixed rows; the scan reads nothing else. */ +function mockDue(duePoll: GitOpsApplicationRow[], dueRetry: GitOpsApplicationRow[] = []): void { + vi.spyOn(GitOpsStore.getInstance(), 'listSourcesDueForPoll').mockReturnValue(duePoll); + vi.spyOn(GitOpsStore.getInstance(), 'listApplicationsDueForRetry').mockReturnValue(dueRetry); +} + +function spyOnReconcile() { + return vi.spyOn(GitSourceService.getInstance(), 'reconcile'); +} + +/** Run the next scheduled tick and let the evaluations it fires settle. */ +async function advanceOneTick(): Promise { + await vi.advanceTimersByTimeAsync(TICK_MS); +} + +beforeAll(async () => { + tmpDir = await setupTestDb(); + GitOpsStore.resetForTests(); +}); + +afterAll(() => { + cleanupTestDb(tmpDir); +}); + +beforeEach(() => { + vi.useFakeTimers(); + SourceController.resetForTests(); + controller = SourceController.getInstance(); +}); + +afterEach(() => { + controller.stop(); + vi.restoreAllMocks(); + vi.useRealTimers(); +}); + +describe('SourceController', () => { + it('evaluates a source whose poll interval is due', async () => { + mockDue([directApplicationFixture('app-poll', 'poll-web')]); + const reconcile = spyOnReconcile().mockResolvedValue(okResult); + + controller.start(); + await advanceOneTick(); + + expect(reconcile).toHaveBeenCalledWith(expect.objectContaining({ + intent: 'fetch', + applicationId: 'app-poll', + stackName: 'poll-web', + trigger: 'poll', + })); + }); + + it('evaluates an application whose retry_at has arrived, tagged as a retry trigger', async () => { + const app = { ...directApplicationFixture('app-retry', 'retry-web'), retry_at: Date.now() - 1_000 }; + mockDue([], [app]); + const reconcile = spyOnReconcile().mockResolvedValue(okResult); + + controller.start(); + await advanceOneTick(); + + expect(reconcile).toHaveBeenCalledWith(expect.objectContaining({ + applicationId: 'app-retry', + trigger: 'retry', + })); + }); + + it('evaluates an application due for both poll and retry exactly once', async () => { + const app = { ...directApplicationFixture('app-both', 'both-web'), retry_at: Date.now() - 1_000 }; + mockDue([app], [app]); + const reconcile = spyOnReconcile().mockResolvedValue(okResult); + + controller.start(); + await advanceOneTick(); + + expect(reconcile).toHaveBeenCalledTimes(1); + }); + + it('does not re-evaluate an application still in flight from a previous tick', async () => { + mockDue([directApplicationFixture('app-slow', 'slow-web')]); + let settleFirstCall!: (result: ReconcileResult) => void; + const firstCall = new Promise((resolve) => { settleFirstCall = resolve; }); + const reconcile = spyOnReconcile().mockReturnValue(firstCall); + + controller.start(); + await advanceOneTick(); + expect(reconcile).toHaveBeenCalledTimes(1); + + // A second tick fires while the first evaluation is still pending. + await advanceOneTick(); + expect(reconcile).toHaveBeenCalledTimes(1); + + settleFirstCall(okResult); + await Promise.resolve(); + await Promise.resolve(); + + // Now that the first evaluation has settled, a later tick may pick it up again. + await advanceOneTick(); + expect(reconcile).toHaveBeenCalledTimes(2); + }); + + it('recovers on the next tick after a store query throws, rather than dying permanently', async () => { + mockDue([directApplicationFixture('app-recovers', 'recovers-web')]); + vi.spyOn(GitOpsStore.getInstance(), 'listSourcesDueForPoll').mockImplementationOnce(() => { + throw new Error('database is locked'); + }); + const reconcile = spyOnReconcile().mockResolvedValue(okResult); + vi.spyOn(console, 'error').mockImplementation(() => {}); + + controller.start(); + await advanceOneTick(); + expect(reconcile).not.toHaveBeenCalled(); + + await advanceOneTick(); + expect(reconcile).toHaveBeenCalledTimes(1); + }); + + it('releases the in-flight slot for an application whose reconcile rejects', async () => { + mockDue([directApplicationFixture('app-rejects', 'rejects-web')]); + const reconcile = spyOnReconcile().mockRejectedValue(new Error('boom')); + + controller.start(); + await advanceOneTick(); + await advanceOneTick(); + + expect(reconcile).toHaveBeenCalledTimes(2); + }); + + it('does not evaluate anything after stop', async () => { + mockDue([directApplicationFixture('app-stopped', 'stopped-web')]); + const reconcile = spyOnReconcile().mockResolvedValue(okResult); + + controller.start(); + controller.stop(); + await advanceOneTick(); + await advanceOneTick(); + + expect(reconcile).not.toHaveBeenCalled(); + }); + + it('does not double-arm when start() is called reentrantly from within an in-flight evaluation', async () => { + mockDue([directApplicationFixture('app-reentrant-start', 'reentrant-start-web')]); + // tick() nulls `timer` before scanning, so a start() call landing + // synchronously during that scan must not see a false "not running" + // reading and arm a second timer. + spyOnReconcile().mockImplementation(() => { + controller.start(); + return Promise.resolve(okResult); + }); + + controller.start(); + await advanceOneTick(); + + expect(vi.getTimerCount()).toBe(1); + }); + + it('logs rather than silently skipping an application with no stack_name', async () => { + mockDue([{ ...directApplicationFixture('app-no-stack', 'no-stack-web'), stack_name: null }]); + const reconcile = spyOnReconcile().mockResolvedValue(okResult); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + + controller.start(); + await advanceOneTick(); + + expect(reconcile).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('app-no-stack')); + }); + + it('restartPolling never leaves two timers running', () => { + controller.start(); + controller.restartPolling(); + controller.restartPolling(); + + expect(vi.getTimerCount()).toBe(1); + }); + + it('start is a no-op when already running', async () => { + mockDue([directApplicationFixture('app-double-start', 'double-start-web')]); + const reconcile = spyOnReconcile().mockResolvedValue(okResult); + + controller.start(); + controller.start(); + await advanceOneTick(); + + expect(reconcile).toHaveBeenCalledTimes(1); + }); +}); diff --git a/backend/src/__tests__/webhooks-git-source.test.ts b/backend/src/__tests__/webhooks-git-source.test.ts index 5ce0832c..b0bd2a7a 100644 --- a/backend/src/__tests__/webhooks-git-source.test.ts +++ b/backend/src/__tests__/webhooks-git-source.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; import request from 'supertest'; import jwt from 'jsonwebtoken'; +import crypto from 'crypto'; import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb'; let tmpDir: string; @@ -237,4 +238,142 @@ describe('node-aware Git source webhooks', () => { expect(history[0].status).toBe('success'); expect(history[0].error).toMatch(/debounced/i); }); + + it('forwards a provider delivery id to a remote node under the webhook namespace', async () => { + const db = DatabaseService.getInstance(); + const remoteNodeId = db.addNode({ + name: 'remote-delivery-id-webhook', + type: 'remote', + compose_dir: '/tmp', + is_default: false, + api_url: 'http://remote-delivery.example', + api_token: 'remote-token', + }); + const webhookId = db.addWebhook({ + node_id: remoteNodeId, + name: 'delivery id remote git', + stack_name: 'remote-stack', + action: 'git-pull', + secret: WebhookService.getInstance().generateSecret(), + enabled: true, + }); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response(JSON.stringify({ status: 'success', message: 'Fetched.' }), { status: 200 }), + ); + + const webhook = db.getWebhook(webhookId)!; + const deliverySourceId = db.getGlobalSettings().delivery_source_id; + const result = await WebhookService.getInstance().execute( + webhook, + 'git-pull', + 'test', + undefined, + 'provider-delivery-1', + ); + + expect(result.success).toBe(true); + expect(fetchSpy).toHaveBeenCalledWith( + 'http://remote-delivery.example/api/stacks/remote-stack/git-source/webhook-pull', + expect.objectContaining({ + method: 'POST', + body: expect.stringContaining(`"deliveryId":"webhook:${deliverySourceId}:${webhookId}:provider-delivery-1"`), + }), + ); + }); + + it('passes the same producer-scoped delivery identity to a local Git source', async () => { + const db = DatabaseService.getInstance(); + const nodeId = db.getDefaultNode()!.id; + const webhookId = db.addWebhook({ + node_id: nodeId, + name: 'delivery id local git', + stack_name: 'local-delivery-stack', + action: 'git-pull', + secret: WebhookService.getInstance().generateSecret(), + enabled: true, + }); + const webhook = db.getWebhook(webhookId)!; + const deliverySourceId = db.getGlobalSettings().delivery_source_id; + const { FileSystemService } = await import('../services/FileSystemService'); + const { GitSourceService } = await import('../services/GitSourceService'); + const stacksSpy = vi.spyOn(FileSystemService.prototype, 'getStacks') + .mockResolvedValue(['local-delivery-stack']); + const pullSpy = vi.spyOn(GitSourceService.getInstance(), 'handleWebhookPull') + .mockResolvedValue({ status: 'success', message: 'Fetched.' }); + + try { + const result = await WebhookService.getInstance().execute( + webhook, + 'git-pull', + 'test', + undefined, + 'provider-delivery-2', + ); + + expect(result).toEqual({ success: true, duration_ms: expect.any(Number) }); + expect(pullSpy).toHaveBeenCalledWith( + 'local-delivery-stack', + true, + `webhook:${deliverySourceId}:${webhookId}:provider-delivery-2`, + ); + + const secondWebhookId = db.addWebhook({ + node_id: nodeId, + name: 'second delivery id local git', + stack_name: 'local-delivery-stack', + action: 'git-pull', + secret: webhook.secret, + enabled: true, + }); + pullSpy.mockClear(); + await WebhookService.getInstance().execute( + db.getWebhook(secondWebhookId)!, + 'git-pull', + 'test', + undefined, + 'provider-delivery-2', + ); + expect(pullSpy).toHaveBeenCalledWith( + 'local-delivery-stack', + true, + `webhook:${deliverySourceId}:${secondWebhookId}:provider-delivery-2`, + ); + + db.updateGlobalSetting('delivery_source_id', 'second-control-source'); + pullSpy.mockClear(); + await WebhookService.getInstance().execute( + webhook, + 'git-pull', + 'test', + undefined, + 'provider-delivery-2', + ); + expect(pullSpy).toHaveBeenCalledWith( + 'local-delivery-stack', + true, + `webhook:second-control-source:${webhookId}:provider-delivery-2`, + ); + db.updateGlobalSetting('delivery_source_id', deliverySourceId!); + + const oversizedDeliveryId = 'x'.repeat(300); + const boundedId = crypto.createHash('sha256').update(oversizedDeliveryId).digest('hex'); + pullSpy.mockClear(); + await WebhookService.getInstance().execute( + webhook, + 'git-pull', + 'test', + undefined, + oversizedDeliveryId, + ); + expect(pullSpy).toHaveBeenCalledWith( + 'local-delivery-stack', + true, + `webhook:${deliverySourceId}:${webhookId}:sha256:${boundedId}`, + ); + } finally { + if (deliverySourceId) db.updateGlobalSetting('delivery_source_id', deliverySourceId); + stacksSpy.mockRestore(); + pullSpy.mockRestore(); + } + }); }); diff --git a/backend/src/__tests__/webhooks-trigger.test.ts b/backend/src/__tests__/webhooks-trigger.test.ts index 977e1b8c..a03dd246 100644 --- a/backend/src/__tests__/webhooks-trigger.test.ts +++ b/backend/src/__tests__/webhooks-trigger.test.ts @@ -251,6 +251,55 @@ describe('POST /api/webhooks/:id/trigger: authenticated happy path', () => { expect(res.body).toMatchObject({ action: 'start' }); }); + it('extracts a recognized provider delivery header and passes it through to execute', async () => { + const { id, secret } = createWebhook({ action: 'stop' }); + const body = '{}'; + const executeSpy = vi.spyOn(WebhookService.getInstance(), 'execute').mockResolvedValue({ success: true, duration_ms: 0 }); + + try { + await request(app) + .post(`/api/webhooks/${id}/trigger`) + .set('Content-Type', 'application/json') + .set('X-Webhook-Signature', sign(body, secret)) + .set('X-GitHub-Delivery', 'gh-delivery-123') + .send(body); + + expect(executeSpy).toHaveBeenCalledWith( + expect.objectContaining({ id }), + 'stop', + expect.anything(), + true, + 'gh-delivery-123', + ); + } finally { + executeSpy.mockRestore(); + } + }); + + it('passes no delivery id through when the caller sends no recognized header', async () => { + const { id, secret } = createWebhook({ action: 'stop' }); + const body = '{}'; + const executeSpy = vi.spyOn(WebhookService.getInstance(), 'execute').mockResolvedValue({ success: true, duration_ms: 0 }); + + try { + await request(app) + .post(`/api/webhooks/${id}/trigger`) + .set('Content-Type', 'application/json') + .set('X-Webhook-Signature', sign(body, secret)) + .send(body); + + expect(executeSpy).toHaveBeenCalledWith( + expect.objectContaining({ id }), + 'stop', + expect.anything(), + true, + undefined, + ); + } finally { + executeSpy.mockRestore(); + } + }); + it('rejects an unknown action override with 400 after the signature passes (L2)', async () => { const { id, secret } = createWebhook(); const body = '{"action":"nuke-the-cluster"}'; diff --git a/backend/src/bootstrap/shutdown.ts b/backend/src/bootstrap/shutdown.ts index 82c62a03..f5951cbd 100644 --- a/backend/src/bootstrap/shutdown.ts +++ b/backend/src/bootstrap/shutdown.ts @@ -9,6 +9,7 @@ import { FleetSyncRetryService } from '../services/FleetSyncRetryService'; import { SuppressionRetractionRetryService } from '../services/SuppressionRetractionRetryService'; import { DockerEventManager } from '../services/DockerEventManager'; import { ImageUpdateService } from '../services/ImageUpdateService'; +import { SourceController } from '../services/gitops/SourceController'; import { SchedulerService } from '../services/SchedulerService'; import { MfaService } from '../services/MfaService'; import { MeshService } from '../services/MeshService'; @@ -49,6 +50,9 @@ export function installShutdownHandlers(server: Server): void { try { ImageUpdateService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] ImageUpdateService cleanup failed:', (e as Error).message); } + try { SourceController.getInstance().stop(); } catch (e) { + console.warn('[Shutdown] SourceController cleanup failed:', (e as Error).message); + } try { SchedulerService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] SchedulerService cleanup failed:', (e as Error).message); } diff --git a/backend/src/bootstrap/startup.ts b/backend/src/bootstrap/startup.ts index e9453541..d7b97608 100644 --- a/backend/src/bootstrap/startup.ts +++ b/backend/src/bootstrap/startup.ts @@ -28,10 +28,11 @@ import { applyPilotModeCapabilityFilter } from '../services/CapabilityRegistry'; import { PilotTunnelManager } from '../services/PilotTunnelManager'; import { PilotMetrics } from '../services/PilotMetrics'; import { invalidateRemoteMetaCache } from '../helpers/cacheInvalidation'; -import { sweepStaleTempDirs as sweepStaleGitTempDirs, sweepGitManifestOrphans } from '../services/GitSourceService'; +import { GitSourceService, sweepStaleTempDirs as sweepStaleGitTempDirs, sweepGitManifestOrphans } from '../services/GitSourceService'; import { assertCreatesSettled, reclassifyInterruptedOperations, resolveInterruptedCreates } from '../services/gitops/createRecovery'; import { loadMigrationManifests, migrateDirectGitStacks, migrateInlineBlueprints } from '../services/gitops/migrate'; import { setGitOpsEventSink } from '../services/gitops/publish'; +import { SourceController } from '../services/gitops/SourceController'; import { NotificationService } from '../services/NotificationService'; import { sanitizeForLog } from '../utils/safeLog'; import { PORT } from '../helpers/constants'; @@ -98,6 +99,33 @@ function clearSelfContainerNotificationRouting(): void { } } +/** + * Reconcile-attempt recovery, then the managed-area sweep, in that fixed + * order: an attempt reserved but never settled (a crash between the two) + * must be resolved from durable state before the sweep or + * SourceController's own timer (started later in startServer, after + * registry delivery recovery settles per AUD-36) can race a recovery pass + * over the same attempts. Exported so this ordering is directly testable + * without driving the rest of startServer's unrelated service + * initialization. + */ +export async function runGitOpsSourceRecovery(): Promise { + try { + await GitSourceService.getInstance().recoverUnsettledReconcileAttempts(); + } catch (err) { + console.error('[GitSource] Reconcile-attempt recovery failed:', err instanceof Error ? err.stack ?? err.message : String(err)); + } + + // The managed-area sweep follows. It preserves anything whose ownership it + // cannot prove, so a failure here can only leave files behind, never remove + // the wrong ones, and retrying next boot is safe. + try { + await sweepGitManifestOrphans(); + } catch (err) { + console.warn('[GitManifest] Managed-area sweep failed:', err instanceof Error ? err.message : String(err)); + } +} + /** * Run the startup sequence: stack-directory migration, service initialization, * background watchdogs, then bind the HTTP server. The caller passes the @@ -233,14 +261,9 @@ export async function startServer(server: Server): Promise { console.error('[GitOps] Migration of pre-existing blueprints failed:', err instanceof Error ? err.stack ?? err.message : String(err)); } - // The managed-area sweep follows. It preserves anything whose ownership it - // cannot prove, so a failure here can only leave files behind, never remove - // the wrong ones, and retrying next boot is safe. - try { - await sweepGitManifestOrphans(); - } catch (err) { - console.warn('[GitManifest] Managed-area sweep failed:', err instanceof Error ? err.message : String(err)); - } + // Both steps must settle before SourceController starts below; see that + // function's own doc comment for why. + await runGitOpsSourceRecovery(); // Registry delivery recovery sweeps must settle before any mutation-capable // producer starts (AUD-36). @@ -287,6 +310,7 @@ export async function startServer(server: Server): Promise { FleetSyncRetryService.getInstance().start(); SuppressionRetractionRetryService.getInstance().start(); ImageUpdateService.getInstance().start(); + SourceController.getInstance().start(); SchedulerService.getInstance().start(); MfaService.getInstance().start(); MeshService.getInstance().start().catch((err) => { diff --git a/backend/src/helpers/proxyExemptPaths.ts b/backend/src/helpers/proxyExemptPaths.ts index 9dab00f0..96c8cf84 100644 --- a/backend/src/helpers/proxyExemptPaths.ts +++ b/backend/src/helpers/proxyExemptPaths.ts @@ -13,6 +13,7 @@ export const PROXY_EXEMPT_PREFIXES: readonly string[] = [ '/api/fleet/', '/api/webhooks', '/api/meta', + '/api/build-info', ]; /** Returns true when the path should bypass the remote proxy (handled locally). */ diff --git a/backend/src/helpers/selfUpdateCompose.ts b/backend/src/helpers/selfUpdateCompose.ts index 2c5f9c17..2ce0f100 100644 --- a/backend/src/helpers/selfUpdateCompose.ts +++ b/backend/src/helpers/selfUpdateCompose.ts @@ -104,13 +104,50 @@ export function isSenchoDevFloatingTag(imageRef: string): boolean { // A digest pin disqualifies the reference (e.g., `@sha256:...`) if (ref.includes('@sha256:') || ref.startsWith('sha256:')) return false; - // Extract the tag using the same logic as classifyImagePin. + const tag = extractTagFromRef(ref); + return tag === 'dev'; +} + +/** + * Build channel of a running or declared image, derived from the image + * reference alone. This is the canonical build identity classifier used by + * `/api/build-info` and the shell/About surfaces: it answers "is this a dev, + * preview, or stable build" from the ref, independent of the packaged semver + * (a dev image carries the last released version, so version alone cannot + * identify it). + * + * - dev repo (`ghcr.io/studio-saelix/sencho-dev`): always `'dev'`. An + * arbitrary `:dev-` tag still reads `dev` (a deliberate operator + * choice); the immutable revision is surfaced separately from the digest. + * - stable repos (`ghcr.io/studio-saelix/sencho-hardened`, `saelix/sencho`, + * `ghcr.io/studio-saelix/sencho`): `pr-` and `preview-` tags are + * `'preview'` (CI builds on the stable repo that are not releases); + * everything else is `'stable'`. + * - any other repository: `'unknown'`. + */ +export type BuildChannel = 'stable' | 'dev' | 'preview' | 'unknown'; + +export function classifyBuildChannel(imageRef: string): BuildChannel { + const repository = normalizeImageRepository(imageRef); + if (repository === 'ghcr.io/studio-saelix/sencho-dev') return 'dev'; + if ( + repository === 'ghcr.io/studio-saelix/sencho-hardened' || + repository === 'saelix/sencho' || + repository === 'ghcr.io/studio-saelix/sencho' + ) { + const tag = extractTagFromRef(imageRef.trim()); + if (tag && (/^pr-\d+$/.test(tag) || /^preview-[0-9a-f]{7,40}$/.test(tag))) return 'preview'; + return 'stable'; + } + return 'unknown'; +} + +/** Extract the tag portion of an image ref (`.../repo:tag`), or '' when absent. */ +function extractTagFromRef(ref: string): string { const lastSlash = ref.lastIndexOf('/'); const lastColon = ref.lastIndexOf(':'); // A colon after the last slash is a tag separator; before it is a registry port. - const tag = lastColon > lastSlash ? ref.slice(lastColon + 1) : ''; - - return tag === 'dev'; + return lastColon > lastSlash ? ref.slice(lastColon + 1) : ''; } /** diff --git a/backend/src/index.ts b/backend/src/index.ts index 84dd41ca..129b293f 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -19,6 +19,7 @@ import { mfaRouter } from './routes/mfa'; import { ssoRouter } from './routes/sso'; import { licenseRouter, systemUpdateRouter } from './routes/license'; import { imageChannelRouter } from './routes/imageChannel'; +import { buildInfoRouter } from './routes/buildInfo'; import { webhooksRouter } from './routes/webhooks'; import { usersRouter } from './routes/users'; import { gitSourcesRouter, stackGitSourceRouter } from './routes/gitSources'; @@ -116,6 +117,7 @@ app.use('/api/', createRemoteProxyMiddleware()); app.use('/api/license', licenseRouter); app.use('/api/license/image-channel', imageChannelRouter); +app.use('/api/build-info', buildInfoRouter); app.use('/api/system', systemUpdateRouter); app.use('/api/permissions', permissionsRouter); app.use('/api/convert', convertRouter); diff --git a/backend/src/routes/buildInfo.ts b/backend/src/routes/buildInfo.ts new file mode 100644 index 00000000..7d999166 --- /dev/null +++ b/backend/src/routes/buildInfo.ts @@ -0,0 +1,47 @@ +import { Router, type Request, type Response } from 'express'; +import { requireUserSession } from '../middleware/tierGates'; +import { classifyImageChannel, type ImageChannel } from '../helpers/imageChannel'; +import type { BuildChannel } from '../helpers/selfUpdateCompose'; +import SelfIdentityService from '../services/SelfIdentityService'; + +export const buildInfoRouter = Router(); + +/** Wire shape of GET /api/build-info. `restricted: true` implies `imageRef` and + * `revision` are nulled for a hardened image viewed by a non-admin. */ +interface BuildInfoResponse { + version: string | null; + channel: BuildChannel; + imageChannel: ImageChannel; + imageRef: string | null; + imageId: string | null; + revision: string | null; + restricted: boolean; +} + +// Canonical runtime build identity of the control instance. Proxy-exempt (see +// helpers/proxyExemptPaths.ts) so it is always served by the local hub, never +// forwarded to a remote node. The running image reference can carry a private +// registry/repository name, so the endpoint requires a human session and +// redacts hardened-image references to non-admins via `restricted: true` (the +// UI shows "Restricted", never "Unknown", when set). +buildInfoRouter.get('/', async (req: Request, res: Response): Promise => { + if (!requireUserSession(req, res)) return; + const service = SelfIdentityService.getInstance(); + // Await the detached revision enrichment so a transient null is never the + // settled value of a successful response. Bounded by the inspect timeout and + // never rejects, so this adds at most a short wait on the first read. + await service.whenRevisionResolved(); + const identity = service.getBuildInfo(); + const isAdmin = req.user?.role === 'admin'; + const imageChannel = identity.imageRef ? classifyImageChannel(identity.imageRef) : 'unknown'; + const restricted = !isAdmin && imageChannel === 'hardened'; + res.json({ + version: identity.version, + channel: identity.channel, + imageChannel, + imageRef: restricted ? null : identity.imageRef, + imageId: identity.imageId, + revision: restricted ? null : identity.revision, + restricted, + } satisfies BuildInfoResponse); +}); \ No newline at end of file diff --git a/backend/src/routes/gitSources.ts b/backend/src/routes/gitSources.ts index cef891f0..0e6f9054 100644 --- a/backend/src/routes/gitSources.ts +++ b/backend/src/routes/gitSources.ts @@ -5,7 +5,7 @@ import { GitProjectManifestService } from '../services/GitProjectManifestService import { FileSystemService } from '../services/FileSystemService'; import { DatabaseService } from '../services/DatabaseService'; import { CryptoService } from '../services/CryptoService'; -import { requirePermission } from '../middleware/permissions'; +import { checkPermission, requirePermission } from '../middleware/permissions'; import { classifySourceRow, satisfiesGitOpsRead } from '../services/gitops/readAuth'; import { NOT_APPLICABLE_REVISION, projectStackRevision, stackResourceSet } from '../helpers/gitopsResponse'; import { respondWithHistory } from '../helpers/gitopsHistoryPage'; @@ -27,6 +27,8 @@ import { assertSafeOutboundHostname, resolveSafeOutboundHostname, UnsafeOutbound const MAX_BRANCH_LENGTH = REF_MAX_LEN; const MAX_ENV_PATH_LENGTH = 1024; const MAX_TOKEN_LENGTH = 8192; +const MAX_SUSPEND_REASON_LENGTH = 512; +const MAX_WEBHOOK_DELIVERY_ID_LENGTH = 512; /** * Shared handler for the "browse repository" compose-file picker: validate the @@ -584,14 +586,28 @@ stackGitSourceRouter.post('/:stackName/git-source/webhook-pull', async (req: Req return; } if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return; + const deliveryId = req.body?.deliveryId; + if ( + deliveryId !== undefined + && (typeof deliveryId !== 'string' || !deliveryId.trim() || deliveryId.length > MAX_WEBHOOK_DELIVERY_ID_LENGTH) + ) { + res.status(400).json({ error: 'deliveryId must be a non-empty string of at most 512 characters' }); + return; + } try { - const source = GitSourceService.getInstance().get(stackName); + const service = GitSourceService.getInstance(); + const source = service.get(stackName); if (!source) { res.status(404).json({ error: 'No Git source configured for this stack', status: 'error' }); return; } - if (source.auto_apply_on_webhook && source.auto_deploy_on_apply && !requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return; - const result = await GitSourceService.getInstance().handleWebhookPull(stackName); + const normalizedDeliveryId = deliveryId?.trim(); + const deployAuthorized = checkPermission(req, 'stack:deploy', 'stack', stackName); + if (service.webhookDeliveryRequiresDeploy(stackName, normalizedDeliveryId) && !deployAuthorized) { + requirePermission(req, res, 'stack:deploy', 'stack', stackName); + return; + } + const result = await service.handleWebhookPull(stackName, deployAuthorized, normalizedDeliveryId); // Map the outcome to a real HTTP status so a Git provider sees a 4xx on // failure instead of a 200 with an error body (which it would read as // "delivered fine, stop retrying"). @@ -616,6 +632,64 @@ stackGitSourceRouter.post('/:stackName/git-source/dismiss-pending', async (req: } }); +stackGitSourceRouter.post('/:stackName/git-source/suspend', async (req: Request, res: Response): Promise => { + const stackName = req.params.stackName as string; + if (!isValidStackName(stackName)) { + res.status(400).json({ error: 'Invalid stack name' }); + return; + } + if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return; + const { reason: rawReason } = req.body ?? {}; + const reason = typeof rawReason === 'string' ? rawReason : undefined; + if (reason !== undefined && reason.length > MAX_SUSPEND_REASON_LENGTH) { + res.status(400).json({ error: 'reason is too long' }); + return; + } + try { + const result = await GitSourceService.getInstance().suspend(stackName, { + actor: req.user?.username ?? 'unknown', + reason, + }); + res.json(result); + } catch (error) { + sendGitSourceError(res, error); + } +}); + +stackGitSourceRouter.post('/:stackName/git-source/resume', async (req: Request, res: Response): Promise => { + const stackName = req.params.stackName as string; + if (!isValidStackName(stackName)) { + res.status(400).json({ error: 'Invalid stack name' }); + return; + } + if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return; + try { + const result = await GitSourceService.getInstance().resume(stackName, { + actor: req.user?.username ?? 'unknown', + }); + res.json(result); + } catch (error) { + sendGitSourceError(res, error); + } +}); + +stackGitSourceRouter.post('/:stackName/git-source/retry', async (req: Request, res: Response): Promise => { + const stackName = req.params.stackName as string; + if (!isValidStackName(stackName)) { + res.status(400).json({ error: 'Invalid stack name' }); + return; + } + if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return; + try { + const result = await GitSourceService.getInstance().retry(stackName, { + actor: req.user?.username ?? 'unknown', + }); + res.json(result); + } catch (error) { + sendGitSourceError(res, error); + } +}); + // Edit-mode repo browse for an existing stack: gated by stack:edit so a user who // can edit (but not create) stacks can re-pick files, and reuses the stored token // when the request omits one. diff --git a/backend/src/routes/meta.ts b/backend/src/routes/meta.ts index 93dcba01..3e6d8402 100644 --- a/backend/src/routes/meta.ts +++ b/backend/src/routes/meta.ts @@ -1,9 +1,10 @@ import { Router, type Request, type Response } from 'express'; import { getActiveCapabilities, getSenchoVersion } from '../services/CapabilityRegistry'; import { classifyImageChannel } from '../helpers/imageChannel'; -import { isRepinBlocked } from '../helpers/selfUpdateCompose'; +import { classifyBuildChannel, isRepinBlocked } from '../helpers/selfUpdateCompose'; import { MeshService } from '../services/MeshService'; import SelfUpdateService from '../services/SelfUpdateService'; +import SelfIdentityService from '../services/SelfIdentityService'; // Captured at boot. Exposed via /api/health and /api/meta so the Fleet update // overlay can distinguish a brand-new process from the old one still mid-pull. @@ -41,6 +42,7 @@ metaRouter.get('/meta', async (_req: Request, res: Response): Promise => { const updateError = selfUpdate.getLastError(); const pin = await selfUpdate.getPinInfo({ cacheOnly: true }); const updateBlocked = pin ? isRepinBlocked(pin.pinKind) : false; + const runningRef = SelfIdentityService.getInstance().getBuildInfo().imageRef; res.json({ version: getSenchoVersion(), capabilities: getActiveCapabilities(), @@ -50,6 +52,10 @@ metaRouter.get('/meta', async (_req: Request, res: Response): Promise => { imagePinKind: pin.pinKind, imageChannel: classifyImageChannel(pin.composeImageRef), } : {}), + // Bounded build channel of the RUNNING image (stable|dev|preview|unknown). + // Like imagePinKind, this is a non-sensitive enum; no image reference is + // ever exposed on this public endpoint. + ...(runningRef ? { buildChannel: classifyBuildChannel(runningRef) } : {}), updateBlocked, ...(updateError ? { updateError: 'update_failed' } : {}), }); diff --git a/backend/src/routes/webhooks.ts b/backend/src/routes/webhooks.ts index f9b97d62..42f32e3c 100644 --- a/backend/src/routes/webhooks.ts +++ b/backend/src/routes/webhooks.ts @@ -12,6 +12,34 @@ function isWebhookAction(value: unknown): value is WebhookAction { return typeof value === 'string' && (VALID_WEBHOOK_ACTIONS as readonly string[]).includes(value); } +// Recognized per-delivery identity headers, in priority order. This +// endpoint is a generic HMAC-signed trigger, not a provider-specific +// receiver, so a well-known provider header is the only delivery identity +// available, and only when the caller happens to send one. The value is a +// stable delivery identity available. WebhookService namespaces the value by +// control instance and configured webhook before it reaches GitSourceService, +// so two producers cannot collide on the same provider-assigned id. +// +// Each header is still the provider's actual per-delivery identity rather +// than a webhook- or connection-level id that stays constant across every +// delivery from that source: GitHub's X-GitHub-Delivery GUID changes per +// delivery (it is stable only across redeliveries of the same delivery); +// GitLab's is Webhook-ID, the modern name for its Idempotency-Key, not +// X-Gitlab-Event-UUID, which tracks recursive-trigger chains and can +// repeat across genuinely distinct events; Bitbucket's is X-Request-UUID, +// not X-Hook-UUID, which identifies the webhook configuration itself. +// Picking the wrong one would deduplicate genuinely distinct pushes. +const DELIVERY_ID_HEADERS = ['x-github-delivery', 'webhook-id', 'idempotency-key', 'x-request-uuid', 'x-webhook-delivery-id'] as const; + +function deliveryIdFromHeaders(headers: Request['headers']): string | undefined { + for (const name of DELIVERY_ID_HEADERS) { + const value = headers[name]; + const first = Array.isArray(value) ? value[0] : value; + if (first) return first; + } + return undefined; +} + export const webhooksRouter = Router(); webhooksRouter.get('/', authMiddleware, async (req: Request, res: Response): Promise => { @@ -193,6 +221,7 @@ webhooksRouter.post('/:id/trigger', webhookTriggerLimiter, async (req: Request, action = overrideAction; } const triggerSource = req.headers['user-agent'] || req.ip || null; + const deliveryId = deliveryIdFromHeaders(req.headers); // Execute asynchronously; return 202 immediately. res.status(202).json({ message: 'Webhook accepted', action }); @@ -202,7 +231,7 @@ webhooksRouter.post('/:id/trigger', webhookTriggerLimiter, async (req: Request, // dispatch the action still completes and recordExecution swallows the // FK error from the CASCADE. atomic is unconditionally true, so the // deploy/pull paths always run in atomic mode here. - svc.execute(webhook, action, triggerSource, true).catch(err => { + svc.execute(webhook, action, triggerSource, true, deliveryId).catch(err => { console.error(`[Webhooks] Execution error for webhook ${id}:`, err); }); } catch (error) { diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index 7c58b36e..027ee7fc 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -1987,6 +1987,24 @@ export class DatabaseService { // existing installs already have. New installs get it from the // CREATE TABLE; older DBs need the additive column here. maybeAddCol('gitops_applications', 'source_suspended_reason', 'TEXT NULL'); + // Portable accepted-generation contract fields. Additive and + // nullable: existing generation rows decode these as an explicit + // limitation rather than invented evidence. + maybeAddCol('gitops_generations', 'portable_manifest_json', 'TEXT NULL'); + maybeAddCol('gitops_generations', 'compose_inputs_json', 'TEXT NULL'); + maybeAddCol('gitops_generations', 'source_policy_evidence_json', 'TEXT NULL'); + maybeAddCol('gitops_generations', 'security_policy_evidence_json', 'TEXT NULL'); + maybeAddCol('gitops_generations', 'support_requirements_json', 'TEXT NULL'); + maybeAddCol('gitops_generations', 'compatibility_requirements_json', 'TEXT NULL'); + // Controller-owned bookkeeping (source policy, poll cadence, attempt + // sequence). New installs get these from the CREATE TABLE; older DBs + // need the additive columns here. Existing installations must not + // start unattended polling, so poll_interval_secs and next_poll_at + // stay NULL until an operator (or the migration below) sets one. + maybeAddCol('gitops_applications', 'source_policy', "TEXT NOT NULL DEFAULT 'manual' CHECK (source_policy IN ('manual','review','automatic'))"); + maybeAddCol('gitops_applications', 'poll_interval_secs', 'INTEGER NULL'); + maybeAddCol('gitops_applications', 'next_poll_at', 'INTEGER NULL'); + maybeAddCol('gitops_applications', 'attempt_seq', 'INTEGER NOT NULL DEFAULT 0'); // Distributed API model columns maybeAddCol('nodes', 'api_url', "TEXT DEFAULT ''"); diff --git a/backend/src/services/GitProjectManifestService.ts b/backend/src/services/GitProjectManifestService.ts index df4138eb..e38f8d1e 100644 --- a/backend/src/services/GitProjectManifestService.ts +++ b/backend/src/services/GitProjectManifestService.ts @@ -51,6 +51,10 @@ export const MANAGED_ROOT_NAME = 'git-managed'; export const MANIFEST_FILENAME = 'manifest.v1.json'; export const PROMOTION_MARKER = 'promotion.json'; export const CANDIDATE_COMPLETE_MARKER = '.candidate-complete'; + +type CandidateClaims = + | { complete: true; dirs: ReadonlySet } + | { complete: false }; export const GENERATIONS_DIR = 'generations'; const DETACH_RECOVERY_MARKER = 'detach-recovery.v1.json'; @@ -421,10 +425,19 @@ export class GitProjectManifestService { /** Atomic manifest write (tmp + rename). */ async writeManifest(stackName: string, manifest: GitProjectManifest): Promise { - const dir = this.managedRoot(stackName); - await fs.promises.mkdir(dir, { recursive: true }); - const target = path.join(dir, MANIFEST_FILENAME); - const tmp = path.join(dir, `${MANIFEST_FILENAME}.tmp`); + // Inline barrier at the mkdir/write/rename sinks (CodeQL path-injection): + // confine the resolved managed directory to the managed area before any + // filesystem call touches it, then confine the filenames joined onto it. + const root = path.resolve(this.managedRoot(stackName)); + if (!root.startsWith(managedAreaBase() + path.sep)) { + throw Object.assign(new Error('Path escapes the managed area'), { code: 'INVALID_PATH' }); + } + const target = path.resolve(root, MANIFEST_FILENAME); + const tmp = path.resolve(root, `${MANIFEST_FILENAME}.tmp`); + if (!target.startsWith(root + path.sep) || !tmp.startsWith(root + path.sep)) { + throw Object.assign(new Error('Path escapes managed project directory'), { code: 'INVALID_PATH' }); + } + await fs.promises.mkdir(root, { recursive: true }); await fs.promises.writeFile(tmp, JSON.stringify(manifest, null, 2), 'utf8'); await fs.promises.rename(tmp, target); } @@ -1268,12 +1281,20 @@ export class GitProjectManifestService { * is finalized; an uncommitted promotion restores the prior generation. * A third state is treated as an operator edit, so recovery declines and * flags migration_required. Interrupted detach snapshots are restored first. + * Complete candidate claims hold the directory basenames that durable state + * still references. Incomplete claims preserve every candidate because + * ownership is uncertain. */ async sweepManagedArea( stackName: string, - opts: { repoUrl: string; branch: string; stackExists: boolean }, + opts: { + repoUrl: string; + branch: string; + stackExists: boolean; + candidateClaims: CandidateClaims; + }, ): Promise { - const { repoUrl, branch, stackExists } = opts; + const { repoUrl, branch, stackExists, candidateClaims } = opts; if (!stackExists) { await this.deleteManagedArea(stackName); return; @@ -1359,44 +1380,64 @@ export class GitProjectManifestService { } } - // Orphan candidates: incomplete or stale. + if (!candidateClaims.complete) { + await this.flagRecoveryRequired( + stackName, + `candidate ownership for ${sanitizeForLog(stackName)} could not be established`, + ); + return; + } + + // With a complete claim inventory, reap candidates that are incomplete + // or stale and unclaimed. const dir = this.generationsDir(stackName); + let entries: fs.Dirent[]; try { - const entries = await fs.promises.readdir(dir, { withFileTypes: true }); - const now = Date.now(); - const areaBase = managedAreaBase(); - for (const entry of entries) { - if (!entry.isDirectory() || !entry.name.startsWith('candidate-')) continue; - const abs = path.resolve(dir, entry.name); - // Inline containment barrier at the removal sink (see - // `managedAreaBase`): the analyzer credits this literal - // comparison, not the positional check below it. - if (!abs.startsWith(areaBase + path.sep)) { - console.warn(`[GitManifest] refusing to reap orphan candidate ${sanitizeForLog(entry.name)} for ${sanitizeForLog(stackName)}: it resolves outside the managed area`); - continue; - } - // Same positional barrier as generation pruning: the boot sweep - // reaps candidate directories nobody claims, which is precisely - // the kind of unattended delete a planted link would steer. - if (!await isRealPathAtManagedLocation(abs)) { - console.warn(`[GitManifest] refusing to reap orphan candidate ${sanitizeForLog(entry.name)} for ${sanitizeForLog(stackName)}: it is not at its own location in the managed area`); - continue; - } - const complete = await fs.promises - .access(path.join(abs, CANDIDATE_COMPLETE_MARKER)) - .then(() => true) - .catch(() => false); - if (!complete) { - await fs.promises.rm(abs, { recursive: true, force: true }); - continue; - } - const st = await fs.promises.stat(abs); - if (now - st.mtimeMs > ORPHAN_CANDIDATE_AGE_MS) { - await fs.promises.rm(abs, { recursive: true, force: true }); - } + entries = await fs.promises.readdir(dir, { withFileTypes: true }); + } catch (e) { + if ((e as NodeJS.ErrnoException).code === 'ENOENT') return; + throw e; + } + const now = Date.now(); + const areaBase = managedAreaBase(); + for (const entry of entries) { + if (!entry.isDirectory() || !entry.name.startsWith('candidate-')) continue; + // A claimed candidate is still needed regardless of age or + // completeness. Claims come from application pointers, the + // source pending record, and generation rows linked to + // unsettled attempts. Together they are the durable ownership + // record that makes recovery before cleanup safe. + if (candidateClaims.dirs.has(entry.name)) continue; + const abs = path.resolve(dir, entry.name); + // Inline containment barrier at the removal sink (see + // `managedAreaBase`): the analyzer credits this literal + // comparison, not the positional check below it. + if (!abs.startsWith(areaBase + path.sep)) { + console.warn(`[GitManifest] refusing to reap orphan candidate ${sanitizeForLog(entry.name)} for ${sanitizeForLog(stackName)}: it resolves outside the managed area`); + continue; + } + // Same positional barrier as generation pruning: the boot sweep + // reaps candidate directories nobody claims, which is precisely + // the kind of unattended delete a planted link would steer. + if (!await isRealPathAtManagedLocation(abs)) { + console.warn(`[GitManifest] refusing to reap orphan candidate ${sanitizeForLog(entry.name)} for ${sanitizeForLog(stackName)}: it is not at its own location in the managed area`); + continue; + } + let complete = true; + try { + await fs.promises.access(path.join(abs, CANDIDATE_COMPLETE_MARKER)); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== 'ENOENT') throw e; + complete = false; + } + if (!complete) { + await fs.promises.rm(abs, { recursive: true, force: true }); + continue; + } + const st = await fs.promises.stat(abs); + if (now - st.mtimeMs > ORPHAN_CANDIDATE_AGE_MS) { + await fs.promises.rm(abs, { recursive: true, force: true }); } - } catch { - // no generations dir yet } } diff --git a/backend/src/services/GitSourceService.ts b/backend/src/services/GitSourceService.ts index 345e2560..07e23eca 100644 --- a/backend/src/services/GitSourceService.ts +++ b/backend/src/services/GitSourceService.ts @@ -34,7 +34,19 @@ import { nativeGitTransport, verifyFastForward } from './git/nativeGitTransport' import { fingerprintFromKnownHostsLine } from './git/sshTrust'; import { validateCaBundlePem } from './git/caBundle'; import { GitOpsStore } from './gitops/store'; -import { GitOpsTransitions, GitOpsTransitionError } from './gitops/transitions'; +import type { GitOpsHistoryCursor } from './gitops/history'; +import { projectApplication } from './gitops/derive'; +import { outcomeFromSourceFacet, isNextAction, isReconcileOutcome, type ReconcileOutcome, type ReconcileResult } from './gitops/outcomes'; +import { coalesceKey, deliveryKey, type ReconcileRequest, type ReconcileTrigger } from './gitops/triggers'; +import { classifyFailure } from './gitops/backoff'; +import { BlueprintTargetAdapter, type AcceptedGeneration, type DispatchContext, type DispatchResult } from './gitops/handoff'; +import { + GitOpsTransitions, + GitOpsTransitionError, + type EventEnvelope, + type ReconcileDeliveryIntent, +} from './gitops/transitions'; +import { decodeGitOpsJson, isRecord, GitOpsJsonError } from './gitops/json'; import { buildCreateCheckpointRow, buildDirectApplicationRow, @@ -43,7 +55,7 @@ import { newGitOpsId, stackManagedRoot, } from './gitops/directApplication'; -import type { GitOpsApplicationRow } from './gitops/types'; +import type { GitOpsApplicationRow, GitOpsHistoryRow } from './gitops/types'; import { appliedRelPathFor, candidateRelPathForSha, deleteStagingMarker, readStagingMarker, validateCandidateRelPath, writeStagingMarker } from './gitops/createStagingMarker'; import { cleanupUnclaimedManagedRoot, removeOperationOwnedPaths } from './gitops/createCleanup'; import { managedAreaBase } from './gitops/managedPaths'; @@ -291,6 +303,37 @@ export interface GitApplyOpts { requirePlanFingerprint?: boolean; } +type GitApplyResult = { + applied: boolean; + deployed: boolean; + deployError?: string; + recoveryId?: string; +}; + +type WorkOutcome = + | { status: 'fulfilled'; value: T } + | { status: 'rejected'; reason: unknown }; + +type SharedExecution = WorkOutcome & { result: ReconcileResult }; + +type LeaderCompletion = { + execution: SharedExecution; + settled: boolean; +}; + +/** + * In-process executions keyed by a request-derived execution key: the + * operation id each reservation minted, and the promise a later joiner awaits + * instead of repeating the work. + */ +type InFlightMap = Map> }>; + +type ExecutionSubmission = + | { kind: 'executed'; execution: SharedExecution } + | { kind: 'replayed'; result: ReconcileResult }; + +type WebhookPullResult = { status: 'success' | 'skipped' | 'error'; message: string }; + // ─── Constants ─────────────────────────────────────────────────────────────── const TEMP_DIR_PREFIX = 'sencho-git-'; @@ -878,13 +921,13 @@ export class GitSourceService { last_debounce_at: existing?.last_debounce_at ?? null, }); - if (configChanged) { + const app = this.gitopsApplicationFor(input.stackName); + if (configChanged || !app) { db.clearGitSourcePending(input.stackName); } - const app = this.gitopsApplicationFor(input.stackName); const envelope = this.gitopsEnvelope(crypto.randomUUID(), 'system:git-source', 'configure'); - if (!app && !existing && !this.gitopsNameHeld(input.stackName)) { + if (!app && !this.gitopsNameHeld(input.stackName)) { // Linking a stack that already exists. Nothing is fetched or // accepted yet, so the application starts live with no desired // commit and the projection asks for a fetch. @@ -1885,29 +1928,68 @@ export class GitSourceService { // ─── Pull / apply ──────────────────────────────────────────────────────── + /** + * A short, log-friendly token for an operation id. A reserved id + * (`:attempt:` or `::`) + * repeats the same prefix across every attempt on one stack, so only the + * suffix after the last colon tells two attempts apart. A plain UUID has + * no such suffix, so its leading characters are used instead. + */ + private static shortOperationId(operationId: string): string { + const lastColon = operationId.lastIndexOf(':'); + return lastColon === -1 ? operationId.slice(0, 8) : operationId.slice(lastColon + 1); + } + public async pull(stackName: string, opts: { actor?: string } = {}): Promise { // Guarded by the per-stack mutex (see withStackLock). Without this, a // concurrent delete-source + pull can land a pending row on a stack // whose config row has just been removed. const actor = opts.actor ?? 'unknown'; - return this.withStackLock(stackName, async () => { + + // Reservation and coalescing need a real gitops application to attach a + // durable attempt to, the same definition pullLocked itself uses to + // decide whether it has any gitops bookkeeping to do at all. + const gitopsApp = this.gitopsApplicationFor(stackName); + const doPull = this.doPullWork(stackName, actor, gitopsApp?.id); + if (!gitopsApp) { + this.refuseUntrackedSource(stackName, actor, 'fetch', true); + return doPull(); + } + + const request: ReconcileRequest = { intent: 'fetch', applicationId: gitopsApp.id, stackName, trigger: 'manual', actor }; + const submission = await this.submitExecution( + this.inFlightFetches, + request, + doPull, + (outcome) => this.fetchExecutionResult(stackName, outcome), + ); + return GitSourceService.valueFromSubmission(submission); + } + + /** + * The shared fetch work closure: the per-stack lock plus pullLocked + * itself, reporting a real failure to the server console and to the + * stack's activity feed here rather than in each producer, so it is + * recorded exactly once no matter which fetch-intent producer (pull(), + * reconcile()) owns the reservation that ends up leading. Kept as one + * factory so pull() and reconcile()'s fetch path run the literal same + * closure shape, which is what lets them join the same coalescing map + * (inFlightFetches) in the first place: two producers can only coalesce + * onto one execution if that execution really is the same work. + */ + private doPullWork(stackName: string, actor: string, applicationId?: string): (operationId?: string) => Promise { + return (operationId?: string) => this.withStackLock(stackName, async () => { try { - return await this.pullLocked(stackName, actor); + if (applicationId) this.assertLiveApplication(stackName, applicationId); + return await this.pullLocked(stackName, actor, operationId); } catch (e) { + console.error(`[GitSource] fetch failed for ${sanitizeForLog(stackName)}:`, e instanceof Error ? e.message : String(e)); this.recordGitActivity(stackName, 'git_pull_failed', `Git pull failed for ${stackName}`, actor, 'error'); throw e; } }); } - /** - * Body of pull(); assumes the caller already holds the per-stack lock. - * handleWebhookPull calls this directly so that its debounce re-check, - * this fetch, and the apply all run inside the single lock that - * handleWebhookPull holds. Without that, a concurrent webhook fan-out - * reads last_debounce_at while it is still unset on every request, slips - * past the gate, and clones once per request. - */ /** * The GitOps application tracking this stack, or null when there is none. * @@ -1977,12 +2059,23 @@ export class GitSourceService { }); } - private async pullLocked(stackName: string, actor: string): Promise { + private async pullLocked(stackName: string, actor: string, operationId?: string): Promise { const db = DatabaseService.getInstance(); const src = db.getGitSource(stackName); if (!src) throw new GitSourceError('GIT_ERROR', 'No Git source configured for this stack.'); const gitopsApp = this.gitopsApplicationFor(stackName); - const gitopsOperationId = crypto.randomUUID(); + // fetchStarted has its own suspension guard, but recordGitOps swallows + // its rejection (so a fetch that already touched the filesystem is never + // failed out from under itself), which would let a suspended source keep + // cloning and staging pending updates. Stop before any of that starts. + if (gitopsApp?.suspended_at) { + throw new GitSourceError('OPERATION_IN_FLIGHT', `Reconciliation is suspended for ${stackName}.`); + } + // A caller that reserved a durable attempt for this fetch passes its own + // operation id in, so the attempt and every stage of gitops evidence it + // produces share one identity. A direct low-level call that reserved + // nothing still gets an id of its own. + const gitopsOperationId = operationId ?? crypto.randomUUID(); const gitopsEnv = this.gitopsEnvelope(gitopsOperationId, actor, 'pull'); // A fetch that starts and never terminates is worse than one that is // never recorded: fetchStarted refuses to open a second operation, so @@ -2078,7 +2171,6 @@ export class GitSourceService { const prior = priorRead; if (prior) manifestSummary = manifestSvc.summaryFrom(prior); - const operationId = crypto.randomUUID(); let plan: GitChangePlan | null = null; if (materialization.value?.inventory) { plan = await this.computeChangePlan({ @@ -2180,7 +2272,13 @@ export class GitSourceService { { fingerprint: plan?.fingerprint ?? '', schemaVersion: GIT_CHANGE_PLAN_SCHEMA_VERSION, - operationId, + // This fetch attempt's own id, not an independent one: + // applyLockedBody falls back to pending.operationId for + // its gitops transitions when its caller reserved no + // attempt, so that fallback has to inherit the real fetch + // attempt's lineage rather than an identity nothing else + // knows about. + operationId: gitopsOperationId, reviewedLive: plan ? this.reviewedLiveFromPlan(plan) : [], }, ), @@ -2196,7 +2294,7 @@ export class GitSourceService { this.recordGitActivity( stackName, 'git_plan_blocked', - `Git plan blocked for ${stackName} (${shortSha}, op ${operationId.slice(0, 8)}, plan ${fpPrefix})`, + `Git plan blocked for ${stackName} (${shortSha}, op ${GitSourceService.shortOperationId(gitopsOperationId)}, plan ${fpPrefix})`, actor, 'warning', ); @@ -2204,7 +2302,7 @@ export class GitSourceService { this.recordGitActivity( stackName, 'git_pull_ready', - `Git pull ready for ${stackName} (${shortSha}, op ${operationId.slice(0, 8)}, plan ${fpPrefix})`, + `Git pull ready for ${stackName} (${shortSha}, op ${GitSourceService.shortOperationId(gitopsOperationId)}, plan ${fpPrefix})`, actor, ); } @@ -2243,11 +2341,892 @@ export class GitSourceService { stackName: string, commitSha: string, opts: GitApplyOpts = {}, - ): Promise<{ applied: boolean; deployed: boolean; deployError?: string; recoveryId?: string }> { - return this.withStackLock(stackName, () => this.applyWithSharedLock(stackName, commitSha, { - ...opts, - requirePlanFingerprint: opts.requirePlanFingerprint !== false, - })); + ): Promise { + // Resolved once, with applyLockedBody's own formula (its shouldDeploy), + // so the coalesce key below and the deploy behavior actually executed + // can never disagree: two calls that resolve to different deploy + // behavior must never share a coalesce key, or one could silently + // receive the other's deployed/not-deployed result. + const resolvedDeploy = opts.deploy ?? DatabaseService.getInstance().getGitSource(stackName)?.auto_deploy_on_apply ?? false; + const finalOpts: GitApplyOpts = { ...opts, deploy: resolvedDeploy, requirePlanFingerprint: opts.requirePlanFingerprint !== false }; + const doApply = (applicationId?: string, operationId?: string) => this.withStackLock( + stackName, + async () => { + if (applicationId) this.assertLiveApplication(stackName, applicationId); + return this.applyWithSharedLock(stackName, commitSha, finalOpts, operationId); + }, + ); + + // Same reasoning as pull(): reservation needs a real gitops + // application to attach a durable attempt to. + const gitopsApp = this.gitopsApplicationFor(stackName); + if (!gitopsApp) { + this.refuseUntrackedSource(stackName, opts.actor ?? 'unknown', 'apply', true); + return doApply(); + } + + const request: ReconcileRequest = { + intent: 'apply', + applicationId: gitopsApp.id, + stackName, + trigger: 'manual', + actor: opts.actor ?? 'unknown', + commitSha, + planFingerprint: opts.planFingerprint ?? '', + deploy: resolvedDeploy, + }; + // A policy-bypassing apply must never coalesce with anything else: + // bypassPolicy changes behavior but is not part of the coalesce key (a + // plain policy-gate bypass has no natural identity to key on), so + // joining could hand a non-bypassing caller someone else's bypassed + // result, or silently drop an admin's explicit bypass onto a request + // that never asked for one. A call-local key suffix guarantees this + // call can neither join an existing leader nor be joined by a later one. + const baseKey = GitSourceService.applyExecutionKey(request, finalOpts.requirePlanFingerprint === true); + const key = opts.bypassPolicy + ? `${baseKey}:policy-bypass:${crypto.randomUUID()}` + : baseKey; + const submission = await this.submitExecution( + this.inFlightApplies, + request, + (operationId) => doApply(gitopsApp.id, operationId), + (outcome) => this.applyExecutionResult(stackName, outcome), + key, + ); + return GitSourceService.valueFromSubmission(submission); + } + + /** + * Outcomes the application row already reflects truthfully, so reconcile + * trusts the derived result over a classified fallback. Most throw sites + * in pullLocked/applyLockedBody fire before any transition opens (missing + * config, stale commitSha, lock contention) and leave the row saying + * nothing about the failure, which is what the fallback covers; + * 'suspended' belongs here because its guards throw only when the row + * already, correctly, says so. + */ + private static readonly FAILURE_REFLECTED_OUTCOMES: ReadonlySet = new Set([ + 'failed_previous_intact', + 'retry_scheduled', + 'recovery_required', + 'blocked', + 'suspended', + ]); + + /** The settled result for a stack that carries no GitOps application at all. */ + private static noApplicationResult(): ReconcileResult { + return { outcome: 'unknown', reason: 'No GitOps application exists for this stack.', nextAction: 'none' }; + } + + /** + * The settled result for a request naming an application id that is no + * longer the live one for its stack. Failing closed here (rather than + * proceeding against whatever application now holds the stack name) + * keeps a request from settling against an application it never named. + */ + private static staleApplicationResult(): ReconcileResult { + return { + outcome: 'unknown', + reason: 'The live application for this stack no longer matches the requested application id.', + nextAction: 'none', + }; + } + + /** Refuse work when the application resolved before locking is no longer live. */ + private assertLiveApplication(stackName: string, applicationId: string): void { + if (GitOpsStore.getInstance().getLiveDirectApplication(stackName)?.id === applicationId) return; + throw new GitSourceError('OPERATION_IN_FLIGHT', GitSourceService.staleApplicationResult().reason); + } + + /** + * Shared fetch and apply executions. Manual routes, controller requests, + * and webhooks all use these maps so producer choice cannot create a + * duplicate side effect or a different normalized result. + */ + private readonly inFlightFetches: InFlightMap = new Map(); + private readonly inFlightApplies: InFlightMap = new Map(); + private readonly inFlightWebhookDeliveries = new Map }>(); + + /** + * The controller-facing entry point: one normalized submission in, + * one normalized result out, for any trigger (manual, poll, retry, + * API, config change, startup, resume). + * + * Every submission against a real application row gets a durable + * attempt, reserved before any side effect and settled with the + * normalized result once execution finishes -- reserved but never + * settled is exactly what startup recovery looks for after a crash. + * A submission naming an application id that does not exist at all + * (a fabricated id, or a stack with no GitOps application) reserves + * nothing: there is no row to attach a durable attempt to, and the + * identity/no-application guards below already produce a truthful + * result for it without doing any work worth protecting. + */ + public async reconcile(request: ReconcileRequest): Promise { + if (!GitOpsStore.getInstance().getApplication(request.applicationId)) { + // request.applicationId does not exist as any row, so it can + // never equal a real live application's id: the identity guard + // alone already produces the truthful result, without a stack + // lock or any work worth protecting. + const liveApp = GitOpsStore.getInstance().getLiveDirectApplication(request.stackName); + return liveApp ? GitSourceService.staleApplicationResult() : GitSourceService.noApplicationResult(); + } + + if (request.intent === 'fetch') { + return this.reconcileFetch(request); + } + + const submission = await this.submitExecution( + this.inFlightApplies, + request, + (operationId) => this.withStackLock(request.stackName, async () => { + this.assertLiveApplication(request.stackName, request.applicationId); + return this.applyWithSharedLock(request.stackName, request.commitSha, { + actor: request.actor, + deploy: request.deploy, + planFingerprint: request.planFingerprint, + requirePlanFingerprint: false, + }, operationId); + }), + (outcome) => this.applyExecutionResult(request.stackName, outcome), + GitSourceService.applyExecutionKey(request, false), + ); + return GitSourceService.resultFromSubmission(submission); + } + + private static applyExecutionKey( + request: ReconcileRequest & { intent: 'apply' }, + requirePlanFingerprint: boolean, + ): string { + return `${coalesceKey(request)}:fingerprint-${requirePlanFingerprint ? 'required' : 'optional'}`; + } + + /** + * The in-flight execution in `map` whose reservation minted exactly this + * operation id, regardless of which coalesce key it is running under. + * Coalesce keys and operation ids are not co-extensive (an apply's key + * includes its commitSha/planFingerprint/deploy, which a shared external + * delivery id does not carry), so two submissions can collide on operation + * id while running under different keys. + */ + private static findByOperationId(map: InFlightMap, operationId: string): Promise> | undefined { + for (const entry of map.values()) { + if (entry.operationId === operationId) return entry.promise; + } + return undefined; + } + + private static resultFromSubmission(submission: ExecutionSubmission): ReconcileResult { + return submission.kind === 'replayed' ? submission.result : submission.execution.result; + } + + /** Preserve manual route return and throw behavior after settlement is attempted. */ + private static valueFromSubmission(submission: ExecutionSubmission): T { + if (submission.kind === 'replayed') { + throw new GitSourceError('GIT_ERROR', 'This operation was already recorded and cannot be replayed as a new manual request.'); + } + if (submission.execution.status === 'rejected') throw submission.execution.reason; + return submission.execution.value; + } + + /** + * Reserve one submission, join equivalent work when possible, and attempt + * to settle each reservation from the leader's single normalized result. + * A failed leader settlement leaves its followers unsettled for recovery. + */ + private async submitExecution( + map: InFlightMap, + request: ReconcileRequest, + work: (operationId: string) => Promise, + normalize: (outcome: WorkOutcome) => ReconcileResult, + key = coalesceKey(request), + deliveryIntent?: ReconcileDeliveryIntent, + ): Promise> { + const leader = map.get(key); + const { envelope, reserved } = this.reserveOwnAttemptOrFailClosed(request, leader?.operationId, deliveryIntent); + + if (leader && (reserved || leader.operationId === envelope.operationId)) { + const completion = await leader.promise; + if (reserved && completion.settled) { + this.settleAttempt(request.applicationId, envelope, completion.execution.result); + } + return { kind: 'executed', execution: completion.execution }; + } + + if (!reserved) { + const byOperationId = GitSourceService.findByOperationId(map, envelope.operationId); + if (byOperationId) { + return { kind: 'executed', execution: (await byOperationId).execution }; + } + return { + kind: 'replayed', + result: this.resolveAlreadyReservedAttempt( + request.applicationId, + envelope.operationId, + request.actor, + request.trigger, + ), + }; + } + + const promise = this.captureExecution(request, envelope, work, normalize) + .then((execution): LeaderCompletion => ({ + execution, + settled: this.settleAttempt(request.applicationId, envelope, execution.result), + })); + const entry = { operationId: envelope.operationId, promise }; + map.set(key, entry); + try { + const completion = await promise; + return { kind: 'executed', execution: completion.execution }; + } finally { + if (map.get(key) === entry) map.delete(key); + } + } + + /** Capture raw producer behavior and compute one normalized result for all followers. */ + private async captureExecution( + request: ReconcileRequest, + envelope: EventEnvelope, + work: (operationId: string) => Promise, + normalize: (outcome: WorkOutcome) => ReconcileResult, + ): Promise> { + let outcome: WorkOutcome; + try { + outcome = { status: 'fulfilled', value: await work(envelope.operationId) }; + } catch (reason) { + outcome = { status: 'rejected', reason: reason ?? new Error('Rejected with no reason.') }; + } + try { + return { ...outcome, result: normalize(outcome) }; + } catch (e) { + console.error( + '[GitSource] Failed to derive a settlement result for attempt %s on application %s:', + sanitizeForLog(envelope.operationId), + sanitizeForLog(request.applicationId), + e instanceof Error ? e.message : String(e), + ); + return { + ...outcome, + result: { outcome: 'unknown', reason: 'This attempt could not be resolved.', nextAction: 'none' }, + }; + } + } + + /** + * Reserve this request's durable attempt, failing closed when the + * reservation bookkeeping itself fails: no fetch, apply, promotion, or + * deploy may run without a durable record of it, so a failure here stops + * the operation rather than letting it proceed as an untracked side + * effect. Two failure shapes, reported differently: an application torn + * down in the window between resolving it and reserving against it (a + * GitOpsTransitionError from requireApp) can never succeed on retry, so + * it gets its own message; anything else (a transient DB error) is worth + * retrying. The refusal is recorded to the stack's own activity history + * as well as the server console, since it is itself an event an operator + * needs to see later. Callers (an HTTP route, or handleWebhookPull's own + * try/catch) already handle a thrown GitSourceError the same way they + * handle any other failure from the work itself. + */ + private reserveOwnAttemptOrFailClosed( + request: ReconcileRequest, + followerOf: string | undefined, + deliveryIntent?: ReconcileDeliveryIntent, + ): { envelope: EventEnvelope; reserved: boolean } { + try { + return this.reserveOwnAttempt(request, followerOf, deliveryIntent); + } catch (e) { + console.error( + `[GitSource] Failed to reserve a durable attempt for application ${sanitizeForLog(request.applicationId)}; refusing to proceed without one:`, + e instanceof Error ? e.stack ?? e.message : String(e), + ); + const isFetch = request.intent === 'fetch'; + this.recordGitActivity( + request.stackName, + isFetch ? 'git_pull_failed' : 'git_apply_failed', + `Git ${request.intent} for ${request.stackName} was refused: could not durably record the attempt.`, + request.actor, + 'error', + ); + if (e instanceof GitOpsTransitionError) { + throw new GitSourceError( + 'GIT_ERROR', + `This stack's GitOps tracking is unavailable; reconfigure the source before ${isFetch ? 'pulling' : 'applying'} again.`, + ); + } + throw new GitSourceError('GIT_ERROR', 'Could not durably record this operation. Please try again.'); + } + } + + /** Refuse any configured source whose work cannot receive a durable attempt. */ + private refuseUntrackedSource(stackName: string, actor: string, intent: 'fetch' | 'apply', recordActivity: boolean): void { + if (!DatabaseService.getInstance().getGitSource(stackName)) return; + const wasDetached = GitOpsStore.getInstance().hasDetachedDirectApplication(stackName); + const message = wasDetached + ? `This stack's GitOps tracking was removed but its Git source configuration still exists; delete the Git source configuration to finish detaching before ${intent === 'fetch' ? 'pulling' : 'applying'} again.` + : `This stack's GitOps tracking is unavailable; reconfigure the source before ${intent === 'fetch' ? 'pulling' : 'applying'} again.`; + if (recordActivity) { + this.recordGitActivity(stackName, intent === 'fetch' ? 'git_pull_failed' : 'git_apply_failed', message, actor, 'error'); + } + throw new GitSourceError('GIT_ERROR', message); + } + + /** + * Reserve this submission's own durable attempt. A request carrying a + * stable external delivery id (webhook redelivery) reserves under a + * producer-namespaced key derived from it, so a redelivery of the same + * event reuses the same operation id and reports `reserved: false` + * rather than minting a second attempt. Any other submission has no + * such stable identity, so its operation id is freshly allocated from + * the row's own attemptSeq, which is always a first-time reservation. + */ + private reserveOwnAttempt( + request: ReconcileRequest, + followerOf: string | undefined, + deliveryIntent?: ReconcileDeliveryIntent, + ): { envelope: EventEnvelope; reserved: boolean } { + const tx = GitOpsTransitions.getInstance(); + if (request.deliveryId) { + const operationId = deliveryKey(request.trigger, request.intent, request.deliveryId); + const envelope = this.gitopsEnvelope(operationId, request.actor, request.trigger); + const { reserved } = tx.reserveReconcileAttempt(request.applicationId, envelope, followerOf, deliveryIntent); + return { envelope, reserved }; + } + const allocated = tx.allocateReconcileAttempt(request.applicationId, request.actor, request.trigger, Date.now(), followerOf); + return { + envelope: this.gitopsEnvelope(allocated.operationId, request.actor, request.trigger), + reserved: allocated.reserved, + }; + } + + /** + * Settle a durable attempt with its already-computed result, tolerating + * a settlement failure rather than letting it turn a correctly-computed + * result (up to and including a real fetch or apply that already + * touched the filesystem) into a thrown error for the caller. The + * attempt is left unsettled on this path, which is exactly the signal + * startup recovery looks for, so nothing here is lost, only deferred. + */ + private settleAttempt(applicationId: string, envelope: EventEnvelope, result: ReconcileResult): boolean { + try { + const { settled } = GitOpsTransitions.getInstance().settleReconcileAttempt(applicationId, envelope, result); + return settled || !!GitOpsStore.getInstance().getSettledAttempt(applicationId, envelope.operationId); + } catch (e) { + console.error( + '[GitSource] Failed to settle reconcile attempt %s for application %s:', + sanitizeForLog(envelope.operationId), + sanitizeForLog(applicationId), + e instanceof Error ? e.message : String(e), + ); + return false; + } + } + + /** + * A submission whose operation id was already reserved elsewhere, with + * no leader for it running in this process: a settled row means a + * duplicate delivery arrived after its original attempt finished, so + * its stored result is returned rather than repeating the work. No + * settled row means the original attempt was orphaned by a crash (in + * this process or another); either way this call must not re-execute a + * fetch or apply someone else may already have run, so it resolves + * from whatever is already durably recorded, settling when that + * yields a real answer and otherwise reporting truthfully that the + * outcome is not yet known rather than guessing one. + */ + private resolveAlreadyReservedAttempt(applicationId: string, operationId: string, actor: string | null, trigger: string): ReconcileResult { + try { + const store = GitOpsStore.getInstance(); + const settled = store.getSettledAttempt(applicationId, operationId); + if (settled) return GitSourceService.resultFromSettledAttempt(settled); + return this.settleFromDurableState(applicationId, operationId, actor, trigger); + } catch (e) { + console.error( + '[GitSource] Failed to resolve already-reserved attempt %s for application %s:', + sanitizeForLog(operationId), + sanitizeForLog(applicationId), + e instanceof Error ? e.message : String(e), + ); + return { outcome: 'unknown', reason: 'This attempt could not be resolved from durable state.', nextAction: 'none' }; + } + } + + /** + * Resolve one reconcile attempt purely from what is already recorded, + * never by re-executing a fetch or apply, settling it durably when that + * yields a real answer. A follower is settled from its leader; anything + * else is derived from the application's current row state. A follower + * whose leader is still unresolved is left unsettled for a later call + * (a future recovery pass, or the leader itself finally settling) to + * resolve, for the reason resolveFollowerOutcome states. + */ + private settleFromDurableState( + applicationId: string, + operationId: string, + actor: string | null, + trigger: string, + ): ReconcileResult { + const started = GitOpsStore.getInstance().getStartedAttempt(applicationId, operationId); + // Prefer the reservation's own recorded actor/trigger over this + // call's, so the settled row's audit trail reflects who and what + // actually reserved the attempt rather than whoever happened to + // resolve it later. + const envelope: EventEnvelope = { + operationId, + actor: started?.actor ?? actor, + trigger: started?.trigger ?? trigger, + at: Date.now(), + }; + const followerOf = started ? GitSourceService.followerOfFromRow(started) : undefined; + if (followerOf) { + const outcome = this.resolveFollowerOutcome(applicationId, followerOf); + if (!outcome.known) { + return { outcome: 'unknown', reason: 'This attempt is waiting on its leader to settle.', nextAction: 'none' }; + } + if (!this.settleAttempt(applicationId, envelope, outcome.result)) { + return GitSourceService.durableResolutionFailureResult(); + } + return outcome.result; + } + const result = this.deriveResultForApplication(applicationId); + if (!this.settleAttempt(applicationId, envelope, result)) { + return GitSourceService.durableResolutionFailureResult(); + } + return result; + } + + private static durableResolutionFailureResult(): ReconcileResult { + return { outcome: 'unknown', reason: 'This attempt could not be durably resolved.', nextAction: 'none' }; + } + + /** + * A follower's outcome from its leader alone: the leader's settled + * result when it has one. When the leader has no settled row but its + * own reservation genuinely exists, its fate is still unresolved + * (`known: false`) and must not be guessed at independently, since it + * could settle to something else later and the follower would then + * durably disagree with it. Only when the leader's own reservation + * cannot be found at all (nothing durable to ever wait for) does + * independent derivation apply, logged distinctly since it means the + * leader/follower agreement invariant could not be honored here. + */ + private resolveFollowerOutcome( + applicationId: string, + leaderOperationId: string, + ): { known: true; result: ReconcileResult } | { known: false } { + const store = GitOpsStore.getInstance(); + const leaderSettled = store.getSettledAttempt(applicationId, leaderOperationId); + if (leaderSettled) return { known: true, result: GitSourceService.resultFromSettledAttempt(leaderSettled) }; + if (store.getStartedAttempt(applicationId, leaderOperationId)) return { known: false }; + console.error( + `[GitSource] follower's leader ${sanitizeForLog(leaderOperationId)} has no recorded reservation for application ${sanitizeForLog(applicationId)}; deriving independently`, + ); + return { known: true, result: this.deriveResultForApplication(applicationId) }; + } + + /** + * The current truthful result for an application, independent of any + * specific attempt. Fails closed on a superseded application id for the + * same reason every live execution revalidates identity under its lock. + */ + private deriveResultForApplication(applicationId: string): ReconcileResult { + const store = GitOpsStore.getInstance(); + const app = store.getApplication(applicationId); + if (!app?.stack_name) return GitSourceService.noApplicationResult(); + if (store.getLiveDirectApplication(app.stack_name)?.id !== applicationId) { + return GitSourceService.staleApplicationResult(); + } + return this.deriveReconcileResult(app.stack_name); + } + + /** The follower-link operation id recorded on a reservation, if any. */ + private static followerOfFromRow(row: GitOpsHistoryRow): string | undefined { + const decoded = decodeGitOpsJson(row.after_json); + if (!isRecord(decoded)) throw new GitOpsJsonError('reserved attempt metadata must be an object'); + if (!('followerOf' in decoded)) return undefined; + if (typeof decoded.followerOf !== 'string') { + throw new GitOpsJsonError('reserved attempt followerOf must be a string'); + } + return decoded.followerOf; + } + + private static deliveryIntentFromStartedAttempt(row: GitOpsHistoryRow): ReconcileDeliveryIntent { + const decoded = decodeGitOpsJson(row.after_json); + if (!isRecord(decoded) || !isRecord(decoded.deliveryIntent)) { + throw new GitOpsJsonError('reserved webhook attempt has no delivery intent'); + } + const { autoApply, deploy } = decoded.deliveryIntent; + if (typeof autoApply !== 'boolean' || typeof deploy !== 'boolean') { + throw new GitOpsJsonError('reserved webhook attempt has an invalid delivery intent'); + } + return GitSourceService.deliveryIntent(autoApply, deploy); + } + + private static deliveryIntent(autoApply: boolean, deploy: boolean): ReconcileDeliveryIntent { + return autoApply ? { autoApply: true, deploy } : { autoApply: false, deploy: false }; + } + + /** + * Decode a settled attempt's recorded result back into a + * ReconcileResult. Unreadable JSON and a well-formed-but-wrong-shaped + * payload are both logged: a corrupt or unexpected settled row is a + * storage or encoding bug an operator needs to see, not a routine + * response variation, matching decodeHistoryDelta's own rule for this + * exact column. + */ + private static resultFromSettledAttempt(row: GitOpsHistoryRow): ReconcileResult { + const unreadable: ReconcileResult = { + outcome: 'unknown', + reason: 'The settled attempt result could not be read.', + nextAction: 'none', + }; + + let decoded: unknown; + try { + decoded = decodeGitOpsJson(row.after_json); + } catch (e) { + if (!(e instanceof GitOpsJsonError)) throw e; + console.error(`[GitSource] settled attempt ${sanitizeForLog(row.operation_id)} is not decodable JSON: ${e.message}`); + return unreadable; + } + if ( + !isRecord(decoded) + || !isReconcileOutcome(decoded.outcome) + || typeof decoded.reason !== 'string' + || !isNextAction(decoded.nextAction) + ) { + console.error(`[GitSource] settled attempt ${sanitizeForLog(row.operation_id)} decoded to an unexpected shape`); + return unreadable; + } + return { + outcome: decoded.outcome, + reason: decoded.reason, + nextAction: decoded.nextAction, + retryAt: typeof decoded.retryAt === 'number' ? decoded.retryAt : undefined, + commitSha: typeof decoded.commitSha === 'string' ? decoded.commitSha : undefined, + }; + } + + /** + * Startup recovery: settle every reconcile attempt that reserved but + * never settled, most likely because the process crashed between the + * two. Never re-executes a fetch or apply. Must run before + * SourceController starts, so no live poll or retry tick can race a + * recovery pass over the same attempts. + * + * Pages by cursor rather than by "still unsettled" status, so a row + * this run cannot recover never blocks the rest of the backlog; see + * listUnsettledReconcileAttempts for why that matters. + * + * Two passes. Pass 1 settles every independent (non-follower) attempt + * by deriving the application's current truthful state, and defers + * every follower rather than settling it yet, so its leader (which + * can only be earlier in this same backlog, since a follower's own + * reservation records that its leader was already in flight) gets a + * chance to settle first. Pass 2 then settles each deferred follower + * from its leader's now-settled result, so a leader and its followers + * always agree; a follower whose leader is still unresolved is left + * for a later recovery run, per resolveFollowerOutcome. + * + * One row failing to recover (a transient DB error, an application + * deleted between listing and processing) is isolated: logged and + * counted, never allowed to block any other row. + */ + public async recoverUnsettledReconcileAttempts(pageSize = 200): Promise { + const store = GitOpsStore.getInstance(); + const tx = GitOpsTransitions.getInstance(); + let recovered = 0; + let failed = 0; + let stillWaiting = 0; + const deferredFollowers: { row: GitOpsHistoryRow; followerOf: string }[] = []; + + const settle = (row: GitOpsHistoryRow, result: ReconcileResult): void => { + tx.settleReconcileAttempt( + row.application_id, + { operationId: row.operation_id, actor: row.actor, trigger: row.trigger, at: Date.now() }, + result, + ); + recovered++; + }; + const noteFailure = (row: GitOpsHistoryRow, e: unknown): void => { + failed++; + console.error( + `[GitSource] Failed to recover reconcile attempt ${sanitizeForLog(row.operation_id)} for application ${sanitizeForLog(row.application_id)}:`, + e instanceof Error ? e.message : String(e), + ); + }; + + // Last-resort guard: the cursor advances strictly past every page, so + // the loop is already bounded by the size of the backlog itself. + const MAX_PAGES = 10_000; + let cursor: GitOpsHistoryCursor | undefined; + let pagesRead = 0; + for (; pagesRead < MAX_PAGES; pagesRead++) { + const page = store.listUnsettledReconcileAttempts(pageSize, cursor); + if (page.length === 0) break; + const last = page[page.length - 1]; + cursor = { createdAt: last.created_at, id: last.id }; + for (const row of page) { + try { + const followerOf = GitSourceService.followerOfFromRow(row); + if (followerOf) { + deferredFollowers.push({ row, followerOf }); + continue; + } + settle(row, this.deriveResultForApplication(row.application_id)); + } catch (e) { + noteFailure(row, e); + } + } + if (page.length < pageSize) break; + } + if (pagesRead === MAX_PAGES) { + console.warn(`[GitSource] Reconcile-attempt recovery stopped at its per-run page cap (${MAX_PAGES} pages); remaining rows will be retried on the next startup.`); + } + + for (const { row, followerOf } of deferredFollowers) { + try { + const outcome = this.resolveFollowerOutcome(row.application_id, followerOf); + if (!outcome.known) { + stillWaiting++; + continue; + } + settle(row, outcome.result); + } catch (e) { + noteFailure(row, e); + } + } + + if (recovered > 0 || failed > 0 || stillWaiting > 0) { + console.log(`[GitSource] Reconcile-attempt recovery: ${recovered} settled, ${failed} could not be recovered, ${stillWaiting} still waiting on their leader.`); + } + } + + /** Run fetch-intent reconcile through the same durable execution as pull(). */ + private async reconcileFetch(request: ReconcileRequest & { intent: 'fetch' }): Promise { + const liveApp = GitOpsStore.getInstance().getLiveDirectApplication(request.stackName); + if (!liveApp) return GitSourceService.noApplicationResult(); + if (liveApp.id !== request.applicationId) return GitSourceService.staleApplicationResult(); + + const submission = await this.submitExecution( + this.inFlightFetches, + request, + this.doPullWork(request.stackName, request.actor, request.applicationId), + (outcome) => this.fetchExecutionResult(request.stackName, outcome), + ); + return GitSourceService.resultFromSubmission(submission); + } + + /** Normalize one shared fetch execution for its leader and all followers. */ + private fetchExecutionResult(stackName: string, outcome: WorkOutcome): ReconcileResult { + return this.finalizeReconcileOutcome(stackName, outcome.status === 'rejected' ? outcome.reason : undefined); + } + + /** Normalize one shared apply execution for its leader and all followers. */ + private applyExecutionResult(stackName: string, outcome: WorkOutcome): ReconcileResult { + if (outcome.status === 'fulfilled' && outcome.value.deployError) { + return { + outcome: 'recovery_required', + reason: `The source applied, but the deploy failed: ${outcome.value.deployError}`, + nextAction: 'view_target_results', + }; + } + return this.finalizeReconcileOutcome(stackName, outcome.status === 'rejected' ? outcome.reason : undefined); + } + + /** + * The result derived from the application's own row state, unless a + * failure occurred that the derived state does not already reflect, in + * which case the failure itself is classified instead. + */ + private finalizeReconcileOutcome(stackName: string, failure: unknown): ReconcileResult { + const derived = this.deriveReconcileResult(stackName); + if (failure === undefined || GitSourceService.FAILURE_REFLECTED_OUTCOMES.has(derived.outcome)) { + return derived; + } + return this.reconcileFailureResult(failure); + } + + /** + * A truthful fallback for a reconcile failure the application row does + * not yet reflect. Routes through the same classifyFailure disposition + * table the controller's own retry/backoff logic uses, so an unretryable + * failure is never reported with nextAction: 'retry'. + */ + private reconcileFailureResult(failure: unknown): ReconcileResult { + if (!(failure instanceof GitSourceError)) { + return { + outcome: 'failed_previous_intact', + reason: 'The reconcile attempt failed unexpectedly.', + nextAction: 'retry', + }; + } + const disposition = classifyFailure({ + kind: 'git_source_error', + code: failure.code, + transportReason: failure.extras?.transportReason, + }); + switch (disposition.class) { + case 'supersession': + return { outcome: 'superseded', reason: failure.message, nextAction: 'none' }; + case 'permanent': + return { outcome: 'failed_previous_intact', reason: failure.message, nextAction: 'configure_credentials' }; + case 'operator_action_required': + return { outcome: 'blocked', reason: failure.message, nextAction: 'resolve_conflict' }; + case 'reconcile': + return { outcome: 'unknown', reason: failure.message, nextAction: 'none' }; + // 'degraded'/'target_*'/'blocked' are not reachable from a + // git_source_error classification today, but are grouped with + // 'transient' so this switch stays exhaustive if that changes. + case 'transient': + case 'degraded': + case 'target_permanent': + case 'target_transient': + case 'target_mutation_failed': + case 'blocked': + return { outcome: 'failed_previous_intact', reason: failure.message, nextAction: 'retry' }; + } + } + + private deriveReconcileResult(stackName: string): ReconcileResult { + const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName); + if (!app) { + return GitSourceService.noApplicationResult(); + } + const projection = projectApplication(app.id, false); + if (projection.targetMode === 'not_applicable') { + if (projection.limitations.some((l) => l.code === 'application_row_missing')) { + return { + outcome: 'recovery_required', + reason: 'The application this reconcile was resolved from is no longer present.', + nextAction: 'view_target_results', + }; + } + return GitSourceService.noApplicationResult(); + } + return outcomeFromSourceFacet(projection.facets.source); + } + + /** + * Stop acting on a source without forgetting anything about it: no new + * fetch, acceptance, or dispatch until resumed (enforced by the + * suspended_at checks at the top of pullLocked/applyLockedBody, not by + * this method itself). Takes the per-stack mutex not to reject a + * concurrent fetch or apply, but because sourceSuspended interrupts and + * clears any in-flight operation's active state, which would corrupt a + * genuinely running apply's own terminal transition; a suspend queued + * behind one instead takes effect once that work settles. + * + * A refused suspend is surfaced as a real error rather than swallowed: + * silently no-op'ing here would leave an operator believing a source is + * suspended when it is not, which is the same false-safety failure this + * method exists to prevent. + */ + public async suspend(stackName: string, opts: { actor: string; reason?: string }): Promise { + return this.withStackLock(stackName, async () => { + const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName); + if (!app) return GitSourceService.noApplicationResult(); + const envelope = this.gitopsEnvelope(crypto.randomUUID(), opts.actor, 'suspend'); + const reason = opts.reason?.trim() || 'Suspended by operator.'; + try { + GitOpsTransitions.getInstance().sourceSuspended(app.id, reason, envelope); + } catch (error) { + if (error instanceof GitOpsTransitionError) { + throw new GitSourceError('OPERATION_IN_FLIGHT', `Cannot suspend ${stackName}: ${error.message}`); + } + throw error; + } + return this.deriveReconcileResult(stackName); + }); + } + + /** + * Resume acting on a source. Does not itself fetch; the next scheduled + * poll, retry, or manual reconcile picks the source back up. + * + * Unlike suspend(), a refused resume is tolerated rather than surfaced. + * The result is read back from the row after the attempted write, so a + * resume that did not take (already not suspended, the application + * vanished, a transient persistence failure) still truthfully reports + * {outcome:'suspended', nextAction:'resume'} rather than a false + * "resumed". The caller cannot be told the source is unsuspended when it + * is not, so there is no false-safety risk to mirror suspend()'s rethrow. + */ + public async resume(stackName: string, opts: { actor: string }): Promise { + return this.withStackLock(stackName, async () => { + const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName); + if (!app) return GitSourceService.noApplicationResult(); + const envelope = this.gitopsEnvelope(crypto.randomUUID(), opts.actor, 'resume'); + this.recordGitOps(stackName, 'source resume', () => { + GitOpsTransitions.getInstance().sourceUnsuspended(app.id, envelope); + }); + return this.deriveReconcileResult(stackName); + }); + } + + /** + * An explicit, operator-initiated re-evaluation: resolves the live + * application server-side rather than trusting a caller-supplied id, for + * callers that hold only a stack name, and drives a fresh fetch-intent + * reconcile through it. + * + * The 'retry' trigger is recorded on the durable attempt for audit and + * recovery. It does not yet change fetch behavior; a later permanent- + * failure gate can use it to authorize an explicit operator retry. + */ + public async retry(stackName: string, opts: { actor: string }): Promise { + const app = GitOpsStore.getInstance().getLiveDirectApplication(stackName); + if (!app) return GitSourceService.noApplicationResult(); + return this.reconcile({ + intent: 'fetch', + applicationId: app.id, + stackName, + trigger: 'retry', + actor: opts.actor, + }); + } + + /** + * Route an accepted generation to its target. Blueprint mode always + * blocks (BlueprintTargetAdapter; rollout orchestration does not exist + * yet). Direct mode has no separate generation-based promotion pipeline + * today, so it dispatches by driving the same reconcile()/apply path a + * manual or webhook apply already uses, translating the normalized + * ReconcileResult into the narrower dispatched/blocked shape a target + * adapter reports. + */ + public async dispatchAcceptedGeneration( + generation: AcceptedGeneration, + context: DispatchContext, + opts: { trigger: ReconcileTrigger; actor: string }, + ): Promise { + if (context.targetMode === 'blueprint') { + return new BlueprintTargetAdapter().dispatch(generation, context); + } + const app = GitOpsStore.getInstance().getApplication(generation.applicationId); + if (!app?.stack_name) { + return { status: 'blocked', reason: 'No Direct stack is bound to this application.' }; + } + const source = DatabaseService.getInstance().getGitSource(app.stack_name); + const result = await this.reconcile({ + intent: 'apply', + applicationId: generation.applicationId, + stackName: app.stack_name, + trigger: opts.trigger, + actor: opts.actor, + commitSha: generation.commitSha, + planFingerprint: generation.changePlanFingerprint ?? '', + deploy: source?.auto_deploy_on_apply ?? false, + }); + // 'converged' is not produced by reconcile() today (it requires + // target + health evidence this source-only path does not have), + // but it is a declared success member of ReconcileOutcome; treating + // only 'no_source_change' as success would silently misreport it as + // blocked the day a broader derivation starts emitting it. + if (result.outcome === 'no_source_change' || result.outcome === 'converged') { + return { status: 'dispatched' }; + } + return { status: 'blocked', reason: result.reason }; } /** @@ -2260,6 +3239,7 @@ export class GitSourceService { stackName: string, commitSha: string, opts: GitApplyOpts, + operationId?: string, ): Promise<{ applied: boolean; deployed: boolean; deployError?: string; recoveryId?: string }> { const nodeId = NodeRegistry.getInstance().getDefaultNodeId(); const lock = await StackOpLockService.getInstance().runExclusive( @@ -2267,7 +3247,7 @@ export class GitSourceService { stackName, 'git_apply', opts.actor ?? 'system:git-source', - () => this.applyLocked(stackName, commitSha, opts), + () => this.applyLocked(stackName, commitSha, opts, operationId), getRegistryDeliveryLockContext(), ); if (!lock.ran) { @@ -2292,6 +3272,7 @@ export class GitSourceService { stackName: string, commitSha: string, opts: GitApplyOpts, + operationId?: string, ): Promise<{ applied: boolean; deployed: boolean; deployError?: string; recoveryId?: string }> { const started: { app: GitOpsApplicationRow | null; env: ReturnType | null; settled: boolean } = { app: null, @@ -2299,7 +3280,7 @@ export class GitSourceService { settled: false, }; try { - return await this.applyLockedBody(stackName, commitSha, opts, started); + return await this.applyLockedBody(stackName, commitSha, opts, started, operationId); } catch (e) { if (started.app && started.env && !started.settled) { const app = started.app; @@ -2321,11 +3302,18 @@ export class GitSourceService { commitSha: string, opts: GitApplyOpts, started: { app: GitOpsApplicationRow | null; env: ReturnType | null; settled: boolean }, + operationId?: string, ): Promise<{ applied: boolean; deployed: boolean; deployError?: string; recoveryId?: string }> { const diag = isDebugEnabled(); const db = DatabaseService.getInstance(); const src = db.getGitSource(stackName); if (!src) throw new GitSourceError('GIT_ERROR', 'No Git source configured for this stack.'); + // Same reasoning as pullLocked's guard: applyStarted's own suspension + // check is swallowed by recordGitOps once promotion is already + // underway, so stop the promotion before it starts, not after. + if (this.gitopsApplicationFor(stackName)?.suspended_at) { + throw new GitSourceError('OPERATION_IN_FLIGHT', `Reconciliation is suspended for ${stackName}.`); + } if (!src.pending_commit_sha || !src.pending_compose_content) { throw new GitSourceError('GIT_ERROR', 'No pending pull to apply. Fetch the source again.'); @@ -2383,7 +3371,16 @@ export class GitSourceService { sanitizeForLog(stackName), sanitizeForLog(commitSha.slice(0, 7)), ); } - const gitopsEnv = this.gitopsEnvelope(pending.operationId, actor, 'apply'); + // A caller that reserved a durable attempt for this apply passes its own + // operation id in, so the attempt and its gitops transitions + // (applyStarted/applied/applyFailed) share one identity. A caller that + // reserved none falls back to the fetch-time pending.operationId, as + // every caller did before reservation existed. The activity messages + // below reuse this same value, not pending.operationId directly, so + // they report the id the durable evidence for this apply actually + // carries rather than the earlier fetch attempt's. + const applyOperationId = operationId ?? pending.operationId; + const gitopsEnv = this.gitopsEnvelope(applyOperationId, actor, 'apply'); if (gitopsApp && gitopsGenerationId) { this.recordGitOps(stackName, 'apply start', () => { GitOpsTransitions.getInstance().applyStarted(gitopsApp.id, gitopsGenerationId, gitopsEnv); @@ -2421,8 +3418,14 @@ export class GitSourceService { pending.candidateRelPath, ); } - const dataDir = process.env.DATA_DIR || path.join(process.cwd(), 'data'); - const candidateAbs = path.join(dataDir, 'git-managed', String(nodeId), stackName, pending.candidateRelPath); + const managedRoot = path.resolve(stackManagedRoot(stackName)); + const pathReason = validateCandidateRelPath(pending.candidateRelPath, managedRoot); + if (pathReason) throw new GitSourceError('GIT_ERROR', pathReason); + // Inline barrier at the access sink (CodeQL path-injection). + const candidateAbs = path.resolve(managedRoot, pending.candidateRelPath); + if (!candidateAbs.startsWith(managedRoot + path.sep)) { + throw new GitSourceError('GIT_ERROR', 'candidateRelPath escapes the managed root'); + } try { await fsPromises.access(candidateAbs); } catch (accessErr: unknown) { @@ -2490,7 +3493,7 @@ export class GitSourceService { { plan: publicPlan, planFingerprint: plan.fingerprint }, ); } - const blockedPlanActivity = `Git plan blocked for ${stackName} (${commitSha.slice(0, 7)}, op ${pending.operationId.slice(0, 8)}, plan ${plan.fingerprint.slice(0, 12)})`; + const blockedPlanActivity = `Git plan blocked for ${stackName} (${commitSha.slice(0, 7)}, op ${GitSourceService.shortOperationId(applyOperationId)}, plan ${plan.fingerprint.slice(0, 12)})`; if (plan.blocked) { this.upsertGitPlanDrift(stackName, plan); db.setGitSourceLastPlan(stackName, plan.fingerprint, 'blocked'); @@ -2629,7 +3632,7 @@ export class GitSourceService { this.recordGitActivity( stackName, 'git_apply_rolled_back', - `Git apply rolled back for ${stackName} (${commitSha.slice(0, 7)}, op ${pending.operationId.slice(0, 8)}, plan ${plan.fingerprint.slice(0, 12)})`, + `Git apply rolled back for ${stackName} (${commitSha.slice(0, 7)}, op ${GitSourceService.shortOperationId(applyOperationId)}, plan ${plan.fingerprint.slice(0, 12)})`, actor, 'warning', ); @@ -2638,7 +3641,7 @@ export class GitSourceService { this.recordGitActivity( stackName, 'git_apply_failed', - `Git apply failed for ${stackName} (${commitSha.slice(0, 7)}, op ${pending.operationId.slice(0, 8)}, plan ${plan.fingerprint.slice(0, 12)})`, + `Git apply failed for ${stackName} (${commitSha.slice(0, 7)}, op ${GitSourceService.shortOperationId(applyOperationId)}, plan ${plan.fingerprint.slice(0, 12)})`, actor, 'error', ); @@ -2651,7 +3654,7 @@ export class GitSourceService { this.recordGitActivity( stackName, 'git_apply', - `Git apply succeeded for ${stackName} (${commitSha.slice(0, 7)}, op ${pending.operationId.slice(0, 8)}, plan ${plan.fingerprint.slice(0, 12)})`, + `Git apply succeeded for ${stackName} (${commitSha.slice(0, 7)}, op ${GitSourceService.shortOperationId(applyOperationId)}, plan ${plan.fingerprint.slice(0, 12)})`, actor, ); // Promotion has committed and rewritten the authoritative Compose @@ -3259,20 +4262,19 @@ export class GitSourceService { } rowInserted = true; - const operationId = crypto.randomUUID(); if (completeProjectManifest && materialization.value && recordedCreatePlan) { db.setGitSourceLastPlan(input.stackName, recordedCreatePlan.fingerprint, 'applied'); this.recordGitActivity( input.stackName, 'git_create', - `Git create succeeded for ${input.stackName} (${fetched.commitSha.slice(0, 7)}, op ${operationId.slice(0, 8)}, plan ${recordedCreatePlan.fingerprint.slice(0, 12)})`, + `Git create succeeded for ${input.stackName} (${fetched.commitSha.slice(0, 7)}, op ${GitSourceService.shortOperationId(gitopsOperationId)}, plan ${recordedCreatePlan.fingerprint.slice(0, 12)})`, 'system:git-source', ); } else { this.recordGitActivity( input.stackName, 'git_create', - `Git create succeeded for ${input.stackName} (${fetched.commitSha.slice(0, 7)}, op ${operationId.slice(0, 8)})`, + `Git create succeeded for ${input.stackName} (${fetched.commitSha.slice(0, 7)}, op ${GitSourceService.shortOperationId(gitopsOperationId)})`, 'system:git-source', ); } @@ -3445,6 +4447,75 @@ export class GitSourceService { } } + /** + * The candidate directory basenames (e.g. "candidate-") the boot + * sweep must not reap for this stack: a row still points at each one, so + * it is still needed no matter how old or how incomplete it looks on + * disk. Direct-mode only, matching the candidate/generation model itself. + */ + private claimedCandidateDirsFor(stackName: string): { dirs: Set; complete: boolean } { + const store = GitOpsStore.getInstance(); + const claimed = new Set(); + let complete = true; + const app = store.getLiveDirectApplication(stackName); + if (app) { + // candidate_generation_id names the currently staged candidate. + // accepted_generation_id is set by applySourceAcceptanceMutation + // and never cleared, so after an ordinary apply it names an + // already-promoted generation whose candidate directory has been + // moved away (harmless to check, just not load-bearing). It earns + // its place for the sourceAccepted-committed-but-targetApplied- + // not-yet-committed window, where the generation is accepted and + // genuinely still unpromoted on disk. Nothing calls sourceAccepted + // or targetApplied yet, so that is forward-looking coverage rather + // than dead code. + for (const generationId of [app.candidate_generation_id, app.accepted_generation_id]) { + if (!generationId) continue; + const generation = store.getGeneration(generationId); + if (generation) { + claimed.add(path.basename(generation.candidate_dir)); + } else { + complete = false; + console.warn( + `[GitSource] Generation claimant ${sanitizeForLog(generationId)} for ${sanitizeForLog(stackName)} could not be resolved during the sweep.`, + ); + } + } + try { + for (const generation of store.listGenerationsClaimedByUnsettledAttempts(app.id)) { + claimed.add(path.basename(generation.candidate_dir)); + } + } catch (e) { + complete = false; + console.warn( + `[GitSource] Could not read unsettled-attempt candidate claims for ${sanitizeForLog(stackName)} during the sweep:`, + e instanceof Error ? e.message : String(e), + ); + } + } + // The pending blob's own candidateRelPath is a third, independent + // claimant: it is written outside the transaction that mints a + // generation, so a candidate can be staged and recorded as pending + // with no generation row at all (fetchedInvalid) or with no live + // application to read a pointer from (a stack whose boot migration + // failed). A decode failure must not abort the sweep; it only means + // this extra claim is unavailable. + try { + const src = DatabaseService.getInstance().getGitSource(stackName); + if (src?.pending_compose_content) { + const pending = this.decodePendingCompose(src.pending_compose_content); + if (pending.candidateRelPath) claimed.add(path.basename(pending.candidateRelPath)); + } + } catch (e) { + complete = false; + console.warn( + `[GitSource] Could not read the pending candidate reference for ${sanitizeForLog(stackName)} while computing sweep claimants:`, + e instanceof Error ? e.message : String(e), + ); + } + return { dirs: claimed, complete }; + } + public async sweepOrphans(): Promise { const fsSvc = FileSystemService.getInstance(); const manifestSvc = GitProjectManifestService.getInstance(); @@ -3482,9 +4553,17 @@ export class GitSourceService { }); continue; } - await this.withStackLock(row.stack_name, () => - manifestSvc.sweepManagedArea(row.stack_name, { repoUrl: row.repo_url, branch: row.branch, stackExists: true }), - ); + await this.withStackLock(row.stack_name, () => { + const claims = this.claimedCandidateDirsFor(row.stack_name); + return manifestSvc.sweepManagedArea(row.stack_name, { + repoUrl: row.repo_url, + branch: row.branch, + stackExists: true, + candidateClaims: claims.complete + ? { complete: true, dirs: claims.dirs } + : { complete: false }, + }); + }); } catch (e) { console.error(`[GitManifest] sweep failed for ${row.stack_name}:`, (e as Error).message); } @@ -3566,81 +4645,221 @@ export class GitSourceService { // ─── Webhook-triggered pull ────────────────────────────────────────────── + /** Whether this delivery's effective intent requires deploy permission. */ + public webhookDeliveryRequiresDeploy(stackName: string, deliveryId?: string): boolean { + const source = DatabaseService.getInstance().getGitSource(stackName); + if (!source) return false; + const app = this.gitopsApplicationFor(stackName); + const started = app && deliveryId + ? GitOpsStore.getInstance().getStartedAttempt( + app.id, + deliveryKey('webhook', 'fetch', deliveryId), + ) + : undefined; + if (started) return GitSourceService.deliveryIntentFromStartedAttempt(started).deploy; + return source.auto_apply_on_webhook && source.auto_deploy_on_apply; + } + /** - * Invoked by the webhook dispatcher. Returns a short status string to - * record in webhook_executions. Enforces the per-source debounce. + * Invoked by the webhook dispatcher. A provider-scoped delivery id resolves + * redeliveries durably. Debounce still rate-limits new deliveries and is + * the only deduplication mechanism when no stable id is available. The + * caller must explicitly pass whether the principal is authorized to + * execute a requested deploy. */ - public async handleWebhookPull(stackName: string): Promise<{ status: 'success' | 'skipped' | 'error'; message: string }> { - // Run the whole critical section under a single lock acquisition so a - // concurrent fan-out (N webhooks for one push) serializes AND re-reads - // last_debounce_at after acquiring the lock. The first request stamps - // the window; every queued duplicate then sees the stamp and skips - // instead of cloning again. The debounce is still stamped only after a - // successful fetch, so a transient failure stays immediately retriable. - return this.withStackLock<{ status: 'success' | 'skipped' | 'error'; message: string }>(stackName, async () => { - const diag = isDebugEnabled(); - const db = DatabaseService.getInstance(); - const src = db.getGitSource(stackName); - if (!src) { - return { status: 'error', message: 'No Git source configured for this stack.' }; - } + public async handleWebhookPull( + stackName: string, + deployAuthorized: boolean, + deliveryId?: string, + ): Promise { + if (!deliveryId) return this.handleWebhookPullOnce(stackName, undefined, deployAuthorized); + const key = `${stackName}:${deliveryId}`; + const leader = this.inFlightWebhookDeliveries.get(key); + if (leader) return leader.promise; - const now = Date.now(); - if (src.last_debounce_at !== null && (now - src.last_debounce_at) < WEBHOOK_DEBOUNCE_MS) { - if (diag) console.log(`[GitSource:diag] webhook debounced stack=${stackName} age=${now - src.last_debounce_at}ms`); - return { status: 'skipped', message: 'Rate limited (debounced).' }; + const promise = this.handleWebhookPullOnce(stackName, deliveryId, deployAuthorized); + const entry = { promise }; + this.inFlightWebhookDeliveries.set(key, entry); + try { + return await promise; + } finally { + if (this.inFlightWebhookDeliveries.get(key) === entry) { + this.inFlightWebhookDeliveries.delete(key); } + } + } - let pullResult: PullResult; + private async handleWebhookPullOnce( + stackName: string, + deliveryId: string | undefined, + deployAuthorized: boolean, + ): Promise { + const actor = 'system:webhook'; + const diag = isDebugEnabled(); + const deliverySuffix = deliveryId ? ` (delivery ${sanitizeForLog(deliveryId)})` : ''; + const db = DatabaseService.getInstance(); + const src = db.getGitSource(stackName); + if (!src) return { status: 'error', message: 'No Git source configured for this stack.' }; + + const gitopsApp = this.gitopsApplicationFor(stackName); + const startedDelivery = gitopsApp && deliveryId + ? GitOpsStore.getInstance().getStartedAttempt( + gitopsApp.id, + deliveryKey('webhook', 'fetch', deliveryId), + ) + : undefined; + const existingDelivery = !!startedDelivery; + const now = Date.now(); + if (!existingDelivery && src.last_debounce_at !== null && (now - src.last_debounce_at) < WEBHOOK_DEBOUNCE_MS) { + if (diag) console.log(`[GitSource:diag] webhook debounced stack=${stackName} age=${now - src.last_debounce_at}ms`); + return { status: 'skipped', message: 'Rate limited (debounced).' }; + } + if (!existingDelivery && gitopsApp?.suspended_at) { + return { status: 'skipped', message: 'Reconciliation is suspended for this source.' }; + } + if (!gitopsApp) { try { - pullResult = await this.pullLocked(stackName, 'system:webhook'); + this.refuseUntrackedSource(stackName, actor, 'fetch', false); } catch (e) { - const msg = e instanceof GitSourceError ? `${e.code}: ${e.message}` : (e as Error).message; - const scrubbed = scrubCredentials(msg); - this.recordGitActivity(stackName, 'git_pull_failed', `Git pull failed for ${stackName}`, 'system:webhook', 'error'); - console.error(`[GitSource] Webhook pull failed for ${sanitizeForLog(stackName)}: ${sanitizeForLog(scrubbed)}`); - return { status: 'error', message: scrubbed }; + const msg = e instanceof GitSourceError ? e.message : (e as Error).message; + console.warn(`[GitSource] Webhook delivery skipped for ${sanitizeForLog(stackName)}: ${sanitizeForLog(msg)}`); + return { status: 'skipped', message: msg }; } + return { status: 'error', message: 'GitOps tracking is unavailable for this source.' }; + } + + let deliveryIntent = GitSourceService.deliveryIntent( + src.auto_apply_on_webhook, + src.auto_deploy_on_apply, + ); + if (startedDelivery) { try { - // Only burn the debounce window once the fetch actually produced - // something. A transient network failure should be retriable - // immediately rather than locked out for the debounce interval. - db.touchGitSourceDebounce(stackName); - if (!pullResult.validation.ok) { - // Webhooks are unattended, so always leave a server-side - // breadcrumb; the caller only sees the HTTP status. - console.warn(`[GitSource] Webhook pull validation failed for ${sanitizeForLog(stackName)}: ${sanitizeForLog(pullResult.validation.error ?? 'unknown')}`); - return { status: 'error', message: `Validation failed: ${pullResult.validation.error}` }; - } - - if (!src.auto_apply_on_webhook) { - if (diag) console.log(`[GitSource:diag] webhook pending-only stack=${stackName} sha=${pullResult.commitSha.slice(0, 7)}`); - return { status: 'success', message: `Pending update ready at ${pullResult.commitSha.slice(0, 7)}.` }; - } - - const applied = await this.applyWithSharedLock(stackName, pullResult.commitSha, { - deploy: src.auto_deploy_on_apply, - actor: 'system:webhook', - requirePlanFingerprint: false, - }); - if (applied.deployError) { - // Apply wrote to disk but deploy failed. Surface it so the - // webhook_executions row records a degraded outcome instead - // of a clean success. - return { status: 'error', message: `Applied commit ${pullResult.commitSha.slice(0, 7)} but deploy failed: ${applied.deployError}` }; - } - const suffix = applied.deployed ? ' and deployed' : ''; - return { status: 'success', message: `Applied commit ${pullResult.commitSha.slice(0, 7)}${suffix}.` }; + deliveryIntent = GitSourceService.deliveryIntentFromStartedAttempt(startedDelivery); } catch (e) { - const msg = e instanceof GitSourceError ? `${e.code}: ${e.message}` : (e as Error).message; - const scrubbed = scrubCredentials(msg); - // Unattended path: record the failure server-side so an operator - // can diagnose without diag mode, since the Git provider only - // logs the HTTP status. - console.error(`[GitSource] Webhook pull failed for ${sanitizeForLog(stackName)}: ${sanitizeForLog(scrubbed)}`); - return { status: 'error', message: scrubbed }; + console.error( + '[GitSource] Could not recover webhook delivery intent for %s%s:', + sanitizeForLog(stackName), + deliverySuffix, + e instanceof Error ? e.message : String(e), + ); + return { status: 'error', message: 'Could not recover the original webhook delivery intent.' }; } - }); + } + if (deliveryIntent.deploy && !deployAuthorized) { + return { status: 'error', message: 'Deploy permission is required for this webhook delivery.' }; + } + + let pullResult: PullResult | undefined; + let fetchResult: ReconcileResult | undefined; + let replayedFetch: boolean; + try { + const request: ReconcileRequest = { + intent: 'fetch', + applicationId: gitopsApp.id, + stackName, + trigger: 'webhook', + actor, + deliveryId, + }; + const submission = await this.submitExecution( + this.inFlightFetches, + request, + this.doPullWork(stackName, actor, gitopsApp.id), + (outcome) => this.fetchExecutionResult(stackName, outcome), + coalesceKey(request), + deliveryId ? deliveryIntent : undefined, + ); + fetchResult = GitSourceService.resultFromSubmission(submission); + replayedFetch = submission.kind === 'replayed'; + if (submission.kind === 'executed') { + if (submission.execution.status === 'rejected') throw submission.execution.reason; + pullResult = submission.execution.value; + } + } catch (e) { + return this.webhookFailure(stackName, deliverySuffix, 'pull', e); + } + + if (fetchResult && !pullResult) { + const replayResult = GitSourceService.webhookResultFromReconcile(fetchResult); + if (replayResult.status !== 'success') return replayResult; + } + if (!replayedFetch) db.touchGitSourceDebounce(stackName); + if (pullResult && !pullResult.validation.ok) { + console.warn(`[GitSource] Webhook pull validation failed for ${sanitizeForLog(stackName)}: ${sanitizeForLog(pullResult.validation.error ?? 'unknown')}`); + return { status: 'error', message: `Validation failed: ${pullResult.validation.error}` }; + } + + const commitSha = pullResult?.commitSha ?? fetchResult?.commitSha; + if (!commitSha) { + return { status: 'error', message: fetchResult?.reason ?? 'The fetch completed without a candidate commit.' }; + } + const currentSource = db.getGitSource(stackName); + if (!currentSource) return { status: 'error', message: 'The Git source configuration was removed during reconciliation.' }; + if (!deliveryIntent.autoApply) { + if (diag) console.log(`[GitSource:diag] webhook pending-only stack=${stackName} sha=${commitSha.slice(0, 7)}`); + return { status: 'success', message: `Pending update ready at ${commitSha.slice(0, 7)}.` }; + } + + try { + const request: ReconcileRequest = { + intent: 'apply', + applicationId: gitopsApp.id, + stackName, + trigger: 'webhook', + actor, + deliveryId, + commitSha, + planFingerprint: '', + deploy: deliveryIntent.deploy, + }; + const submission = await this.submitExecution( + this.inFlightApplies, + request, + (operationId) => this.withStackLock(stackName, async () => { + this.assertLiveApplication(stackName, gitopsApp.id); + return this.applyWithSharedLock(stackName, commitSha, { + deploy: deliveryIntent.deploy, + actor, + requirePlanFingerprint: false, + }, operationId); + }), + (outcome) => this.applyExecutionResult(stackName, outcome), + GitSourceService.applyExecutionKey(request, false), + ); + if (submission.kind === 'replayed') { + return GitSourceService.webhookResultFromReconcile(submission.result); + } + if (submission.execution.status === 'rejected') throw submission.execution.reason; + const applied = submission.execution.value; + if (applied.deployError) { + return { status: 'error', message: `Applied commit ${commitSha.slice(0, 7)} but deploy failed: ${applied.deployError}` }; + } + const suffix = applied.deployed ? ' and deployed' : ''; + return { status: 'success', message: `Applied commit ${commitSha.slice(0, 7)}${suffix}.` }; + } catch (e) { + return this.webhookFailure(stackName, deliverySuffix, 'apply', e); + } + } + + private webhookFailure(stackName: string, deliverySuffix: string, phase: 'pull' | 'apply', error: unknown): WebhookPullResult { + const msg = error instanceof GitSourceError ? `${error.code}: ${error.message}` : error instanceof Error ? error.message : String(error); + const scrubbed = scrubCredentials(msg); + console.error(`[GitSource] Webhook ${phase} failed for ${sanitizeForLog(stackName)}${deliverySuffix}: ${sanitizeForLog(scrubbed)}`); + return { status: 'error', message: scrubbed }; + } + + private static webhookResultFromReconcile(result: ReconcileResult): WebhookPullResult { + switch (result.outcome) { + case 'converged': + case 'no_source_change': + case 'candidate_already_fetched': + case 'pending_review': + return { status: 'success', message: result.reason }; + case 'suspended': + return { status: 'skipped', message: result.reason }; + default: + return { status: 'error', message: result.reason }; + } } // ─── Change plan helpers ───────────────────────────────────────────────── diff --git a/backend/src/services/SelfIdentityService.ts b/backend/src/services/SelfIdentityService.ts index e134bb01..29454db5 100644 --- a/backend/src/services/SelfIdentityService.ts +++ b/backend/src/services/SelfIdentityService.ts @@ -1,5 +1,10 @@ import fs from 'fs/promises'; import DockerController from './DockerController'; +import { classifyBuildChannel, isSenchoDevRepository, type BuildChannel } from '../helpers/selfUpdateCompose'; +import { defaultInspectImage } from './selfDevBuildDetect'; +import { parseImageRef, selectLocalRepoDigest } from './registry-api'; +import { getSenchoVersion } from './CapabilityRegistry'; +import { withTimeout } from '../utils/withTimeout'; /** * Identifies the Docker resources that belong to the running Sencho container @@ -20,17 +25,32 @@ import DockerController from './DockerController'; * stays in its empty state, every `isOwn*()` returns false, and today's * behavior is preserved. */ +/** Canonical runtime build identity of the running Sencho container. */ +export interface BuildInfo { + version: string | null; + channel: BuildChannel; + /** The image reference the running container was started with, null when unknown. */ + imageRef: string | null; + /** Running image sha256 hex (no prefix), null when unknown. */ + imageId: string | null; + /** Validated registry digest or pinned `dev-` tag, null when unknown. */ + revision: string | null; +} + class SelfIdentityService { private static instance: SelfIdentityService; private containerId: string | null = null; private containerName: string | null = null; private composeProjectName: string | null = null; private imageIdHex: string | null = null; + private imageRef: string | null = null; + private revision: string | null = null; private networkIds = new Set(); private networkNames = new Set(); private volumeNames = new Set(); private initialized = false; private initializePromise: Promise | null = null; + private enrichmentPromise: Promise | null = null; public static getInstance(): SelfIdentityService { if (!SelfIdentityService.instance) { @@ -58,6 +78,14 @@ class SelfIdentityService { this.containerName = (info.Name || '').replace(/^\//, '') || null; this.composeProjectName = info.Config?.Labels?.['com.docker.compose.project'] ?? null; this.imageIdHex = SelfIdentityService.stripSha(info.Image ?? '') || null; + this.imageRef = info.Config?.Image ?? null; + // Bounded revision enrichment runs detached so it never blocks the callers + // awaiting initialize() (Docker event monitoring, resources discovery). Core + // identity above is already captured; enrichment only adds the registry + // digest / pinned dev- and is failure-isolated. The promise is retained + // so a reader that needs the settled revision can await it (see + // whenRevisionResolved) instead of observing a transient null. + this.enrichmentPromise = this.enrichRevision(this.imageRef, this.imageIdHex); const nets = info.NetworkSettings?.Networks ?? {}; for (const [name, net] of Object.entries(nets)) { @@ -118,6 +146,64 @@ class SelfIdentityService { } } + /** + * Canonical runtime build identity. All fields are captured fields or derived + * synchronously from them; this read never triggers a Docker call. `revision` + * is populated by the detached enrichment step fired during initialize() and + * reads null until that resolves (or if it fails). + */ + getBuildInfo(): BuildInfo { + return { + version: getSenchoVersion(), + channel: this.imageRef ? classifyBuildChannel(this.imageRef) : 'unknown', + imageRef: this.imageRef, + imageId: this.imageIdHex, + revision: this.revision, + }; + } + + /** + * Resolves once the detached revision enrichment has settled (success or + * failure), or immediately when none was started. Awaiting cannot hang or + * throw (enrichment is bounded and failure-isolated). A reader that needs + * the final `revision` awaits this before getBuildInfo() so a successful + * response never freezes a transient null. + */ + async whenRevisionResolved(): Promise { + if (this.enrichmentPromise) await this.enrichmentPromise; + } + + /** + * Resolve the immutable revision from the running image. For a dev-repo image + * carrying a pinned `dev-` tag, the tag itself is the revision. Otherwise + * the running image's `RepoDigests` are inspected for a digest matching the + * running reference. Any failure (inspect rejection, timeout, no matching + * digest) leaves `revision` null; enrichment never throws to the caller. + */ + private async enrichRevision(imageRef: string | null, imageIdHex: string | null): Promise { + try { + let revision: string | null = null; + if (imageRef && isSenchoDevRepository(imageRef)) { + const tag = parseImageRef(imageRef)?.tag; + if (tag && /^dev-[0-9a-f]{7,40}$/.test(tag)) revision = tag; + } + if (!revision && imageRef && imageIdHex) { + revision = await this.resolveDigestRevision(imageRef, imageIdHex); + } + this.revision = revision; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + console.warn('[SelfIdentity] build revision enrichment failed:', message); + } + } + + private async resolveDigestRevision(imageRef: string, imageIdHex: string): Promise { + const parsed = parseImageRef(imageRef); + if (!parsed) return null; + const inspected = await withTimeout(defaultInspectImage(imageIdHex), 2000, 'build revision inspect'); + return selectLocalRepoDigest(inspected.RepoDigests ?? [], parsed); + } + /** True when the given container ID or name matches the running Sencho container. Accepts short or full IDs. */ isOwnContainer(idOrName: string): boolean { if (!idOrName) return false; @@ -205,11 +291,14 @@ class SelfIdentityService { this.containerName = null; this.composeProjectName = null; this.imageIdHex = null; + this.imageRef = null; + this.revision = null; this.networkIds.clear(); this.networkNames.clear(); this.volumeNames.clear(); this.initialized = false; this.initializePromise = null; + this.enrichmentPromise = null; } private static stripSha(s: string): string { diff --git a/backend/src/services/WebhookService.ts b/backend/src/services/WebhookService.ts index e462a606..1ecf50b5 100644 --- a/backend/src/services/WebhookService.ts +++ b/backend/src/services/WebhookService.ts @@ -20,6 +20,7 @@ type ExecutionResult = { success: boolean; error?: string; duration_ms: number } type ExecutionStatus = 'success' | 'failure'; const REMOTE_WEBHOOK_REQUEST_TIMEOUT_MS = 30_000; +const MAX_PROVIDER_DELIVERY_ID_LENGTH = 256; // Maps a webhook lifecycle action to the per-stack lock action. 'pull' updates, // so it locks as 'update'; 'git-pull' is excluded (it locks inside GitSourceService). @@ -96,6 +97,7 @@ export class WebhookService { action: string, triggerSource: string | null, atomic?: boolean, + deliveryId?: string, ): Promise { if (webhook.id === undefined) { throw new Error('Webhook must be loaded from the database before execution'); @@ -110,11 +112,34 @@ export class WebhookService { return { success: false, error, duration_ms: 0 }; } + const scopedDeliveryId = action === 'git-pull' + ? WebhookService.scopedDeliveryId( + DatabaseService.getInstance().getGlobalSettings().delivery_source_id, + webhookId, + deliveryId, + ) + : undefined; + if (node.type === 'remote') { - return this.executeRemote(webhookId, nodeId, webhook.stack_name, action, triggerSource, atomic); + return this.executeRemote(webhookId, nodeId, webhook.stack_name, action, triggerSource, atomic, scopedDeliveryId); } - return this.executeLocal(webhookId, nodeId, webhook.stack_name, action, triggerSource, atomic); + return this.executeLocal(webhookId, nodeId, webhook.stack_name, action, triggerSource, atomic, scopedDeliveryId); + } + + /** Stable external identity, isolated to one configured webhook producer. */ + private static scopedDeliveryId( + deliverySourceId: string | undefined, + webhookId: number, + deliveryId: string | undefined, + ): string | undefined { + const normalized = deliveryId?.trim(); + if (!normalized) return undefined; + if (!deliverySourceId) throw new Error('Webhook delivery source identity is not configured'); + const bounded = normalized.length <= MAX_PROVIDER_DELIVERY_ID_LENGTH + ? normalized + : `sha256:${crypto.createHash('sha256').update(normalized).digest('hex')}`; + return `webhook:${deliverySourceId}:${webhookId}:${bounded}`; } public maskSecret(secret: string): string { @@ -129,6 +154,7 @@ export class WebhookService { action: string, triggerSource: string | null, atomic?: boolean, + deliveryId?: string, ): Promise { const stacks = await FileSystemService.getInstance(nodeId).getStacks(); if (!stacks.includes(stackName)) { @@ -142,7 +168,7 @@ export class WebhookService { // git-pull pulls then deploys through GitSourceService, which holds // the per-stack lock itself; locking here too would self-conflict. if (action === 'git-pull') { - return this.executeLocalGitPull(webhookId, stackName, action, triggerSource, startTime); + return this.executeLocalGitPull(webhookId, stackName, action, triggerSource, startTime, deliveryId); } const lockAction = WEBHOOK_LOCK_ACTION[action]; if (!lockAction) throw new Error(`Unknown action: ${action}`); @@ -225,8 +251,9 @@ export class WebhookService { action: string, triggerSource: string | null, startTime: number, + deliveryId?: string, ): Promise { - const result = await GitSourceService.getInstance().handleWebhookPull(stackName); + const result = await GitSourceService.getInstance().handleWebhookPull(stackName, true, deliveryId); const durationMs = Date.now() - startTime; if (result.status === 'error') { this.recordExecution(webhookId, action, 'failure', triggerSource, durationMs, result.message); @@ -255,6 +282,7 @@ export class WebhookService { action: string, triggerSource: string | null, atomic?: boolean, + deliveryId?: string, ): Promise { const startTime = Date.now(); try { @@ -263,7 +291,9 @@ export class WebhookService { : action === 'pull' ? 'update' : action; - const body = atomic === undefined ? undefined : { atomic }; + const body = action === 'git-pull' + ? { ...(atomic === undefined ? {} : { atomic }), ...(deliveryId ? { deliveryId } : {}) } + : atomic === undefined ? undefined : { atomic }; const response = await this.remoteStackRequest(nodeId, stackName, endpoint, 'POST', body); const durationMs = Date.now() - startTime; const payload = await response.json().catch(() => ({})) as { error?: string; message?: string; status?: string }; diff --git a/backend/src/services/gitops/SourceController.ts b/backend/src/services/gitops/SourceController.ts new file mode 100644 index 00000000..a551b96f --- /dev/null +++ b/backend/src/services/gitops/SourceController.ts @@ -0,0 +1,158 @@ +import { GitOpsStore } from './store'; +import { GitSourceService } from '../GitSourceService'; +import type { GitOpsApplicationRow } from './types'; +import { sanitizeForLog } from '../../utils/safeLog'; + +/** + * Background driver for unattended GitOps reconciliation: polls sources on + * their configured interval and re-evaluates applications whose retry_at + * has arrived, driving each through GitSourceService.reconcile(). + * + * Scope for this delivery: a tick only issues a fetch-intent reconcile, the + * same "detect and stage a candidate" step a manual pull performs. It does + * not evaluate source policy or drive automatic acceptance/dispatch for a + * newly staged candidate, and it does not yet resume a failure at the + * specific stage it failed at (fetch vs. dispatch); every retry re-issues + * a fetch. Both are real gaps against the source policy design, not silent + * omissions: automatic-policy acceptance and stage-aware retry are follow-on + * work once dispatchAcceptedGeneration() has a caller that decides when a + * staged candidate should be accepted. + * + * One self-rescheduling timer drives the scan, matching ImageUpdateService. + * The re-arm always runs, even when a scan throws, so one bad tick (a + * locked database, a transient store error) never permanently stops the + * driver. The per-application in-flight set, not the timer, is what keeps + * one busy application from blocking another: the tick never awaits any + * evaluation before rescheduling, so a slow application only pauses itself. + */ +export class SourceController { + private static instance: SourceController; + + private static readonly TICK_INTERVAL_MS = 60_000; + + private timer: NodeJS.Timeout | null = null; + private polling = false; + // Bumped by cancelPending(), so by stop() and restartPolling(). tick() has + // no internal await point today, so nothing can currently call either one + // mid-tick; this is a second, currently-redundant line of defense against a + // stale timer firing, kept cheap on purpose for when stage-aware retry (see + // above) gives evaluate() a real yield point. + private scheduleGeneration = 0; + private readonly inFlight = new Set(); + + private constructor() { } + + static getInstance(): SourceController { + if (!SourceController.instance) { + SourceController.instance = new SourceController(); + } + return SourceController.instance; + } + + /** Test-only: replace the singleton so timer/in-flight state never leaks between tests. */ + static resetForTests(): void { + SourceController.instance = new SourceController(); + } + + start(): void { + // Guards on `polling`, not `timer`: tick() nulls `timer` before it + // scans (so a stale timer reference can never block a restart), which + // would otherwise let a start() call landing during that scan see a + // false "not running" reading and arm a second timer. + if (this.polling) return; + this.polling = true; + this.armNext(); + } + + stop(): void { + this.cancelPending(); + this.polling = false; + } + + /** + * Reschedule the next tick without restarting: always clears any pending + * timer first, so calling this any number of times in a row never leaves + * more than one timer armed. + */ + restartPolling(): void { + this.cancelPending(); + if (this.polling) { + this.armNext(); + } + } + + isPolling(): boolean { + return this.polling; + } + + /** Clear any armed timer and invalidate the tick it would have run. */ + private cancelPending(): void { + this.scheduleGeneration++; + if (this.timer) { + clearTimeout(this.timer); + this.timer = null; + } + } + + private armNext(): void { + const gen = this.scheduleGeneration; + this.timer = setTimeout(() => { void this.tick(gen); }, SourceController.TICK_INTERVAL_MS); + this.timer.unref(); + } + + private async tick(gen: number): Promise { + if (!this.polling || gen !== this.scheduleGeneration) return; + this.timer = null; + try { + this.scan(); + } catch (e) { + console.error('[SourceController] scan failed:', e instanceof Error ? e.message : String(e)); + } finally { + if (this.polling && gen === this.scheduleGeneration) { + this.armNext(); + } + } + } + + /** + * Fire an evaluation for every due application without waiting for any + * of them. An application already in the in-flight set is left for a + * later tick instead of being queued behind its own still-running + * evaluation. A row due for both poll and retry is evaluated once. + */ + private scan(): void { + const now = Date.now(); + const store = GitOpsStore.getInstance(); + const due = new Map(); + for (const app of store.listSourcesDueForPoll(now)) due.set(app.id, app); + for (const app of store.listApplicationsDueForRetry(now)) due.set(app.id, app); + + for (const app of due.values()) { + if (this.inFlight.has(app.id)) continue; + this.inFlight.add(app.id); + this.evaluate(app).finally(() => this.inFlight.delete(app.id)); + } + } + + private async evaluate(app: GitOpsApplicationRow): Promise { + if (!app.stack_name) { + console.warn(`[SourceController] Skipping ${sanitizeForLog(app.id)}: direct-mode application has no stack_name.`); + return; + } + const isRetry = app.retry_at !== null && app.retry_at <= Date.now(); + try { + await GitSourceService.getInstance().reconcile({ + intent: 'fetch', + applicationId: app.id, + stackName: app.stack_name, + trigger: isRetry ? 'retry' : 'poll', + actor: 'system:source-controller', + }); + } catch (e) { + console.error( + `[SourceController] evaluation failed for ${sanitizeForLog(app.id)}:`, + e instanceof Error ? e.message : String(e), + ); + } + } +} diff --git a/backend/src/services/gitops/backoff.ts b/backend/src/services/gitops/backoff.ts new file mode 100644 index 00000000..a1e2ff22 --- /dev/null +++ b/backend/src/services/gitops/backoff.ts @@ -0,0 +1,140 @@ +import type { GitSourceErrorCode } from '../GitSourceService'; +import type { TransportFailureReason } from '../git/errors'; + +/** + * Everything a controller attempt can fail with. `git_source_error` covers + * every GitSourceError the fetch/apply path can throw, transport-classified + * or not. The remaining kinds cover dispatch-stage failures that have no + * GitSourceError at all (target binding, target availability, a deploy or + * health failure after a successful apply, Blueprint evaluation, and an + * interrupted or unknown-completion operation). + */ +export type FailureEvidence = + | { kind: 'git_source_error'; code: GitSourceErrorCode; transportReason?: TransportFailureReason } + | { kind: 'policy_unavailable' } + | { kind: 'persistence_unavailable' } + | { kind: 'target_binding_invalid' } + | { kind: 'target_unavailable' } + | { kind: 'target_mutation_failed' } + | { kind: 'blueprint_unavailable' } + | { kind: 'interrupted' }; + +export type FailureDisposition = + /** A newer revision replaced the one being worked on; re-resolve, not backoff. */ + | { class: 'supersession' } + /** Retryable. retryCeiling bounds how many attempts before it escalates to permanent. */ + | { class: 'transient'; retryCeiling: number } + /** Will never succeed by retrying; needs a configuration, environment, or credential change. */ + | { class: 'permanent' } + /** A human decision is required (review a plan, resolve a conflict); not retried automatically. */ + | { class: 'operator_action_required' } + /** Evidence could not be produced (e.g. scanner unavailable); held for review, not retried blind. */ + | { class: 'degraded' } + /** The target itself will never accept this generation without a configuration change. */ + | { class: 'target_permanent' } + /** The target is temporarily unreachable; retryable at the target/dispatch stage only. */ + | { class: 'target_transient' } + /** Applied but deploy or health failed: never refetch or reapply, only redeploy. */ + | { class: 'target_mutation_failed' } + /** Blocked pending a capability this program does not yet provide (e.g. Blueprint rollout). */ + | { class: 'blocked' } + /** Ambiguous or interrupted; reconcile from durable state, never blind retry. */ + | { class: 'reconcile' }; + +export const DEFAULT_TRANSIENT_CEILING = 8; +export const LOW_TRANSIENT_CEILING = 3; + +const TRANSIENT_DEFAULT: FailureDisposition = { class: 'transient', retryCeiling: DEFAULT_TRANSIENT_CEILING }; +const TRANSIENT_LOW: FailureDisposition = { class: 'transient', retryCeiling: LOW_TRANSIENT_CEILING }; +const PERMANENT: FailureDisposition = { class: 'permanent' }; + +/** + * Total over every TransportFailureReason except `exit`, which is generic + * and needs the classified GitSourceErrorCode (see CODE_DISPOSITION) to + * tell a transient network condition from a rate limit from an + * unrecognized error. Adding a new reason to the source union without + * adding it here fails the build. + */ +const REASON_DISPOSITION: Record, FailureDisposition> = { + 'tip-changed': { class: 'supersession' }, + timeout: TRANSIENT_DEFAULT, + 'target-unresolved': TRANSIENT_DEFAULT, + 'invalid-url': PERMANENT, + 'unsafe-target': PERMANENT, + 'invalid-ref': PERMANENT, + 'redirect-scope': PERMANENT, + 'git-missing': PERMANENT, + 'git-old': PERMANENT, + size: PERMANENT, + 'ssh-auth-required': PERMANENT, + 'ref-not-found': PERMANENT, + 'unsupported-ref': PERMANENT, +}; + +/** + * Total over every GitSourceErrorCode. Used directly when there is no + * transport reason (a plan/validation/file/operation-conflict error), and + * as the exit-reason fallback (RATE_LIMITED, NETWORK_TIMEOUT, and GIT_ERROR + * only ever arise from an `exit` transport reason). Adding a new code + * without adding it here fails the build. + */ +const CODE_DISPOSITION: Record = { + REPO_NOT_FOUND: PERMANENT, + AUTH_FAILED: PERMANENT, + REF_NOT_FOUND: PERMANENT, + REF_DELETED: PERMANENT, + UNSUPPORTED_REF: PERMANENT, + SSH_HOST_KEY_FAILED: PERMANENT, + FILE_NOT_FOUND: { class: 'operator_action_required' }, + RATE_LIMITED: TRANSIENT_DEFAULT, + NETWORK_TIMEOUT: TRANSIENT_DEFAULT, + GIT_ERROR: TRANSIENT_LOW, + STALE_PLAN: { class: 'operator_action_required' }, + PLAN_FINGERPRINT_REQUIRED: { class: 'operator_action_required' }, + PLAN_BLOCKED: { class: 'operator_action_required' }, + LEGACY_PENDING: { class: 'operator_action_required' }, + PLAN_UNAVAILABLE: { class: 'operator_action_required' }, + OPERATION_IN_FLIGHT: { class: 'reconcile' }, +}; + +export function classifyFailure(evidence: FailureEvidence): FailureDisposition { + switch (evidence.kind) { + case 'git_source_error': + if (evidence.transportReason && evidence.transportReason !== 'exit') { + return REASON_DISPOSITION[evidence.transportReason]; + } + return CODE_DISPOSITION[evidence.code]; + case 'policy_unavailable': + return { class: 'degraded' }; + case 'persistence_unavailable': + return TRANSIENT_DEFAULT; + case 'target_binding_invalid': + return { class: 'target_permanent' }; + case 'target_unavailable': + return { class: 'target_transient' }; + case 'target_mutation_failed': + return { class: 'target_mutation_failed' }; + case 'blueprint_unavailable': + return { class: 'blocked' }; + case 'interrupted': + return { class: 'reconcile' }; + } +} + +const BASE_DELAY_MS = 60_000; +const MAX_DELAY_MS = 3_600_000; +const JITTER_RATIO = 0.1; + +/** + * Bounded exponential backoff with jitter: 60s * 2^retryCount, capped at one + * hour, with up to +-10% jitter so many sources retrying at once do not + * all land on the same second. A provider-supplied retry floor (e.g. a + * rate-limit Retry-After) takes precedence whenever it is larger than the + * computed delay. + */ +export function nextRetryAt(now: number, retryCount: number, providerFloorMs?: number): number { + const capped = Math.min(BASE_DELAY_MS * 2 ** retryCount, MAX_DELAY_MS); + const jittered = capped + capped * JITTER_RATIO * (Math.random() * 2 - 1); + const delay = providerFloorMs !== undefined ? Math.max(jittered, providerFloorMs) : jittered; + return now + delay; +} diff --git a/backend/src/services/gitops/blueprintProducers.ts b/backend/src/services/gitops/blueprintProducers.ts index 7dfc5f17..8dfc5013 100644 --- a/backend/src/services/gitops/blueprintProducers.ts +++ b/backend/src/services/gitops/blueprintProducers.ts @@ -341,7 +341,7 @@ export function commitBlueprintDelete(blueprintId: number, actor: string | null) } /** A Blueprint application before anything has been asked of it. */ -export function blankInlineApplication(id: string, blueprintId: number, at: number) { +export function blankInlineApplication(id: string, blueprintId: number, at: number): GitOpsApplicationRow { return { id, lifecycle_key: `blueprint:${blueprintId}`, @@ -382,6 +382,10 @@ export function blankInlineApplication(id: string, blueprintId: number, at: numb pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, diff --git a/backend/src/services/gitops/directApplication.ts b/backend/src/services/gitops/directApplication.ts index c3693dd9..2ad5d303 100644 --- a/backend/src/services/gitops/directApplication.ts +++ b/backend/src/services/gitops/directApplication.ts @@ -157,6 +157,10 @@ export function buildDirectApplicationRow(args: { pause_at: null, pause_reason: null, source_suspended_reason: null, + source_policy: 'manual', + poll_interval_secs: null, + next_poll_at: null, + attempt_seq: 0, partial_json: null, failure_stage: null, failure_class: null, @@ -217,6 +221,12 @@ export function buildGenerationRow(args: { actor: args.actor, previous_generation_id: null, redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, created_at: args.at, }; } diff --git a/backend/src/services/gitops/handoff.ts b/backend/src/services/gitops/handoff.ts new file mode 100644 index 00000000..cc8e0f8f --- /dev/null +++ b/backend/src/services/gitops/handoff.ts @@ -0,0 +1,149 @@ +import type { RepoIdentity } from './repoIdentity'; +import type { RefKind } from '../git/types'; +import type { GitOpsGenerationRow } from './types'; + +/** + * A portable projection of the managed-project manifest: authored file set + * and per-file content digests, without node id, stack name, or generation + * directory. The real GitProjectManifest carries those identity fields; + * this is deliberately narrower. + */ +export type PortableManifest = { + files: Array<{ path: string; role: string; contentSha256?: string | null }>; +}; + +/** Authored Compose invocation shape, without a target project name. */ +export type ComposeInputs = { + composeFileOrder: string[]; + profiles?: string[]; + contextDir?: string | null; +}; + +/** + * One accepted generation, described purely by what it contains. Direct and + * Blueprint dispatch consume the exact same shape; current target mode, + * binding revision, and any execution-local path travel separately in + * DispatchContext, re-read under the dispatch lock rather than carried + * here, so a stale acceptance can never authorize a routing decision made + * after it. + */ +export type AcceptedGeneration = { + contractVersion: 1; + generationId: string; + applicationId: string; + repoIdentity: RepoIdentity; + configuredRef: string; + commitSha: string; + resolvedRefKind: RefKind | null; + manifestVersion: number; + portableManifest: PortableManifest | null; + composeInputs: ComposeInputs | null; + materializationFingerprint: string; + changePlanFingerprint: string | null; + validationOk: boolean; + sourcePolicyEvidence: unknown | null; + securityPolicyEvidence: unknown | null; + supportRequirements: unknown | null; + compatibilityRequirements: unknown | null; + /** Capability metadata only; never a secret value. Not yet populated by any producer. */ + secretCapability: unknown | null; + trigger: string; + actor: string | null; + operationId: string; + previousGenerationId: string | null; + /** Why some field above could not be proven, recorded honestly rather than guessed. */ + limitations: string[]; +}; + +function parseOptionalJson(raw: string | null, limitationLabel: string, limitations: string[]): T | null { + if (raw === null) { + limitations.push(limitationLabel); + return null; + } + try { + return JSON.parse(raw) as T; + } catch { + limitations.push(`${limitationLabel}_unparseable`); + return null; + } +} + +/** + * Build the portable accepted-generation contract from a persisted + * generation row. A legacy row predating one of the portable-contract + * columns decodes that field as null with an explicit limitation recorded, + * never as invented evidence. + */ +export function buildAcceptedGeneration(row: GitOpsGenerationRow): AcceptedGeneration { + let repoIdentity: RepoIdentity; + try { + repoIdentity = JSON.parse(row.repo_identity_json) as RepoIdentity; + } catch { + throw new Error(`Generation ${row.id} has an unparseable repo_identity_json; refusing to build an accepted-generation contract from corrupt evidence.`); + } + + const limitations: string[] = JSON.parse(row.redacted_limitations_json) as string[]; + const portableManifest = parseOptionalJson(row.portable_manifest_json, 'portable_manifest_missing', limitations); + const composeInputs = parseOptionalJson(row.compose_inputs_json, 'compose_inputs_missing', limitations); + const sourcePolicyEvidence = parseOptionalJson(row.source_policy_evidence_json, 'source_policy_evidence_missing', limitations); + const securityPolicyEvidence = parseOptionalJson(row.security_policy_evidence_json, 'security_policy_evidence_missing', limitations); + const supportRequirements = parseOptionalJson(row.support_requirements_json, 'support_requirements_missing', limitations); + const compatibilityRequirements = parseOptionalJson(row.compatibility_requirements_json, 'compatibility_requirements_missing', limitations); + + return { + contractVersion: 1, + generationId: row.id, + applicationId: row.application_id, + repoIdentity, + configuredRef: row.configured_ref, + commitSha: row.commit_sha, + resolvedRefKind: row.resolved_ref_kind, + manifestVersion: row.manifest_version, + portableManifest, + composeInputs, + materializationFingerprint: row.materialization_fingerprint, + changePlanFingerprint: row.change_plan_fingerprint, + validationOk: row.validation_ok === 1, + sourcePolicyEvidence, + securityPolicyEvidence, + supportRequirements, + compatibilityRequirements, + secretCapability: null, + trigger: row.trigger, + actor: row.actor, + operationId: row.operation_id, + previousGenerationId: row.previous_generation_id, + limitations, + }; +} + +/** + * Current target mode and binding, re-read under the dispatch lock rather + * than carried on AcceptedGeneration, so a routing decision is always made + * from the current state, never a value an earlier acceptance froze. + */ +export type DispatchContext = { + targetMode: 'direct' | 'blueprint'; + nodeId: number | null; + bindingRevision: string | null; +}; + +export type DispatchResult = + | { status: 'dispatched' } + | { status: 'blocked'; reason: string }; + +export interface TargetAdapter { + dispatch(generation: AcceptedGeneration, context: DispatchContext): Promise; +} + +/** + * Fails closed until Blueprint rollout orchestration exists. Never + * inspects selectors, target sets, or placement: an accepted generation + * for a Blueprint-mode application is evaluated the same as Direct, but + * dispatch stops here. + */ +export class BlueprintTargetAdapter implements TargetAdapter { + async dispatch(_generation: AcceptedGeneration, _context: DispatchContext): Promise { + return { status: 'blocked', reason: 'Blueprint rollout orchestration is not yet implemented.' }; + } +} diff --git a/backend/src/services/gitops/history.ts b/backend/src/services/gitops/history.ts index bee1642f..65ef7005 100644 --- a/backend/src/services/gitops/history.ts +++ b/backend/src/services/gitops/history.ts @@ -70,6 +70,8 @@ export type GitOpsHistoryStage = | 'rollout_unpaused' | 'source_accepted' | 'source_conflict_blocker' + | 'source_reconcile_started' + | 'source_reconcile_settled' | 'source_retry_scheduled' | 'source_suspended' | 'source_unsuspended' diff --git a/backend/src/services/gitops/migrate.ts b/backend/src/services/gitops/migrate.ts index 002a3652..aa7a6e6e 100644 --- a/backend/src/services/gitops/migrate.ts +++ b/backend/src/services/gitops/migrate.ts @@ -226,6 +226,12 @@ function migrateAccepted( actor: envelope.actor, previous_generation_id: null, redacted_limitations_json: '[]', + portable_manifest_json: null, + compose_inputs_json: null, + source_policy_evidence_json: null, + security_policy_evidence_json: null, + support_requirements_json: null, + compatibility_requirements_json: null, created_at: envelope.at, }; store.insertGeneration(generation); diff --git a/backend/src/services/gitops/outcomes.ts b/backend/src/services/gitops/outcomes.ts new file mode 100644 index 00000000..0b72babb --- /dev/null +++ b/backend/src/services/gitops/outcomes.ts @@ -0,0 +1,211 @@ +import type { SourceFacet } from './types'; + +/** + * Normalized reconcile outcomes. Silence is not an acceptable GitOps + * result: every attempt settles into exactly one of these, never a bare + * success/failure boolean. + * + * `converged` is deliberately never produced by outcomeFromSourceFacet: it + * requires target and health evidence this source-only projection does not + * have, and "no source change" is not proof of full convergence. A later + * composition over source + target + health facets is what may report it. + */ +export type ReconcileOutcome = + | 'converged' + | 'no_source_change' + | 'candidate_already_fetched' + | 'pending_review' + | 'suspended' + | 'retry_scheduled' + | 'blocked' + | 'superseded' + | 'failed_previous_intact' + | 'recovery_required' + | 'unknown'; + +export type NextAction = + | 'none' + | 'review' + | 'resume' + | 'retry' + | 'resolve_conflict' + | 'configure_credentials' + | 'view_target_results'; + +export type ReconcileResult = { + outcome: ReconcileOutcome; + reason: string; + nextAction: NextAction; + retryAt?: number; + commitSha?: string; +}; + +/** Every ReconcileOutcome member, for runtime validation of a value read back from storage. */ +const RECONCILE_OUTCOMES: ReadonlySet = new Set([ + 'converged', + 'no_source_change', + 'candidate_already_fetched', + 'pending_review', + 'suspended', + 'retry_scheduled', + 'blocked', + 'superseded', + 'failed_previous_intact', + 'recovery_required', + 'unknown', +]); + +/** Every NextAction member, for runtime validation of a value read back from storage. */ +const NEXT_ACTIONS: ReadonlySet = new Set([ + 'none', + 'review', + 'resume', + 'retry', + 'resolve_conflict', + 'configure_credentials', + 'view_target_results', +]); + +export function isReconcileOutcome(value: unknown): value is ReconcileOutcome { + return typeof value === 'string' && RECONCILE_OUTCOMES.has(value); +} + +export function isNextAction(value: unknown): value is NextAction { + return typeof value === 'string' && NEXT_ACTIONS.has(value); +} + +function commitShaOf(facet: Extract): string | undefined { + return facet.desiredCommitSha ?? facet.fetchedCommitSha ?? undefined; +} + +/** + * Derive the normalized outcome of a settled source reconcile attempt from + * the existing source-facet projection, rather than re-deriving status from + * raw application-row fields. Keeps the outcome vocabulary and the + * projection's own status vocabulary from silently drifting apart. + */ +export function outcomeFromSourceFacet(facet: SourceFacet): ReconcileResult { + switch (facet.status) { + case 'not_applicable': + return { outcome: 'unknown', reason: 'No GitOps application exists for this stack.', nextAction: 'none' }; + + case 'never_reconciled': + return { outcome: 'unknown', reason: 'The source has never been reconciled.', nextAction: 'none' }; + + case 'checking_fetching': + case 'applying': + return { + outcome: 'unknown', + reason: 'A reconcile operation is currently in flight; no settled result yet.', + nextAction: 'none', + commitSha: commitShaOf(facet), + }; + + case 'source_reconcile_required': + return { + outcome: 'unknown', + reason: 'The source has advanced but reconciliation has not evaluated it yet.', + nextAction: 'none', + commitSha: commitShaOf(facet), + }; + + case 'application_generation_accepted': + return { + outcome: 'no_source_change', + reason: 'The configured ref still resolves to the accepted generation. This is not proof of full convergence.', + nextAction: 'none', + commitSha: commitShaOf(facet), + }; + + case 'candidate_ready': + return { + outcome: 'candidate_already_fetched', + reason: 'A candidate generation is already staged and awaiting acceptance.', + nextAction: 'none', + commitSha: commitShaOf(facet), + }; + + case 'source_review_pending': + return { + outcome: 'pending_review', + reason: 'A candidate is staged and requires explicit review before acceptance.', + nextAction: 'review', + commitSha: commitShaOf(facet), + }; + + case 'source_conflict_blocker': + return { + outcome: 'blocked', + reason: 'A local conflict is blocking the candidate from being accepted.', + nextAction: 'resolve_conflict', + commitSha: commitShaOf(facet), + }; + + case 'source_superseded': + return { + outcome: 'superseded', + reason: 'A newer revision superseded this candidate before it was accepted.', + nextAction: 'none', + commitSha: commitShaOf(facet), + }; + + case 'source_retry_scheduled': + return { + outcome: 'retry_scheduled', + reason: `A previous attempt failed transiently; retry ${facet.retryCount + 1} is scheduled.`, + nextAction: 'none', + retryAt: facet.retryAt, + commitSha: commitShaOf(facet), + }; + + case 'source_suspended': + return { + outcome: 'suspended', + reason: facet.suspendedReason + ? `Reconciliation is suspended: ${facet.suspendedReason}` + : 'Reconciliation is suspended.', + nextAction: 'resume', + commitSha: commitShaOf(facet), + }; + + case 'source_failed': + return { + outcome: 'failed_previous_intact', + reason: `The ${facet.failureStage} stage failed (${facet.failureClass}). The previously accepted generation is unchanged.`, + nextAction: facet.retryAt !== null ? 'retry' : 'configure_credentials', + retryAt: facet.retryAt ?? undefined, + commitSha: commitShaOf(facet), + }; + + case 'source_unknown': + return { + outcome: 'recovery_required', + reason: `An operation was interrupted at ${facet.interruptedStage} and its outcome is unproven.`, + nextAction: 'view_target_results', + commitSha: commitShaOf(facet), + }; + + case 'recovery_required': + return { + outcome: 'recovery_required', + reason: 'Recovery from an earlier failed mutation is still outstanding.', + nextAction: 'view_target_results', + commitSha: commitShaOf(facet), + }; + + case 'recovery_failed': + return { + outcome: 'recovery_required', + reason: `Recovery itself failed (${facet.failureClass}); this needs operator attention.`, + nextAction: 'view_target_results', + commitSha: commitShaOf(facet), + }; + + case 'not_live': + return { + outcome: 'unknown', + reason: `The application is ${facet.lifecycleStatus}, not live; there is nothing to reconcile.`, + nextAction: 'none', + }; + } +} diff --git a/backend/src/services/gitops/schema.ts b/backend/src/services/gitops/schema.ts index 1c311e24..d881ed3e 100644 --- a/backend/src/services/gitops/schema.ts +++ b/backend/src/services/gitops/schema.ts @@ -100,6 +100,16 @@ CREATE TABLE IF NOT EXISTS gitops_applications ( -- target rows, so suspending a source can never clobber an unrelated -- rollout pause reason (or the reverse). source_suspended_reason TEXT NULL, + -- Controller-owned bookkeeping. NULL poll_interval_secs inherits the + -- global default; 0 disables polling for this application. next_poll_at + -- is the durable scheduling cursor. attempt_seq is allocated + -- transactionally per submission lacking a stable external delivery id. + source_policy TEXT NOT NULL DEFAULT 'manual' CHECK ( + source_policy IN ('manual','review','automatic') + ), + poll_interval_secs INTEGER NULL, + next_poll_at INTEGER NULL, + attempt_seq INTEGER NOT NULL DEFAULT 0, partial_json TEXT NULL, failure_stage TEXT NULL CHECK ( failure_stage IS NULL OR failure_stage IN ( @@ -177,6 +187,17 @@ CREATE TABLE IF NOT EXISTS gitops_generations ( actor TEXT NULL, previous_generation_id TEXT NULL, redacted_limitations_json TEXT NOT NULL DEFAULT '[]', + -- Portable accepted-generation contract (content only: no node id, local + -- path, target mode, or secret value). Additive and nullable so existing + -- rows decode as an explicit limitation rather than invented evidence; a + -- legacy pending candidate lacking these must be re-evaluated before it + -- can be accepted or dispatched. + portable_manifest_json TEXT NULL, + compose_inputs_json TEXT NULL, + source_policy_evidence_json TEXT NULL, + security_policy_evidence_json TEXT NULL, + support_requirements_json TEXT NULL, + compatibility_requirements_json TEXT NULL, created_at INTEGER NOT NULL ); CREATE INDEX IF NOT EXISTS idx_gitops_gen_app_created @@ -453,6 +474,10 @@ CREATE INDEX IF NOT EXISTS idx_gitops_history_node ON gitops_history(node_id); CREATE INDEX IF NOT EXISTS idx_gitops_history_trigger ON gitops_history(trigger); CREATE INDEX IF NOT EXISTS idx_gitops_history_actor ON gitops_history(actor); CREATE INDEX IF NOT EXISTS idx_gitops_history_outcome ON gitops_history(outcome); +-- listUnsettledReconcileAttempts filters on stage and orders by created_at; +-- without this, that query (run on every startup, ahead of the server +-- listening) scans and sorts the whole table. +CREATE INDEX IF NOT EXISTS idx_gitops_history_stage_created ON gitops_history(stage, created_at); CREATE INDEX IF NOT EXISTS idx_gitops_history_repo_ref ON gitops_history(repo_url, configured_ref); CREATE INDEX IF NOT EXISTS idx_gitops_history_stack_created diff --git a/backend/src/services/gitops/store.ts b/backend/src/services/gitops/store.ts index 4069d6a2..f5199f92 100644 --- a/backend/src/services/gitops/store.ts +++ b/backend/src/services/gitops/store.ts @@ -1,5 +1,6 @@ import type Database from 'better-sqlite3'; import { DatabaseService } from '../DatabaseService'; +import type { GitOpsHistoryCursor } from './history'; import { decodeArtifactEvidenceJson, decodeGitOpsApprovedTargetEffectJson, @@ -16,6 +17,7 @@ import type { GitOpsCreateCheckpointRow, GitOpsCreatePhase, GitOpsGenerationRow, + GitOpsHistoryRow, GitOpsIntentRevisionRow, GitOpsRolloutCandidateRow, GitOpsTargetCurrentRow, @@ -131,6 +133,29 @@ export class GitOpsStore { ).get(stackName) as GitOpsApplicationRow | undefined; } + /** + * Whether a detached Direct application exists for this stack name, + * distinct from "no application was ever created" (the legitimate + * pre-migration case, where a stack's git-source config predates + * GitOps tracking entirely). `detached` only, matching + * getDetachedDirectApplication above and for the same reason: + * `deleted` is not a safe signal here. Two production paths tombstone + * an application as `deleted` while deliberately preserving its + * git-source row so a future upsert or migration can rebuild from it + * (`gitops/createRecovery.ts`'s checkpointless-create sweep, and + * `gitops/migrate.ts`'s `tombstoned_missing_stack` outcome) -- + * treating that state as a refusal would be permanent and + * unrecoverable, since neither upsert() nor migration currently mints + * a fresh application once a matching migration checkpoint exists. + * `detach()` itself deletes the git-source row in the same transaction + * as tombstoning (`detached`), so the window this method exists to + * catch (tracking removed, config surviving) is a crash between those + * two writes, not routine operation. + */ + hasDetachedDirectApplication(stackName: string): boolean { + return this.getDetachedDirectApplication(stackName) !== undefined; + } + /** Direct applications that never reached their success boundary. */ listCreatingDirectApplications(): GitOpsApplicationRow[] { return this.db().prepare( @@ -144,6 +169,25 @@ export class GitOpsStore { return this.db().prepare('SELECT * FROM gitops_generations WHERE id = ?').get(id) as GitOpsGenerationRow | undefined; } + /** Generations whose creating reconcile attempt has not durably settled. */ + listGenerationsClaimedByUnsettledAttempts(applicationId: string): GitOpsGenerationRow[] { + return this.db().prepare( + `SELECT DISTINCT generation.* + FROM gitops_generations generation + JOIN gitops_history started + ON started.application_id = generation.application_id + AND started.operation_id = generation.operation_id + AND started.stage = 'source_reconcile_started' + WHERE generation.application_id = ? + AND NOT EXISTS ( + SELECT 1 FROM gitops_history settled + WHERE settled.application_id = started.application_id + AND settled.operation_id = started.operation_id + AND settled.stage = 'source_reconcile_settled' + )`, + ).all(applicationId) as GitOpsGenerationRow[]; + } + getArtifactSet(id: string): GitOpsArtifactSetRow | undefined { return this.db().prepare('SELECT * FROM gitops_artifact_sets WHERE id = ?').get(id) as GitOpsArtifactSetRow | undefined; } @@ -195,6 +239,126 @@ export class GitOpsStore { ).all() as GitOpsApplicationRow[]; } + /** + * The settled result for one exact reconcile attempt, or undefined when + * that attempt has not settled (or never existed). Used to recover a + * reservation that already completed rather than repeating the work. + */ + getSettledAttempt(applicationId: string, operationId: string): GitOpsHistoryRow | undefined { + return this.db().prepare( + `SELECT * FROM gitops_history + WHERE application_id = ? AND operation_id = ? AND stage = 'source_reconcile_settled' + LIMIT 1`, + ).get(applicationId, operationId) as GitOpsHistoryRow | undefined; + } + + /** + * The reservation row for one exact reconcile attempt, or undefined when + * it was never reserved. Used to recover this attempt's own recorded + * follower link (if any), so a follower can be settled from its + * leader's actual result rather than derived independently of it. + */ + getStartedAttempt(applicationId: string, operationId: string): GitOpsHistoryRow | undefined { + return this.db().prepare( + `SELECT * FROM gitops_history + WHERE application_id = ? AND operation_id = ? AND stage = 'source_reconcile_started' + LIMIT 1`, + ).get(applicationId, operationId) as GitOpsHistoryRow | undefined; + } + + /** + * Every reservation with no matching settled row, oldest first: an + * attempt that started but never recorded a result, most likely because + * the process crashed between reservation and settlement. Startup + * recovery reconciles these from durable stage evidence rather than + * leaving them silently open forever. + * + * `after` pages strictly forward by (created_at, id), the same cursor + * shape `queryHistoryRows` uses, and is load-bearing rather than + * cosmetic: a row a caller cannot settle (its application vanished, a DB + * error) stays unsettled forever by definition, so without a cursor it + * would occupy the same "oldest N" window on every future call and hide + * every genuinely recoverable row behind it once the backlog exceeds one + * page. + */ + listUnsettledReconcileAttempts(limit = 200, after?: GitOpsHistoryCursor): GitOpsHistoryRow[] { + const clauses = ["started.stage = 'source_reconcile_started'"]; + const params: Array = []; + if (after) { + clauses.push('(started.created_at > ? OR (started.created_at = ? AND started.id > ?))'); + params.push(after.createdAt, after.createdAt, after.id); + } + params.push(limit); + return this.db().prepare( + `SELECT started.* FROM gitops_history started + WHERE ${clauses.join(' AND ')} + AND NOT EXISTS ( + SELECT 1 FROM gitops_history settled + WHERE settled.application_id = started.application_id + AND settled.operation_id = started.operation_id + AND settled.stage = 'source_reconcile_settled' + ) + ORDER BY started.created_at ASC, started.id ASC + LIMIT ?`, + ).all(...params) as GitOpsHistoryRow[]; + } + + /** + * The most recently settled attempt for an application, for API and UI + * projection. Distinct from getSettledAttempt, which looks up one exact + * operation rather than the newest one. + */ + latestSettledAttempt(applicationId: string): GitOpsHistoryRow | undefined { + // rowid (SQLite's implicit insertion-order key), not the id column: id + // is a random UUID and does not sort by recency the way rowid does, so + // it cannot break a created_at tie between two attempts settled within + // the same millisecond. + return this.db().prepare( + `SELECT * FROM gitops_history + WHERE application_id = ? AND stage = 'source_reconcile_settled' + ORDER BY created_at DESC, rowid DESC + LIMIT 1`, + ).get(applicationId) as GitOpsHistoryRow | undefined; + } + + /** + * Direct sources whose poll time has arrived: active, not suspended, no + * operation in flight. Blueprint-mode applications are never polled here + * -- source evaluation for them is blocked at the evaluation boundary + * until an application-keyed source engine exists for that mode. + */ + listSourcesDueForPoll(now: number, limit = 200): GitOpsApplicationRow[] { + return this.db().prepare( + `SELECT * FROM gitops_applications + WHERE target_mode = 'direct' + AND lifecycle_status = 'active' + AND suspended_at IS NULL + AND active_operation_stage IS NULL + AND next_poll_at IS NOT NULL + AND next_poll_at <= ? + ORDER BY next_poll_at ASC + LIMIT ?`, + ).all(now, limit) as GitOpsApplicationRow[]; + } + + /** + * Applications with a scheduled retry that has come due: not suspended, + * no operation in flight. Poll eligibility and retry eligibility are + * deliberately separate queries, since a retry can be due on an + * application whose poll cadence would not otherwise select it yet. + */ + listApplicationsDueForRetry(now: number, limit = 200): GitOpsApplicationRow[] { + return this.db().prepare( + `SELECT * FROM gitops_applications + WHERE retry_at IS NOT NULL + AND retry_at <= ? + AND suspended_at IS NULL + AND active_operation_stage IS NULL + ORDER BY retry_at ASC + LIMIT ?`, + ).all(now, limit) as GitOpsApplicationRow[]; + } + /** Every live target on one node, across all applications. */ listActiveTargetsForNode(nodeId: number): GitOpsTargetCurrentRow[] { return this.db().prepare( @@ -365,12 +529,13 @@ export class GitOpsStore { intent_revision_id, rollout_candidate_id, rollout_generation_id, source_acceptance_ref, placement_approval_ref, rollout_authorization_ref, legacy_combined_approval_ref, preflight_fingerprint, latest_operation_id, active_operation_id, active_operation_stage, - active_operation_at, active_generation_id, pause_at, pause_reason, source_suspended_reason, partial_json, + active_operation_at, active_generation_id, pause_at, pause_reason, source_suspended_reason, + source_policy, poll_interval_secs, next_poll_at, attempt_seq, partial_json, failure_stage, failure_class, failure_at, retry_at, retry_count, suspended_at, recovery_ref, recovery_phase, interruption_stage, interruption_at, interruption_operation_id, interruption_generation_id, evidence_fresh_at, evidence_limitations_json, created_at, updated_at - ) VALUES (${Array(56).fill('?').join(', ')})`, + ) VALUES (${Array(60).fill('?').join(', ')})`, ).run( row.id, row.lifecycle_key, row.lifecycle_status, row.target_mode, row.stack_name, row.blueprint_id, row.configured_repo_url, row.repo_identity_json, row.configured_ref, row.compose_paths_json, @@ -380,7 +545,8 @@ export class GitOpsStore { row.intent_revision_id, row.rollout_candidate_id, row.rollout_generation_id, row.source_acceptance_ref, row.placement_approval_ref, row.rollout_authorization_ref, row.legacy_combined_approval_ref, row.preflight_fingerprint, row.latest_operation_id, row.active_operation_id, row.active_operation_stage, - row.active_operation_at, row.active_generation_id, row.pause_at, row.pause_reason, row.source_suspended_reason, row.partial_json, + row.active_operation_at, row.active_generation_id, row.pause_at, row.pause_reason, row.source_suspended_reason, + row.source_policy, row.poll_interval_secs, row.next_poll_at, row.attempt_seq, row.partial_json, row.failure_stage, row.failure_class, row.failure_at, row.retry_at, row.retry_count, row.suspended_at, row.recovery_ref, row.recovery_phase, row.interruption_stage, row.interruption_at, row.interruption_operation_id, row.interruption_generation_id, row.evidence_fresh_at, @@ -394,13 +560,18 @@ export class GitOpsStore { id, application_id, commit_sha, repo_url, configured_ref, resolved_ref_kind, repo_identity_json, manifest_version, candidate_dir, applied_dir, expected_invocation_json, materialization_fingerprint, validation_ok, plan_blocked, change_plan_fingerprint, - operation_id, trigger, actor, previous_generation_id, redacted_limitations_json, created_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + operation_id, trigger, actor, previous_generation_id, redacted_limitations_json, + portable_manifest_json, compose_inputs_json, source_policy_evidence_json, + security_policy_evidence_json, support_requirements_json, compatibility_requirements_json, + created_at + ) VALUES (${Array(27).fill('?').join(', ')})`, ).run( row.id, row.application_id, row.commit_sha, row.repo_url, row.configured_ref, row.resolved_ref_kind, row.repo_identity_json, row.manifest_version, row.candidate_dir, row.applied_dir, row.expected_invocation_json, row.materialization_fingerprint, row.validation_ok, row.plan_blocked, row.change_plan_fingerprint, row.operation_id, row.trigger, row.actor, row.previous_generation_id, row.redacted_limitations_json, + row.portable_manifest_json, row.compose_inputs_json, row.source_policy_evidence_json, + row.security_policy_evidence_json, row.support_requirements_json, row.compatibility_requirements_json, row.created_at, ); } diff --git a/backend/src/services/gitops/transitions.ts b/backend/src/services/gitops/transitions.ts index 5ebf736a..91fb286e 100644 --- a/backend/src/services/gitops/transitions.ts +++ b/backend/src/services/gitops/transitions.ts @@ -28,6 +28,10 @@ export type EventEnvelope = { at: number; }; +export type ReconcileDeliveryIntent = + | { autoApply: false; deploy: false } + | { autoApply: true; deploy: boolean }; + export type AppliedArgs = { applicationId: string; generationId: string; @@ -893,6 +897,102 @@ export class GitOpsTransitions { }); } + /** + * Reserve a durable attempt before any side effect: a bare history + * insert in its own transaction, deliberately not through mutateApp, so + * nothing about the application row changes. A `reserved: false` return + * means this exact (application, operation) already reserved -- the + * caller reconstructs from durable state rather than repeating work. + * + * `followerOf` records that this reservation joined another running + * attempt. Recovery uses the link to settle the follower from its leader's + * result. `deliveryIntent` preserves the original webhook apply and deploy + * decision so redelivery cannot change behavior with later settings. + */ + reserveReconcileAttempt( + applicationId: string, + envelope: EventEnvelope, + followerOf?: string, + deliveryIntent?: ReconcileDeliveryIntent, + ): { reserved: boolean } { + return this.raw().transaction(() => ({ + reserved: this.insertReconcileReservation(this.requireApp(applicationId), envelope, followerOf, deliveryIntent), + }))(); + } + + /** + * Allocate the next attemptSeq for a submission with no stable external + * delivery identity, and reserve its durable attempt in the same + * transaction, so two concurrent submissions can never mint the same + * operation id. Unlike reserveReconcileAttempt, this does write one + * column of application state (attempt_seq) -- allocation is the one + * thing here that is not a bare history insert, since a fresh id has to + * come from somewhere durable. Only the allocated id's uniqueness is + * load-bearing; its embedded sequence number is for traceability. + */ + allocateReconcileAttempt( + applicationId: string, + actor: string | null, + trigger: string, + at: number, + followerOf?: string, + ): { operationId: string; reserved: boolean } { + return this.raw().transaction(() => { + const app = this.requireApp(applicationId); + const seq = app.attempt_seq + 1; + this.raw().prepare('UPDATE gitops_applications SET attempt_seq = ? WHERE id = ?').run(seq, applicationId); + const operationId = `${applicationId}:attempt:${seq}`; + const envelope: EventEnvelope = { operationId, actor, trigger, at }; + return { operationId, reserved: this.insertReconcileReservation(app, envelope, followerOf) }; + })(); + } + + /** + * The reservation row itself, shared by reserveReconcileAttempt and + * allocateReconcileAttempt: a bare history insert whose dedupe index is + * what makes a repeat reservation report false rather than recording a + * second attempt. + */ + private insertReconcileReservation( + app: GitOpsApplicationRow, + envelope: EventEnvelope, + followerOf: string | undefined, + deliveryIntent?: ReconcileDeliveryIntent, + ): boolean { + return this.history(app, envelope, { + stage: 'source_reconcile_started', + outcome: 'committed', + before: {}, + after: { + ...(followerOf ? { followerOf } : {}), + ...(deliveryIntent ? { deliveryIntent } : {}), + }, + }) !== null; + } + + /** + * Settle a reserved attempt with its normalized outcome, the same way: + * a bare history insert, not a state mutation. Safe to call more than + * once for the same operation; a repeat is a no-op via the history + * dedupe index, so the first settled result is never overwritten. + */ + settleReconcileAttempt( + applicationId: string, + envelope: EventEnvelope, + result: { outcome: string; reason: string; nextAction: string; retryAt?: number; commitSha?: string }, + ): { settled: boolean } { + return this.raw().transaction(() => { + const app = this.requireApp(applicationId); + const historyId = this.history(app, envelope, { + stage: 'source_reconcile_settled', + outcome: 'committed', + before: {}, + after: { ...result }, + }); + return { settled: historyId !== null }; + })(); + } + /** * Pause a rollout, application-wide or on one target. * @@ -2319,7 +2419,8 @@ export class GitOpsTransitions { legacy_combined_approval_ref=?, preflight_fingerprint=?, latest_operation_id=?, active_operation_id=?, active_operation_stage=?, active_operation_at=?, active_generation_id=?, - pause_at=?, pause_reason=?, source_suspended_reason=?, partial_json=?, + pause_at=?, pause_reason=?, source_suspended_reason=?, + source_policy=?, poll_interval_secs=?, next_poll_at=?, attempt_seq=?, partial_json=?, failure_stage=?, failure_class=?, failure_at=?, retry_at=?, retry_count=?, suspended_at=?, recovery_ref=?, recovery_phase=?, interruption_stage=?, interruption_at=?, interruption_operation_id=?, @@ -2336,7 +2437,8 @@ export class GitOpsTransitions { app.legacy_combined_approval_ref, app.preflight_fingerprint, app.latest_operation_id, app.active_operation_id, app.active_operation_stage, app.active_operation_at, app.active_generation_id, - app.pause_at, app.pause_reason, app.source_suspended_reason, app.partial_json, + app.pause_at, app.pause_reason, app.source_suspended_reason, + app.source_policy, app.poll_interval_secs, app.next_poll_at, app.attempt_seq, app.partial_json, app.failure_stage, app.failure_class, app.failure_at, app.retry_at, app.retry_count, app.suspended_at, app.recovery_ref, app.recovery_phase, app.interruption_stage, app.interruption_at, app.interruption_operation_id, diff --git a/backend/src/services/gitops/triggers.ts b/backend/src/services/gitops/triggers.ts new file mode 100644 index 00000000..16c7c512 --- /dev/null +++ b/backend/src/services/gitops/triggers.ts @@ -0,0 +1,77 @@ +/** + * Normalized reconciliation triggers for the GitOps source controller. + * + * A trigger only authorizes evaluation; it is not proof anything changed. + * `manual`, `webhook`, `poll`, and `retry` have current execution producers. + * The remaining values are typed ahead of later deliveries so those callers + * extend this union instead of inventing a parallel one. + */ +export type ReconcileTrigger = + | 'manual' + | 'api' + | 'webhook' + | 'poll' + | 'retry' + | 'config_change' + | 'startup' + | 'resume' + | 'provider_event' + | 'schedule' + | 'binding_change'; + +/** + * One normalized submission to the controller. `dismiss` is deliberately + * not a reconcile intent: it changes candidate state but does not + * authorize source evaluation. + */ +export type ReconcileRequest = + | { + intent: 'fetch'; + applicationId: string; + stackName: string; + trigger: ReconcileTrigger; + actor: string; + deliveryId?: string; + } + | { + intent: 'apply'; + applicationId: string; + stackName: string; + trigger: ReconcileTrigger; + actor: string; + commitSha: string; + planFingerprint: string; + deploy: boolean; + deliveryId?: string; + }; + +/** + * The in-process joining key for concurrent evaluations of the same work. + * A fetch has only one live outcome per application regardless of trigger, + * so any two fetch submissions for the same application and stack join. An + * apply is identified by exactly what it would do: two applies join only + * when they target the same commit, the same plan fingerprint, and the + * same deploy choice. Two applies that differ in any of those must never + * join, or one request could silently receive another request's result. + * + * Both the fetch and the apply form carry the stack name alongside the + * applicationId, so a caller that pairs a live applicationId with the + * wrong stackName can never join a leader evaluating the right one. + */ +export function coalesceKey(request: ReconcileRequest): string { + if (request.intent === 'fetch') { + return `${request.applicationId}:${request.stackName}:fetch`; + } + return `${request.applicationId}:${request.stackName}:apply:${request.commitSha}:${request.planFingerprint}:${request.deploy}`; +} + +/** + * A producer-namespaced key for an external delivery, so the same delivery + * id from two different trigger sources is never treated as one delivery. + * Also namespaced by intent: a webhook that both fetches and applies under + * one delivery id must reserve two distinct attempts, not have the apply's + * reservation collide with the fetch's and silently never run. + */ +export function deliveryKey(trigger: ReconcileTrigger, intent: ReconcileRequest['intent'], deliveryId: string): string { + return `${trigger}:${intent}:${deliveryId}`; +} diff --git a/backend/src/services/gitops/types.ts b/backend/src/services/gitops/types.ts index c9f1ffa8..3d3790dc 100644 --- a/backend/src/services/gitops/types.ts +++ b/backend/src/services/gitops/types.ts @@ -73,6 +73,11 @@ export type GitOpsApplicationRow = { pause_reason: string | null; /** sourceSuspended/sourceUnsuspended's own reason field; independent of pause_reason. */ source_suspended_reason: string | null; + /** Controller-owned. See gitops/SourceController.ts. */ + source_policy: 'manual' | 'review' | 'automatic'; + poll_interval_secs: number | null; + next_poll_at: number | null; + attempt_seq: number; partial_json: string | null; failure_stage: ApplicationFailureStage | null; failure_class: string | null; @@ -161,6 +166,13 @@ export type GitOpsGenerationRow = { actor: string | null; previous_generation_id: string | null; redacted_limitations_json: string; + /** Portable accepted-generation contract fields. See gitops/handoff.ts. */ + portable_manifest_json: string | null; + compose_inputs_json: string | null; + source_policy_evidence_json: string | null; + security_policy_evidence_json: string | null; + support_requirements_json: string | null; + compatibility_requirements_json: string | null; created_at: number; }; diff --git a/backend/src/services/selfDevBuildDetect.ts b/backend/src/services/selfDevBuildDetect.ts index 65cf4cfa..2ee64ef6 100644 --- a/backend/src/services/selfDevBuildDetect.ts +++ b/backend/src/services/selfDevBuildDetect.ts @@ -27,13 +27,17 @@ export type SelfDevBuildDetectResult = | { kind: 'inconclusive'; reason: string }; /** The subset of `docker image inspect` output the detector reads. */ -interface InspectedImage { +export interface InspectedImage { RepoDigests: string[]; Os: string; Architecture: string; } -async function defaultInspectImage(imageId: string): Promise { +/** Bounded read of `docker image inspect` (RepoDigests, OS, architecture) for a + * resolved image. Reused by `SelfIdentityService` for revision enrichment; + * callers wrap it so any rejection stays isolated from the fields already + * captured. */ +export async function defaultInspectImage(imageId: string): Promise { const inspect = await DockerController.getInstance().getDocker().getImage(`sha256:${imageId}`).inspect(); return { RepoDigests: inspect.RepoDigests ?? [], Os: inspect.Os, Architecture: inspect.Architecture }; } diff --git a/docs/features/licensing.mdx b/docs/features/licensing.mdx index fc0f82dc..67574fc3 100644 --- a/docs/features/licensing.mdx +++ b/docs/features/licensing.mdx @@ -108,7 +108,7 @@ The **Plan** card lists: - **License** (Community only): a link to view the AGPLv3 source on GitHub. - **Recovery Vault** (Admiral only): confirms the entitlement is included in your subscription. - **Hardened Build** (Admiral only): a **Switch to Hardened** button; see [Switching to Hardened Build](#switching-to-hardened-build) below. -- **Current image** and **Channel**: the image reference and channel (`Community` or `Hardened`) this control plane is currently running. +- **Current image** and **Channel**: the image reference and channel (`Community` or `Hardened`) this control plane is currently running. These rows reflect the running build, not the configured target, so a compose edit does not change them until the container is recreated. A hardened image is shown as `Restricted` to non-administrators. - **Customer**, **Product**, and **License key** (active licenses only): purchase metadata and your key masked to its last four characters (`****-****-****-XXXX`). The full key is never re-displayed after activation. ## Switching to Hardened Build diff --git a/docs/operations/verifying-images.mdx b/docs/operations/verifying-images.mdx index 07d25f12..27376e32 100644 --- a/docs/operations/verifying-images.mdx +++ b/docs/operations/verifying-images.mdx @@ -154,3 +154,17 @@ Maintainers publish these from open PRs for external validation. They are unsign |---|---|---| | `pr-` | `saelix/sencho:pr-1526` | Each re-run of the preview workflow for that PR | | `preview-` | `saelix/sencho:preview-abc1234` | Never (immutable per build) | + +## Reading your running build + +Sencho surfaces the build it is actually running in **Settings → About → Build** (and in the **Channel** and **Current image** rows of **Settings → Admiral Account**). These fields come from the running container's identity, not from the compose file, so a compose edit does not change them until the container is recreated. + +| Field | Meaning | +|---|---| +| **Version** | The packaged semantic version of the build. A dev build keeps the last stable version here, which is why the Channel row matters. | +| **Channel** | The build track of the running image: `Dev`, `Preview`, `Stable`, or `Unknown`. A `dev` or `dev-` image reads `Dev`; a `pr-` or `preview-` image reads `Preview`; a release image reads `Stable`. | +| **Current image** | The image reference the container was started with, for example `ghcr.io/studio-saelix/sencho-dev:dev-a1b2c3d`. | +| **Revision** | The immutable digest this build resolves to, or the pinned `dev-` tag when the running image is on the integration track. | +| **Image ID** | The first twelve characters of the running image's sha256 identifier; click to copy the full id. | + +These fields describe the **control plane** instance you are logged in to, not remote nodes. When identity metadata cannot be determined, the reference fields read `Unknown` rather than guessing. A hardened image viewed by a non-administrator shows `Restricted` for the reference fields instead of `Unknown`. diff --git a/docs/reference/settings.mdx b/docs/reference/settings.mdx index 8ca93db6..b0cd8c8b 100644 --- a/docs/reference/settings.mdx +++ b/docs/reference/settings.mdx @@ -169,8 +169,8 @@ Activate, view, or deactivate the license for this Sencho control plane, and see | **Sencho Admiral** | The active license on this control plane, with a tier badge. Community instances see an upgrade prompt here instead. | | **Recovery Vault** | Whether the current subscription includes Recovery Vault entitlement. | | **Hardened Build** | Switches this control plane between the Community image channel and the Admiral Hardened Build channel. Review entitlement and registry access before switching; see [Plans](/features/licensing#feature-breakdown) for what Hardened Build changes. | -| **Current image** | The image reference this control plane is currently running, so you can confirm which channel took effect after a switch. | -| **Channel** | The active image channel (Community or Hardened). | +| **Current image** | The image reference this control plane is currently running, so you can confirm which channel took effect after a switch. A hardened image is shown as `Restricted` to non-administrators. | +| **Channel** | The running image channel this control plane is on: Community, Hardened, or Unknown. Reflects the running build, not the configured target. | | **Customer** | The customer name on file with Lemon Squeezy (paid plans only). | | **Product** | The product (paid plans only). | | **License key** | The active key, masked to the last four characters. | @@ -753,9 +753,15 @@ Displays instance information at a glance. | Field | Description | |-------|-------------| -| **Version** | Current Sencho version. | +| **Version** | The semantic version of this control plane instance. | +| **Channel** | The build track this control plane is running: `Dev`, `Preview`, `Stable`, or `Unknown`. A dev build is identified here even when its packaged version still matches the previous stable release. | +| **Current image** | The image reference this control plane was started with. A compose edit changes the configured target until the container is recreated; this row always shows the running image, never the configured one. | +| **Revision** | The immutable digest or pinned `dev-` tag this build resolves to, or `Unknown` when it cannot be determined. | +| **Image ID** | The first twelve characters of the running image's sha256 identifier. Click to copy the full id. Hidden when no image identity is available. | | **Tier** | Community or Admiral badge. | | **Plan Status** | active, trial, expired, or community (Admiral entitlement state, not the AGPL software license). | | **Instance ID** | First eight characters of the unique identifier for this Sencho control plane (used by the license server to identify it). | +The Build rows describe the **control plane** instance (the Sencho you are logged in to), not remote nodes. When identity metadata is unavailable, the reference rows read `Unknown` rather than inferring a value. For a hardened image seen by a non-administrator, the reference fields read `Restricted` instead. See [Verifying images](/operations/verifying-images) for how the build tracks and immutable tags relate. + The **Links** section contains Source code, AGPLv3 License, Licensing documentation, and Changelog links. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 57df0365..0b43fe75 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -9,6 +9,7 @@ import { retryDomain } from './lib/preferences/syncBus'; import { toast } from './components/ui/toast-store'; import { NodeProvider } from './context/NodeContext'; import { LicenseProvider } from './context/LicenseContext'; +import { BuildInfoProvider } from './context/BuildInfoProvider'; import { Login } from './components/Login'; import { Setup } from './components/Setup'; import EditorLayout from './components/EditorLayout'; @@ -103,11 +104,13 @@ function AppContent() { )} - - {/* Portal lives inside LicenseProvider so the editor surface and its - portalled overlays can read license state via useLicense(). - Outer DeployFeedbackProvider is still an ancestor through App. */} - + + + {/* Portal lives inside LicenseProvider so the editor surface and its + portalled overlays can read license state via useLicense(). + Outer DeployFeedbackProvider is still an ancestor through App. */} + + diff --git a/frontend/src/components/EditorLayout.tsx b/frontend/src/components/EditorLayout.tsx index dbd41f94..fd47142f 100644 --- a/frontend/src/components/EditorLayout.tsx +++ b/frontend/src/components/EditorLayout.tsx @@ -22,6 +22,7 @@ import { useOverlayState } from './EditorLayout/hooks/useOverlayState'; import { useStackActions, NODE_SWITCH_PENDING_TOKEN } from './EditorLayout/hooks/useStackActions'; import { useSelectedStackLiveRefresh } from './EditorLayout/hooks/useSelectedStackLiveRefresh'; import { useTheme } from '@/hooks/use-theme'; +import { useBuildInfo } from '@/hooks/useBuildInfo'; import { ThemeQuickSwitch } from './theme/ThemeQuickSwitch'; import { useNotifications } from './EditorLayout/hooks/useNotifications'; import { useContainerStats } from './EditorLayout/hooks/useContainerStats'; @@ -455,6 +456,7 @@ export default function EditorLayout() { const stackMuteActions = useStackMuteActions(stackDisplayName, openMuteRulesWithPrefill); const { isDarkMode } = useTheme(); + const { buildInfo } = useBuildInfo(); // ---- Mobile shell (below md) --------------------------------------------- // Desktop renders the persistent sidebar + workspace untouched. On a phone we @@ -977,6 +979,7 @@ export default function EditorLayout() { const sidebarEl = ( openSettings('nodes')} diff --git a/frontend/src/components/MobileTabBar.test.tsx b/frontend/src/components/MobileTabBar.test.tsx index ad917026..162a9914 100644 --- a/frontend/src/components/MobileTabBar.test.tsx +++ b/frontend/src/components/MobileTabBar.test.tsx @@ -1,15 +1,37 @@ -import { describe, it, expect, vi } from 'vitest'; +import { describe, it, expect, vi, beforeEach } from 'vitest'; import { render, screen, fireEvent } from '@testing-library/react'; import { Home, Radar, Clock } from 'lucide-react'; import { MobileTabBar } from './MobileTabBar'; +import { useBuildInfo } from '@/hooks/useBuildInfo'; +import type { BuildInfo } from '@/context/BuildInfoProvider'; import type { NavItem } from './EditorLayout/hooks/useViewNavigationState'; +vi.mock('@/hooks/useBuildInfo', () => ({ + useBuildInfo: vi.fn(() => ({ buildInfo: null, status: 'ready', retry: vi.fn() })), +})); + +const mockUseBuildInfo = vi.mocked(useBuildInfo); + const allItems: NavItem[] = [ { value: 'dashboard', label: 'Home', icon: Home }, { value: 'fleet', label: 'Fleet', icon: Radar }, { value: 'scheduled-ops', label: 'Schedules', icon: Clock }, ]; +function buildInfo(channel: BuildInfo['channel']): BuildInfo { + return { + version: '0.97.1', + channel, + imageChannel: 'community', + imageRef: channel === 'dev' ? 'ghcr.io/studio-saelix/sencho-dev:dev' : 'ghcr.io/studio-saelix/sencho:0.97.1', + imageId: 'a'.repeat(64), + revision: null, + restricted: false, + }; +} + +const noPill = { buildInfo: null, status: 'ready' as const, retry: vi.fn() }; + function renderBar(over: Partial> = {}) { const props: React.ComponentProps = { navItems: allItems, @@ -76,3 +98,52 @@ describe('MobileTabBar', () => { expect(screen.getByRole('button', { name: 'Stacks' })).not.toHaveAttribute('aria-current'); }); }); + +describe('MobileTabBar build-identity pill', () => { + beforeEach(() => { + mockUseBuildInfo.mockReturnValue(noPill); + }); + + it('shows a text DEV pill for a dev build, not an interactive control', () => { + mockUseBuildInfo.mockReturnValue({ buildInfo: buildInfo('dev'), status: 'ready', retry: vi.fn() }); + renderBar(); + expect(screen.getByText('DEV')).toBeInTheDocument(); + // The pill is a plain span: it adds no competing tap target in the tab row. + expect(screen.queryByRole('button', { name: 'DEV' })).not.toBeInTheDocument(); + }); + + it('shows a text PREVIEW pill for a preview build', () => { + mockUseBuildInfo.mockReturnValue({ buildInfo: buildInfo('preview'), status: 'ready', retry: vi.fn() }); + renderBar(); + expect(screen.getByText('PREVIEW')).toBeInTheDocument(); + expect(screen.queryByRole('button', { name: 'PREVIEW' })).not.toBeInTheDocument(); + }); + + it('renders no pill for a stable build', () => { + mockUseBuildInfo.mockReturnValue({ buildInfo: buildInfo('stable'), status: 'ready', retry: vi.fn() }); + renderBar(); + expect(screen.queryByText('DEV')).not.toBeInTheDocument(); + expect(screen.queryByText('PREVIEW')).not.toBeInTheDocument(); + }); + + it('keeps the tab touch targets present alongside the pill', () => { + mockUseBuildInfo.mockReturnValue({ buildInfo: buildInfo('dev'), status: 'ready', retry: vi.fn() }); + renderBar(); + expect(screen.getByRole('button', { name: 'Home' })).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'Stacks' })).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'Settings' })).toBeInTheDocument(); + }); + + it('lays the pill in-flow as a non-overlapping sibling of the tabs', () => { + mockUseBuildInfo.mockReturnValue({ buildInfo: buildInfo('dev'), status: 'ready', retry: vi.fn() }); + renderBar(); + const pill = screen.getByText('DEV'); + // In-flow (flex sibling), not absolutely positioned over the Settings tab. + expect(pill).not.toHaveClass('absolute'); + expect(pill).toHaveClass('self-center', 'shrink-0'); + // Sibling of the tab buttons inside the nav, so flexbox reserves its own + // region rather than letting it overlap the rightmost tab. + const settingsTab = screen.getByRole('button', { name: 'Settings' }); + expect(pill.parentElement).toBe(settingsTab.parentElement); + }); +}); diff --git a/frontend/src/components/MobileTabBar.tsx b/frontend/src/components/MobileTabBar.tsx index 8edb096b..b9952a7a 100644 --- a/frontend/src/components/MobileTabBar.tsx +++ b/frontend/src/components/MobileTabBar.tsx @@ -1,6 +1,7 @@ import { Home, Layers, Radar, Clock, Settings as SettingsIcon } from 'lucide-react'; import type { LucideIcon } from 'lucide-react'; import { cn } from '@/lib/utils'; +import { useBuildInfo } from '@/hooks/useBuildInfo'; import type { NavItem, ActiveView } from './EditorLayout/hooks/useViewNavigationState'; import type { MobileView } from './EditorLayout/mobile-surface'; @@ -45,7 +46,12 @@ export function MobileTabBar({ onNavigate, onSettings, }: MobileTabBarProps) { + const { buildInfo } = useBuildInfo(); + const has = (value: ActiveView) => navItems.some(i => i.value === value); + const channel = buildInfo?.channel; + const showPill = channel === 'dev' || channel === 'preview'; + const pillLabel = channel === 'dev' ? 'DEV' : 'PREVIEW'; const tabs: Tab[] = [ { id: 'home', label: 'Home', icon: Home }, @@ -79,7 +85,7 @@ export function MobileTabBar({ aria-label="Primary mobile" data-sn-glass="mobile-tabbar" className={cn( - 'md:hidden flex shrink-0 items-stretch', + 'md:hidden relative flex shrink-0 items-stretch', 'border-t border-hairline', 'bg-[color-mix(in_oklch,var(--card)_70%,transparent)] backdrop-blur-md backdrop-saturate-150', 'pb-[max(8px,env(safe-area-inset-bottom))]', @@ -108,6 +114,17 @@ export function MobileTabBar({ ); })} + {showPill ? ( + + {pillLabel} + + ) : null} ); } diff --git a/frontend/src/components/settings/AboutSection.tsx b/frontend/src/components/settings/AboutSection.tsx index 83c8016e..6a1d6bd0 100644 --- a/frontend/src/components/settings/AboutSection.tsx +++ b/frontend/src/components/settings/AboutSection.tsx @@ -1,5 +1,11 @@ +import { useState } from 'react'; import { useLicense } from '@/context/LicenseContext'; +import { useBuildInfo } from '@/hooks/useBuildInfo'; import { TierBadge } from '@/components/TierBadge'; +import { Badge } from '@/components/ui/badge'; +import { FlaskConical } from 'lucide-react'; +import { copyToClipboard } from '@/lib/clipboard'; +import { toast } from '@/components/ui/toast-store'; import { TogglePill } from '@/components/ui/toggle-pill'; import { useWhatsNewPreference } from '@/hooks/useWhatsNewPreference'; import { whatsNewEntries } from '@/whats-new/entries'; @@ -15,16 +21,110 @@ import { const linkClassName = 'font-mono text-[10px] leading-3 uppercase tracking-[0.18em] text-brand hover:text-brand/80 transition-colors'; +const mono = 'font-mono text-sm text-stat-value'; + +function BuildChannelChip({ label }: { label: string }) { + if (label === 'Dev') { + return ( + + Dev + + ); + } + if (label === 'Preview') { + return ( + + Preview + + ); + } + return {label}; +} + export function AboutSection() { const { license } = useLicense(); + const { buildInfo, status } = useBuildInfo(); const { enabled: whatsNewEnabled, setEnabled: setWhatsNewEnabled } = useWhatsNewPreference(); + const [copied, setCopied] = useState(false); + + // Loading surfaces a placeholder and error surfaces "Unknown" (truthful); + // the reference and revision fields below read "Restricted" for a redacted + // hardened image, before their null-check. + const channelLabel = (() => { + if (status === 'loading') return '…'; + if (status === 'error' || !buildInfo) return 'Unknown'; + switch (buildInfo.channel) { + case 'dev': return 'Dev'; + case 'preview': return 'Preview'; + case 'stable': return 'Stable'; + default: return 'Unknown'; + } + })(); + const resolveLabel = (value: string | null | undefined) => + buildInfo?.restricted + ? 'Restricted' + : status === 'loading' + ? '…' + : status === 'error' + ? 'Unknown' + : value ?? 'Unknown'; + + const imageRefLabel = resolveLabel(buildInfo?.imageRef); + const revisionLabel = resolveLabel(buildInfo?.revision); + const imageIdLabel = + status === 'loading' ? '…' + : status === 'error' || !buildInfo?.imageId ? 'Unknown' + : `sha256:${buildInfo.imageId.slice(0, 12)}`; + + const copyImageId = async () => { + if (!buildInfo?.imageId) return; + try { + await copyToClipboard(`sha256:${buildInfo.imageId}`); + setCopied(true); + window.setTimeout(() => setCopied(false), 1500); + } catch { + toast.error('Could not copy the image id.'); + } + }; return (
- v{__APP_VERSION__} + v{buildInfo?.version ?? __APP_VERSION__} + + + + + {imageRefLabel} + + + {revisionLabel} + + {imageIdLabel !== 'Unknown' && imageIdLabel !== '…' ? ( + + + + ) : null} diff --git a/frontend/src/components/settings/LicenseSection.tsx b/frontend/src/components/settings/LicenseSection.tsx index d1981dc8..7a2eb023 100644 --- a/frontend/src/components/settings/LicenseSection.tsx +++ b/frontend/src/components/settings/LicenseSection.tsx @@ -3,6 +3,7 @@ import { Button } from '@/components/ui/button'; import { Input } from '@/components/ui/input'; import { toast } from '@/components/ui/toast-store'; import { useLicense } from '@/context/LicenseContext'; +import { useBuildInfo } from '@/hooks/useBuildInfo'; import { TierBadge } from '@/components/TierBadge'; import { Crown, CheckCircle, XCircle, Clock, ExternalLink, @@ -47,8 +48,8 @@ function formatChannel(channel: ImageChannel): string { return 'Community'; case 'hardened': return 'Hardened'; - default: - return 'Custom'; + case 'unknown': + return 'Unknown'; } } @@ -64,6 +65,7 @@ function getTierMastheadValue(tier?: string): string { export function LicenseSection() { const { license, isPaid, activate, deactivate } = useLicense(); + const { buildInfo, status: buildInfoStatus } = useBuildInfo(); const [licenseKeyInput, setLicenseKeyInput] = useState(''); const [isActivating, setIsActivating] = useState(false); const [isDeactivating, setIsDeactivating] = useState(false); @@ -253,16 +255,28 @@ export function LicenseSection() { ) : null} - {channelStatus?.composeImageRef ?? formatChannel(channelStatus?.channel ?? 'unknown')} + {buildInfo?.restricted + ? 'Restricted' + : buildInfoStatus === 'loading' + ? '…' + : buildInfoStatus === 'error' + ? 'Unknown' + : buildInfo?.imageRef ?? 'Unknown'} - {formatChannel(channelStatus?.channel ?? 'unknown')} + + {buildInfoStatus === 'loading' + ? '…' + : buildInfoStatus === 'error' || !buildInfo + ? 'Unknown' + : formatChannel(buildInfo.imageChannel)} + {channelStatus?.operation?.state === 'failed' ? ( ({ useWhatsNewPreference: () => ({ enabled: true, setEnabled: mockSetEnabled, hasUnseen: false, markSeen: vi.fn() }), })); +vi.mock('@/hooks/useBuildInfo', () => ({ + useBuildInfo: vi.fn(() => ({ buildInfo: null, status: 'ready', retry: vi.fn() })), +})); +import { useBuildInfo } from '@/hooks/useBuildInfo'; +import type { BuildInfo } from '@/context/BuildInfoProvider'; + +const { mockCopyToClipboard, mockToastError } = vi.hoisted(() => ({ + mockCopyToClipboard: vi.fn(), + mockToastError: vi.fn(), +})); +vi.mock('@/lib/clipboard', () => ({ copyToClipboard: mockCopyToClipboard })); +vi.mock('@/components/ui/toast-store', () => ({ + toast: { error: mockToastError, success: vi.fn(), info: vi.fn(), loading: vi.fn(), dismiss: vi.fn() }, +})); + +const mockUseBuildInfo = vi.mocked(useBuildInfo); + +function buildInfo(over: Partial = {}): BuildInfo { + return { + version: '0.97.1', + channel: 'dev', + imageChannel: 'community', + imageRef: 'ghcr.io/studio-saelix/sencho-dev:dev', + imageId: 'a'.repeat(64), + revision: 'dev-abc1234', + restricted: false, + ...over, + }; +} + // The shipped entries.json is empty, so populate it here; the empty state has its own file. vi.mock('@/whats-new/entries', () => ({ whatsNewEntries: [{ id: 'entry-a', title: 'A feature', blurb: 'Does a thing.' }], @@ -91,3 +121,49 @@ describe('AboutSection Preferences', () => { expect(mockSetEnabled).toHaveBeenCalledWith(false); }); }); + +describe('AboutSection Build identity', () => { + it('shows the runtime channel, current image, revision and version', () => { + mockUseBuildInfo.mockReturnValue({ buildInfo: buildInfo(), status: 'ready', retry: vi.fn() }); + render(); + expect(screen.getByText('Dev')).toBeTruthy(); + expect(screen.getByText('ghcr.io/studio-saelix/sencho-dev:dev')).toBeTruthy(); + expect(screen.getByText('dev-abc1234')).toBeTruthy(); + expect(screen.getByText('v0.97.1')).toBeTruthy(); + }); + + it('labels redacted hardened reference fields Restricted, not Unknown', () => { + mockUseBuildInfo.mockReturnValue({ + buildInfo: buildInfo({ channel: 'stable', restricted: true, imageRef: null, revision: null }), + status: 'ready', + retry: vi.fn(), + }); + render(); + expect(screen.getByText('Stable')).toBeTruthy(); + expect(screen.getAllByText('Restricted').length).toBeGreaterThanOrEqual(2); + expect(screen.queryByText('Unknown')).toBeNull(); + }); + + it('shows Unknown for reference fields when build info is unavailable', () => { + mockUseBuildInfo.mockReturnValue({ buildInfo: null, status: 'error', retry: vi.fn() }); + render(); + expect(screen.getAllByText('Unknown').length).toBeGreaterThan(0); + }); + + it('wraps long image and revision tokens so they do not overflow', () => { + mockUseBuildInfo.mockReturnValue({ buildInfo: buildInfo(), status: 'ready', retry: vi.fn() }); + render(); + expect(screen.getByText('ghcr.io/studio-saelix/sencho-dev:dev')).toHaveClass('break-all'); + expect(screen.getByText('dev-abc1234')).toHaveClass('break-all'); + }); + + it('surfaces an error toast when copying the image id fails', async () => { + mockUseBuildInfo.mockReturnValue({ buildInfo: buildInfo(), status: 'ready', retry: vi.fn() }); + mockCopyToClipboard.mockRejectedValue(new Error('clipboard blocked')); + render(); + + await userEvent.click(screen.getByRole('button', { name: /sha256:/ })); + expect(mockCopyToClipboard).toHaveBeenCalledWith(`sha256:${'a'.repeat(64)}`); + expect(mockToastError).toHaveBeenCalledWith('Could not copy the image id.'); + }); +}); diff --git a/frontend/src/components/settings/__tests__/AboutSection.whatsNewEmpty.test.tsx b/frontend/src/components/settings/__tests__/AboutSection.whatsNewEmpty.test.tsx index 181a3c7d..8c0b92fa 100644 --- a/frontend/src/components/settings/__tests__/AboutSection.whatsNewEmpty.test.tsx +++ b/frontend/src/components/settings/__tests__/AboutSection.whatsNewEmpty.test.tsx @@ -43,6 +43,10 @@ vi.mock('@/hooks/useWhatsNewPreference', () => ({ useWhatsNewPreference: () => ({ enabled: true, setEnabled: vi.fn(), hasUnseen: false, markSeen: vi.fn() }), })); +vi.mock('@/hooks/useBuildInfo', () => ({ + useBuildInfo: () => ({ buildInfo: null, status: 'ready', retry: vi.fn() }), +})); + describe("AboutSection with no What's New entries authored", () => { it('hides the Preferences section entirely, so no toggle describes an absent icon', () => { render(); diff --git a/frontend/src/components/settings/__tests__/LicenseSection.test.tsx b/frontend/src/components/settings/__tests__/LicenseSection.test.tsx index 257bc0fe..d8d99605 100644 --- a/frontend/src/components/settings/__tests__/LicenseSection.test.tsx +++ b/frontend/src/components/settings/__tests__/LicenseSection.test.tsx @@ -1,6 +1,7 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import { render, screen } from '@testing-library/react'; import type { LicenseInfo } from '@/context/LicenseContext'; +import type { BuildInfoContextType } from '@/context/BuildInfoProvider'; const useLicenseMock = vi.fn(); @@ -12,6 +13,12 @@ vi.mock('../MastheadStatsContext', () => ({ useMastheadStats: () => {}, })); +const useBuildInfoMock = vi.fn<() => BuildInfoContextType>(() => ({ buildInfo: null, status: 'ready', retry: vi.fn() })); + +vi.mock('@/hooks/useBuildInfo', () => ({ + useBuildInfo: () => useBuildInfoMock(), +})); + vi.mock('@/components/TierBadge', () => ({ TierBadge: () => tier, })); @@ -135,3 +142,72 @@ describe('LicenseSection pricing link', () => { expect(screen.getByText('See pricing')).toBeTruthy(); }); }); + +describe('LicenseSection build rows (running identity consistency)', () => { + beforeEach(() => { + useLicenseMock.mockReset(); + useBuildInfoMock.mockReset(); + useBuildInfoMock.mockReturnValue({ buildInfo: null, status: 'ready', retry: vi.fn() }); + mockLicense(baseLicense()); + }); + + it('renders the running Community channel and image ref, regardless of configured target', () => { + useBuildInfoMock.mockReturnValue({ + buildInfo: { + version: '0.97.1', + channel: 'stable', + imageChannel: 'community', + imageRef: 'ghcr.io/studio-saelix/sencho:0.97.1', + imageId: 'a'.repeat(64), + revision: null, + restricted: false, + }, + status: 'ready', + retry: vi.fn(), + }); + render(); + // The configured/compose target is hardened, but the running build is + // Community: the row must show the running image, never the target. + expect(screen.getByText('ghcr.io/studio-saelix/sencho:0.97.1')).toBeTruthy(); + expect(screen.queryByText('Hardened')).toBeNull(); + }); + + it('renders a hardened running image as Hardened channel and Restricted image, not Unknown', () => { + useBuildInfoMock.mockReturnValue({ + buildInfo: { + version: '0.97.1', + channel: 'stable', + imageChannel: 'hardened', + imageRef: null, + imageId: 'b'.repeat(64), + revision: null, + restricted: true, + }, + status: 'ready', + retry: vi.fn(), + }); + render(); + expect(screen.getByText('Hardened')).toBeTruthy(); + expect(screen.getByText('Restricted')).toBeTruthy(); + expect(screen.queryByText('Unknown')).toBeNull(); + }); + + it('labels an unclassifiable running image Channel Unknown, never Custom', () => { + useBuildInfoMock.mockReturnValue({ + buildInfo: { + version: '0.97.1', + channel: 'unknown', + imageChannel: 'unknown', + imageRef: null, + imageId: null, + revision: null, + restricted: false, + }, + status: 'ready', + retry: vi.fn(), + }); + render(); + expect(screen.getAllByText('Unknown').length).toBeGreaterThan(0); + expect(screen.queryByText('Custom')).toBeNull(); + }); +}); diff --git a/frontend/src/components/sidebar/SidebarBrand.test.tsx b/frontend/src/components/sidebar/SidebarBrand.test.tsx new file mode 100644 index 00000000..19b6f59a --- /dev/null +++ b/frontend/src/components/sidebar/SidebarBrand.test.tsx @@ -0,0 +1,72 @@ +import { describe, it, expect } from 'vitest'; +import { render, screen } from '@testing-library/react'; +import { SidebarBrand } from './SidebarBrand'; +import { chipDetail } from './chipDetail'; +import type { BuildInfo } from '@/context/BuildInfoProvider'; + +function info(channel: BuildInfo['channel']): BuildInfo { + return { + version: '0.97.1', + channel, + imageChannel: 'community', + imageRef: channel === 'dev' ? 'ghcr.io/studio-saelix/sencho-dev:dev' : 'ghcr.io/studio-saelix/sencho:0.97.1', + imageId: 'a'.repeat(64), + revision: null, + restricted: false, + }; +} + +describe('SidebarBrand build-identity chip', () => { + it('shows a DEV text chip for a dev build', () => { + render(); + const chip = screen.getByText('DEV'); + expect(chip).toBeInTheDocument(); + // Text is the cue, not color alone. + expect(chip.tagName).toBe('SPAN'); + expect(chip.textContent).toContain('DEV'); + }); + + it('shows a PREVIEW text chip for a preview build', () => { + render(); + expect(screen.getByText('PREVIEW')).toBeInTheDocument(); + }); + + it('renders no chip for a stable build', () => { + render(); + expect(screen.queryByText('DEV')).not.toBeInTheDocument(); + expect(screen.queryByText('PREVIEW')).not.toBeInTheDocument(); + }); + + it('renders no chip when build info is unavailable', () => { + render(); + expect(screen.queryByText('DEV')).not.toBeInTheDocument(); + expect(screen.queryByText('PREVIEW')).not.toBeInTheDocument(); + }); + + it('prefers the runtime version when available', () => { + render(); + expect(screen.getByText('v0.97.1')).toBeInTheDocument(); + }); +}); + +describe('chipDetail', () => { + it('reads Restricted for a redacted hardened reference', () => { + const b: BuildInfo = { ...info('stable'), restricted: true, imageRef: null, revision: null }; + expect(chipDetail(b)).toBe('Restricted'); + }); + + it('combines the reference and revision when both are present', () => { + const b: BuildInfo = { ...info('dev'), revision: 'dev-abc1234' }; + expect(chipDetail(b)).toBe('ghcr.io/studio-saelix/sencho-dev:dev · dev-abc1234'); + }); + + it('reads the reference alone when the revision is unknown', () => { + const b: BuildInfo = { ...info('dev'), revision: null }; + expect(chipDetail(b)).toBe('ghcr.io/studio-saelix/sencho-dev:dev'); + }); + + it('reads Unknown when the reference is absent and not restricted', () => { + const b: BuildInfo = { ...info('dev'), imageRef: null }; + expect(chipDetail(b)).toBe('Unknown'); + }); +}); \ No newline at end of file diff --git a/frontend/src/components/sidebar/SidebarBrand.tsx b/frontend/src/components/sidebar/SidebarBrand.tsx index 8e5f0a78..08d867e5 100644 --- a/frontend/src/components/sidebar/SidebarBrand.tsx +++ b/frontend/src/components/sidebar/SidebarBrand.tsx @@ -1,8 +1,17 @@ +import { FlaskConical } from 'lucide-react'; +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/components/ui/tooltip'; +import type { BuildInfo } from '@/context/BuildInfoProvider'; +import { chipDetail } from './chipDetail'; + interface SidebarBrandProps { isDarkMode: boolean; + buildInfo?: BuildInfo | null; } -export function SidebarBrand({ isDarkMode }: SidebarBrandProps) { +export function SidebarBrand({ isDarkMode, buildInfo }: SidebarBrandProps) { + const channel = buildInfo?.channel; + const showChip = channel === 'dev' || channel === 'preview'; + return (
-
+
Sencho - v{__APP_VERSION__} + v{buildInfo?.version ?? __APP_VERSION__} + {showChip ? ( + + + + + {channel === 'dev' ? : null} + {channel === 'dev' ? 'DEV' : 'PREVIEW'} + + + + {chipDetail(buildInfo)} + + + + ) : null}
); diff --git a/frontend/src/components/sidebar/StackSidebar.tsx b/frontend/src/components/sidebar/StackSidebar.tsx index 206d36cf..c4749b3b 100644 --- a/frontend/src/components/sidebar/StackSidebar.tsx +++ b/frontend/src/components/sidebar/StackSidebar.tsx @@ -11,10 +11,12 @@ import { StackList, type StackListProps } from './StackList'; import type { FilterChip } from './sidebar-types'; import type { BulkAction } from '@/hooks/useBulkStackActions'; import type { SidebarActivitySummary } from './useSidebarActivitySummary'; +import type { BuildInfo } from '@/context/BuildInfoProvider'; import { isStacksListSettled } from './stacksLoadUi'; export interface StackSidebarProps { isDarkMode: boolean; + buildInfo?: BuildInfo | null; nodeSwitcherSlot: ReactNode; createStackSlot: ReactNode | null; onScan: () => void; @@ -46,7 +48,7 @@ export interface StackSidebarProps { export function StackSidebar(props: StackSidebarProps) { const { - isDarkMode, nodeSwitcherSlot, createStackSlot, onScan, isScanning, canCreate, + isDarkMode, buildInfo, nodeSwitcherSlot, createStackSlot, onScan, isScanning, canCreate, searchQuery, onSearchChange, filterChip, filterCounts, onFilterChipChange, list, activitySummary, onActivityAction, bulkMode, selectedFiles, onToggleBulkMode, onToggleSelect, onClearSelection, onBulkAction, @@ -80,7 +82,7 @@ export function StackSidebar(props: StackSidebarProps) { its kicker chip), so the in-sidebar brand and node rows are redundant there and hidden to save vertical space. */}
- +
{nodeSwitcherSlot}
{canCreate && createStackSlot !== null && ( diff --git a/frontend/src/components/sidebar/chipDetail.ts b/frontend/src/components/sidebar/chipDetail.ts new file mode 100644 index 00000000..e98ed8ae --- /dev/null +++ b/frontend/src/components/sidebar/chipDetail.ts @@ -0,0 +1,11 @@ +import type { BuildInfo } from '@/context/BuildInfoProvider'; + +/** Detail shown under the DEV/PREVIEW chip, or the truthful Unknown / Restricted + * states when the running reference is unavailable or redacted for this user. */ +export function chipDetail(buildInfo: BuildInfo | null | undefined): string { + if (buildInfo?.restricted) return 'Restricted'; + if (buildInfo?.imageRef) { + return buildInfo.revision ? `${buildInfo.imageRef} · ${buildInfo.revision}` : buildInfo.imageRef; + } + return 'Unknown'; +} \ No newline at end of file diff --git a/frontend/src/context/BuildInfoProvider.test.tsx b/frontend/src/context/BuildInfoProvider.test.tsx new file mode 100644 index 00000000..f9b498ee --- /dev/null +++ b/frontend/src/context/BuildInfoProvider.test.tsx @@ -0,0 +1,135 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { render, screen, act } from '@testing-library/react'; +import type { ReactNode } from 'react'; +import { BuildInfoProvider } from './BuildInfoProvider'; +import { useBuildInfo } from '@/hooks/useBuildInfo'; + +const { currentUserRef } = vi.hoisted(() => ({ + currentUserRef: { value: null as { username: string; role: string } | null }, +})); + +vi.mock('./AuthContext', () => ({ + useAuth: () => ({ user: currentUserRef.value }), +})); + +vi.mock('@/lib/api', () => ({ + apiFetch: vi.fn(), +})); +import { apiFetch } from '@/lib/api'; + +const restrictedViewer = { + version: '0.97.1', + channel: 'stable', + imageChannel: 'hardened', + imageRef: null, + imageId: 'b'.repeat(64), + revision: null, + restricted: true, +}; + +const adminCommunity = { + version: '0.97.1', + channel: 'dev', + imageChannel: 'community', + imageRef: 'ghcr.io/studio-saelix/sencho-dev:dev', + imageId: 'a'.repeat(64), + revision: null, + restricted: false, +}; + +function json(data: unknown, status = 200): Response { + return new Response(JSON.stringify(data), { status }); +} + +function Harness() { + const { buildInfo, status } = useBuildInfo(); + return
{status}:{buildInfo?.imageRef ?? 'none'}
; +} + +function wrapper({ children }: { children: ReactNode }) { + return {children}; +} + +async function flush() { + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); +} + +describe('BuildInfoProvider', () => { + beforeEach(() => { + currentUserRef.value = null; + vi.mocked(apiFetch).mockReset(); + }); + + it('clears permission-filtered state and refetches when the auth identity changes', async () => { + currentUserRef.value = { username: 'admin', role: 'admin' }; + vi.mocked(apiFetch).mockResolvedValue(json(adminCommunity)); + const { rerender } = render(, { wrapper }); + await flush(); + expect(screen.getByTestId('s').textContent).toContain('ready'); + expect(screen.getByTestId('s').textContent).toContain('ghcr.io/studio-saelix/sencho-dev:dev'); + + // A viewer session must never see the admin-cached ref: the provider + // clears immediately and rejects the stale in-flight completion. + currentUserRef.value = { username: 'viewer', role: 'viewer' }; + vi.mocked(apiFetch).mockResolvedValue(json(restrictedViewer)); + rerender(); + // Cleared synchronously on the identity change, before the refetch lands. + expect(screen.getByTestId('s').textContent).toBe('loading:none'); + await flush(); + expect(screen.getByTestId('s').textContent).toBe('ready:none'); + }); + + it('rejects a stale in-flight completion from a prior identity', async () => { + currentUserRef.value = { username: 'admin', role: 'admin' }; + let resolveAdmin!: (r: Response) => void; + vi.mocked(apiFetch).mockReturnValue(new Promise((res) => { resolveAdmin = res; })); + const { rerender } = render(, { wrapper }); + + currentUserRef.value = { username: 'viewer', role: 'viewer' }; + vi.mocked(apiFetch).mockResolvedValue(json(restrictedViewer)); + rerender(); + await flush(); + expect(screen.getByTestId('s').textContent).toBe('ready:none'); + + // The old admin request resolving later must not surface its ref. + await act(async () => { resolveAdmin(json(adminCommunity)); }); + expect(screen.getByTestId('s').textContent).toBe('ready:none'); + }); + + it('shares one fetch across two consumers', async () => { + currentUserRef.value = { username: 'admin', role: 'admin' }; + vi.mocked(apiFetch).mockResolvedValue(json(adminCommunity)); + render( + + + + , + ); + await flush(); + expect(apiFetch).toHaveBeenCalledTimes(1); + expect(apiFetch).toHaveBeenCalledWith('/build-info', expect.objectContaining({ localOnly: true })); + expect(screen.getAllByTestId('s')).toHaveLength(2); + for (const el of screen.getAllByTestId('s')) { + expect(el.textContent).toContain('ghcr.io/studio-saelix/sencho-dev:dev'); + } + }); + + it('surfaces error as Unknown and retries on focus', async () => { + currentUserRef.value = { username: 'admin', role: 'admin' }; + vi.mocked(apiFetch).mockRejectedValue(new Error('down')); + render(, { wrapper }); + await flush(); + expect(screen.getByTestId('s').textContent).toBe('error:none'); + + vi.mocked(apiFetch).mockResolvedValue(json(adminCommunity)); + await act(async () => { + window.dispatchEvent(new Event('focus')); + await Promise.resolve(); + await Promise.resolve(); + }); + expect(screen.getByTestId('s').textContent).toBe('ready:ghcr.io/studio-saelix/sencho-dev:dev'); + }); +}); \ No newline at end of file diff --git a/frontend/src/context/BuildInfoProvider.tsx b/frontend/src/context/BuildInfoProvider.tsx new file mode 100644 index 00000000..614a124b --- /dev/null +++ b/frontend/src/context/BuildInfoProvider.tsx @@ -0,0 +1,103 @@ +import { createContext, useCallback, useEffect, useRef, useState, type ReactNode } from 'react'; +import { apiFetch } from '@/lib/api'; +import { useAuth } from './AuthContext'; + +export type BuildChannel = 'stable' | 'dev' | 'preview' | 'unknown'; +export type ImageChannel = 'community' | 'hardened' | 'unknown'; +export type BuildInfoStatus = 'loading' | 'ready' | 'error'; + +/** Canonical runtime build identity of the control instance. Mirrors the + * /api/build-info response: imageRef and revision are nulled for hardened + * images when the viewer is not an admin, with `restricted` marking that + * redaction (the UI shows "Restricted", never "Unknown", when set). + * `restricted === true` implies `imageRef === null && revision === null`. */ +export interface BuildInfo { + version: string | null; + channel: BuildChannel; + imageChannel: ImageChannel; + imageRef: string | null; + imageId: string | null; + revision: string | null; + restricted: boolean; +} + +export interface BuildInfoContextType { + buildInfo: BuildInfo | null; + status: BuildInfoStatus; + retry: () => void; +} + +// eslint-disable-next-line react-refresh/only-export-components +export const BuildInfoContext = createContext(undefined); + +/** Single owner of the control-instance build identity. Mounted inside the + * authenticated subtree. One shared fetch (localOnly, so it always targets the + * local hub); permission-filtered state is cleared the moment the auth identity + * or role changes, and stale in-flight completions from a prior session are + * rejected by a generation counter. Focus retries an error so it never sticks + * failed; consumers show a placeholder while loading and "Unknown" on error. */ +export function BuildInfoProvider({ children }: { children: ReactNode }) { + const { user } = useAuth(); + + const [buildInfo, setBuildInfo] = useState(null); + const [status, setStatus] = useState('loading'); + + const generationRef = useRef(0); + const statusRef = useRef('loading'); + const identityRef = useRef(null); + + const load = useCallback(async () => { + const gen = ++generationRef.current; + setStatus('loading'); + try { + const res = await apiFetch('/build-info', { localOnly: true }); + if (gen !== generationRef.current) return; + if (res.ok) { + const data = (await res.json()) as BuildInfo; + if (gen !== generationRef.current) return; + setBuildInfo(data); + setStatus('ready'); + } else { + console.error(`[BuildInfo] fetch returned ${res.status}`); + setStatus('error'); + } + } catch (err) { + if (gen !== generationRef.current) return; + console.error('[BuildInfo] fetch failed', err); + setStatus('error'); + } + }, []); + + // Reload whenever the signed-in identity or role changes. Clearing state + // synchronously (before the replacement fetch resolves) guarantees a + // hardened admin's reference never surfaces in a viewer session. + useEffect(() => { + const identity = user ? `${user.username}:${user.role}` : null; + if (identity !== identityRef.current) { + identityRef.current = identity; + setBuildInfo(null); + setStatus('loading'); + void load(); + } + }, [user, load]); + + useEffect(() => { + statusRef.current = status; + }, [status]); + + // A transient failure (auth expiry, hub restart) should not stick: retry the + // moment the tab regains focus. + useEffect(() => { + const onFocus = () => { + if (statusRef.current === 'error') void load(); + }; + window.addEventListener('focus', onFocus); + return () => window.removeEventListener('focus', onFocus); + }, [load]); + + return ( + void load() }}> + {children} + + ); +} \ No newline at end of file diff --git a/frontend/src/hooks/useBuildInfo.ts b/frontend/src/hooks/useBuildInfo.ts new file mode 100644 index 00000000..7b13f06a --- /dev/null +++ b/frontend/src/hooks/useBuildInfo.ts @@ -0,0 +1,13 @@ +import { useContext } from 'react'; +import { BuildInfoContext, type BuildInfoContextType } from '@/context/BuildInfoProvider'; + +/** Consumer of the single shared control-instance build identity. The shell, + * About, and Admiral Account all read the same fetched BuildInfo by reference + * instead of issuing their own fetches. */ +export function useBuildInfo(): BuildInfoContextType { + const context = useContext(BuildInfoContext); + if (context === undefined) { + throw new Error('useBuildInfo must be used within a BuildInfoProvider'); + } + return context; +} \ No newline at end of file