fix(git-sources): harden webhook delivery, transport errors, and clone limits (#1249)

* fix(git-sources): harden webhook delivery, transport errors, and clone limits

Map webhook-pull outcomes to real HTTP status codes (200 success, 202
debounced, 404 no source, 422 failure) instead of always returning 200, so a
Git provider and any monitoring on it can tell when a delivery actually failed.

Close a concurrent webhook fan-out gap: the debounce window is now re-checked
inside the per-stack lock, so simultaneous deliveries for one push run a single
clone instead of one per request. The whole pull/apply critical section runs
under a single lock acquisition.

Unwrap fetch transport causes (ENOTFOUND, ECONNREFUSED, ECONNRESET, TLS) so a
clone failure surfaces an actionable, host-qualified message instead of a bare
"fetch failed".

Cap how many bytes a single clone may download to protect the host disk;
operators can tune it with GITSOURCE_MAX_CLONE_BYTES (default 100 MB).

Log webhook pull failures server-side, since the webhook path is unattended.

* test(git-sources): assert surfaced host via toContain to satisfy CodeQL

* fix(git-sources): bound per-file read, treat debounced webhooks as non-failure, correct clone-cap docs

* docs(git-sources): correct clone-cap comment to describe a download bound, not disk

backend/src/__tests__/git-source-http.test.ts

@@ -9,7 +9,7 @@
  */
 import { describe, it, expect, vi } from 'vitest';
 import type { Response } from 'express';
-import { gitSourceStatus, sendGitSourceError } from '../utils/gitSourceHttp';
+import { gitSourceStatus, sendGitSourceError, webhookPullStatus } from '../utils/gitSourceHttp';
 import { GitSourceError } from '../services/GitSourceService';
 
 describe('gitSourceStatus', () => {
@@ -32,6 +32,20 @@ describe('gitSourceStatus', () => {
     });
 });
 
+describe('webhookPullStatus', () => {
+    it('maps a successful pull/apply to 200', () => {
+        expect(webhookPullStatus('success')).toBe(200);
+    });
+
+    it('maps a debounced (skipped) pull to 202', () => {
+        expect(webhookPullStatus('skipped')).toBe(202);
+    });
+
+    it('maps a failed pull/apply to 422, never 200', () => {
+        expect(webhookPullStatus('error')).toBe(422);
+    });
+});
+
 describe('sendGitSourceError', () => {
     function mockRes() {
         const res = { status: vi.fn(), json: vi.fn() } as unknown as Response;

backend/src/__tests__/git-source-routes.test.ts

@@ -11,12 +11,36 @@
  * Service-layer logic (encryption, error mapping, mutex, pending lifecycle)
  * is covered in git-source-service.test.ts.
  */
-import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
 import request from 'supertest';
 import jwt from 'jsonwebtoken';
 import fs from 'fs';
 import path from 'path';
 import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
+import { DatabaseService } from '../services/DatabaseService';
+import { GitSourceService } from '../services/GitSourceService';
+
+function seedGitSource(stackName: string): void {
+    DatabaseService.getInstance().upsertGitSource({
+        stack_name: stackName,
+        repo_url: 'https://github.com/example/repo.git',
+        branch: 'main',
+        compose_path: 'compose.yaml',
+        sync_env: false,
+        env_path: null,
+        auth_type: 'none',
+        encrypted_token: null,
+        auto_apply_on_webhook: false,
+        auto_deploy_on_apply: false,
+        last_applied_commit_sha: null,
+        last_applied_content_hash: null,
+        pending_commit_sha: null,
+        pending_compose_content: null,
+        pending_env_content: null,
+        pending_fetched_at: null,
+        last_debounce_at: null,
+    });
+}
 
 let tmpDir: string;
 let app: import('express').Express;
@@ -291,6 +315,55 @@ describe('POST /api/stacks/from-git', () => {
     });
 });
 
+describe('POST /api/stacks/:stackName/git-source/webhook-pull status codes', () => {
+    it('returns 404 (not 200) when the stack has no Git source configured', async () => {
+        const res = await request(app)
+            .post('/api/stacks/existing-stack/git-source/webhook-pull')
+            .set('Authorization', `Bearer ${adminToken()}`);
+        expect(res.status).toBe(404);
+        expect(res.body.error).toMatch(/no git source/i);
+    });
+
+    it('returns 401 without auth', async () => {
+        const res = await request(app).post('/api/stacks/existing-stack/git-source/webhook-pull');
+        expect(res.status).toBe(401);
+    });
+
+    it('maps a failed pull to 422 (not 200) so a Git provider sees the failure', async () => {
+        seedGitSource('webhook-status-422');
+        const pullSpy = vi.spyOn(GitSourceService.getInstance(), 'handleWebhookPull')
+            .mockResolvedValue({ status: 'error', message: 'Validation failed: bad compose' });
+        const res = await request(app)
+            .post('/api/stacks/webhook-status-422/git-source/webhook-pull')
+            .set('Authorization', `Bearer ${adminToken()}`);
+        expect(res.status).toBe(422);
+        expect(res.body.status).toBe('error');
+        pullSpy.mockRestore();
+    });
+
+    it('maps a debounced pull to 202', async () => {
+        seedGitSource('webhook-status-202');
+        const pullSpy = vi.spyOn(GitSourceService.getInstance(), 'handleWebhookPull')
+            .mockResolvedValue({ status: 'skipped', message: 'Rate limited (debounced).' });
+        const res = await request(app)
+            .post('/api/stacks/webhook-status-202/git-source/webhook-pull')
+            .set('Authorization', `Bearer ${adminToken()}`);
+        expect(res.status).toBe(202);
+        pullSpy.mockRestore();
+    });
+
+    it('maps a successful pull to 200', async () => {
+        seedGitSource('webhook-status-200');
+        const pullSpy = vi.spyOn(GitSourceService.getInstance(), 'handleWebhookPull')
+            .mockResolvedValue({ status: 'success', message: 'Pending update ready at abc1234.' });
+        const res = await request(app)
+            .post('/api/stacks/webhook-status-200/git-source/webhook-pull')
+            .set('Authorization', `Bearer ${adminToken()}`);
+        expect(res.status).toBe(200);
+        pullSpy.mockRestore();
+    });
+});
+
 describe('GET /api/git-sources', () => {
     it('returns 200 and a JSON array for an authenticated admin', async () => {
         const res = await request(app)

backend/src/__tests__/git-source-service.test.ts

@@ -362,6 +362,66 @@ describe('GitSourceService error mapping', () => {
         await expect(svc().fetchFromGit(fetchParams)).rejects.toMatchObject({ code: 'NETWORK_TIMEOUT' });
     });
 
+    it('maps a bare "fetch failed" TypeError with an ENOTFOUND cause to NETWORK_TIMEOUT', async () => {
+        // Node's global fetch() reports DNS failure as TypeError('fetch failed')
+        // with the real reason on err.cause. Without cause-unwrapping this fell
+        // through to a useless GIT_ERROR: "fetch failed".
+        mockGitClone.mockRejectedValueOnce(
+            new TypeError('fetch failed', {
+                cause: Object.assign(new Error('getaddrinfo ENOTFOUND github.com'), { code: 'ENOTFOUND' }),
+            }),
+        );
+        await expect(svc().fetchFromGit(fetchParams)).rejects.toMatchObject({ code: 'NETWORK_TIMEOUT' });
+    });
+
+    it('maps a "fetch failed" TypeError with an ECONNREFUSED cause to NETWORK_TIMEOUT', async () => {
+        mockGitClone.mockRejectedValueOnce(
+            new TypeError('fetch failed', {
+                cause: Object.assign(new Error('connect ECONNREFUSED 127.0.0.1:443'), { code: 'ECONNREFUSED' }),
+            }),
+        );
+        await expect(svc().fetchFromGit(fetchParams)).rejects.toMatchObject({ code: 'NETWORK_TIMEOUT' });
+    });
+
+    it('surfaces the host instead of bare "fetch failed" in transport errors', async () => {
+        mockGitClone.mockRejectedValueOnce(
+            new TypeError('fetch failed', {
+                cause: Object.assign(new Error('getaddrinfo ENOTFOUND'), { code: 'ENOTFOUND' }),
+            }),
+        );
+        try {
+            await svc().fetchFromGit(fetchParams);
+            expect.fail('should have thrown');
+        } catch (e) {
+            const err = e as Error;
+            expect(err.message).not.toMatch(/^fetch failed$/i);
+            expect(err.message).toContain('github.com');
+        }
+    });
+
+    it('unwraps a nested fetch cause chain to find the transport code', async () => {
+        mockGitClone.mockRejectedValueOnce(
+            new TypeError('fetch failed', {
+                cause: new TypeError('terminated', {
+                    cause: Object.assign(new Error('reset'), { code: 'ECONNRESET' }),
+                }),
+            }),
+        );
+        await expect(svc().fetchFromGit(fetchParams)).rejects.toMatchObject({ code: 'NETWORK_TIMEOUT' });
+    });
+
+    it('maps a TLS certificate "fetch failed" cause to a certificate GIT_ERROR', async () => {
+        mockGitClone.mockRejectedValueOnce(
+            new TypeError('fetch failed', {
+                cause: Object.assign(new Error('self-signed certificate'), { code: 'DEPTH_ZERO_SELF_SIGNED_CERT' }),
+            }),
+        );
+        await expect(svc().fetchFromGit(fetchParams)).rejects.toMatchObject({
+            code: 'GIT_ERROR',
+            message: expect.stringMatching(/certificate/i),
+        });
+    });
+
     it('surfaces FILE_NOT_FOUND when the compose path is missing from the clone', async () => {
         mockGitClone.mockImplementation(async () => { /* clone empty repo */ });
         mockGitLog.mockResolvedValue([{ oid: 'deadbeef' }]);
@@ -384,6 +444,95 @@ describe('GitSourceService error mapping', () => {
     });
 });
 
+describe('countingBodyIterator (clone size cap)', () => {
+    function chunkStream(...sizes: number[]): AsyncIterableIterator<Uint8Array> {
+        async function* gen(): AsyncIterableIterator<Uint8Array> {
+            for (const s of sizes) yield new Uint8Array(s);
+        }
+        return gen();
+    }
+
+    it('passes chunks through unchanged while under the cap', async () => {
+        const { countingBodyIterator } = await import('../services/GitSourceService');
+        const controller = new AbortController();
+        const state = { exceeded: false, received: 0 };
+        const out: number[] = [];
+        for await (const c of countingBodyIterator(chunkStream(10, 20, 30), controller, 1000, state)) {
+            out.push(c.byteLength);
+        }
+        expect(out).toEqual([10, 20, 30]);
+        expect(state.exceeded).toBe(false);
+        expect(state.received).toBe(60);
+        expect(controller.signal.aborted).toBe(false);
+    });
+
+    it('aborts the transport and throws once the cumulative size exceeds the cap', async () => {
+        const { countingBodyIterator } = await import('../services/GitSourceService');
+        const controller = new AbortController();
+        const state = { exceeded: false, received: 0 };
+        await expect((async () => {
+            for await (const _c of countingBodyIterator(chunkStream(60, 60), controller, 100, state)) {
+                void _c;
+            }
+        })()).rejects.toThrow(/maximum allowed size/i);
+        expect(state.exceeded).toBe(true);
+        expect(controller.signal.aborted).toBe(true);
+    });
+});
+
+describe('GitSourceService.fetchFromGit (size limits)', () => {
+    const svc = () => GitSourceService.getInstance();
+    const fetchParams = {
+        repoUrl: 'https://github.com/example/repo.git',
+        branch: 'main',
+        composePath: 'compose.yaml',
+    };
+
+    it('rejects a compose file larger than the per-file read cap', async () => {
+        // The download cap bounds the compressed pack, not a single decompressed
+        // file, so readRepoFile guards the in-memory read by file size.
+        mockSuccessfulClone();
+        const { promises: fsp } = await import('fs');
+        const lstatSpy = vi.spyOn(fsp, 'lstat').mockResolvedValue({
+            isSymbolicLink: () => false,
+            size: 11 * 1024 * 1024,
+        } as Awaited<ReturnType<typeof fsp.lstat>>);
+
+        await expect(svc().fetchFromGit(fetchParams)).rejects.toMatchObject({
+            code: 'GIT_ERROR',
+            message: expect.stringMatching(/too large/i),
+        });
+
+        lstatSpy.mockRestore();
+    });
+
+    it('surfaces a clone-size error when the download exceeds the cap', async () => {
+        // Drive the real size-counting transport the service injected into
+        // git.clone, with a tiny cap, and confirm fetchFromGit reports it as a
+        // clone-size error rather than a generic transport failure.
+        process.env.GITSOURCE_MAX_CLONE_BYTES = '8';
+        const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(
+            new Response(new Uint8Array(64), { status: 200 }),
+        );
+        mockGitClone.mockImplementation(async (args: {
+            http: { request: (r: { url: string; method: string; headers: Record<string, string> }) => Promise<{ body: AsyncIterableIterator<Uint8Array> }> };
+        }) => {
+            const resp = await args.http.request({ url: 'https://example.test/info/refs', method: 'GET', headers: {} });
+            for await (const chunk of resp.body) { void chunk; }
+        });
+
+        try {
+            await expect(svc().fetchFromGit(fetchParams)).rejects.toMatchObject({
+                code: 'GIT_ERROR',
+                message: expect.stringMatching(/exceeds the maximum clone size/i),
+            });
+        } finally {
+            delete process.env.GITSOURCE_MAX_CLONE_BYTES;
+            fetchSpy.mockRestore();
+        }
+    });
+});
+
 describe('GitSourceService pending lifecycle', () => {
     it('dismissPending clears pending columns', async () => {
         mockSuccessfulClone();
@@ -439,6 +588,64 @@ describe('GitSourceService.handleWebhookPull debounce', () => {
         expect(result.status).toBe('error');
         expect(result.message).toMatch(/no git source/i);
     });
+
+    it('runs a single clone for a concurrent webhook fan-out', async () => {
+        // The original failure: N webhooks for one push each ran a full clone
+        // because the debounce gate was read before the per-stack lock. The
+        // gate now lives inside the lock, so the first request stamps the
+        // window and the rest skip.
+        const sha = 'eeee555eeee555eeee555eeee555eeee555eeee5';
+        mockSuccessfulClone({ sha });
+        const svc = GitSourceService.getInstance();
+        const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true });
+        await svc.upsert({
+            stackName: 'fanout-stack',
+            repoUrl: 'https://github.com/example/repo.git',
+            branch: 'main',
+            composePath: 'compose.yaml',
+            syncEnv: false,
+            envPath: null,
+            authType: 'none',
+            autoApplyOnWebhook: false,
+            autoDeployOnApply: false,
+        });
+        // upsert performs a dry-run fetch; clear that call so we count only
+        // the clones triggered by the webhook fan-out below.
+        mockGitClone.mockClear();
+
+        const results = await Promise.all(
+            Array.from({ length: 5 }, () => svc.handleWebhookPull('fanout-stack')),
+        );
+
+        expect(mockGitClone.mock.calls.length).toBe(1);
+        expect(results.filter(r => r.status === 'success')).toHaveLength(1);
+        expect(results.filter(r => r.status === 'skipped')).toHaveLength(4);
+        validateSpy.mockRestore();
+    });
+
+    it('returns error when the pulled compose fails validation', async () => {
+        mockSuccessfulClone();
+        const svc = GitSourceService.getInstance();
+        await svc.upsert({
+            stackName: 'webhook-validate-fail',
+            repoUrl: 'https://github.com/example/repo.git',
+            branch: 'main',
+            composePath: 'compose.yaml',
+            syncEnv: false,
+            envPath: null,
+            authType: 'none',
+            autoApplyOnWebhook: false,
+            autoDeployOnApply: false,
+        });
+        // upsert runs a dry-run fetch but not validateCompose, so the stub only
+        // affects the webhook pull below.
+        const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: false, error: 'bad compose' });
+
+        const result = await svc.handleWebhookPull('webhook-validate-fail');
+        expect(result.status).toBe('error');
+        expect(result.message).toMatch(/validation failed/i);
+        validateSpy.mockRestore();
+    });
 });
 
 describe('GitSourceService per-stack mutex', () => {

backend/src/__tests__/webhooks-git-source.test.ts

@@ -217,4 +217,35 @@ describe('node-aware Git source webhooks', () => {
         expect(history[0].error).toMatch(/timed out/i);
         vi.useRealTimers();
     });
+
+    it('records a debounced (202 skipped) remote git-pull as success, not failure', async () => {
+        const db = DatabaseService.getInstance();
+        const remoteNodeId = db.addNode({
+            name: 'remote-debounce-webhook',
+            type: 'remote',
+            compose_dir: '/tmp',
+            is_default: false,
+            api_url: 'http://remote-debounce.example',
+            api_token: 'remote-token',
+        });
+        const webhookId = db.addWebhook({
+            node_id: remoteNodeId,
+            name: 'debounced remote git',
+            stack_name: 'remote-stack',
+            action: 'git-pull',
+            secret: WebhookService.getInstance().generateSecret(),
+            enabled: true,
+        });
+        // 202 Accepted + status "skipped" is a debounce, not a failure.
+        vi.spyOn(globalThis, 'fetch').mockResolvedValue(
+            new Response(JSON.stringify({ status: 'skipped', message: 'Rate limited (debounced).' }), { status: 202 }),
+        );
+
+        const result = await WebhookService.getInstance().execute(db.getWebhook(webhookId)!, 'git-pull', 'test');
+
+        expect(result.success).toBe(true);
+        const history = db.getWebhookExecutions(webhookId);
+        expect(history[0].status).toBe('success');
+        expect(history[0].error).toMatch(/debounced/i);
+    });
 });