fix(git-sources): harden webhook delivery, transport errors, and clone limits (#1249)

* fix(git-sources): harden webhook delivery, transport errors, and clone limits

Map webhook-pull outcomes to real HTTP status codes (200 success, 202
debounced, 404 no source, 422 failure) instead of always returning 200, so a
Git provider and any monitoring on it can tell when a delivery actually failed.

Close a concurrent webhook fan-out gap: the debounce window is now re-checked
inside the per-stack lock, so simultaneous deliveries for one push run a single
clone instead of one per request. The whole pull/apply critical section runs
under a single lock acquisition.

Unwrap fetch transport causes (ENOTFOUND, ECONNREFUSED, ECONNRESET, TLS) so a
clone failure surfaces an actionable, host-qualified message instead of a bare
"fetch failed".

Cap how many bytes a single clone may download to protect the host disk;
operators can tune it with GITSOURCE_MAX_CLONE_BYTES (default 100 MB).

Log webhook pull failures server-side, since the webhook path is unattended.

* test(git-sources): assert surfaced host via toContain to satisfy CodeQL

* fix(git-sources): bound per-file read, treat debounced webhooks as non-failure, correct clone-cap docs

* docs(git-sources): correct clone-cap comment to describe a download bound, not disk
This commit is contained in:
Anso
2026-05-29 09:43:37 -04:00
committed by GitHub
parent b33a0e8422
commit 2844f606cd
10 changed files with 722 additions and 166 deletions
+15 -1
View File
@@ -9,7 +9,7 @@
*/
import { describe, it, expect, vi } from 'vitest';
import type { Response } from 'express';
import { gitSourceStatus, sendGitSourceError } from '../utils/gitSourceHttp';
import { gitSourceStatus, sendGitSourceError, webhookPullStatus } from '../utils/gitSourceHttp';
import { GitSourceError } from '../services/GitSourceService';
describe('gitSourceStatus', () => {
@@ -32,6 +32,20 @@ describe('gitSourceStatus', () => {
});
});
describe('webhookPullStatus', () => {
it('maps a successful pull/apply to 200', () => {
expect(webhookPullStatus('success')).toBe(200);
});
it('maps a debounced (skipped) pull to 202', () => {
expect(webhookPullStatus('skipped')).toBe(202);
});
it('maps a failed pull/apply to 422, never 200', () => {
expect(webhookPullStatus('error')).toBe(422);
});
});
describe('sendGitSourceError', () => {
function mockRes() {
const res = { status: vi.fn(), json: vi.fn() } as unknown as Response;
@@ -11,12 +11,36 @@
* Service-layer logic (encryption, error mapping, mutex, pending lifecycle)
* is covered in git-source-service.test.ts.
*/
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
import request from 'supertest';
import jwt from 'jsonwebtoken';
import fs from 'fs';
import path from 'path';
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
import { DatabaseService } from '../services/DatabaseService';
import { GitSourceService } from '../services/GitSourceService';
function seedGitSource(stackName: string): void {
DatabaseService.getInstance().upsertGitSource({
stack_name: stackName,
repo_url: 'https://github.com/example/repo.git',
branch: 'main',
compose_path: 'compose.yaml',
sync_env: false,
env_path: null,
auth_type: 'none',
encrypted_token: null,
auto_apply_on_webhook: false,
auto_deploy_on_apply: false,
last_applied_commit_sha: null,
last_applied_content_hash: null,
pending_commit_sha: null,
pending_compose_content: null,
pending_env_content: null,
pending_fetched_at: null,
last_debounce_at: null,
});
}
let tmpDir: string;
let app: import('express').Express;
@@ -291,6 +315,55 @@ describe('POST /api/stacks/from-git', () => {
});
});
describe('POST /api/stacks/:stackName/git-source/webhook-pull status codes', () => {
it('returns 404 (not 200) when the stack has no Git source configured', async () => {
const res = await request(app)
.post('/api/stacks/existing-stack/git-source/webhook-pull')
.set('Authorization', `Bearer ${adminToken()}`);
expect(res.status).toBe(404);
expect(res.body.error).toMatch(/no git source/i);
});
it('returns 401 without auth', async () => {
const res = await request(app).post('/api/stacks/existing-stack/git-source/webhook-pull');
expect(res.status).toBe(401);
});
it('maps a failed pull to 422 (not 200) so a Git provider sees the failure', async () => {
seedGitSource('webhook-status-422');
const pullSpy = vi.spyOn(GitSourceService.getInstance(), 'handleWebhookPull')
.mockResolvedValue({ status: 'error', message: 'Validation failed: bad compose' });
const res = await request(app)
.post('/api/stacks/webhook-status-422/git-source/webhook-pull')
.set('Authorization', `Bearer ${adminToken()}`);
expect(res.status).toBe(422);
expect(res.body.status).toBe('error');
pullSpy.mockRestore();
});
it('maps a debounced pull to 202', async () => {
seedGitSource('webhook-status-202');
const pullSpy = vi.spyOn(GitSourceService.getInstance(), 'handleWebhookPull')
.mockResolvedValue({ status: 'skipped', message: 'Rate limited (debounced).' });
const res = await request(app)
.post('/api/stacks/webhook-status-202/git-source/webhook-pull')
.set('Authorization', `Bearer ${adminToken()}`);
expect(res.status).toBe(202);
pullSpy.mockRestore();
});
it('maps a successful pull to 200', async () => {
seedGitSource('webhook-status-200');
const pullSpy = vi.spyOn(GitSourceService.getInstance(), 'handleWebhookPull')
.mockResolvedValue({ status: 'success', message: 'Pending update ready at abc1234.' });
const res = await request(app)
.post('/api/stacks/webhook-status-200/git-source/webhook-pull')
.set('Authorization', `Bearer ${adminToken()}`);
expect(res.status).toBe(200);
pullSpy.mockRestore();
});
});
describe('GET /api/git-sources', () => {
it('returns 200 and a JSON array for an authenticated admin', async () => {
const res = await request(app)
@@ -362,6 +362,66 @@ describe('GitSourceService error mapping', () => {
await expect(svc().fetchFromGit(fetchParams)).rejects.toMatchObject({ code: 'NETWORK_TIMEOUT' });
});
it('maps a bare "fetch failed" TypeError with an ENOTFOUND cause to NETWORK_TIMEOUT', async () => {
// Node's global fetch() reports DNS failure as TypeError('fetch failed')
// with the real reason on err.cause. Without cause-unwrapping this fell
// through to a useless GIT_ERROR: "fetch failed".
mockGitClone.mockRejectedValueOnce(
new TypeError('fetch failed', {
cause: Object.assign(new Error('getaddrinfo ENOTFOUND github.com'), { code: 'ENOTFOUND' }),
}),
);
await expect(svc().fetchFromGit(fetchParams)).rejects.toMatchObject({ code: 'NETWORK_TIMEOUT' });
});
it('maps a "fetch failed" TypeError with an ECONNREFUSED cause to NETWORK_TIMEOUT', async () => {
mockGitClone.mockRejectedValueOnce(
new TypeError('fetch failed', {
cause: Object.assign(new Error('connect ECONNREFUSED 127.0.0.1:443'), { code: 'ECONNREFUSED' }),
}),
);
await expect(svc().fetchFromGit(fetchParams)).rejects.toMatchObject({ code: 'NETWORK_TIMEOUT' });
});
it('surfaces the host instead of bare "fetch failed" in transport errors', async () => {
mockGitClone.mockRejectedValueOnce(
new TypeError('fetch failed', {
cause: Object.assign(new Error('getaddrinfo ENOTFOUND'), { code: 'ENOTFOUND' }),
}),
);
try {
await svc().fetchFromGit(fetchParams);
expect.fail('should have thrown');
} catch (e) {
const err = e as Error;
expect(err.message).not.toMatch(/^fetch failed$/i);
expect(err.message).toContain('github.com');
}
});
it('unwraps a nested fetch cause chain to find the transport code', async () => {
mockGitClone.mockRejectedValueOnce(
new TypeError('fetch failed', {
cause: new TypeError('terminated', {
cause: Object.assign(new Error('reset'), { code: 'ECONNRESET' }),
}),
}),
);
await expect(svc().fetchFromGit(fetchParams)).rejects.toMatchObject({ code: 'NETWORK_TIMEOUT' });
});
it('maps a TLS certificate "fetch failed" cause to a certificate GIT_ERROR', async () => {
mockGitClone.mockRejectedValueOnce(
new TypeError('fetch failed', {
cause: Object.assign(new Error('self-signed certificate'), { code: 'DEPTH_ZERO_SELF_SIGNED_CERT' }),
}),
);
await expect(svc().fetchFromGit(fetchParams)).rejects.toMatchObject({
code: 'GIT_ERROR',
message: expect.stringMatching(/certificate/i),
});
});
it('surfaces FILE_NOT_FOUND when the compose path is missing from the clone', async () => {
mockGitClone.mockImplementation(async () => { /* clone empty repo */ });
mockGitLog.mockResolvedValue([{ oid: 'deadbeef' }]);
@@ -384,6 +444,95 @@ describe('GitSourceService error mapping', () => {
});
});
describe('countingBodyIterator (clone size cap)', () => {
function chunkStream(...sizes: number[]): AsyncIterableIterator<Uint8Array> {
async function* gen(): AsyncIterableIterator<Uint8Array> {
for (const s of sizes) yield new Uint8Array(s);
}
return gen();
}
it('passes chunks through unchanged while under the cap', async () => {
const { countingBodyIterator } = await import('../services/GitSourceService');
const controller = new AbortController();
const state = { exceeded: false, received: 0 };
const out: number[] = [];
for await (const c of countingBodyIterator(chunkStream(10, 20, 30), controller, 1000, state)) {
out.push(c.byteLength);
}
expect(out).toEqual([10, 20, 30]);
expect(state.exceeded).toBe(false);
expect(state.received).toBe(60);
expect(controller.signal.aborted).toBe(false);
});
it('aborts the transport and throws once the cumulative size exceeds the cap', async () => {
const { countingBodyIterator } = await import('../services/GitSourceService');
const controller = new AbortController();
const state = { exceeded: false, received: 0 };
await expect((async () => {
for await (const _c of countingBodyIterator(chunkStream(60, 60), controller, 100, state)) {
void _c;
}
})()).rejects.toThrow(/maximum allowed size/i);
expect(state.exceeded).toBe(true);
expect(controller.signal.aborted).toBe(true);
});
});
describe('GitSourceService.fetchFromGit (size limits)', () => {
const svc = () => GitSourceService.getInstance();
const fetchParams = {
repoUrl: 'https://github.com/example/repo.git',
branch: 'main',
composePath: 'compose.yaml',
};
it('rejects a compose file larger than the per-file read cap', async () => {
// The download cap bounds the compressed pack, not a single decompressed
// file, so readRepoFile guards the in-memory read by file size.
mockSuccessfulClone();
const { promises: fsp } = await import('fs');
const lstatSpy = vi.spyOn(fsp, 'lstat').mockResolvedValue({
isSymbolicLink: () => false,
size: 11 * 1024 * 1024,
} as Awaited<ReturnType<typeof fsp.lstat>>);
await expect(svc().fetchFromGit(fetchParams)).rejects.toMatchObject({
code: 'GIT_ERROR',
message: expect.stringMatching(/too large/i),
});
lstatSpy.mockRestore();
});
it('surfaces a clone-size error when the download exceeds the cap', async () => {
// Drive the real size-counting transport the service injected into
// git.clone, with a tiny cap, and confirm fetchFromGit reports it as a
// clone-size error rather than a generic transport failure.
process.env.GITSOURCE_MAX_CLONE_BYTES = '8';
const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(
new Response(new Uint8Array(64), { status: 200 }),
);
mockGitClone.mockImplementation(async (args: {
http: { request: (r: { url: string; method: string; headers: Record<string, string> }) => Promise<{ body: AsyncIterableIterator<Uint8Array> }> };
}) => {
const resp = await args.http.request({ url: 'https://example.test/info/refs', method: 'GET', headers: {} });
for await (const chunk of resp.body) { void chunk; }
});
try {
await expect(svc().fetchFromGit(fetchParams)).rejects.toMatchObject({
code: 'GIT_ERROR',
message: expect.stringMatching(/exceeds the maximum clone size/i),
});
} finally {
delete process.env.GITSOURCE_MAX_CLONE_BYTES;
fetchSpy.mockRestore();
}
});
});
describe('GitSourceService pending lifecycle', () => {
it('dismissPending clears pending columns', async () => {
mockSuccessfulClone();
@@ -439,6 +588,64 @@ describe('GitSourceService.handleWebhookPull debounce', () => {
expect(result.status).toBe('error');
expect(result.message).toMatch(/no git source/i);
});
it('runs a single clone for a concurrent webhook fan-out', async () => {
// The original failure: N webhooks for one push each ran a full clone
// because the debounce gate was read before the per-stack lock. The
// gate now lives inside the lock, so the first request stamps the
// window and the rest skip.
const sha = 'eeee555eeee555eeee555eeee555eeee555eeee5';
mockSuccessfulClone({ sha });
const svc = GitSourceService.getInstance();
const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true });
await svc.upsert({
stackName: 'fanout-stack',
repoUrl: 'https://github.com/example/repo.git',
branch: 'main',
composePath: 'compose.yaml',
syncEnv: false,
envPath: null,
authType: 'none',
autoApplyOnWebhook: false,
autoDeployOnApply: false,
});
// upsert performs a dry-run fetch; clear that call so we count only
// the clones triggered by the webhook fan-out below.
mockGitClone.mockClear();
const results = await Promise.all(
Array.from({ length: 5 }, () => svc.handleWebhookPull('fanout-stack')),
);
expect(mockGitClone.mock.calls.length).toBe(1);
expect(results.filter(r => r.status === 'success')).toHaveLength(1);
expect(results.filter(r => r.status === 'skipped')).toHaveLength(4);
validateSpy.mockRestore();
});
it('returns error when the pulled compose fails validation', async () => {
mockSuccessfulClone();
const svc = GitSourceService.getInstance();
await svc.upsert({
stackName: 'webhook-validate-fail',
repoUrl: 'https://github.com/example/repo.git',
branch: 'main',
composePath: 'compose.yaml',
syncEnv: false,
envPath: null,
authType: 'none',
autoApplyOnWebhook: false,
autoDeployOnApply: false,
});
// upsert runs a dry-run fetch but not validateCompose, so the stub only
// affects the webhook pull below.
const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: false, error: 'bad compose' });
const result = await svc.handleWebhookPull('webhook-validate-fail');
expect(result.status).toBe('error');
expect(result.message).toMatch(/validation failed/i);
validateSpy.mockRestore();
});
});
describe('GitSourceService per-stack mutex', () => {
@@ -217,4 +217,35 @@ describe('node-aware Git source webhooks', () => {
expect(history[0].error).toMatch(/timed out/i);
vi.useRealTimers();
});
it('records a debounced (202 skipped) remote git-pull as success, not failure', async () => {
const db = DatabaseService.getInstance();
const remoteNodeId = db.addNode({
name: 'remote-debounce-webhook',
type: 'remote',
compose_dir: '/tmp',
is_default: false,
api_url: 'http://remote-debounce.example',
api_token: 'remote-token',
});
const webhookId = db.addWebhook({
node_id: remoteNodeId,
name: 'debounced remote git',
stack_name: 'remote-stack',
action: 'git-pull',
secret: WebhookService.getInstance().generateSecret(),
enabled: true,
});
// 202 Accepted + status "skipped" is a debounce, not a failure.
vi.spyOn(globalThis, 'fetch').mockResolvedValue(
new Response(JSON.stringify({ status: 'skipped', message: 'Rate limited (debounced).' }), { status: 202 }),
);
const result = await WebhookService.getInstance().execute(db.getWebhook(webhookId)!, 'git-pull', 'test');
expect(result.success).toBe(true);
const history = db.getWebhookExecutions(webhookId);
expect(history[0].status).toBe('success');
expect(history[0].error).toMatch(/debounced/i);
});
});
+10 -3
View File
@@ -7,7 +7,7 @@ import { checkPermission, requirePermission } from '../middleware/permissions';
import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
import { triggerPostDeployScan } from '../helpers/policyGate';
import { isValidGitSourcePath, isValidStackName } from '../utils/validation';
import { sendGitSourceError } from '../utils/gitSourceHttp';
import { sendGitSourceError, webhookPullStatus } from '../utils/gitSourceHttp';
import { sanitizeForLog } from '../utils/safeLog';
// Reasonable upper bounds so a caller cannot flood the service with huge
@@ -260,9 +260,16 @@ stackGitSourceRouter.post('/:stackName/git-source/webhook-pull', async (req: Req
if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return;
try {
const source = GitSourceService.getInstance().get(stackName);
if (source?.auto_apply_on_webhook && source.auto_deploy_on_apply && !requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return;
if (!source) {
res.status(404).json({ error: 'No Git source configured for this stack', status: 'error' });
return;
}
if (source.auto_apply_on_webhook && source.auto_deploy_on_apply && !requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return;
const result = await GitSourceService.getInstance().handleWebhookPull(stackName);
res.json(result);
// Map the outcome to a real HTTP status so a Git provider sees a 4xx on
// failure instead of a 200 with an error body (which it would read as
// "delivered fine, stop retrying").
res.status(webhookPullStatus(result.status)).json(result);
} catch (error) {
sendGitSourceError(res, error);
}
+337 -157
View File
@@ -74,7 +74,51 @@ function responseBodyIterator(body: ReadableStream<Uint8Array> | null): AsyncIte
return iterate();
}
function createAbortableGitHttp(signal: AbortSignal): HttpClient {
/**
* Mutable counter shared across every request in one clone so the size cap
* is cumulative, and so the caller can tell a size abort from a timeout via
* `exceeded` rather than the thrown error (which isomorphic-git may wrap).
*/
interface CloneSizeState {
exceeded: boolean;
received: number;
}
/**
* Wrap a response-body iterator with a cumulative byte counter shared
* across every request in one clone. When the running total crosses
* `maxBytes`, flip `state.exceeded`, abort the transport (closing the
* socket so the download stops), and throw to unwind the stream. The
* caller distinguishes a size abort from a timeout/transport abort via
* `state.exceeded` rather than the thrown error, which isomorphic-git may
* wrap.
*/
export function countingBodyIterator(
src: AsyncIterableIterator<Uint8Array>,
controller: AbortController,
maxBytes: number,
state: CloneSizeState,
): AsyncIterableIterator<Uint8Array> {
async function* iterate(): AsyncIterableIterator<Uint8Array> {
for await (const chunk of src) {
state.received += chunk.byteLength;
if (state.received > maxBytes) {
state.exceeded = true;
controller.abort();
throw new Error('Clone exceeded the maximum allowed size');
}
yield chunk;
}
}
return iterate();
}
function createAbortableGitHttp(
controller: AbortController,
maxBytes: number,
state: CloneSizeState,
): HttpClient {
const signal = controller.signal;
return {
async request(request: GitHttpRequest): Promise<GitHttpResponse> {
if (signal.aborted) {
@@ -94,7 +138,7 @@ function createAbortableGitHttp(signal: AbortSignal): HttpClient {
statusCode: response.status,
statusMessage: response.statusText,
headers: Object.fromEntries(response.headers.entries()),
body: responseBodyIterator(response.body),
body: countingBodyIterator(responseBodyIterator(response.body), controller, maxBytes, state),
};
},
};
@@ -215,6 +259,31 @@ const DEFAULT_FETCH_TIMEOUT_MS = 30_000;
const TEMP_DIR_MAX_AGE_MS = 60 * 60 * 1000; // 1 hour
const WEBHOOK_DEBOUNCE_MS = 10_000;
// Ceiling on how many bytes a single clone may download (the compressed pack
// from the Git host) before it is aborted. This bounds network transfer and
// abuse, not the decompressed on-disk checkout; it is paired with
// MAX_REPO_FILE_BYTES and the 30s timeout. Generous default; operators with a
// legitimately large monorepo can raise it via GITSOURCE_MAX_CLONE_BYTES.
const DEFAULT_MAX_CLONE_BYTES = 100 * 1024 * 1024; // 100 MB
// Per-file ceiling for the compose/env file read into memory after the clone.
// These files are KB-scale in practice; the clone byte cap bounds the
// compressed download, not the decompressed working tree, so this guards the
// in-memory read against a single huge (or highly compressible) file.
const MAX_REPO_FILE_BYTES = 10 * 1024 * 1024; // 10 MB
function maxCloneBytes(): number {
const raw = process.env.GITSOURCE_MAX_CLONE_BYTES;
const n = raw ? Number(raw) : NaN;
return Number.isFinite(n) && n > 0 ? Math.floor(n) : DEFAULT_MAX_CLONE_BYTES;
}
function formatBytes(bytes: number): string {
if (bytes >= 1024 * 1024) return `${Math.round(bytes / (1024 * 1024))} MB`;
if (bytes >= 1024) return `${Math.round(bytes / 1024)} KB`;
return `${bytes} B`;
}
// ─── Credential scrubbing ────────────────────────────────────────────────────
/**
@@ -245,6 +314,59 @@ export function repoHost(url: string): string {
}
}
/**
* Node's global `fetch()` reports every transport-level failure as a bare
* `TypeError('fetch failed')` and hides the real reason on `error.cause`
* (occasionally nested one level deeper through undici). Walk the cause
* chain for the first Node error code so DNS / connection / TLS failures
* can be translated into an actionable message instead of "fetch failed",
* which reads like an internal Sencho bug.
*/
function findCauseCode(err: unknown): { code?: string } {
let cur: unknown = err;
for (let depth = 0; depth < 5 && cur; depth++) {
const code = (cur as { code?: unknown }).code;
if (typeof code === 'string' && code) {
return { code };
}
cur = (cur as { cause?: unknown }).cause;
}
return {};
}
/**
* Translate a Node transport error code into a GitSourceError with a
* host-qualified, user-actionable message. Returns null for codes we do
* not specifically recognise so the caller can fall through to its generic
* handling. `host` is the bare hostname from `repoHost()` (never carries a
* credential), so it is safe to surface.
*/
function transportError(code: string, host: string): GitSourceError | null {
const dest = host && host !== 'unknown' ? ` ${host}` : ' the repository host';
switch (code) {
case 'ENOTFOUND':
case 'EAI_AGAIN':
return new GitSourceError('NETWORK_TIMEOUT', `Could not resolve${dest}. Check the repository URL and your network or DNS.`);
case 'ECONNREFUSED':
return new GitSourceError('NETWORK_TIMEOUT', `Connection refused by${dest}.`);
case 'ECONNRESET':
return new GitSourceError('NETWORK_TIMEOUT', `Connection to${dest} was reset. Retry; if it persists, check the host.`);
case 'ETIMEDOUT':
case 'UND_ERR_CONNECT_TIMEOUT':
case 'UND_ERR_HEADERS_TIMEOUT':
case 'UND_ERR_BODY_TIMEOUT':
return new GitSourceError('NETWORK_TIMEOUT', `Timed out reaching${dest}.`);
case 'DEPTH_ZERO_SELF_SIGNED_CERT':
case 'SELF_SIGNED_CERT_IN_CHAIN':
case 'UNABLE_TO_VERIFY_LEAF_SIGNATURE':
case 'CERT_HAS_EXPIRED':
case 'ERR_TLS_CERT_ALTNAME_INVALID':
return new GitSourceError('GIT_ERROR', `TLS certificate error reaching${dest} (${code}). The host certificate could not be verified.`);
default:
return null;
}
}
/**
* Git LFS stores large files as small pointer stubs in the working tree.
* The pointer is a short text file that always begins with this line.
@@ -297,6 +419,12 @@ async function readRepoFile(rootDir: string, relPath: string, label: string): Pr
if (stat.isSymbolicLink()) {
throw new GitSourceError('FILE_NOT_FOUND', `${label} cannot be a symbolic link.`);
}
// Bound the in-memory read. The clone byte cap only limits the compressed
// download; a single decompressed file can still be large, so reject an
// oversized compose/env file before reading it into a string.
if (stat.size > MAX_REPO_FILE_BYTES) {
throw new GitSourceError('GIT_ERROR', `${label} is too large (${formatBytes(stat.size)}); the maximum is ${formatBytes(MAX_REPO_FILE_BYTES)}.`);
}
let real;
try {
@@ -526,6 +654,8 @@ export class GitSourceService {
// fetches do not keep sockets and packfile streams alive.
let timer: NodeJS.Timeout | undefined;
const controller = new AbortController();
const maxBytes = maxCloneBytes();
const sizeState = { exceeded: false, received: 0 };
const timeout = new Promise<never>((_, reject) => {
timer = setTimeout(() => {
controller.abort();
@@ -536,7 +666,7 @@ export class GitSourceService {
await Promise.race([
git.clone({
fs: { promises: fsPromises },
http: createAbortableGitHttp(controller.signal),
http: createAbortableGitHttp(controller, maxBytes, sizeState),
dir,
url: repoUrl,
ref: branch,
@@ -548,7 +678,15 @@ export class GitSourceService {
timeout,
]);
} catch (e) {
throw this.mapGitError(e as Error, Boolean(token));
// A size abort surfaces as a generic transport error once
// isomorphic-git unwinds, so detect it via the shared flag.
if (sizeState.exceeded) {
throw new GitSourceError(
'GIT_ERROR',
`Repository exceeds the maximum clone size of ${formatBytes(maxBytes)}.`,
);
}
throw this.mapGitError(e as Error, Boolean(token), repoHost(repoUrl));
} finally {
if (timer) clearTimeout(timer);
}
@@ -619,7 +757,7 @@ export class GitSourceService {
}
}
private mapGitError(err: Error, hasToken: boolean): GitSourceError {
private mapGitError(err: Error, hasToken: boolean, host = 'unknown'): GitSourceError {
const raw = scrubCredentials(err.message || String(err));
const code = (err as Error & { code?: string }).code;
// isomorphic-git's HttpError exposes the numeric status on .data; inspect
@@ -649,6 +787,18 @@ export class GitSourceService {
return new GitSourceError('REPO_NOT_FOUND', 'Repository not found, or it is private. Add a Personal Access Token if the repo is private.');
}
// Transport failures: Node's fetch() throws a bare "fetch failed"
// TypeError with the real reason (ENOTFOUND, ECONNREFUSED, TLS, ...)
// on err.cause. Translate the underlying code before falling through
// to the generic branches, which only see the useless "fetch failed".
if (!statusCode) {
const cause = findCauseCode(err);
if (cause.code) {
const mapped = transportError(cause.code, host);
if (mapped) return mapped;
}
}
// Fallbacks for errors without a numeric status attached (e.g. git CLI
// output, DNS/lib errors, or future isomorphic-git transports that do
// not populate err.data.statusCode). Kept as defense-in-depth.
@@ -770,62 +920,69 @@ export class GitSourceService {
// ─── Pull / apply ────────────────────────────────────────────────────────
public async pull(stackName: string): Promise<PullResult> {
// Guarded by the same per-stack mutex as apply(). Without this, a
// concurrent delete-source + pull can land a pending row on a
// stack whose config row has just been removed; the DELETE clears
// the row but the subsequent setGitSourcePending re-inserts via
// UPDATE failing silently (no row), and a later upsert would
// inherit stale pending columns on read.
return this.withStackLock(stackName, async () => {
const db = DatabaseService.getInstance();
const src = db.getGitSource(stackName);
if (!src) throw new GitSourceError('GIT_ERROR', 'No Git source configured for this stack.');
// Guarded by the per-stack mutex (see withStackLock). Without this, a
// concurrent delete-source + pull can land a pending row on a stack
// whose config row has just been removed.
return this.withStackLock(stackName, () => this.pullLocked(stackName));
}
const diag = isDebugEnabled();
if (diag) {
console.log(`[GitSource:diag] pull start stack=${stackName} branch=${src.branch} host=${repoHost(src.repo_url)}`);
}
/**
* Body of pull(); assumes the caller already holds the per-stack lock.
* handleWebhookPull calls this directly so that its debounce re-check,
* this fetch, and the apply all run inside the single lock that
* handleWebhookPull holds. Without that, a concurrent webhook fan-out
* reads last_debounce_at while it is still unset on every request, slips
* past the gate, and clones once per request.
*/
private async pullLocked(stackName: string): Promise<PullResult> {
const db = DatabaseService.getInstance();
const src = db.getGitSource(stackName);
if (!src) throw new GitSourceError('GIT_ERROR', 'No Git source configured for this stack.');
const token = src.encrypted_token ? this.crypto.decrypt(src.encrypted_token) : null;
const fetched = await this.fetchFromGit({
repoUrl: src.repo_url,
branch: src.branch,
composePath: src.compose_path,
envPath: src.sync_env ? src.env_path : null,
token,
});
const diag = isDebugEnabled();
if (diag) {
console.log(`[GitSource:diag] pull start stack=${stackName} branch=${src.branch} host=${repoHost(src.repo_url)}`);
}
const validation = await this.validateCompose(fetched.composeContent, fetched.envContent);
const disk = await this.readDiskContent(stackName, src.sync_env);
const currentHash = this.hashContent(disk.compose, disk.env);
const hasLocalChanges = src.last_applied_content_hash !== null
&& src.last_applied_content_hash !== currentHash;
// Store pending so a subsequent apply doesn't re-fetch. Compose files
// routinely contain secrets inlined as env interpolations or passwords,
// so encrypt the pending buffers at rest.
db.setGitSourcePending(
stackName,
fetched.commitSha,
this.crypto.encrypt(fetched.composeContent),
fetched.envContent !== null ? this.crypto.encrypt(fetched.envContent) : null,
);
console.log(`[GitSource] Pending update ready for ${stackName} at ${fetched.commitSha.slice(0, 7)} (validation=${validation.ok ? 'ok' : 'fail'}, localEdits=${hasLocalChanges})`);
if (diag) {
console.log(`[GitSource:diag] pull done stack=${stackName} sha=${fetched.commitSha.slice(0, 7)} validation=${validation.ok} localEdits=${hasLocalChanges}`);
}
return {
commitSha: fetched.commitSha,
incomingCompose: fetched.composeContent,
incomingEnv: fetched.envContent,
currentCompose: disk.compose,
currentEnv: disk.env,
validation,
hasLocalChanges,
};
const token = src.encrypted_token ? this.crypto.decrypt(src.encrypted_token) : null;
const fetched = await this.fetchFromGit({
repoUrl: src.repo_url,
branch: src.branch,
composePath: src.compose_path,
envPath: src.sync_env ? src.env_path : null,
token,
});
const validation = await this.validateCompose(fetched.composeContent, fetched.envContent);
const disk = await this.readDiskContent(stackName, src.sync_env);
const currentHash = this.hashContent(disk.compose, disk.env);
const hasLocalChanges = src.last_applied_content_hash !== null
&& src.last_applied_content_hash !== currentHash;
// Store pending so a subsequent apply doesn't re-fetch. Compose files
// routinely contain secrets inlined as env interpolations or passwords,
// so encrypt the pending buffers at rest.
db.setGitSourcePending(
stackName,
fetched.commitSha,
this.crypto.encrypt(fetched.composeContent),
fetched.envContent !== null ? this.crypto.encrypt(fetched.envContent) : null,
);
console.log(`[GitSource] Pending update ready for ${stackName} at ${fetched.commitSha.slice(0, 7)} (validation=${validation.ok ? 'ok' : 'fail'}, localEdits=${hasLocalChanges})`);
if (diag) {
console.log(`[GitSource:diag] pull done stack=${stackName} sha=${fetched.commitSha.slice(0, 7)} validation=${validation.ok} localEdits=${hasLocalChanges}`);
}
return {
commitSha: fetched.commitSha,
incomingCompose: fetched.composeContent,
incomingEnv: fetched.envContent,
currentCompose: disk.compose,
currentEnv: disk.env,
validation,
hasLocalChanges,
};
}
/**
@@ -845,72 +1002,79 @@ export class GitSourceService {
commitSha: string,
opts: { deploy?: boolean; actor?: string; bypassPolicy?: boolean } = {},
): Promise<{ applied: boolean; deployed: boolean; deployError?: string }> {
return this.withStackLock(stackName, async () => {
const diag = isDebugEnabled();
const db = DatabaseService.getInstance();
const src = db.getGitSource(stackName);
if (!src) throw new GitSourceError('GIT_ERROR', 'No Git source configured for this stack.');
return this.withStackLock(stackName, () => this.applyLocked(stackName, commitSha, opts));
}
if (!src.pending_commit_sha || !src.pending_compose_content) {
throw new GitSourceError('GIT_ERROR', 'No pending pull to apply. Fetch the source again.');
/** Body of apply(); assumes the caller already holds the per-stack lock. */
private async applyLocked(
stackName: string,
commitSha: string,
opts: { deploy?: boolean; actor?: string; bypassPolicy?: boolean },
): Promise<{ applied: boolean; deployed: boolean; deployError?: string }> {
const diag = isDebugEnabled();
const db = DatabaseService.getInstance();
const src = db.getGitSource(stackName);
if (!src) throw new GitSourceError('GIT_ERROR', 'No Git source configured for this stack.');
if (!src.pending_commit_sha || !src.pending_compose_content) {
throw new GitSourceError('GIT_ERROR', 'No pending pull to apply. Fetch the source again.');
}
if (src.pending_commit_sha !== commitSha) {
if (diag) console.log('[GitSource:diag] apply sha mismatch stack=%s expected=%s pending=%s', sanitizeForLog(stackName), sanitizeForLog(commitSha.slice(0, 7)), sanitizeForLog(src.pending_commit_sha.slice(0, 7)));
throw new GitSourceError('GIT_ERROR', 'Pending commit has changed since this pull was fetched. Please review the latest diff.');
}
// Pending buffers are stored encrypted; decrypt is a no-op for any
// legacy plaintext rows (isEncrypted check inside CryptoService).
const composeContent = this.crypto.decrypt(src.pending_compose_content);
const envContent = src.pending_env_content !== null
? this.crypto.decrypt(src.pending_env_content)
: null;
// Re-validate before writing.
const validation = await this.validateCompose(composeContent, envContent);
if (!validation.ok) {
if (diag) console.log(`[GitSource:diag] apply validation fail stack=${stackName}`);
throw new GitSourceError('GIT_ERROR', `Compose validation failed: ${validation.error}`);
}
const fsSvc = FileSystemService.getInstance();
await fsSvc.saveStackContent(stackName, composeContent);
if (src.sync_env && envContent !== null) {
await fsSvc.saveEnvContent(stackName, envContent);
}
const hash = this.hashContent(composeContent, envContent);
db.markGitSourceApplied(stackName, commitSha, hash);
const shouldDeploy = opts.deploy ?? src.auto_deploy_on_apply;
if (diag) console.log('[GitSource:diag] apply wrote stack=%s sha=%s deploy=%s', sanitizeForLog(stackName), sanitizeForLog(commitSha.slice(0, 7)), sanitizeForLog(shouldDeploy));
if (shouldDeploy) {
try {
const nodeId = NodeRegistry.getInstance().getDefaultNodeId();
await assertPolicyGateAllows(
stackName,
nodeId,
buildSystemPolicyGateOptions(opts.actor ?? 'git-source', {
bypass: opts.bypassPolicy === true,
auditPath: `/api/stacks/${stackName}/git-source/apply`,
}),
);
await ComposeService.getInstance().deployStack(stackName);
console.log(`[GitSource] Applied and deployed ${stackName} at ${commitSha.slice(0, 7)}`);
return { applied: true, deployed: true };
} catch (e) {
// File is on disk, DB is marked applied. Returning the
// error separately lets the UI flag it as a partial
// success rather than rolling back the disk.
const scrubbed = scrubCredentials((e as Error).message || String(e));
console.error(`[GitSource] Auto-deploy failed for ${stackName}: ${scrubbed}`);
return { applied: true, deployed: false, deployError: scrubbed };
}
if (src.pending_commit_sha !== commitSha) {
if (diag) console.log('[GitSource:diag] apply sha mismatch stack=%s expected=%s pending=%s', sanitizeForLog(stackName), sanitizeForLog(commitSha.slice(0, 7)), sanitizeForLog(src.pending_commit_sha.slice(0, 7)));
throw new GitSourceError('GIT_ERROR', 'Pending commit has changed since this pull was fetched. Please review the latest diff.');
}
// Pending buffers are stored encrypted; decrypt is a no-op for any
// legacy plaintext rows (isEncrypted check inside CryptoService).
const composeContent = this.crypto.decrypt(src.pending_compose_content);
const envContent = src.pending_env_content !== null
? this.crypto.decrypt(src.pending_env_content)
: null;
// Re-validate before writing.
const validation = await this.validateCompose(composeContent, envContent);
if (!validation.ok) {
if (diag) console.log(`[GitSource:diag] apply validation fail stack=${stackName}`);
throw new GitSourceError('GIT_ERROR', `Compose validation failed: ${validation.error}`);
}
const fsSvc = FileSystemService.getInstance();
await fsSvc.saveStackContent(stackName, composeContent);
if (src.sync_env && envContent !== null) {
await fsSvc.saveEnvContent(stackName, envContent);
}
const hash = this.hashContent(composeContent, envContent);
db.markGitSourceApplied(stackName, commitSha, hash);
const shouldDeploy = opts.deploy ?? src.auto_deploy_on_apply;
if (diag) console.log('[GitSource:diag] apply wrote stack=%s sha=%s deploy=%s', sanitizeForLog(stackName), sanitizeForLog(commitSha.slice(0, 7)), sanitizeForLog(shouldDeploy));
if (shouldDeploy) {
try {
const nodeId = NodeRegistry.getInstance().getDefaultNodeId();
await assertPolicyGateAllows(
stackName,
nodeId,
buildSystemPolicyGateOptions(opts.actor ?? 'git-source', {
bypass: opts.bypassPolicy === true,
auditPath: `/api/stacks/${stackName}/git-source/apply`,
}),
);
await ComposeService.getInstance().deployStack(stackName);
console.log(`[GitSource] Applied and deployed ${stackName} at ${commitSha.slice(0, 7)}`);
return { applied: true, deployed: true };
} catch (e) {
// File is on disk, DB is marked applied. Returning the
// error separately lets the UI flag it as a partial
// success rather than rolling back the disk.
const scrubbed = scrubCredentials((e as Error).message || String(e));
console.error(`[GitSource] Auto-deploy failed for ${stackName}: ${scrubbed}`);
return { applied: true, deployed: false, deployError: scrubbed };
}
}
console.log(`[GitSource] Applied ${stackName} at ${commitSha.slice(0, 7)}`);
return { applied: true, deployed: false };
});
}
console.log(`[GitSource] Applied ${stackName} at ${commitSha.slice(0, 7)}`);
return { applied: true, deployed: false };
}
public dismissPending(stackName: string): void {
@@ -1035,47 +1199,63 @@ export class GitSourceService {
* record in webhook_executions. Enforces the per-source debounce.
*/
public async handleWebhookPull(stackName: string): Promise<{ status: 'success' | 'skipped' | 'error'; message: string }> {
const diag = isDebugEnabled();
const db = DatabaseService.getInstance();
const src = db.getGitSource(stackName);
if (!src) {
return { status: 'error', message: 'No Git source configured for this stack.' };
}
const now = Date.now();
if (src.last_debounce_at !== null && (now - src.last_debounce_at) < WEBHOOK_DEBOUNCE_MS) {
if (diag) console.log(`[GitSource:diag] webhook debounced stack=${stackName} age=${now - src.last_debounce_at}ms`);
return { status: 'skipped', message: 'Rate limited (debounced).' };
}
try {
const pullResult = await this.pull(stackName);
// Only burn the debounce window once the fetch actually produced
// something. A transient network failure should be retriable
// immediately rather than locked out for the debounce interval.
db.touchGitSourceDebounce(stackName);
if (!pullResult.validation.ok) {
return { status: 'error', message: `Validation failed: ${pullResult.validation.error}` };
// Run the whole critical section under a single lock acquisition so a
// concurrent fan-out (N webhooks for one push) serializes AND re-reads
// last_debounce_at after acquiring the lock. The first request stamps
// the window; every queued duplicate then sees the stamp and skips
// instead of cloning again. The debounce is still stamped only after a
// successful fetch, so a transient failure stays immediately retriable.
return this.withStackLock<{ status: 'success' | 'skipped' | 'error'; message: string }>(stackName, async () => {
const diag = isDebugEnabled();
const db = DatabaseService.getInstance();
const src = db.getGitSource(stackName);
if (!src) {
return { status: 'error', message: 'No Git source configured for this stack.' };
}
if (!src.auto_apply_on_webhook) {
if (diag) console.log(`[GitSource:diag] webhook pending-only stack=${stackName} sha=${pullResult.commitSha.slice(0, 7)}`);
return { status: 'success', message: `Pending update ready at ${pullResult.commitSha.slice(0, 7)}.` };
const now = Date.now();
if (src.last_debounce_at !== null && (now - src.last_debounce_at) < WEBHOOK_DEBOUNCE_MS) {
if (diag) console.log(`[GitSource:diag] webhook debounced stack=${stackName} age=${now - src.last_debounce_at}ms`);
return { status: 'skipped', message: 'Rate limited (debounced).' };
}
const applied = await this.apply(stackName, pullResult.commitSha, { deploy: src.auto_deploy_on_apply });
if (applied.deployError) {
// Apply wrote to disk but deploy failed. Surface it so the
// webhook_executions row records a degraded outcome instead
// of a clean success.
return { status: 'error', message: `Applied commit ${pullResult.commitSha.slice(0, 7)} but deploy failed: ${applied.deployError}` };
try {
const pullResult = await this.pullLocked(stackName);
// Only burn the debounce window once the fetch actually produced
// something. A transient network failure should be retriable
// immediately rather than locked out for the debounce interval.
db.touchGitSourceDebounce(stackName);
if (!pullResult.validation.ok) {
// Webhooks are unattended, so always leave a server-side
// breadcrumb; the caller only sees the HTTP status.
console.warn(`[GitSource] Webhook pull validation failed for ${sanitizeForLog(stackName)}: ${sanitizeForLog(pullResult.validation.error ?? 'unknown')}`);
return { status: 'error', message: `Validation failed: ${pullResult.validation.error}` };
}
if (!src.auto_apply_on_webhook) {
if (diag) console.log(`[GitSource:diag] webhook pending-only stack=${stackName} sha=${pullResult.commitSha.slice(0, 7)}`);
return { status: 'success', message: `Pending update ready at ${pullResult.commitSha.slice(0, 7)}.` };
}
const applied = await this.applyLocked(stackName, pullResult.commitSha, { deploy: src.auto_deploy_on_apply });
if (applied.deployError) {
// Apply wrote to disk but deploy failed. Surface it so the
// webhook_executions row records a degraded outcome instead
// of a clean success.
return { status: 'error', message: `Applied commit ${pullResult.commitSha.slice(0, 7)} but deploy failed: ${applied.deployError}` };
}
const suffix = applied.deployed ? ' and deployed' : '';
return { status: 'success', message: `Applied commit ${pullResult.commitSha.slice(0, 7)}${suffix}.` };
} catch (e) {
const msg = e instanceof GitSourceError ? `${e.code}: ${e.message}` : (e as Error).message;
const scrubbed = scrubCredentials(msg);
// Unattended path: record the failure server-side so an operator
// can diagnose without diag mode, since the Git provider only
// logs the HTTP status.
console.error(`[GitSource] Webhook pull failed for ${sanitizeForLog(stackName)}: ${sanitizeForLog(scrubbed)}`);
return { status: 'error', message: scrubbed };
}
const suffix = applied.deployed ? ' and deployed' : '';
return { status: 'success', message: `Applied commit ${pullResult.commitSha.slice(0, 7)}${suffix}.` };
} catch (e) {
const msg = e instanceof GitSourceError ? `${e.code}: ${e.message}` : (e as Error).message;
return { status: 'error', message: scrubCredentials(msg) };
}
});
}
// ─── Concurrency ─────────────────────────────────────────────────────────
+11 -4
View File
@@ -183,15 +183,18 @@ export class WebhookService {
}
const skipped = result.status === 'skipped';
// A debounced pull is rate-limited, not failed (the route answers 202
// Accepted). Record it as a success carrying the debounce note so it
// does not pollute the webhook's failure history.
this.recordExecution(
webhookId,
action,
skipped ? 'failure' : 'success',
'success',
triggerSource,
durationMs,
skipped ? result.message : null,
);
return { success: !skipped, error: skipped ? result.message : undefined, duration_ms: durationMs };
return { success: true, error: undefined, duration_ms: durationMs };
}
private async executeRemote(
@@ -214,13 +217,17 @@ export class WebhookService {
const durationMs = Date.now() - startTime;
const payload = await response.json().catch(() => ({})) as { error?: string; message?: string; status?: string };
if (!response.ok || payload.status === 'error' || payload.status === 'skipped') {
if (!response.ok || payload.status === 'error') {
const error = payload.error || payload.message || `Remote ${action} failed with status ${response.status}`;
this.recordExecution(webhookId, action, 'failure', triggerSource, durationMs, error);
return { success: false, error, duration_ms: durationMs };
}
this.recordExecution(webhookId, action, 'success', triggerSource, durationMs, null);
// A debounced remote pull comes back 202 with status "skipped": it was
// accepted and rate-limited, not failed. Record it as a success with
// the debounce note rather than failure noise.
const skipped = payload.status === 'skipped';
this.recordExecution(webhookId, action, 'success', triggerSource, durationMs, skipped ? (payload.message ?? null) : null);
return { success: true, duration_ms: durationMs };
} catch (err) {
const durationMs = Date.now() - startTime;
+25
View File
@@ -27,6 +27,31 @@ export function gitSourceStatus(code: GitSourceErrorCode): number {
}
}
/**
* Map a webhook-pull outcome to an HTTP status so a Git provider (and any
* monitoring on top of it) can tell delivery succeeded, was a no-op, or
* failed by status code alone, not just by parsing the JSON body. The
* missing-source case is handled by the route as a 404 before this runs.
*
* success -> 200 applied / pending update ready
* skipped -> 202 accepted but debounced (no work done this call)
* error -> 422 request understood, the pull/apply/deploy failed
*/
export function webhookPullStatus(status: 'success' | 'skipped' | 'error'): number {
switch (status) {
case 'success': return 200;
case 'skipped': return 202;
case 'error': return 422;
default: {
// Exhaustiveness guard: if a new status is added to the union without a
// case here, this becomes a compile error instead of returning undefined.
const _exhaustive: never = status;
void _exhaustive;
return 500;
}
}
}
export function sendGitSourceError(res: Response, err: unknown): void {
if (err instanceof GitSourceError) {
res.status(gitSourceStatus(err.code)).json({ error: err.message, code: err.code });