mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-08 09:54:26 +00:00
fix(git-sources): harden webhook delivery, transport errors, and clone limits (#1249)
* fix(git-sources): harden webhook delivery, transport errors, and clone limits Map webhook-pull outcomes to real HTTP status codes (200 success, 202 debounced, 404 no source, 422 failure) instead of always returning 200, so a Git provider and any monitoring on it can tell when a delivery actually failed. Close a concurrent webhook fan-out gap: the debounce window is now re-checked inside the per-stack lock, so simultaneous deliveries for one push run a single clone instead of one per request. The whole pull/apply critical section runs under a single lock acquisition. Unwrap fetch transport causes (ENOTFOUND, ECONNREFUSED, ECONNRESET, TLS) so a clone failure surfaces an actionable, host-qualified message instead of a bare "fetch failed". Cap how many bytes a single clone may download to protect the host disk; operators can tune it with GITSOURCE_MAX_CLONE_BYTES (default 100 MB). Log webhook pull failures server-side, since the webhook path is unattended. * test(git-sources): assert surfaced host via toContain to satisfy CodeQL * fix(git-sources): bound per-file read, treat debounced webhooks as non-failure, correct clone-cap docs * docs(git-sources): correct clone-cap comment to describe a download bound, not disk
This commit is contained in:
@@ -9,7 +9,7 @@
|
||||
*/
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import type { Response } from 'express';
|
||||
import { gitSourceStatus, sendGitSourceError } from '../utils/gitSourceHttp';
|
||||
import { gitSourceStatus, sendGitSourceError, webhookPullStatus } from '../utils/gitSourceHttp';
|
||||
import { GitSourceError } from '../services/GitSourceService';
|
||||
|
||||
describe('gitSourceStatus', () => {
|
||||
@@ -32,6 +32,20 @@ describe('gitSourceStatus', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('webhookPullStatus', () => {
|
||||
it('maps a successful pull/apply to 200', () => {
|
||||
expect(webhookPullStatus('success')).toBe(200);
|
||||
});
|
||||
|
||||
it('maps a debounced (skipped) pull to 202', () => {
|
||||
expect(webhookPullStatus('skipped')).toBe(202);
|
||||
});
|
||||
|
||||
it('maps a failed pull/apply to 422, never 200', () => {
|
||||
expect(webhookPullStatus('error')).toBe(422);
|
||||
});
|
||||
});
|
||||
|
||||
describe('sendGitSourceError', () => {
|
||||
function mockRes() {
|
||||
const res = { status: vi.fn(), json: vi.fn() } as unknown as Response;
|
||||
|
||||
@@ -11,12 +11,36 @@
|
||||
* Service-layer logic (encryption, error mapping, mutex, pending lifecycle)
|
||||
* is covered in git-source-service.test.ts.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
|
||||
import request from 'supertest';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
|
||||
import { DatabaseService } from '../services/DatabaseService';
|
||||
import { GitSourceService } from '../services/GitSourceService';
|
||||
|
||||
function seedGitSource(stackName: string): void {
|
||||
DatabaseService.getInstance().upsertGitSource({
|
||||
stack_name: stackName,
|
||||
repo_url: 'https://github.com/example/repo.git',
|
||||
branch: 'main',
|
||||
compose_path: 'compose.yaml',
|
||||
sync_env: false,
|
||||
env_path: null,
|
||||
auth_type: 'none',
|
||||
encrypted_token: null,
|
||||
auto_apply_on_webhook: false,
|
||||
auto_deploy_on_apply: false,
|
||||
last_applied_commit_sha: null,
|
||||
last_applied_content_hash: null,
|
||||
pending_commit_sha: null,
|
||||
pending_compose_content: null,
|
||||
pending_env_content: null,
|
||||
pending_fetched_at: null,
|
||||
last_debounce_at: null,
|
||||
});
|
||||
}
|
||||
|
||||
let tmpDir: string;
|
||||
let app: import('express').Express;
|
||||
@@ -291,6 +315,55 @@ describe('POST /api/stacks/from-git', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('POST /api/stacks/:stackName/git-source/webhook-pull status codes', () => {
|
||||
it('returns 404 (not 200) when the stack has no Git source configured', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/existing-stack/git-source/webhook-pull')
|
||||
.set('Authorization', `Bearer ${adminToken()}`);
|
||||
expect(res.status).toBe(404);
|
||||
expect(res.body.error).toMatch(/no git source/i);
|
||||
});
|
||||
|
||||
it('returns 401 without auth', async () => {
|
||||
const res = await request(app).post('/api/stacks/existing-stack/git-source/webhook-pull');
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
|
||||
it('maps a failed pull to 422 (not 200) so a Git provider sees the failure', async () => {
|
||||
seedGitSource('webhook-status-422');
|
||||
const pullSpy = vi.spyOn(GitSourceService.getInstance(), 'handleWebhookPull')
|
||||
.mockResolvedValue({ status: 'error', message: 'Validation failed: bad compose' });
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/webhook-status-422/git-source/webhook-pull')
|
||||
.set('Authorization', `Bearer ${adminToken()}`);
|
||||
expect(res.status).toBe(422);
|
||||
expect(res.body.status).toBe('error');
|
||||
pullSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('maps a debounced pull to 202', async () => {
|
||||
seedGitSource('webhook-status-202');
|
||||
const pullSpy = vi.spyOn(GitSourceService.getInstance(), 'handleWebhookPull')
|
||||
.mockResolvedValue({ status: 'skipped', message: 'Rate limited (debounced).' });
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/webhook-status-202/git-source/webhook-pull')
|
||||
.set('Authorization', `Bearer ${adminToken()}`);
|
||||
expect(res.status).toBe(202);
|
||||
pullSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('maps a successful pull to 200', async () => {
|
||||
seedGitSource('webhook-status-200');
|
||||
const pullSpy = vi.spyOn(GitSourceService.getInstance(), 'handleWebhookPull')
|
||||
.mockResolvedValue({ status: 'success', message: 'Pending update ready at abc1234.' });
|
||||
const res = await request(app)
|
||||
.post('/api/stacks/webhook-status-200/git-source/webhook-pull')
|
||||
.set('Authorization', `Bearer ${adminToken()}`);
|
||||
expect(res.status).toBe(200);
|
||||
pullSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
describe('GET /api/git-sources', () => {
|
||||
it('returns 200 and a JSON array for an authenticated admin', async () => {
|
||||
const res = await request(app)
|
||||
|
||||
@@ -362,6 +362,66 @@ describe('GitSourceService error mapping', () => {
|
||||
await expect(svc().fetchFromGit(fetchParams)).rejects.toMatchObject({ code: 'NETWORK_TIMEOUT' });
|
||||
});
|
||||
|
||||
it('maps a bare "fetch failed" TypeError with an ENOTFOUND cause to NETWORK_TIMEOUT', async () => {
|
||||
// Node's global fetch() reports DNS failure as TypeError('fetch failed')
|
||||
// with the real reason on err.cause. Without cause-unwrapping this fell
|
||||
// through to a useless GIT_ERROR: "fetch failed".
|
||||
mockGitClone.mockRejectedValueOnce(
|
||||
new TypeError('fetch failed', {
|
||||
cause: Object.assign(new Error('getaddrinfo ENOTFOUND github.com'), { code: 'ENOTFOUND' }),
|
||||
}),
|
||||
);
|
||||
await expect(svc().fetchFromGit(fetchParams)).rejects.toMatchObject({ code: 'NETWORK_TIMEOUT' });
|
||||
});
|
||||
|
||||
it('maps a "fetch failed" TypeError with an ECONNREFUSED cause to NETWORK_TIMEOUT', async () => {
|
||||
mockGitClone.mockRejectedValueOnce(
|
||||
new TypeError('fetch failed', {
|
||||
cause: Object.assign(new Error('connect ECONNREFUSED 127.0.0.1:443'), { code: 'ECONNREFUSED' }),
|
||||
}),
|
||||
);
|
||||
await expect(svc().fetchFromGit(fetchParams)).rejects.toMatchObject({ code: 'NETWORK_TIMEOUT' });
|
||||
});
|
||||
|
||||
it('surfaces the host instead of bare "fetch failed" in transport errors', async () => {
|
||||
mockGitClone.mockRejectedValueOnce(
|
||||
new TypeError('fetch failed', {
|
||||
cause: Object.assign(new Error('getaddrinfo ENOTFOUND'), { code: 'ENOTFOUND' }),
|
||||
}),
|
||||
);
|
||||
try {
|
||||
await svc().fetchFromGit(fetchParams);
|
||||
expect.fail('should have thrown');
|
||||
} catch (e) {
|
||||
const err = e as Error;
|
||||
expect(err.message).not.toMatch(/^fetch failed$/i);
|
||||
expect(err.message).toContain('github.com');
|
||||
}
|
||||
});
|
||||
|
||||
it('unwraps a nested fetch cause chain to find the transport code', async () => {
|
||||
mockGitClone.mockRejectedValueOnce(
|
||||
new TypeError('fetch failed', {
|
||||
cause: new TypeError('terminated', {
|
||||
cause: Object.assign(new Error('reset'), { code: 'ECONNRESET' }),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
await expect(svc().fetchFromGit(fetchParams)).rejects.toMatchObject({ code: 'NETWORK_TIMEOUT' });
|
||||
});
|
||||
|
||||
it('maps a TLS certificate "fetch failed" cause to a certificate GIT_ERROR', async () => {
|
||||
mockGitClone.mockRejectedValueOnce(
|
||||
new TypeError('fetch failed', {
|
||||
cause: Object.assign(new Error('self-signed certificate'), { code: 'DEPTH_ZERO_SELF_SIGNED_CERT' }),
|
||||
}),
|
||||
);
|
||||
await expect(svc().fetchFromGit(fetchParams)).rejects.toMatchObject({
|
||||
code: 'GIT_ERROR',
|
||||
message: expect.stringMatching(/certificate/i),
|
||||
});
|
||||
});
|
||||
|
||||
it('surfaces FILE_NOT_FOUND when the compose path is missing from the clone', async () => {
|
||||
mockGitClone.mockImplementation(async () => { /* clone empty repo */ });
|
||||
mockGitLog.mockResolvedValue([{ oid: 'deadbeef' }]);
|
||||
@@ -384,6 +444,95 @@ describe('GitSourceService error mapping', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('countingBodyIterator (clone size cap)', () => {
|
||||
function chunkStream(...sizes: number[]): AsyncIterableIterator<Uint8Array> {
|
||||
async function* gen(): AsyncIterableIterator<Uint8Array> {
|
||||
for (const s of sizes) yield new Uint8Array(s);
|
||||
}
|
||||
return gen();
|
||||
}
|
||||
|
||||
it('passes chunks through unchanged while under the cap', async () => {
|
||||
const { countingBodyIterator } = await import('../services/GitSourceService');
|
||||
const controller = new AbortController();
|
||||
const state = { exceeded: false, received: 0 };
|
||||
const out: number[] = [];
|
||||
for await (const c of countingBodyIterator(chunkStream(10, 20, 30), controller, 1000, state)) {
|
||||
out.push(c.byteLength);
|
||||
}
|
||||
expect(out).toEqual([10, 20, 30]);
|
||||
expect(state.exceeded).toBe(false);
|
||||
expect(state.received).toBe(60);
|
||||
expect(controller.signal.aborted).toBe(false);
|
||||
});
|
||||
|
||||
it('aborts the transport and throws once the cumulative size exceeds the cap', async () => {
|
||||
const { countingBodyIterator } = await import('../services/GitSourceService');
|
||||
const controller = new AbortController();
|
||||
const state = { exceeded: false, received: 0 };
|
||||
await expect((async () => {
|
||||
for await (const _c of countingBodyIterator(chunkStream(60, 60), controller, 100, state)) {
|
||||
void _c;
|
||||
}
|
||||
})()).rejects.toThrow(/maximum allowed size/i);
|
||||
expect(state.exceeded).toBe(true);
|
||||
expect(controller.signal.aborted).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('GitSourceService.fetchFromGit (size limits)', () => {
|
||||
const svc = () => GitSourceService.getInstance();
|
||||
const fetchParams = {
|
||||
repoUrl: 'https://github.com/example/repo.git',
|
||||
branch: 'main',
|
||||
composePath: 'compose.yaml',
|
||||
};
|
||||
|
||||
it('rejects a compose file larger than the per-file read cap', async () => {
|
||||
// The download cap bounds the compressed pack, not a single decompressed
|
||||
// file, so readRepoFile guards the in-memory read by file size.
|
||||
mockSuccessfulClone();
|
||||
const { promises: fsp } = await import('fs');
|
||||
const lstatSpy = vi.spyOn(fsp, 'lstat').mockResolvedValue({
|
||||
isSymbolicLink: () => false,
|
||||
size: 11 * 1024 * 1024,
|
||||
} as Awaited<ReturnType<typeof fsp.lstat>>);
|
||||
|
||||
await expect(svc().fetchFromGit(fetchParams)).rejects.toMatchObject({
|
||||
code: 'GIT_ERROR',
|
||||
message: expect.stringMatching(/too large/i),
|
||||
});
|
||||
|
||||
lstatSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('surfaces a clone-size error when the download exceeds the cap', async () => {
|
||||
// Drive the real size-counting transport the service injected into
|
||||
// git.clone, with a tiny cap, and confirm fetchFromGit reports it as a
|
||||
// clone-size error rather than a generic transport failure.
|
||||
process.env.GITSOURCE_MAX_CLONE_BYTES = '8';
|
||||
const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(
|
||||
new Response(new Uint8Array(64), { status: 200 }),
|
||||
);
|
||||
mockGitClone.mockImplementation(async (args: {
|
||||
http: { request: (r: { url: string; method: string; headers: Record<string, string> }) => Promise<{ body: AsyncIterableIterator<Uint8Array> }> };
|
||||
}) => {
|
||||
const resp = await args.http.request({ url: 'https://example.test/info/refs', method: 'GET', headers: {} });
|
||||
for await (const chunk of resp.body) { void chunk; }
|
||||
});
|
||||
|
||||
try {
|
||||
await expect(svc().fetchFromGit(fetchParams)).rejects.toMatchObject({
|
||||
code: 'GIT_ERROR',
|
||||
message: expect.stringMatching(/exceeds the maximum clone size/i),
|
||||
});
|
||||
} finally {
|
||||
delete process.env.GITSOURCE_MAX_CLONE_BYTES;
|
||||
fetchSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('GitSourceService pending lifecycle', () => {
|
||||
it('dismissPending clears pending columns', async () => {
|
||||
mockSuccessfulClone();
|
||||
@@ -439,6 +588,64 @@ describe('GitSourceService.handleWebhookPull debounce', () => {
|
||||
expect(result.status).toBe('error');
|
||||
expect(result.message).toMatch(/no git source/i);
|
||||
});
|
||||
|
||||
it('runs a single clone for a concurrent webhook fan-out', async () => {
|
||||
// The original failure: N webhooks for one push each ran a full clone
|
||||
// because the debounce gate was read before the per-stack lock. The
|
||||
// gate now lives inside the lock, so the first request stamps the
|
||||
// window and the rest skip.
|
||||
const sha = 'eeee555eeee555eeee555eeee555eeee555eeee5';
|
||||
mockSuccessfulClone({ sha });
|
||||
const svc = GitSourceService.getInstance();
|
||||
const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: true });
|
||||
await svc.upsert({
|
||||
stackName: 'fanout-stack',
|
||||
repoUrl: 'https://github.com/example/repo.git',
|
||||
branch: 'main',
|
||||
composePath: 'compose.yaml',
|
||||
syncEnv: false,
|
||||
envPath: null,
|
||||
authType: 'none',
|
||||
autoApplyOnWebhook: false,
|
||||
autoDeployOnApply: false,
|
||||
});
|
||||
// upsert performs a dry-run fetch; clear that call so we count only
|
||||
// the clones triggered by the webhook fan-out below.
|
||||
mockGitClone.mockClear();
|
||||
|
||||
const results = await Promise.all(
|
||||
Array.from({ length: 5 }, () => svc.handleWebhookPull('fanout-stack')),
|
||||
);
|
||||
|
||||
expect(mockGitClone.mock.calls.length).toBe(1);
|
||||
expect(results.filter(r => r.status === 'success')).toHaveLength(1);
|
||||
expect(results.filter(r => r.status === 'skipped')).toHaveLength(4);
|
||||
validateSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('returns error when the pulled compose fails validation', async () => {
|
||||
mockSuccessfulClone();
|
||||
const svc = GitSourceService.getInstance();
|
||||
await svc.upsert({
|
||||
stackName: 'webhook-validate-fail',
|
||||
repoUrl: 'https://github.com/example/repo.git',
|
||||
branch: 'main',
|
||||
composePath: 'compose.yaml',
|
||||
syncEnv: false,
|
||||
envPath: null,
|
||||
authType: 'none',
|
||||
autoApplyOnWebhook: false,
|
||||
autoDeployOnApply: false,
|
||||
});
|
||||
// upsert runs a dry-run fetch but not validateCompose, so the stub only
|
||||
// affects the webhook pull below.
|
||||
const validateSpy = vi.spyOn(svc, 'validateCompose').mockResolvedValue({ ok: false, error: 'bad compose' });
|
||||
|
||||
const result = await svc.handleWebhookPull('webhook-validate-fail');
|
||||
expect(result.status).toBe('error');
|
||||
expect(result.message).toMatch(/validation failed/i);
|
||||
validateSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
describe('GitSourceService per-stack mutex', () => {
|
||||
|
||||
@@ -217,4 +217,35 @@ describe('node-aware Git source webhooks', () => {
|
||||
expect(history[0].error).toMatch(/timed out/i);
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it('records a debounced (202 skipped) remote git-pull as success, not failure', async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const remoteNodeId = db.addNode({
|
||||
name: 'remote-debounce-webhook',
|
||||
type: 'remote',
|
||||
compose_dir: '/tmp',
|
||||
is_default: false,
|
||||
api_url: 'http://remote-debounce.example',
|
||||
api_token: 'remote-token',
|
||||
});
|
||||
const webhookId = db.addWebhook({
|
||||
node_id: remoteNodeId,
|
||||
name: 'debounced remote git',
|
||||
stack_name: 'remote-stack',
|
||||
action: 'git-pull',
|
||||
secret: WebhookService.getInstance().generateSecret(),
|
||||
enabled: true,
|
||||
});
|
||||
// 202 Accepted + status "skipped" is a debounce, not a failure.
|
||||
vi.spyOn(globalThis, 'fetch').mockResolvedValue(
|
||||
new Response(JSON.stringify({ status: 'skipped', message: 'Rate limited (debounced).' }), { status: 202 }),
|
||||
);
|
||||
|
||||
const result = await WebhookService.getInstance().execute(db.getWebhook(webhookId)!, 'git-pull', 'test');
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
const history = db.getWebhookExecutions(webhookId);
|
||||
expect(history[0].status).toBe('success');
|
||||
expect(history[0].error).toMatch(/debounced/i);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -7,7 +7,7 @@ import { checkPermission, requirePermission } from '../middleware/permissions';
|
||||
import { invalidateNodeCaches } from '../helpers/cacheInvalidation';
|
||||
import { triggerPostDeployScan } from '../helpers/policyGate';
|
||||
import { isValidGitSourcePath, isValidStackName } from '../utils/validation';
|
||||
import { sendGitSourceError } from '../utils/gitSourceHttp';
|
||||
import { sendGitSourceError, webhookPullStatus } from '../utils/gitSourceHttp';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
|
||||
// Reasonable upper bounds so a caller cannot flood the service with huge
|
||||
@@ -260,9 +260,16 @@ stackGitSourceRouter.post('/:stackName/git-source/webhook-pull', async (req: Req
|
||||
if (!requirePermission(req, res, 'stack:edit', 'stack', stackName)) return;
|
||||
try {
|
||||
const source = GitSourceService.getInstance().get(stackName);
|
||||
if (source?.auto_apply_on_webhook && source.auto_deploy_on_apply && !requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return;
|
||||
if (!source) {
|
||||
res.status(404).json({ error: 'No Git source configured for this stack', status: 'error' });
|
||||
return;
|
||||
}
|
||||
if (source.auto_apply_on_webhook && source.auto_deploy_on_apply && !requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return;
|
||||
const result = await GitSourceService.getInstance().handleWebhookPull(stackName);
|
||||
res.json(result);
|
||||
// Map the outcome to a real HTTP status so a Git provider sees a 4xx on
|
||||
// failure instead of a 200 with an error body (which it would read as
|
||||
// "delivered fine, stop retrying").
|
||||
res.status(webhookPullStatus(result.status)).json(result);
|
||||
} catch (error) {
|
||||
sendGitSourceError(res, error);
|
||||
}
|
||||
|
||||
@@ -74,7 +74,51 @@ function responseBodyIterator(body: ReadableStream<Uint8Array> | null): AsyncIte
|
||||
return iterate();
|
||||
}
|
||||
|
||||
function createAbortableGitHttp(signal: AbortSignal): HttpClient {
|
||||
/**
|
||||
* Mutable counter shared across every request in one clone so the size cap
|
||||
* is cumulative, and so the caller can tell a size abort from a timeout via
|
||||
* `exceeded` rather than the thrown error (which isomorphic-git may wrap).
|
||||
*/
|
||||
interface CloneSizeState {
|
||||
exceeded: boolean;
|
||||
received: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap a response-body iterator with a cumulative byte counter shared
|
||||
* across every request in one clone. When the running total crosses
|
||||
* `maxBytes`, flip `state.exceeded`, abort the transport (closing the
|
||||
* socket so the download stops), and throw to unwind the stream. The
|
||||
* caller distinguishes a size abort from a timeout/transport abort via
|
||||
* `state.exceeded` rather than the thrown error, which isomorphic-git may
|
||||
* wrap.
|
||||
*/
|
||||
export function countingBodyIterator(
|
||||
src: AsyncIterableIterator<Uint8Array>,
|
||||
controller: AbortController,
|
||||
maxBytes: number,
|
||||
state: CloneSizeState,
|
||||
): AsyncIterableIterator<Uint8Array> {
|
||||
async function* iterate(): AsyncIterableIterator<Uint8Array> {
|
||||
for await (const chunk of src) {
|
||||
state.received += chunk.byteLength;
|
||||
if (state.received > maxBytes) {
|
||||
state.exceeded = true;
|
||||
controller.abort();
|
||||
throw new Error('Clone exceeded the maximum allowed size');
|
||||
}
|
||||
yield chunk;
|
||||
}
|
||||
}
|
||||
return iterate();
|
||||
}
|
||||
|
||||
function createAbortableGitHttp(
|
||||
controller: AbortController,
|
||||
maxBytes: number,
|
||||
state: CloneSizeState,
|
||||
): HttpClient {
|
||||
const signal = controller.signal;
|
||||
return {
|
||||
async request(request: GitHttpRequest): Promise<GitHttpResponse> {
|
||||
if (signal.aborted) {
|
||||
@@ -94,7 +138,7 @@ function createAbortableGitHttp(signal: AbortSignal): HttpClient {
|
||||
statusCode: response.status,
|
||||
statusMessage: response.statusText,
|
||||
headers: Object.fromEntries(response.headers.entries()),
|
||||
body: responseBodyIterator(response.body),
|
||||
body: countingBodyIterator(responseBodyIterator(response.body), controller, maxBytes, state),
|
||||
};
|
||||
},
|
||||
};
|
||||
@@ -215,6 +259,31 @@ const DEFAULT_FETCH_TIMEOUT_MS = 30_000;
|
||||
const TEMP_DIR_MAX_AGE_MS = 60 * 60 * 1000; // 1 hour
|
||||
const WEBHOOK_DEBOUNCE_MS = 10_000;
|
||||
|
||||
// Ceiling on how many bytes a single clone may download (the compressed pack
|
||||
// from the Git host) before it is aborted. This bounds network transfer and
|
||||
// abuse, not the decompressed on-disk checkout; it is paired with
|
||||
// MAX_REPO_FILE_BYTES and the 30s timeout. Generous default; operators with a
|
||||
// legitimately large monorepo can raise it via GITSOURCE_MAX_CLONE_BYTES.
|
||||
const DEFAULT_MAX_CLONE_BYTES = 100 * 1024 * 1024; // 100 MB
|
||||
|
||||
// Per-file ceiling for the compose/env file read into memory after the clone.
|
||||
// These files are KB-scale in practice; the clone byte cap bounds the
|
||||
// compressed download, not the decompressed working tree, so this guards the
|
||||
// in-memory read against a single huge (or highly compressible) file.
|
||||
const MAX_REPO_FILE_BYTES = 10 * 1024 * 1024; // 10 MB
|
||||
|
||||
function maxCloneBytes(): number {
|
||||
const raw = process.env.GITSOURCE_MAX_CLONE_BYTES;
|
||||
const n = raw ? Number(raw) : NaN;
|
||||
return Number.isFinite(n) && n > 0 ? Math.floor(n) : DEFAULT_MAX_CLONE_BYTES;
|
||||
}
|
||||
|
||||
function formatBytes(bytes: number): string {
|
||||
if (bytes >= 1024 * 1024) return `${Math.round(bytes / (1024 * 1024))} MB`;
|
||||
if (bytes >= 1024) return `${Math.round(bytes / 1024)} KB`;
|
||||
return `${bytes} B`;
|
||||
}
|
||||
|
||||
// ─── Credential scrubbing ────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
@@ -245,6 +314,59 @@ export function repoHost(url: string): string {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Node's global `fetch()` reports every transport-level failure as a bare
|
||||
* `TypeError('fetch failed')` and hides the real reason on `error.cause`
|
||||
* (occasionally nested one level deeper through undici). Walk the cause
|
||||
* chain for the first Node error code so DNS / connection / TLS failures
|
||||
* can be translated into an actionable message instead of "fetch failed",
|
||||
* which reads like an internal Sencho bug.
|
||||
*/
|
||||
function findCauseCode(err: unknown): { code?: string } {
|
||||
let cur: unknown = err;
|
||||
for (let depth = 0; depth < 5 && cur; depth++) {
|
||||
const code = (cur as { code?: unknown }).code;
|
||||
if (typeof code === 'string' && code) {
|
||||
return { code };
|
||||
}
|
||||
cur = (cur as { cause?: unknown }).cause;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Translate a Node transport error code into a GitSourceError with a
|
||||
* host-qualified, user-actionable message. Returns null for codes we do
|
||||
* not specifically recognise so the caller can fall through to its generic
|
||||
* handling. `host` is the bare hostname from `repoHost()` (never carries a
|
||||
* credential), so it is safe to surface.
|
||||
*/
|
||||
function transportError(code: string, host: string): GitSourceError | null {
|
||||
const dest = host && host !== 'unknown' ? ` ${host}` : ' the repository host';
|
||||
switch (code) {
|
||||
case 'ENOTFOUND':
|
||||
case 'EAI_AGAIN':
|
||||
return new GitSourceError('NETWORK_TIMEOUT', `Could not resolve${dest}. Check the repository URL and your network or DNS.`);
|
||||
case 'ECONNREFUSED':
|
||||
return new GitSourceError('NETWORK_TIMEOUT', `Connection refused by${dest}.`);
|
||||
case 'ECONNRESET':
|
||||
return new GitSourceError('NETWORK_TIMEOUT', `Connection to${dest} was reset. Retry; if it persists, check the host.`);
|
||||
case 'ETIMEDOUT':
|
||||
case 'UND_ERR_CONNECT_TIMEOUT':
|
||||
case 'UND_ERR_HEADERS_TIMEOUT':
|
||||
case 'UND_ERR_BODY_TIMEOUT':
|
||||
return new GitSourceError('NETWORK_TIMEOUT', `Timed out reaching${dest}.`);
|
||||
case 'DEPTH_ZERO_SELF_SIGNED_CERT':
|
||||
case 'SELF_SIGNED_CERT_IN_CHAIN':
|
||||
case 'UNABLE_TO_VERIFY_LEAF_SIGNATURE':
|
||||
case 'CERT_HAS_EXPIRED':
|
||||
case 'ERR_TLS_CERT_ALTNAME_INVALID':
|
||||
return new GitSourceError('GIT_ERROR', `TLS certificate error reaching${dest} (${code}). The host certificate could not be verified.`);
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Git LFS stores large files as small pointer stubs in the working tree.
|
||||
* The pointer is a short text file that always begins with this line.
|
||||
@@ -297,6 +419,12 @@ async function readRepoFile(rootDir: string, relPath: string, label: string): Pr
|
||||
if (stat.isSymbolicLink()) {
|
||||
throw new GitSourceError('FILE_NOT_FOUND', `${label} cannot be a symbolic link.`);
|
||||
}
|
||||
// Bound the in-memory read. The clone byte cap only limits the compressed
|
||||
// download; a single decompressed file can still be large, so reject an
|
||||
// oversized compose/env file before reading it into a string.
|
||||
if (stat.size > MAX_REPO_FILE_BYTES) {
|
||||
throw new GitSourceError('GIT_ERROR', `${label} is too large (${formatBytes(stat.size)}); the maximum is ${formatBytes(MAX_REPO_FILE_BYTES)}.`);
|
||||
}
|
||||
|
||||
let real;
|
||||
try {
|
||||
@@ -526,6 +654,8 @@ export class GitSourceService {
|
||||
// fetches do not keep sockets and packfile streams alive.
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
const controller = new AbortController();
|
||||
const maxBytes = maxCloneBytes();
|
||||
const sizeState = { exceeded: false, received: 0 };
|
||||
const timeout = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => {
|
||||
controller.abort();
|
||||
@@ -536,7 +666,7 @@ export class GitSourceService {
|
||||
await Promise.race([
|
||||
git.clone({
|
||||
fs: { promises: fsPromises },
|
||||
http: createAbortableGitHttp(controller.signal),
|
||||
http: createAbortableGitHttp(controller, maxBytes, sizeState),
|
||||
dir,
|
||||
url: repoUrl,
|
||||
ref: branch,
|
||||
@@ -548,7 +678,15 @@ export class GitSourceService {
|
||||
timeout,
|
||||
]);
|
||||
} catch (e) {
|
||||
throw this.mapGitError(e as Error, Boolean(token));
|
||||
// A size abort surfaces as a generic transport error once
|
||||
// isomorphic-git unwinds, so detect it via the shared flag.
|
||||
if (sizeState.exceeded) {
|
||||
throw new GitSourceError(
|
||||
'GIT_ERROR',
|
||||
`Repository exceeds the maximum clone size of ${formatBytes(maxBytes)}.`,
|
||||
);
|
||||
}
|
||||
throw this.mapGitError(e as Error, Boolean(token), repoHost(repoUrl));
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
}
|
||||
@@ -619,7 +757,7 @@ export class GitSourceService {
|
||||
}
|
||||
}
|
||||
|
||||
private mapGitError(err: Error, hasToken: boolean): GitSourceError {
|
||||
private mapGitError(err: Error, hasToken: boolean, host = 'unknown'): GitSourceError {
|
||||
const raw = scrubCredentials(err.message || String(err));
|
||||
const code = (err as Error & { code?: string }).code;
|
||||
// isomorphic-git's HttpError exposes the numeric status on .data; inspect
|
||||
@@ -649,6 +787,18 @@ export class GitSourceService {
|
||||
return new GitSourceError('REPO_NOT_FOUND', 'Repository not found, or it is private. Add a Personal Access Token if the repo is private.');
|
||||
}
|
||||
|
||||
// Transport failures: Node's fetch() throws a bare "fetch failed"
|
||||
// TypeError with the real reason (ENOTFOUND, ECONNREFUSED, TLS, ...)
|
||||
// on err.cause. Translate the underlying code before falling through
|
||||
// to the generic branches, which only see the useless "fetch failed".
|
||||
if (!statusCode) {
|
||||
const cause = findCauseCode(err);
|
||||
if (cause.code) {
|
||||
const mapped = transportError(cause.code, host);
|
||||
if (mapped) return mapped;
|
||||
}
|
||||
}
|
||||
|
||||
// Fallbacks for errors without a numeric status attached (e.g. git CLI
|
||||
// output, DNS/lib errors, or future isomorphic-git transports that do
|
||||
// not populate err.data.statusCode). Kept as defense-in-depth.
|
||||
@@ -770,62 +920,69 @@ export class GitSourceService {
|
||||
// ─── Pull / apply ────────────────────────────────────────────────────────
|
||||
|
||||
public async pull(stackName: string): Promise<PullResult> {
|
||||
// Guarded by the same per-stack mutex as apply(). Without this, a
|
||||
// concurrent delete-source + pull can land a pending row on a
|
||||
// stack whose config row has just been removed; the DELETE clears
|
||||
// the row but the subsequent setGitSourcePending re-inserts via
|
||||
// UPDATE failing silently (no row), and a later upsert would
|
||||
// inherit stale pending columns on read.
|
||||
return this.withStackLock(stackName, async () => {
|
||||
const db = DatabaseService.getInstance();
|
||||
const src = db.getGitSource(stackName);
|
||||
if (!src) throw new GitSourceError('GIT_ERROR', 'No Git source configured for this stack.');
|
||||
// Guarded by the per-stack mutex (see withStackLock). Without this, a
|
||||
// concurrent delete-source + pull can land a pending row on a stack
|
||||
// whose config row has just been removed.
|
||||
return this.withStackLock(stackName, () => this.pullLocked(stackName));
|
||||
}
|
||||
|
||||
const diag = isDebugEnabled();
|
||||
if (diag) {
|
||||
console.log(`[GitSource:diag] pull start stack=${stackName} branch=${src.branch} host=${repoHost(src.repo_url)}`);
|
||||
}
|
||||
/**
|
||||
* Body of pull(); assumes the caller already holds the per-stack lock.
|
||||
* handleWebhookPull calls this directly so that its debounce re-check,
|
||||
* this fetch, and the apply all run inside the single lock that
|
||||
* handleWebhookPull holds. Without that, a concurrent webhook fan-out
|
||||
* reads last_debounce_at while it is still unset on every request, slips
|
||||
* past the gate, and clones once per request.
|
||||
*/
|
||||
private async pullLocked(stackName: string): Promise<PullResult> {
|
||||
const db = DatabaseService.getInstance();
|
||||
const src = db.getGitSource(stackName);
|
||||
if (!src) throw new GitSourceError('GIT_ERROR', 'No Git source configured for this stack.');
|
||||
|
||||
const token = src.encrypted_token ? this.crypto.decrypt(src.encrypted_token) : null;
|
||||
const fetched = await this.fetchFromGit({
|
||||
repoUrl: src.repo_url,
|
||||
branch: src.branch,
|
||||
composePath: src.compose_path,
|
||||
envPath: src.sync_env ? src.env_path : null,
|
||||
token,
|
||||
});
|
||||
const diag = isDebugEnabled();
|
||||
if (diag) {
|
||||
console.log(`[GitSource:diag] pull start stack=${stackName} branch=${src.branch} host=${repoHost(src.repo_url)}`);
|
||||
}
|
||||
|
||||
const validation = await this.validateCompose(fetched.composeContent, fetched.envContent);
|
||||
const disk = await this.readDiskContent(stackName, src.sync_env);
|
||||
const currentHash = this.hashContent(disk.compose, disk.env);
|
||||
const hasLocalChanges = src.last_applied_content_hash !== null
|
||||
&& src.last_applied_content_hash !== currentHash;
|
||||
|
||||
// Store pending so a subsequent apply doesn't re-fetch. Compose files
|
||||
// routinely contain secrets inlined as env interpolations or passwords,
|
||||
// so encrypt the pending buffers at rest.
|
||||
db.setGitSourcePending(
|
||||
stackName,
|
||||
fetched.commitSha,
|
||||
this.crypto.encrypt(fetched.composeContent),
|
||||
fetched.envContent !== null ? this.crypto.encrypt(fetched.envContent) : null,
|
||||
);
|
||||
|
||||
console.log(`[GitSource] Pending update ready for ${stackName} at ${fetched.commitSha.slice(0, 7)} (validation=${validation.ok ? 'ok' : 'fail'}, localEdits=${hasLocalChanges})`);
|
||||
if (diag) {
|
||||
console.log(`[GitSource:diag] pull done stack=${stackName} sha=${fetched.commitSha.slice(0, 7)} validation=${validation.ok} localEdits=${hasLocalChanges}`);
|
||||
}
|
||||
|
||||
return {
|
||||
commitSha: fetched.commitSha,
|
||||
incomingCompose: fetched.composeContent,
|
||||
incomingEnv: fetched.envContent,
|
||||
currentCompose: disk.compose,
|
||||
currentEnv: disk.env,
|
||||
validation,
|
||||
hasLocalChanges,
|
||||
};
|
||||
const token = src.encrypted_token ? this.crypto.decrypt(src.encrypted_token) : null;
|
||||
const fetched = await this.fetchFromGit({
|
||||
repoUrl: src.repo_url,
|
||||
branch: src.branch,
|
||||
composePath: src.compose_path,
|
||||
envPath: src.sync_env ? src.env_path : null,
|
||||
token,
|
||||
});
|
||||
|
||||
const validation = await this.validateCompose(fetched.composeContent, fetched.envContent);
|
||||
const disk = await this.readDiskContent(stackName, src.sync_env);
|
||||
const currentHash = this.hashContent(disk.compose, disk.env);
|
||||
const hasLocalChanges = src.last_applied_content_hash !== null
|
||||
&& src.last_applied_content_hash !== currentHash;
|
||||
|
||||
// Store pending so a subsequent apply doesn't re-fetch. Compose files
|
||||
// routinely contain secrets inlined as env interpolations or passwords,
|
||||
// so encrypt the pending buffers at rest.
|
||||
db.setGitSourcePending(
|
||||
stackName,
|
||||
fetched.commitSha,
|
||||
this.crypto.encrypt(fetched.composeContent),
|
||||
fetched.envContent !== null ? this.crypto.encrypt(fetched.envContent) : null,
|
||||
);
|
||||
|
||||
console.log(`[GitSource] Pending update ready for ${stackName} at ${fetched.commitSha.slice(0, 7)} (validation=${validation.ok ? 'ok' : 'fail'}, localEdits=${hasLocalChanges})`);
|
||||
if (diag) {
|
||||
console.log(`[GitSource:diag] pull done stack=${stackName} sha=${fetched.commitSha.slice(0, 7)} validation=${validation.ok} localEdits=${hasLocalChanges}`);
|
||||
}
|
||||
|
||||
return {
|
||||
commitSha: fetched.commitSha,
|
||||
incomingCompose: fetched.composeContent,
|
||||
incomingEnv: fetched.envContent,
|
||||
currentCompose: disk.compose,
|
||||
currentEnv: disk.env,
|
||||
validation,
|
||||
hasLocalChanges,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -845,72 +1002,79 @@ export class GitSourceService {
|
||||
commitSha: string,
|
||||
opts: { deploy?: boolean; actor?: string; bypassPolicy?: boolean } = {},
|
||||
): Promise<{ applied: boolean; deployed: boolean; deployError?: string }> {
|
||||
return this.withStackLock(stackName, async () => {
|
||||
const diag = isDebugEnabled();
|
||||
const db = DatabaseService.getInstance();
|
||||
const src = db.getGitSource(stackName);
|
||||
if (!src) throw new GitSourceError('GIT_ERROR', 'No Git source configured for this stack.');
|
||||
return this.withStackLock(stackName, () => this.applyLocked(stackName, commitSha, opts));
|
||||
}
|
||||
|
||||
if (!src.pending_commit_sha || !src.pending_compose_content) {
|
||||
throw new GitSourceError('GIT_ERROR', 'No pending pull to apply. Fetch the source again.');
|
||||
/** Body of apply(); assumes the caller already holds the per-stack lock. */
|
||||
private async applyLocked(
|
||||
stackName: string,
|
||||
commitSha: string,
|
||||
opts: { deploy?: boolean; actor?: string; bypassPolicy?: boolean },
|
||||
): Promise<{ applied: boolean; deployed: boolean; deployError?: string }> {
|
||||
const diag = isDebugEnabled();
|
||||
const db = DatabaseService.getInstance();
|
||||
const src = db.getGitSource(stackName);
|
||||
if (!src) throw new GitSourceError('GIT_ERROR', 'No Git source configured for this stack.');
|
||||
|
||||
if (!src.pending_commit_sha || !src.pending_compose_content) {
|
||||
throw new GitSourceError('GIT_ERROR', 'No pending pull to apply. Fetch the source again.');
|
||||
}
|
||||
if (src.pending_commit_sha !== commitSha) {
|
||||
if (diag) console.log('[GitSource:diag] apply sha mismatch stack=%s expected=%s pending=%s', sanitizeForLog(stackName), sanitizeForLog(commitSha.slice(0, 7)), sanitizeForLog(src.pending_commit_sha.slice(0, 7)));
|
||||
throw new GitSourceError('GIT_ERROR', 'Pending commit has changed since this pull was fetched. Please review the latest diff.');
|
||||
}
|
||||
|
||||
// Pending buffers are stored encrypted; decrypt is a no-op for any
|
||||
// legacy plaintext rows (isEncrypted check inside CryptoService).
|
||||
const composeContent = this.crypto.decrypt(src.pending_compose_content);
|
||||
const envContent = src.pending_env_content !== null
|
||||
? this.crypto.decrypt(src.pending_env_content)
|
||||
: null;
|
||||
|
||||
// Re-validate before writing.
|
||||
const validation = await this.validateCompose(composeContent, envContent);
|
||||
if (!validation.ok) {
|
||||
if (diag) console.log(`[GitSource:diag] apply validation fail stack=${stackName}`);
|
||||
throw new GitSourceError('GIT_ERROR', `Compose validation failed: ${validation.error}`);
|
||||
}
|
||||
|
||||
const fsSvc = FileSystemService.getInstance();
|
||||
await fsSvc.saveStackContent(stackName, composeContent);
|
||||
if (src.sync_env && envContent !== null) {
|
||||
await fsSvc.saveEnvContent(stackName, envContent);
|
||||
}
|
||||
|
||||
const hash = this.hashContent(composeContent, envContent);
|
||||
db.markGitSourceApplied(stackName, commitSha, hash);
|
||||
|
||||
const shouldDeploy = opts.deploy ?? src.auto_deploy_on_apply;
|
||||
if (diag) console.log('[GitSource:diag] apply wrote stack=%s sha=%s deploy=%s', sanitizeForLog(stackName), sanitizeForLog(commitSha.slice(0, 7)), sanitizeForLog(shouldDeploy));
|
||||
|
||||
if (shouldDeploy) {
|
||||
try {
|
||||
const nodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
await assertPolicyGateAllows(
|
||||
stackName,
|
||||
nodeId,
|
||||
buildSystemPolicyGateOptions(opts.actor ?? 'git-source', {
|
||||
bypass: opts.bypassPolicy === true,
|
||||
auditPath: `/api/stacks/${stackName}/git-source/apply`,
|
||||
}),
|
||||
);
|
||||
await ComposeService.getInstance().deployStack(stackName);
|
||||
console.log(`[GitSource] Applied and deployed ${stackName} at ${commitSha.slice(0, 7)}`);
|
||||
return { applied: true, deployed: true };
|
||||
} catch (e) {
|
||||
// File is on disk, DB is marked applied. Returning the
|
||||
// error separately lets the UI flag it as a partial
|
||||
// success rather than rolling back the disk.
|
||||
const scrubbed = scrubCredentials((e as Error).message || String(e));
|
||||
console.error(`[GitSource] Auto-deploy failed for ${stackName}: ${scrubbed}`);
|
||||
return { applied: true, deployed: false, deployError: scrubbed };
|
||||
}
|
||||
if (src.pending_commit_sha !== commitSha) {
|
||||
if (diag) console.log('[GitSource:diag] apply sha mismatch stack=%s expected=%s pending=%s', sanitizeForLog(stackName), sanitizeForLog(commitSha.slice(0, 7)), sanitizeForLog(src.pending_commit_sha.slice(0, 7)));
|
||||
throw new GitSourceError('GIT_ERROR', 'Pending commit has changed since this pull was fetched. Please review the latest diff.');
|
||||
}
|
||||
|
||||
// Pending buffers are stored encrypted; decrypt is a no-op for any
|
||||
// legacy plaintext rows (isEncrypted check inside CryptoService).
|
||||
const composeContent = this.crypto.decrypt(src.pending_compose_content);
|
||||
const envContent = src.pending_env_content !== null
|
||||
? this.crypto.decrypt(src.pending_env_content)
|
||||
: null;
|
||||
|
||||
// Re-validate before writing.
|
||||
const validation = await this.validateCompose(composeContent, envContent);
|
||||
if (!validation.ok) {
|
||||
if (diag) console.log(`[GitSource:diag] apply validation fail stack=${stackName}`);
|
||||
throw new GitSourceError('GIT_ERROR', `Compose validation failed: ${validation.error}`);
|
||||
}
|
||||
|
||||
const fsSvc = FileSystemService.getInstance();
|
||||
await fsSvc.saveStackContent(stackName, composeContent);
|
||||
if (src.sync_env && envContent !== null) {
|
||||
await fsSvc.saveEnvContent(stackName, envContent);
|
||||
}
|
||||
|
||||
const hash = this.hashContent(composeContent, envContent);
|
||||
db.markGitSourceApplied(stackName, commitSha, hash);
|
||||
|
||||
const shouldDeploy = opts.deploy ?? src.auto_deploy_on_apply;
|
||||
if (diag) console.log('[GitSource:diag] apply wrote stack=%s sha=%s deploy=%s', sanitizeForLog(stackName), sanitizeForLog(commitSha.slice(0, 7)), sanitizeForLog(shouldDeploy));
|
||||
|
||||
if (shouldDeploy) {
|
||||
try {
|
||||
const nodeId = NodeRegistry.getInstance().getDefaultNodeId();
|
||||
await assertPolicyGateAllows(
|
||||
stackName,
|
||||
nodeId,
|
||||
buildSystemPolicyGateOptions(opts.actor ?? 'git-source', {
|
||||
bypass: opts.bypassPolicy === true,
|
||||
auditPath: `/api/stacks/${stackName}/git-source/apply`,
|
||||
}),
|
||||
);
|
||||
await ComposeService.getInstance().deployStack(stackName);
|
||||
console.log(`[GitSource] Applied and deployed ${stackName} at ${commitSha.slice(0, 7)}`);
|
||||
return { applied: true, deployed: true };
|
||||
} catch (e) {
|
||||
// File is on disk, DB is marked applied. Returning the
|
||||
// error separately lets the UI flag it as a partial
|
||||
// success rather than rolling back the disk.
|
||||
const scrubbed = scrubCredentials((e as Error).message || String(e));
|
||||
console.error(`[GitSource] Auto-deploy failed for ${stackName}: ${scrubbed}`);
|
||||
return { applied: true, deployed: false, deployError: scrubbed };
|
||||
}
|
||||
}
|
||||
console.log(`[GitSource] Applied ${stackName} at ${commitSha.slice(0, 7)}`);
|
||||
return { applied: true, deployed: false };
|
||||
});
|
||||
}
|
||||
console.log(`[GitSource] Applied ${stackName} at ${commitSha.slice(0, 7)}`);
|
||||
return { applied: true, deployed: false };
|
||||
}
|
||||
|
||||
public dismissPending(stackName: string): void {
|
||||
@@ -1035,47 +1199,63 @@ export class GitSourceService {
|
||||
* record in webhook_executions. Enforces the per-source debounce.
|
||||
*/
|
||||
public async handleWebhookPull(stackName: string): Promise<{ status: 'success' | 'skipped' | 'error'; message: string }> {
|
||||
const diag = isDebugEnabled();
|
||||
const db = DatabaseService.getInstance();
|
||||
const src = db.getGitSource(stackName);
|
||||
if (!src) {
|
||||
return { status: 'error', message: 'No Git source configured for this stack.' };
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
if (src.last_debounce_at !== null && (now - src.last_debounce_at) < WEBHOOK_DEBOUNCE_MS) {
|
||||
if (diag) console.log(`[GitSource:diag] webhook debounced stack=${stackName} age=${now - src.last_debounce_at}ms`);
|
||||
return { status: 'skipped', message: 'Rate limited (debounced).' };
|
||||
}
|
||||
|
||||
try {
|
||||
const pullResult = await this.pull(stackName);
|
||||
// Only burn the debounce window once the fetch actually produced
|
||||
// something. A transient network failure should be retriable
|
||||
// immediately rather than locked out for the debounce interval.
|
||||
db.touchGitSourceDebounce(stackName);
|
||||
if (!pullResult.validation.ok) {
|
||||
return { status: 'error', message: `Validation failed: ${pullResult.validation.error}` };
|
||||
// Run the whole critical section under a single lock acquisition so a
|
||||
// concurrent fan-out (N webhooks for one push) serializes AND re-reads
|
||||
// last_debounce_at after acquiring the lock. The first request stamps
|
||||
// the window; every queued duplicate then sees the stamp and skips
|
||||
// instead of cloning again. The debounce is still stamped only after a
|
||||
// successful fetch, so a transient failure stays immediately retriable.
|
||||
return this.withStackLock<{ status: 'success' | 'skipped' | 'error'; message: string }>(stackName, async () => {
|
||||
const diag = isDebugEnabled();
|
||||
const db = DatabaseService.getInstance();
|
||||
const src = db.getGitSource(stackName);
|
||||
if (!src) {
|
||||
return { status: 'error', message: 'No Git source configured for this stack.' };
|
||||
}
|
||||
|
||||
if (!src.auto_apply_on_webhook) {
|
||||
if (diag) console.log(`[GitSource:diag] webhook pending-only stack=${stackName} sha=${pullResult.commitSha.slice(0, 7)}`);
|
||||
return { status: 'success', message: `Pending update ready at ${pullResult.commitSha.slice(0, 7)}.` };
|
||||
const now = Date.now();
|
||||
if (src.last_debounce_at !== null && (now - src.last_debounce_at) < WEBHOOK_DEBOUNCE_MS) {
|
||||
if (diag) console.log(`[GitSource:diag] webhook debounced stack=${stackName} age=${now - src.last_debounce_at}ms`);
|
||||
return { status: 'skipped', message: 'Rate limited (debounced).' };
|
||||
}
|
||||
|
||||
const applied = await this.apply(stackName, pullResult.commitSha, { deploy: src.auto_deploy_on_apply });
|
||||
if (applied.deployError) {
|
||||
// Apply wrote to disk but deploy failed. Surface it so the
|
||||
// webhook_executions row records a degraded outcome instead
|
||||
// of a clean success.
|
||||
return { status: 'error', message: `Applied commit ${pullResult.commitSha.slice(0, 7)} but deploy failed: ${applied.deployError}` };
|
||||
try {
|
||||
const pullResult = await this.pullLocked(stackName);
|
||||
// Only burn the debounce window once the fetch actually produced
|
||||
// something. A transient network failure should be retriable
|
||||
// immediately rather than locked out for the debounce interval.
|
||||
db.touchGitSourceDebounce(stackName);
|
||||
if (!pullResult.validation.ok) {
|
||||
// Webhooks are unattended, so always leave a server-side
|
||||
// breadcrumb; the caller only sees the HTTP status.
|
||||
console.warn(`[GitSource] Webhook pull validation failed for ${sanitizeForLog(stackName)}: ${sanitizeForLog(pullResult.validation.error ?? 'unknown')}`);
|
||||
return { status: 'error', message: `Validation failed: ${pullResult.validation.error}` };
|
||||
}
|
||||
|
||||
if (!src.auto_apply_on_webhook) {
|
||||
if (diag) console.log(`[GitSource:diag] webhook pending-only stack=${stackName} sha=${pullResult.commitSha.slice(0, 7)}`);
|
||||
return { status: 'success', message: `Pending update ready at ${pullResult.commitSha.slice(0, 7)}.` };
|
||||
}
|
||||
|
||||
const applied = await this.applyLocked(stackName, pullResult.commitSha, { deploy: src.auto_deploy_on_apply });
|
||||
if (applied.deployError) {
|
||||
// Apply wrote to disk but deploy failed. Surface it so the
|
||||
// webhook_executions row records a degraded outcome instead
|
||||
// of a clean success.
|
||||
return { status: 'error', message: `Applied commit ${pullResult.commitSha.slice(0, 7)} but deploy failed: ${applied.deployError}` };
|
||||
}
|
||||
const suffix = applied.deployed ? ' and deployed' : '';
|
||||
return { status: 'success', message: `Applied commit ${pullResult.commitSha.slice(0, 7)}${suffix}.` };
|
||||
} catch (e) {
|
||||
const msg = e instanceof GitSourceError ? `${e.code}: ${e.message}` : (e as Error).message;
|
||||
const scrubbed = scrubCredentials(msg);
|
||||
// Unattended path: record the failure server-side so an operator
|
||||
// can diagnose without diag mode, since the Git provider only
|
||||
// logs the HTTP status.
|
||||
console.error(`[GitSource] Webhook pull failed for ${sanitizeForLog(stackName)}: ${sanitizeForLog(scrubbed)}`);
|
||||
return { status: 'error', message: scrubbed };
|
||||
}
|
||||
const suffix = applied.deployed ? ' and deployed' : '';
|
||||
return { status: 'success', message: `Applied commit ${pullResult.commitSha.slice(0, 7)}${suffix}.` };
|
||||
} catch (e) {
|
||||
const msg = e instanceof GitSourceError ? `${e.code}: ${e.message}` : (e as Error).message;
|
||||
return { status: 'error', message: scrubCredentials(msg) };
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// ─── Concurrency ─────────────────────────────────────────────────────────
|
||||
|
||||
@@ -183,15 +183,18 @@ export class WebhookService {
|
||||
}
|
||||
|
||||
const skipped = result.status === 'skipped';
|
||||
// A debounced pull is rate-limited, not failed (the route answers 202
|
||||
// Accepted). Record it as a success carrying the debounce note so it
|
||||
// does not pollute the webhook's failure history.
|
||||
this.recordExecution(
|
||||
webhookId,
|
||||
action,
|
||||
skipped ? 'failure' : 'success',
|
||||
'success',
|
||||
triggerSource,
|
||||
durationMs,
|
||||
skipped ? result.message : null,
|
||||
);
|
||||
return { success: !skipped, error: skipped ? result.message : undefined, duration_ms: durationMs };
|
||||
return { success: true, error: undefined, duration_ms: durationMs };
|
||||
}
|
||||
|
||||
private async executeRemote(
|
||||
@@ -214,13 +217,17 @@ export class WebhookService {
|
||||
const durationMs = Date.now() - startTime;
|
||||
const payload = await response.json().catch(() => ({})) as { error?: string; message?: string; status?: string };
|
||||
|
||||
if (!response.ok || payload.status === 'error' || payload.status === 'skipped') {
|
||||
if (!response.ok || payload.status === 'error') {
|
||||
const error = payload.error || payload.message || `Remote ${action} failed with status ${response.status}`;
|
||||
this.recordExecution(webhookId, action, 'failure', triggerSource, durationMs, error);
|
||||
return { success: false, error, duration_ms: durationMs };
|
||||
}
|
||||
|
||||
this.recordExecution(webhookId, action, 'success', triggerSource, durationMs, null);
|
||||
// A debounced remote pull comes back 202 with status "skipped": it was
|
||||
// accepted and rate-limited, not failed. Record it as a success with
|
||||
// the debounce note rather than failure noise.
|
||||
const skipped = payload.status === 'skipped';
|
||||
this.recordExecution(webhookId, action, 'success', triggerSource, durationMs, skipped ? (payload.message ?? null) : null);
|
||||
return { success: true, duration_ms: durationMs };
|
||||
} catch (err) {
|
||||
const durationMs = Date.now() - startTime;
|
||||
|
||||
@@ -27,6 +27,31 @@ export function gitSourceStatus(code: GitSourceErrorCode): number {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a webhook-pull outcome to an HTTP status so a Git provider (and any
|
||||
* monitoring on top of it) can tell delivery succeeded, was a no-op, or
|
||||
* failed by status code alone, not just by parsing the JSON body. The
|
||||
* missing-source case is handled by the route as a 404 before this runs.
|
||||
*
|
||||
* success -> 200 applied / pending update ready
|
||||
* skipped -> 202 accepted but debounced (no work done this call)
|
||||
* error -> 422 request understood, the pull/apply/deploy failed
|
||||
*/
|
||||
export function webhookPullStatus(status: 'success' | 'skipped' | 'error'): number {
|
||||
switch (status) {
|
||||
case 'success': return 200;
|
||||
case 'skipped': return 202;
|
||||
case 'error': return 422;
|
||||
default: {
|
||||
// Exhaustiveness guard: if a new status is added to the union without a
|
||||
// case here, this becomes a compile error instead of returning undefined.
|
||||
const _exhaustive: never = status;
|
||||
void _exhaustive;
|
||||
return 500;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function sendGitSourceError(res: Response, err: unknown): void {
|
||||
if (err instanceof GitSourceError) {
|
||||
res.status(gitSourceStatus(err.code)).json({ error: err.message, code: err.code });
|
||||
|
||||
Reference in New Issue
Block a user