feat(gitops): resolve branch, tag, and SHA refs to immutable commits before fetch (#1864)

* feat(gitops): resolve branch, tag, and SHA refs to immutable commits before fetch

The ref model now resolves a configured branch, tag, or full commit SHA to
an immutable commit before any content is downloaded, and records both the
configured and the resolved identity where revision state persists.

- RefKind (branch | tag | sha) is a resolved property, not caller-asserted.
  A bare string resolves branch-first, then tag; a full 40/64-hex SHA
  self-resolves with no remote round-trip. Branch and tag both fetch via a
  bare --branch name; a SHA uses init + shallow fetch + detached checkout.
- A single ls-remote with narrow heads/tags refspecs pins the configured ref
  to an immutable SHA; rev-parse HEAD must equal the resolved SHA or the
  fetch refuses (tip-changed) instead of materializing unreviewed content.
- Error union grew: REF_NOT_FOUND (ref-neutral, replaces BRANCH_NOT_FOUND),
  UNSUPPORTED_REF (a pinned SHA the host will not serve), and a service-level
  REF_DELETED upgrade that fires when a classified REF_NOT_FOUND occurs for a
  source with prior fetch history (a vanished ref reads as delete/force-push,
  not a fresh typo). Status mapping: REF_NOT_FOUND/REF_DELETED to 404,
  UNSUPPORTED_REF to 400.
- Configured-vs-resolved identity is recorded via a nullable resolved_ref_kind
  column on gitops_generations (added to CREATE TABLE and re-added for legacy
  installs through maybeAddCol). The kind is deliberately NOT in the plan
  fingerprint: two sources naming the same commit differently are the same plan.

Docs updated (git-sources feature page, connect-a-git-source tutorial, and the
native-git-transport internal deep-dive) to the ref-neutral naming.

* fix(gitops): harden ref resolution after pre-merge audit

Request peeled annotated-tag refs from ls-remote, detect force-pushes and
ref-kind changes against prior fetch identity, persist resolved kind on
application rows, and add real-git tag/SHA integration coverage plus
ref-neutral UI and operator docs.

* test(gitops): mock verifyFastForward in direct producer suite

The producer tests stub the transport seam but were missing resolved kind
on resolveRef and a verifyFastForward stub, so second pulls tripped the new
ref-continuity checks as REF_DELETED.

* test(git): remove unused buildBareFixtureRepo helper

Fixes backend lint failure after the integration fixture was refactored
to buildRichFixtureRepo without dropping the old wrapper.

* fix(gitops): correct fast-forward ancestry verification under size bounds

Replace the dual shallow-fetch ancestry probe with a single-tip deepen
strategy, keep verifier Git work inside the transport watchdog, and add
real-Git regression coverage for linear advances and rewritten history.

* fix(gitops): bound fast-forward verification with exponential deepen

Replace per-commit deepen loops with exponential steps, cap remote fetch
rounds, and share one deadline across verifier Git calls. Budget exhaustion
now surfaces as a classified timeout instead of REF_DELETED.

* fix(gitops): classify fast-forward probe failures accurately

Normalize verifier probe timeouts and unexpected exit codes into transport
failures, interpret merge-base status 1 as proven non-ancestry only, and
treat shallow stagnation as timeout instead of REF_DELETED.

* fix(gitops): satisfy tsc on probeFailure never returns

* fix(gitops): address Phase E QA findings on ref verification

Remove the fast-forward scratch repo after verification so pull size
caps are not inflated, classify GitHub not-our-ref as UNSUPPORTED_REF,
persist fetched_resolved_ref_kind on create-from-git, and broaden
REF_DELETED copy for retagged tags.
This commit is contained in:
Anso
2026-08-28 20:15:37 +00:00
committed by GitHub
parent 7cd42699d1
commit 48f010475b
38 changed files with 1397 additions and 137 deletions
+337 -56
View File
@@ -10,7 +10,7 @@ import {
writeCredentialHelper,
} from './credentialHelper';
import { isTransportFailure, type TransportFailure } from './errors';
import type { FetchRequest, FetchResult, GitTransport, ResolveRequest } from './types';
import type { FetchRequest, FetchResult, GitTransport, ResolveRequest, ResolveResult } from './types';
/**
* Native git transport: every Git operation is an `execFile`-style spawn of
@@ -590,26 +590,31 @@ async function ensureBinaryReady(hasToken: boolean): Promise<void> {
}
}
function parseLsRemoteLine(line: string, fullRef: string): string | null {
const tabIndex = line.indexOf('\t');
if (tabIndex === -1) return null;
if (line.slice(tabIndex + 1).trim() !== fullRef) return null;
const sha = line.slice(0, tabIndex).trim();
return SHA_PATTERN.test(sha) ? sha.toLowerCase() : null;
interface ResolvedRemoteRefs {
branchSha: string | null;
tagSha: string | null;
}
async function lsRemoteHead(
/**
* Ask the remote where a bare ref name lives. One `ls-remote` with explicit
* refspecs for both namespaces keeps the response tiny (a name matches at
* most a couple of lines even on huge repos), so the stdout cap can never
* truncate the answer we need. For an annotated tag the peeled `^{}` entry
* carries the commit, so it wins over the raw tag-object line; a lightweight
* tag's raw line already points at the commit.
*/
async function lsRemoteRefs(
url: URL,
ref: string,
env: NodeJS.ProcessEnv,
baseArgs: string[],
timeoutMs: number,
hasToken: boolean,
): Promise<string> {
): Promise<ResolvedRemoteRefs> {
let res: RunResult;
try {
res = await runGit(
[...baseArgs, 'ls-remote', '--heads', url.href, `refs/heads/${ref}`],
[...baseArgs, 'ls-remote', url.href, `refs/heads/${ref}`, `refs/tags/${ref}`, `refs/tags/${ref}^{}`],
{ env, timeoutMs: Math.min(timeoutMs, LS_REMOTE_MAX_MS) },
);
} catch (e) {
@@ -623,27 +628,289 @@ async function lsRemoteHead(
if (res.exitCode !== 0) {
throw { transportFailure: true as const, reason: 'exit', stderr: res.stderr, exitCode: res.exitCode, argv: baseArgs, host: url.host, hasToken } satisfies TransportFailure;
}
const fullRef = `refs/heads/${ref}`;
const found: ResolvedRemoteRefs = { branchSha: null, tagSha: null };
for (const line of res.stdout.split(/\r?\n/)) {
const sha = parseLsRemoteLine(line, fullRef);
if (sha) return sha;
const tabIndex = line.indexOf('\t');
if (tabIndex === -1) continue;
const sha = line.slice(0, tabIndex).trim();
if (!SHA_PATTERN.test(sha)) continue;
const full = line.slice(tabIndex + 1).trim();
if (full === `refs/heads/${ref}`) {
found.branchSha = sha.toLowerCase();
} else if (full === `refs/tags/${ref}^{}`) {
found.tagSha = sha.toLowerCase();
} else if (full === `refs/tags/${ref}` && found.tagSha === null) {
found.tagSha = sha.toLowerCase();
}
}
return found;
}
/** Remote fetch rounds allowed after the initial shallow tip fetch. */
const MAX_FF_FETCH_ROUNDS = 14;
/** Per-round deepen step cap for exponential backoff. */
const MAX_FF_DEEPEN_STEP = 2048;
/**
* Whether `descendantSha` is a fast-forward from `ancestorSha` on the remote.
* Distinguishes a normal branch advance from a force-push after ls-remote has
* already resolved the ref to a new tip.
*
* Fetches the descendant tip once, then deepens that shallow boundary with
* exponentially increasing steps until the prior commit is reachable, the
* downloaded history is complete, or a safety budget is exhausted. Operational
* failures throw a classified TransportFailure; only a proven non-fast-forward
* returns false.
*/
export async function verifyFastForward(req: {
repoUrl: string;
ancestorSha: string;
descendantSha: string;
token?: string | null;
timeoutMs?: number;
workspaceRoot: string;
maxBytes: number;
}): Promise<boolean> {
const ancestor = req.ancestorSha.toLowerCase();
const descendant = req.descendantSha.toLowerCase();
if (ancestor === descendant) return true;
const hasToken = Boolean(req.token);
await ensureBinaryReady(hasToken);
const url = assertValidRepoUrl(req.repoUrl, hasToken);
const timeoutMs = req.timeoutMs ?? DEFAULT_TIMEOUT_MS;
const deadline = Date.now() + timeoutMs;
const remainingMs = (): number => Math.max(1, deadline - Date.now());
const assertTimeBudget = (): void => {
if (Date.now() >= deadline) {
throw { transportFailure: true as const, reason: 'timeout', host: url.host, hasToken } satisfies TransportFailure;
}
};
const { env, baseArgs } = await prepareInvocation(req.workspaceRoot, req.token);
const repoDir = path.join(req.workspaceRoot, 'ff-check');
await fs.mkdir(repoDir, { recursive: true });
let sizeExceeded = false;
let activeChild: ChildProcess | undefined;
let breachKill: Promise<void> | undefined;
const watchdog = startSizeWatchdog(req.workspaceRoot, req.maxBytes, () => {
sizeExceeded = true;
breachKill = killTree(activeChild);
});
const throwIfSizeExceeded = (): void => {
if (sizeExceeded) {
throw { transportFailure: true as const, reason: 'size', maxBytes: req.maxBytes, host: url.host, hasToken } satisfies TransportFailure;
}
};
try {
const materialize = async (args: string[]): Promise<RunResult> => {
assertTimeBudget();
let res: RunResult;
try {
res = await runGit(args, {
cwd: repoDir,
env,
timeoutMs: remainingMs(),
onSpawn: (child) => { activeChild = child; },
});
} catch (e) {
throwIfSizeExceeded();
if (isTimeoutError(e)) {
throw { transportFailure: true as const, reason: 'timeout', host: url.host, hasToken } satisfies TransportFailure;
}
throw { transportFailure: true as const, reason: 'exit', stderr: e instanceof Error ? e.message : String(e), argv: args, host: url.host, hasToken } satisfies TransportFailure;
}
throwIfSizeExceeded();
if (res.exitCode !== 0) {
throw { transportFailure: true as const, reason: 'exit', stderr: res.stderr, exitCode: res.exitCode, argv: args, host: url.host, hasToken } satisfies TransportFailure;
}
return res;
};
const probeFailure = (res: RunResult, argv: string[]): never => {
throw {
transportFailure: true as const,
reason: 'exit',
stderr: res.stderr,
exitCode: res.exitCode,
argv,
host: url.host,
hasToken,
} satisfies TransportFailure;
};
const runProbe = async (args: string[]): Promise<RunResult> => {
assertTimeBudget();
try {
const res = await runGit(args, {
cwd: repoDir,
env,
timeoutMs: Math.min(remainingMs(), LS_REMOTE_MAX_MS),
});
throwIfSizeExceeded();
return res;
} catch (e) {
throwIfSizeExceeded();
if (isTimeoutError(e)) {
throw { transportFailure: true as const, reason: 'timeout', host: url.host, hasToken } satisfies TransportFailure;
}
throw {
transportFailure: true as const,
reason: 'exit',
stderr: e instanceof Error ? e.message : String(e),
argv: args,
host: url.host,
hasToken,
} satisfies TransportFailure;
}
};
const isMissingObjectProbe = (res: RunResult): boolean => {
if (res.exitCode === 0) return false;
if (res.exitCode === 1) return true;
const err = res.stderr.toLowerCase();
return err.includes('not a valid object name')
|| err.includes('bad object')
|| err.includes('could not get');
};
await materialize([...baseArgs, 'init']);
await materialize([...baseArgs, 'fetch', '--depth=1', url.href, descendant]);
const countReachable = async (): Promise<number> => {
const argv = [...baseArgs, 'rev-list', '--count', descendant];
const listed = await runProbe(argv);
if (listed.exitCode !== 0) {
return probeFailure(listed, argv);
}
const parsed = Number.parseInt(listed.stdout.trim(), 10);
if (!Number.isFinite(parsed) || parsed < 0) {
throw {
transportFailure: true as const,
reason: 'exit',
stderr: `unexpected rev-list output: ${listed.stdout}`,
exitCode: listed.exitCode,
argv,
host: url.host,
hasToken,
} satisfies TransportFailure;
}
return parsed;
};
const isShallowRepository = async (): Promise<boolean> => {
const argv = [...baseArgs, 'rev-parse', '--is-shallow-repository'];
const shallow = await runProbe(argv);
if (shallow.exitCode !== 0) {
return probeFailure(shallow, argv);
}
const flag = shallow.stdout.trim();
if (flag === 'true') return true;
if (flag === 'false') return false;
throw {
transportFailure: true as const,
reason: 'exit',
stderr: `unexpected shallow-repository output: ${shallow.stdout}`,
exitCode: shallow.exitCode,
argv,
host: url.host,
hasToken,
} satisfies TransportFailure;
};
const isProvenAncestor = async (): Promise<boolean> => {
const argv = [...baseArgs, 'merge-base', '--is-ancestor', ancestor, descendant];
const ancestry = await runProbe(argv);
if (ancestry.exitCode === 0) return true;
if (ancestry.exitCode === 1) return false;
return probeFailure(ancestry, argv);
};
let reachableCount = await countReachable();
let fetchRounds = 1;
let deepenStep = 1;
const assertWithinSizeBudget = async (): Promise<void> => {
const finalSize = await treeSize(req.workspaceRoot).catch((e: unknown) => {
console.warn(`[GitSource:transport] final size measurement failed for ${req.workspaceRoot}, failing closed: ${e instanceof Error ? e.message : String(e)}`);
return -1;
});
if (sizeExceeded || finalSize < 0 || finalSize > req.maxBytes) {
throw { transportFailure: true as const, reason: 'size', maxBytes: req.maxBytes, host: url.host, hasToken } satisfies TransportFailure;
}
};
while (true) {
assertTimeBudget();
const ancestorArgv = [...baseArgs, 'cat-file', '-e', `${ancestor}^{commit}`];
const hasAncestor = await runProbe(ancestorArgv);
if (hasAncestor.exitCode === 0) {
await assertWithinSizeBudget();
return await isProvenAncestor();
}
if (!isMissingObjectProbe(hasAncestor)) {
return probeFailure(hasAncestor, ancestorArgv);
}
if (!(await isShallowRepository())) {
await assertWithinSizeBudget();
return false;
}
if (fetchRounds >= MAX_FF_FETCH_ROUNDS) {
throw { transportFailure: true as const, reason: 'timeout', host: url.host, hasToken } satisfies TransportFailure;
}
const previousCount = reachableCount;
await materialize([...baseArgs, 'fetch', `--deepen=${deepenStep}`, url.href, descendant]);
fetchRounds += 1;
reachableCount = await countReachable();
if (reachableCount <= previousCount) {
if (!(await isShallowRepository())) {
await assertWithinSizeBudget();
return false;
}
throw { transportFailure: true as const, reason: 'timeout', host: url.host, hasToken } satisfies TransportFailure;
}
deepenStep = Math.min(deepenStep * 2, MAX_FF_DEEPEN_STEP);
}
} finally {
watchdog.stop();
await awaitKillConfirmed(breachKill, `size-breach kill for ${url.host}`);
await fs.rm(repoDir, { recursive: true, force: true }).catch((e: unknown) => {
console.warn(`[GitSource:transport] failed to remove fast-forward scratch repo ${repoDir}: ${e instanceof Error ? e.message : String(e)}`);
});
}
throw { transportFailure: true as const, reason: 'ref-not-found', host: url.host, hasToken } satisfies TransportFailure;
}
export const nativeGitTransport: GitTransport = {
async resolveRef(req: ResolveRequest): Promise<{ commitSha: string }> {
async resolveRef(req: ResolveRequest): Promise<ResolveResult> {
const hasToken = Boolean(req.token);
await ensureBinaryReady(hasToken);
const url = assertValidRepoUrl(req.repoUrl, hasToken);
assertValidRef(req.ref, url.host, hasToken);
if (SHA_PATTERN.test(req.ref)) {
// A full SHA is self-resolving: the immutable identity IS the
// value, so there is nothing to look up. Reachability is verified
// at fetch, where the host either serves the object or refuses it
// (classified as UNSUPPORTED_REF).
return { commitSha: req.ref.toLowerCase(), kind: 'sha' };
}
assertValidRef(req.ref, url.host, hasToken);
const { env, baseArgs } = await prepareInvocation(req.workspaceRoot, req.token);
const commitSha = await lsRemoteHead(
const found = await lsRemoteRefs(
url, req.ref, env, baseArgs,
req.timeoutMs ?? DEFAULT_TIMEOUT_MS, hasToken,
);
return { commitSha };
if (found.branchSha) return { commitSha: found.branchSha, kind: 'branch' };
if (found.tagSha) return { commitSha: found.tagSha, kind: 'tag' };
throw { transportFailure: true as const, reason: 'ref-not-found', host: url.host, hasToken } satisfies TransportFailure;
},
async fetchAtCommit(req: FetchRequest): Promise<FetchResult> {
@@ -670,49 +937,63 @@ export const nativeGitTransport: GitTransport = {
});
try {
let cloneResult: RunResult;
try {
cloneResult = await runGit(
[
...baseArgs, 'clone',
'--depth=1', '--single-branch', '--no-tags', '--no-recurse-submodules',
'--branch', req.ref, url.href, checkout,
],
{ cwd: layout.homeDir, env, timeoutMs, onSpawn: (child) => { activeChild = child; } },
);
} catch (e) {
// A size breach wins over the timeout wording: both kills are
// ours, but the operator guidance differs.
// Run each materialization step through one failure mapper. A size
// breach wins over the timeout wording: both kills are ours, but
// the operator guidance differs. runGit resolves on any exit code,
// so a non-zero exit is classified here by its real stderr rather
// than leaking a generic GIT_ERROR upstream.
const materialize = async (args: string[]): Promise<RunResult> => {
let res: RunResult;
try {
res = await runGit(args, {
cwd: layout.homeDir, env, timeoutMs,
onSpawn: (child) => { activeChild = child; },
});
} catch (e) {
// A size breach wins over the timeout wording.
if (sizeExceeded) {
throw { transportFailure: true as const, reason: 'size', maxBytes: req.maxBytes, host: url.host, hasToken } satisfies TransportFailure;
}
if (isTimeoutError(e)) {
throw { transportFailure: true as const, reason: 'timeout', host: url.host, hasToken } satisfies TransportFailure;
}
throw { transportFailure: true as const, reason: 'exit', stderr: e instanceof Error ? e.message : String(e), argv: args, host: url.host, hasToken } satisfies TransportFailure;
}
// A watchdog-triggered SIGKILL settles runGit's promise via the
// child's normal 'close' event (code null -> exitCode -1), not
// a rejection, so this is the common path for an in-flight
// breach and must check sizeExceeded before the generic mapping.
if (sizeExceeded) {
throw { transportFailure: true as const, reason: 'size', maxBytes: req.maxBytes, host: url.host, hasToken } satisfies TransportFailure;
}
if (isTimeoutError(e)) {
throw { transportFailure: true as const, reason: 'timeout', host: url.host, hasToken } satisfies TransportFailure;
if (res.exitCode !== 0) {
throw { transportFailure: true as const, reason: 'exit', stderr: res.stderr, exitCode: res.exitCode, argv: args, host: url.host, hasToken } satisfies TransportFailure;
}
throw { transportFailure: true as const, reason: 'exit', stderr: e instanceof Error ? e.message : String(e), argv: [...baseArgs, 'clone'], host: url.host, hasToken } satisfies TransportFailure;
}
return res;
};
// A watchdog-triggered SIGKILL settles runGit's promise via the
// child's normal 'close' event (code null -> exitCode -1), not a
// rejection, so this branch is the common path for an in-flight
// breach and must check sizeExceeded before the generic mapping.
if (sizeExceeded) {
throw { transportFailure: true as const, reason: 'size', maxBytes: req.maxBytes, host: url.host, hasToken } satisfies TransportFailure;
}
// runGit resolves on any exit code: a failed clone (auth, missing
// repo, TLS) must classify by its real stderr here rather than
// fall through to rev-parse and surface as a generic GIT_ERROR.
if (cloneResult.exitCode !== 0) {
throw {
transportFailure: true as const,
reason: 'exit',
stderr: cloneResult.stderr,
exitCode: cloneResult.exitCode,
argv: [...baseArgs, 'clone'],
host: url.host,
hasToken,
} satisfies TransportFailure;
if (req.refKind === 'sha') {
// `--branch` cannot take a bare SHA, so a pinned commit uses a
// third strategy: init a repo, fetch exactly that object, and
// check it out detached. The host must allow fetching a direct
// SHA (GitHub does by default); a refusal surfaces as a
// non-zero `git fetch` here and classifies as UNSUPPORTED_REF.
await materialize([...baseArgs, 'init', checkout]);
await materialize([...baseArgs, '-C', checkout, 'fetch', '--depth=1', url.href, req.ref]);
await materialize([...baseArgs, '-C', checkout, 'checkout', '--detach', req.ref]);
} else {
// A bare name works for both branches and tags: `--branch`
// detaches at the named ref's commit either way, and passing a
// fully-qualified `refs/tags/<ref>` is rejected by git
// (`Remote branch ... not found`). The resolved kind is
// already pinned by ls-remote, and the rev-parse HEAD
// verification below confirms the checkout matched it.
const branchArg = req.ref;
await materialize([
...baseArgs, 'clone',
'--depth=1', '--single-branch', '--no-tags', '--no-recurse-submodules',
'--branch', branchArg, url.href, checkout,
]);
}
let actual: string;