import is from '@sindresorhus/is'; import { PlatformId } from '../../../constants'; import { logger } from '../../../logger'; import { ExternalHostError } from '../../../types/errors/external-host-error'; import { getCache } from '../../../util/cache/repository'; import type { GithubHttp, GithubHttpOptions } from '../../../util/http/github'; import { parseLinkHeader } from '../../../util/url'; import { getPrBodyStruct } from '../pr-body'; import type { Pr } from '../types'; import { ApiCache } from './api-cache'; import { coerceRestPr } from './common'; import type { ApiPageCache, GhRestPr } from './types'; function removeUrlFields(input: unknown): void { if (is.plainObject(input)) { for (const [key, val] of Object.entries(input)) { if ((key === 'url' || key.endsWith('_url')) && is.string(val)) { delete input[key]; } else { removeUrlFields(val); } } } } function compactPrBodyStructure(input: unknown): void { if (is.plainObject(input)) { if (!input.bodyStruct && is.string(input.body)) { input.bodyStruct = getPrBodyStruct(input.body); delete input.body; } } } function massageGhRestPr(ghPr: GhRestPr): GhRestPr { removeUrlFields(ghPr); delete ghPr.head?.repo?.pushed_at; delete ghPr.base?.repo?.pushed_at; delete ghPr._links; compactPrBodyStructure(ghPr); return ghPr; } function getPrApiCache(): ApiCache { const repoCache = getCache(); repoCache.platform ??= {}; repoCache.platform.github ??= {}; repoCache.platform.github.prCache ??= { items: {} }; const apiPageCache = repoCache.platform.github .prCache as ApiPageCache; const items = Object.values(apiPageCache.items); const firstItem = items?.[0]; if (firstItem?.body) { apiPageCache.items = {}; } else if (firstItem?._links) { for (const ghPr of items) { massageGhRestPr(ghPr); } } const prCache = new ApiCache(apiPageCache); return prCache; } /** * Fetch and return Pull Requests from GitHub repository: * * 1. Synchronize long-term cache. * * 2. Store items in raw format, i.e. exactly what * has been returned by GitHub REST API. * * 3. Convert items to the Renovate format and return. * * In order synchronize ApiCache properly, we handle 3 cases: * * a. We never fetched PR list for this repo before. * If cached PR list is empty, we assume it's the case. * * In this case, we're falling back to quick fetch via * `paginate=true` option (see `util/http/github.ts`). * * b. Some of PRs had changed since last run. * * In this case, we sequentially fetch page by page * until `ApiCache.coerce` function indicates that * no more fresh items can be found in the next page. * * We expect to fetch just one page per run in average, * since it's rare to have more than 100 updated PRs. */ export async function getPrCache( http: GithubHttp, repo: string, username: string | null ): Promise> { const prCache: Record = {}; const prApiCache = getPrApiCache(); const isInitial = is.emptyArray(prApiCache.getItems()); try { let requestsTotal = 0; let apiQuotaAffected = false; let needNextPageFetch = true; let needNextPageSync = true; let pageIdx = 1; while (needNextPageFetch && needNextPageSync) { const opts: GithubHttpOptions = { paginate: false }; if (pageIdx === 1 && isInitial) { // Speed up initial fetch opts.paginate = true; } const perPage = isInitial ? 100 : 20; const urlPath = `repos/${repo}/pulls?per_page=${perPage}&state=all&sort=updated&direction=desc&page=${pageIdx}`; const res = await http.getJson(urlPath, opts); apiQuotaAffected = true; requestsTotal += 1; const { headers: { link: linkHeader }, } = res; let { body: page } = res; if (username) { page = page.filter( (ghPr) => ghPr?.user?.login && ghPr.user.login === username ); } for (const ghPr of page) { massageGhRestPr(ghPr); } needNextPageSync = prApiCache.reconcile(page); needNextPageFetch = !!parseLinkHeader(linkHeader)?.next; if (pageIdx === 1) { needNextPageFetch &&= !opts.paginate; } pageIdx += 1; } logger.debug( { pullsTotal: prApiCache.getItems().length, requestsTotal, apiQuotaAffected, }, `getPrList success` ); } catch (err) /* istanbul ignore next */ { logger.debug({ err }, 'getPrList err'); throw new ExternalHostError(err, PlatformId.Github); } const cacheItems = prApiCache.getItems(); for (const ghPr of cacheItems) { const pr = coerceRestPr(ghPr); if (pr) { prCache[ghPr.number] = pr; } } return prCache; }