import { isEmptyArray, isNonEmptyArray } from '@sindresorhus/is'; import { DateTime } from 'luxon'; import { GlobalConfig } from '../../../config/global.ts'; import { instrument } from '../../../instrumentation/index.ts'; import { logger } from '../../../logger/index.ts'; import { ExternalHostError } from '../../../types/errors/external-host-error.ts'; import { getCache } from '../../../util/cache/repository/index.ts'; import { repoCacheProvider } from '../../../util/http/cache/repository-http-cache-provider.ts'; import type { GithubHttp, GithubHttpOptions, } from '../../../util/http/github.ts'; import { parseLinkHeader } from '../../../util/url.ts'; import { ApiCache } from './api-cache.ts'; import { coerceRestPr } from './common.ts'; import type { ApiPageCache, GhPr, GhRestPr } from './types.ts'; function getPrApiCache(): ApiCache { const repoCache = getCache(); if (!repoCache?.platform?.github?.pullRequestsCache) { logger.debug('PR cache: cached data not found, creating new cache'); repoCache.platform ??= {}; repoCache.platform.github ??= {}; repoCache.platform.github.pullRequestsCache ??= { items: {} }; } const prApiCache = new ApiCache( repoCache.platform.github.pullRequestsCache as ApiPageCache, ); return prApiCache; } /** * Fetch and return Pull Requests from GitHub repository: * * 1. Synchronize long-term cache. * * 2. Store items in raw format, i.e. exactly what * has been returned by GitHub REST API. * * 3. Convert items to the Renovate format and return. * * In order synchronize ApiCache properly, we handle 3 cases: * * a. We never fetched PR list for this repo before. * If cached PR list is empty, we assume it's the case. * * In this case, we're falling back to quick fetch via * `paginate=true` option (see `util/http/github.ts`). * * b. Some of PRs had changed since last run. * * In this case, we sequentially fetch page by page * until the oldest item on an unfiltered page predates * the cache's `lastModified` timestamp. * * We expect to fetch just one page per run in average, * since it's rare to have more than 100 updated PRs. */ export async function getPrCache( http: GithubHttp, repo: string, username?: string, ): Promise> { const prApiCache = getPrApiCache(); const isInitial = isEmptyArray(prApiCache.getItems()); // Snapshot before the loop — reconcile() updates lastModified as it // processes items, so reading it inside the loop would create a moving target. // If lastModified is missing but items exist (populated via updateItem()), // derive cutoff from the newest cached item. let lastModifiedRaw = prApiCache.getLastModified(); if (!lastModifiedRaw && !isInitial) { const items = prApiCache.getItems(); for (const item of items) { if (!lastModifiedRaw || item.updated_at > lastModifiedRaw) { lastModifiedRaw = item.updated_at; } } } const cutoffTime = lastModifiedRaw ? DateTime.fromISO(lastModifiedRaw) : null; try { const maxSyncPages = GlobalConfig.get('prCacheSyncMaxPages', 100); const startTime = Date.now(); let requestsTotal = 0; let apiQuotaAffected = false; let needNextPageFetch = true; let needNextPageSync = true; let pageIdx = 1; await instrument('sync GitHub PR cache', async () => { while (needNextPageFetch && needNextPageSync) { const opts: GithubHttpOptions = { paginate: false, memCache: false }; if (pageIdx === 1) { opts.cacheProvider = repoCacheProvider; if (isInitial) { opts.paginate = true; } } let perPage: number; if (isInitial) { logger.debug('PR cache: initial fetch'); perPage = 100; } else { logger.debug('PR cache: sync fetch'); perPage = 20; } const urlPath = `repos/${repo}/pulls?per_page=${perPage}&state=all&sort=updated&direction=desc&page=${pageIdx}`; const res = await http.getJsonUnchecked(urlPath, opts); apiQuotaAffected = true; requestsTotal += 1; const { headers: { link: linkHeader }, } = res; let { body: page } = res; if (!isInitial && cutoffTime && isNonEmptyArray(page)) { // Advance watermark so next run doesn't re-scan these pages, // even if no Renovate PRs are found. prApiCache.updateLastModified(page[0].updated_at); const oldestOnPage = DateTime.fromISO( page[page.length - 1].updated_at, ); if (oldestOnPage < cutoffTime) { needNextPageSync = false; } } if (username) { const filteredPage = page.filter( (ghPr) => ghPr?.user?.login && ghPr.user.login === username, ); logger.debug( `PR cache: Filtered ${page.length} PRs to ${filteredPage.length} (user=${username})`, ); page = filteredPage; } const items = page.map(coerceRestPr); if (isNonEmptyArray(items)) { prApiCache.reconcile(items); } needNextPageFetch = !!parseLinkHeader(linkHeader)?.next; if (pageIdx === 1) { needNextPageFetch &&= !opts.paginate; } // Safety net: cutoff-based stop should always fire first if ( !isInitial && needNextPageFetch && needNextPageSync && pageIdx >= maxSyncPages ) { logger.warn( { repo, pages: pageIdx }, 'PR cache: hit max sync pages, stopping', ); needNextPageSync = false; } pageIdx += 1; } }); const durationMs = Math.round(Date.now() - startTime); logger.debug( { pullsTotal: prApiCache.getItems().length, requestsTotal, apiQuotaAffected, durationMs, }, `PR cache: getPrList success`, ); } catch (err) /* v8 ignore next */ { logger.debug({ err }, 'PR cache: getPrList err'); throw new ExternalHostError(err, 'github'); } return prApiCache.getItems(); } export function updatePrCache(pr: GhPr): void { const cache = getPrApiCache(); cache.updateItem(pr); }