import is from '@sindresorhus/is'; import { GlobalConfig } from '../../../config/global.ts'; import { logger } from '../../../logger/index.ts'; import { ExternalHostError } from '../../../types/errors/external-host-error.ts'; import * as memCache from '../../cache/memory/index.ts'; import * as packageCache from '../../cache/package/index.ts'; import type { PackageCacheNamespace } from '../../cache/package/types.ts'; import type { GithubGraphqlResponse, GithubHttp, GithubHttpOptions, } from '../../http/github.ts'; import type { HttpResponse } from '../../http/types.ts'; import { regEx } from '../../regex.ts'; import { getApiBaseUrl } from '../url.ts'; import { GithubGraphqlMemoryCacheStrategy } from './cache-strategies/memory-cache-strategy.ts'; import { GithubGraphqlPackageCacheStrategy } from './cache-strategies/package-cache-strategy.ts'; import type { GithubDatasourceItem, GithubGraphqlCacheStrategy, GithubGraphqlDatasourceAdapter, GithubGraphqlPayload, GithubGraphqlRepoParams, GithubGraphqlRepoResponse, GithubPackageConfig, RawQueryResponse, } from './types.ts'; /** * Page sizes to try, from the largest to the smallest. * * Repositories with many tags can exceed the GitHub GraphQL execution * timeout even for small page sizes, hence the low floor. * * @see https://github.com/renovatebot/renovate/discussions/45248 */ const pageSizes = [100, 50, 25, 10, 5] as const; type PageSize = (typeof pageSizes)[number]; /** * We know empirically that certain type of GraphQL errors * can be fixed by shrinking page size. * * @see https://github.com/renovatebot/renovate/issues/16343 */ function isUnknownGraphqlError(err: Error): boolean { const { message } = err; return message.startsWith('Something went wrong while executing your query.'); } function canBeSolvedByShrinking(err: Error): boolean { const errors: Error[] = err instanceof AggregateError ? err.errors : [err]; return errors.some( (e) => err instanceof ExternalHostError || isUnknownGraphqlError(e), ); } export class GithubGraphqlDatasourceFetcher< GraphqlItem, ResultItem extends GithubDatasourceItem, > { static async query( config: GithubPackageConfig, http: GithubHttp, adapter: GithubGraphqlDatasourceAdapter, ): Promise { const instance = new GithubGraphqlDatasourceFetcher( config, http, adapter, ); const items = await instance.getItems(); return items; } private readonly baseUrl: string; private readonly repoOwner: string; private readonly repoName: string; private itemsPerQuery: PageSize = pageSizes[0]; private queryCount = 0; private cursor: string | null = null; private isPersistent: boolean | undefined; private http: GithubHttp; private datasourceAdapter: GithubGraphqlDatasourceAdapter< GraphqlItem, ResultItem >; constructor( packageConfig: GithubPackageConfig, http: GithubHttp, datasourceAdapter: GithubGraphqlDatasourceAdapter, ) { this.http = http; this.datasourceAdapter = datasourceAdapter; const { packageName, registryUrl } = packageConfig; [this.repoOwner, this.repoName] = packageName.split('/'); this.baseUrl = getApiBaseUrl(registryUrl).replace(regEx(/\/v3\/$/), '/'); // Replace for GHE } private getCacheNs(): PackageCacheNamespace { return this.datasourceAdapter.key; } private getCacheKey(): string { return [this.baseUrl, this.repoOwner, this.repoName].join(':'); } private getRawQueryOptions(): GithubHttpOptions { const baseUrl = this.baseUrl; const repository = `${this.repoOwner}/${this.repoName}`; const query = this.datasourceAdapter.query; const variables: GithubGraphqlRepoParams = { owner: this.repoOwner, name: this.repoName, count: this.itemsPerQuery, cursor: this.cursor, }; return { baseUrl, repository, readOnly: true, body: { query, variables }, }; } private async doRawQuery(): Promise< RawQueryResponse> > { const requestOptions = this.getRawQueryOptions(); type GraphqlData = GithubGraphqlRepoResponse; type HttpBody = GithubGraphqlResponse; let httpRes: HttpResponse; try { httpRes = await this.http.postJson('/graphql', requestOptions); } catch (err) { return [null, err]; } const { body } = httpRes; const { data, errors } = body; if (errors?.length) { if (errors.length === 1) { const { message } = errors[0]; const err = new Error(message); return [null, err]; } const errorInstances = errors.map(({ message }) => new Error(message)); const err = new AggregateError(errorInstances); return [null, err]; } if (!data) { const msg = 'GitHub GraphQL datasource: failed to obtain data'; const err = new Error(msg); return [null, err]; } if (!data.repository) { const msg = 'GitHub GraphQL datasource: failed to obtain repository data'; const err = new Error(msg); return [null, err]; } if (!data.repository.payload) { const msg = 'GitHub GraphQL datasource: failed to obtain repository payload data'; const err = new Error(msg); return [null, err]; } this.queryCount += 1; // For values other than explicit `false`, // we assume that items can not be cached. this.isPersistent ??= data.repository.isRepoPrivate === false; const res = data.repository.payload; return [res, null]; } private shrinkPageSize(): boolean { const nextIdx = pageSizes.indexOf(this.itemsPerQuery) + 1; if (nextIdx >= pageSizes.length) { return false; } this.itemsPerQuery = pageSizes[nextIdx]; return true; } private hasReachedQueryLimit(): boolean { return this.queryCount >= 100; } private async doShrinkableQuery(): Promise< GithubGraphqlPayload > { let res: GithubGraphqlPayload | null = null; let err: Error | null = null; while (!res) { [res, err] = await this.doRawQuery(); if (err) { if (!canBeSolvedByShrinking(err)) { throw err; } const shrinkResult = this.shrinkPageSize(); if (!shrinkResult) { throw err; } const { body: _, ...options } = this.getRawQueryOptions(); logger.debug( { options, newSize: this.itemsPerQuery }, 'Shrinking GitHub GraphQL page size after error', ); } } return res; } private _cacheStrategy: GithubGraphqlCacheStrategy | undefined; private cacheStrategy(): GithubGraphqlCacheStrategy { if (this._cacheStrategy) { return this._cacheStrategy; } const cacheNs = this.getCacheNs(); const cacheKey = this.getCacheKey(); const cachePrivatePackages = GlobalConfig.get('cachePrivatePackages'); const skipStabilization = !is.undefined(this.datasourceAdapter.maxItems); this._cacheStrategy = cachePrivatePackages || this.isPersistent ? new GithubGraphqlPackageCacheStrategy( cacheNs, cacheKey, skipStabilization, ) : new GithubGraphqlMemoryCacheStrategy( cacheNs, cacheKey, skipStabilization, ); return this._cacheStrategy; } /** * This method is responsible for data synchronization. * It also detects persistence of the package, based on the first page result. */ private async doPaginatedFetch(): Promise { let hasNextPage = true; let isPaginationDone = false; let nextCursor: string | undefined; let itemCount = 0; const maxItems = this.datasourceAdapter.maxItems; while (hasNextPage && !isPaginationDone && !this.hasReachedQueryLimit()) { if (!is.undefined(maxItems) && itemCount >= maxItems) { break; } const queryResult = await this.doShrinkableQuery(); const resultItems: ResultItem[] = []; for (const node of queryResult.nodes) { const item = this.datasourceAdapter.transform(node); if (!item) { logger.once.info( { packageName: `${this.repoOwner}/${this.repoName}`, baseUrl: this.baseUrl, }, `GitHub GraphQL datasource: skipping empty item`, ); continue; } resultItems.push(item); } itemCount += resultItems.length; // It's important to call `getCacheStrategy()` after `doShrinkableQuery()` // because `doShrinkableQuery()` may change `this.isCacheable`. // // Otherwise, cache items for public packages will never be persisted // in long-term cache. isPaginationDone = await this.cacheStrategy().reconcile(resultItems); hasNextPage = !!queryResult?.pageInfo?.hasNextPage; nextCursor = queryResult?.pageInfo?.endCursor; if (hasNextPage && nextCursor) { this.cursor = nextCursor; } } if (this.isPersistent) { await this.storePersistenceFlag(30); } } private async doCachedQuery(): Promise { await this.loadPersistenceFlag(); if (!this.isPersistent) { await this.doPaginatedFetch(); } const res = await this.cacheStrategy().finalizeAndReturn(); if (res.length) { return res; } delete this.isPersistent; await this.doPaginatedFetch(); return this.cacheStrategy().finalizeAndReturn(); } async loadPersistenceFlag(): Promise { const ns = this.getCacheNs(); const key = `${this.getCacheKey()}:is-persistent`; this.isPersistent = await packageCache.get(ns, key); } async storePersistenceFlag(minutes: number): Promise { const ns = this.getCacheNs(); const key = `${this.getCacheKey()}:is-persistent`; await packageCache.set(ns, key, true, minutes); } /** * This method ensures the only one query is executed * to a particular package during single run. */ private doUniqueQuery(): Promise { const cacheKey = `github-pending:${this.getCacheNs()}:${this.getCacheKey()}`; let resultPromise = memCache.get>(cacheKey); resultPromise ??= this.doCachedQuery(); memCache.set(cacheKey, resultPromise); return resultPromise; } async getItems(): Promise { const res = await this.doUniqueQuery(); return res; } }