import { isArray, isNonEmptyObject, isNullOrUndefined, isPlainObject, isString, } from '@sindresorhus/is'; import { DateTime } from 'luxon'; import { z } from 'zod/v4'; import { GlobalConfig } from '../../config/global.ts'; import { PLATFORM_BAD_CREDENTIALS, PLATFORM_INTEGRATION_UNAUTHORIZED, PLATFORM_RATE_LIMIT_EXCEEDED, REPOSITORY_CHANGED, } from '../../constants/error-messages.ts'; import { logger } from '../../logger/index.ts'; import { ExternalHostError } from '../../types/errors/external-host-error.ts'; import { getCache } from '../cache/repository/index.ts'; import { getEnv } from '../env.ts'; import * as hostRules from '../host-rules.ts'; import { maskToken } from '../mask.ts'; import { coerceObject } from '../object.ts'; import * as p from '../promises.ts'; import { range } from '../range.ts'; import { regEx } from '../regex.ts'; import { LooseArray } from '../schema-utils/index.ts'; import { joinUrlParts, parseLinkHeader, parseUrl } from '../url.ts'; import { findMatchingRule } from './host-rules.ts'; import { HttpBase, type InternalHttpOptions, type InternalJsonUnsafeOptions, } from './http.ts'; import type { GotLegacyError } from './legacy.ts'; import type { GraphqlOptions, HttpMethod, HttpOptions, HttpResponse, } from './types.ts'; const githubBaseUrl = 'https://api.github.com/'; const MAX_PAGINATION_PAGES = 100; let baseUrl = githubBaseUrl; export function setBaseUrl(url: string): void { baseUrl = url; } export interface GithubBaseHttpOptions extends HttpOptions { repository?: string; } export interface GithubHttpOptions extends GithubBaseHttpOptions { paginate?: boolean | string; paginationField?: string; pageLimit?: number; } interface GithubGraphqlRepoData { repository?: T; } export type GithubGraphqlResponse = | { data: T; errors?: never; } | { data?: never; errors: { type?: string; code?: string; message: string; }[]; }; /** * GitHub reports a spent GraphQL budget in two shapes: `RATE_LIMITED` from the * GraphQL API itself, and `graphql_rate_limit` when an app installation has * exhausted its allowance. Both mean "come back later", not "unknown error". */ function isGraphqlRateLimited( errors: { type?: string; code?: string }[] | undefined, ): boolean { return !!errors?.some( (err) => err.type === 'RATE_LIMITED' || err.code === 'graphql_rate_limit', ); } const GithubError = z.object({ field: z.string().optional(), code: z.string().optional(), message: z.string().optional(), }); /** * GitHub usually returns `errors` as an array, but not always - normalize it * into an array of error objects, dropping anything that doesn't fit. */ const GithubErrors = z .union([LooseArray(GithubError), GithubError.transform((error) => [error])]) .catch([]); function handleGotError( err: GotLegacyError, url: string | URL, opts: GithubBaseHttpOptions, ): Error { const path = url.toString(); let message = err.message || ''; const body = err.response?.body; if (isPlainObject(body) && 'message' in body) { message = String(body.message); } if ( err.code === 'ERR_HTTP2_STREAM_ERROR' || err.code === 'ENOTFOUND' || err.code === 'ETIMEDOUT' || err.code === 'EAI_AGAIN' || err.code === 'ECONNRESET' ) { logger.debug({ err }, 'GitHub failure: RequestError'); return new ExternalHostError(err, 'github'); } if (err.name === 'ParseError') { logger.debug({ err }, ''); return new ExternalHostError(err, 'github'); } if (err.statusCode && err.statusCode >= 500 && err.statusCode < 600) { logger.debug({ err }, 'GitHub failure: 5xx'); return new ExternalHostError(err, 'github'); } if ( err.statusCode === 403 && message.startsWith('You have triggered an abuse detection mechanism') ) { logger.debug({ err }, 'GitHub failure: abuse detection'); return new Error(PLATFORM_RATE_LIMIT_EXCEEDED); } if ( err.statusCode === 403 && message.startsWith('You have exceeded a secondary rate limit') ) { logger.warn({ err }, 'GitHub failure: secondary rate limit'); return new Error(PLATFORM_RATE_LIMIT_EXCEEDED); } if (err.statusCode === 403 && message.includes('Upgrade to GitHub Pro')) { logger.debug(`Endpoint: ${path}, needs paid GitHub plan`); return err; } if (err.statusCode === 403 && message.includes('rate limit exceeded')) { logger.debug({ err }, 'GitHub failure: rate limit'); const parsed = parseUrl(baseUrl); const rule = hostRules.find({ url: baseUrl }); if (rule.token || rule.password) { logger.once.warn( 'Rate limit exceeded for api.github.com, even though we are authenticated', ); } else { if (parsed?.hostname === 'api.github.com') { logger.once.warn( { documentationUrl: `${GlobalConfig.get('productLinks').documentation}getting-started/running/#githubcom-token-for-changelogs-and-tools`, }, 'Rate limit exceeded for api.github.com, as no hostRules set for this host. Please set a GITHUB_COM_TOKEN', ); } else { logger.once.warn( { host: parsed!.host }, 'Rate limit exceeded, as no hostRules set for this host', ); } } return new Error(PLATFORM_RATE_LIMIT_EXCEEDED); } if ( err.statusCode === 403 && message.startsWith('Resource not accessible by integration') ) { logger.debug( { err }, 'GitHub failure: Resource not accessible by integration', ); return new Error(PLATFORM_INTEGRATION_UNAUTHORIZED); } if (err.statusCode === 401) { // Warn once for github.com token if unauthorized const hostname = parseUrl(url)?.hostname; // v8 ignore else -- TODO: add test #40625 if (hostname === 'github.com' || hostname === 'api.github.com') { logger.once.warn('github.com token 401 unauthorized'); } if (message.includes('Bad credentials')) { const rateLimit = err.headers?.['x-ratelimit-limit'] ?? -1; logger.debug( { token: maskToken(opts.token), err, }, 'GitHub failure: Bad credentials', ); if (rateLimit === '60') { return new ExternalHostError(err, 'github'); } return new Error(PLATFORM_BAD_CREDENTIALS); } } if (err.statusCode === 422) { if ( message.includes('Review cannot be requested from pull request author') ) { return err; } const errors = GithubErrors.parse(err.body?.errors); if (errors.some((e) => e.field === 'milestone')) { return err; } if (errors.some((e) => e.code === 'invalid')) { logger.debug({ err }, 'Received invalid response - aborting'); return new Error(REPOSITORY_CHANGED); } if ( errors.some((e) => e.message?.startsWith('A pull request already exists')) ) { return err; } logger.debug({ err }, '422 Error thrown from GitHub'); return new ExternalHostError(err, 'github'); } if ( err.statusCode === 410 && err.body?.message === 'Issues are disabled for this repo' ) { return err; } return err; } interface GraphqlPaginatedContent { nodes?: T[]; edges?: T[]; pageInfo: { hasNextPage: boolean; endCursor: string }; } function constructAcceptString(input?: unknown): string { const defaultAccept = 'application/vnd.github.v3+json'; const acceptStrings = isString(input) ? input.split(regEx(/\s*,\s*/)) : []; // TODO: regression of #6736 // v8 ignore else -- TODO: add test #40625 if ( !acceptStrings.some((x) => x === defaultAccept) && (!acceptStrings.some((x) => x.startsWith('application/vnd.github.')) || acceptStrings.length < 2) ) { acceptStrings.push(defaultAccept); } return acceptStrings.join(', '); } const MAX_GRAPHQL_PAGE_SIZE = 100; interface GraphqlPageCacheItem { pageLastResizedAt: string; pageSize: number; } export type GraphqlPageCache = Record; function getGraphqlPageSize( fieldName: string, defaultPageSize = MAX_GRAPHQL_PAGE_SIZE, ): number { const cache = getCache(); const graphqlPageCache = cache?.platform?.github ?.graphqlPageCache as GraphqlPageCache; const cachedRecord = graphqlPageCache?.[fieldName]; if (graphqlPageCache && cachedRecord) { logger.debug( { fieldName, ...cachedRecord }, 'GraphQL page size: found cached value', ); const oldPageSize = cachedRecord.pageSize; const now = DateTime.local(); const then = DateTime.fromISO(cachedRecord.pageLastResizedAt); const expiry = then.plus({ hours: 24 }); if (now > expiry) { const newPageSize = Math.min(oldPageSize * 2, MAX_GRAPHQL_PAGE_SIZE); if (newPageSize < MAX_GRAPHQL_PAGE_SIZE) { const timestamp = now.toISO(); logger.debug( { fieldName, oldPageSize, newPageSize, timestamp }, 'GraphQL page size: expanding', ); cachedRecord.pageLastResizedAt = timestamp; cachedRecord.pageSize = newPageSize; } else { logger.debug( { fieldName, oldPageSize, newPageSize }, 'GraphQL page size: expanded to default page size', ); delete graphqlPageCache[fieldName]; } return newPageSize; } return oldPageSize; } return defaultPageSize; } function setGraphqlPageSize(fieldName: string, newPageSize: number): void { const oldPageSize = getGraphqlPageSize(fieldName); if (newPageSize !== oldPageSize) { const now = DateTime.local(); const pageLastResizedAt = now.toISO(); logger.debug( { fieldName, oldPageSize, newPageSize, timestamp: pageLastResizedAt }, 'GraphQL page size: shrinking', ); const cache = getCache(); cache.platform ??= {}; cache.platform.github ??= {}; cache.platform.github.graphqlPageCache ??= {}; const graphqlPageCache = cache.platform.github .graphqlPageCache as GraphqlPageCache; graphqlPageCache[fieldName] = { pageLastResizedAt, pageSize: newPageSize, }; } } function replaceUrlBase(url: URL, baseUrl: string): URL { const relativeUrl = `${url.pathname}${url.search}`; return new URL(relativeUrl, baseUrl); } function resolvePaginationUrl( url: string, baseUrl: string | undefined, rebasePaginationLinks: boolean, ): URL { const parsedUrl = new URL(url, baseUrl); const rebasePagination = !!baseUrl && rebasePaginationLinks && // Preserve github.com URLs for use cases like release notes parsedUrl.origin !== 'https://api.github.com'; return rebasePagination ? replaceUrlBase(parsedUrl, baseUrl) : parsedUrl; } export class GithubHttp extends HttpBase { protected override get baseUrl(): string | undefined { return baseUrl; } constructor(hostType = 'github', options?: HttpOptions) { super(hostType, options); } protected override extraOptions(): readonly string[] { return super .extraOptions() .concat([ 'pageLimit', 'paginate', 'paginationField', 'repository', ] as (keyof GithubHttpOptions)[]); } protected override processOptions( url: URL, opts: InternalHttpOptions & GithubBaseHttpOptions, ): void { if (!opts.token) { // create a mutable copy of `url` const authUrl = parseUrl(url.toString())!; if (opts.repository) { // set authUrl to https://api.github.com/repos/org/repo or https://gihub.domain.com/api/v3/repos/org/repo authUrl.hash = ''; authUrl.search = ''; authUrl.pathname = joinUrlParts( authUrl.pathname.startsWith('/api/v3') ? '/api/v3' : '', 'repos', `${opts.repository}`, ); } let readOnly = opts.readOnly; const { method = 'get' } = opts; if ( readOnly === undefined && ['get', 'head'].includes(method.toLowerCase()) ) { readOnly = true; } const { token } = findMatchingRule(authUrl.toString(), { hostType: this.hostType, readOnly, }); opts.token = token; } const accept = constructAcceptString(opts.headers?.accept); opts.headers = { ...opts.headers, accept, }; } protected override handleError( url: string | URL, opts: HttpOptions, err: GotLegacyError, ): never { throw handleGotError(err, url, opts); } protected override async requestJsonUnsafe( method: HttpMethod, options: InternalJsonUnsafeOptions, ): Promise> { const httpOptions = coerceObject(options.httpOptions); const resolvedUrl = this.resolveUrl(options.url, httpOptions); const opts = { ...options, url: resolvedUrl, }; const result = await super.requestJsonUnsafe(method, opts); if (httpOptions.paginate) { delete httpOptions.cacheProvider; httpOptions.memCache = false; // Check if result is paginated const pageLimit = httpOptions.pageLimit ?? 10; const linkHeader = parseLinkHeader(result?.headers?.link); const next = linkHeader?.next; const env = getEnv(); if (next?.url) { const baseUrl = httpOptions.baseUrl ?? this.baseUrl; const rebasePaginationLinks = !!env.RENOVATE_X_REBASE_PAGINATION_LINKS; const firstPageUrl = resolvePaginationUrl( next.url, baseUrl, rebasePaginationLinks, ); // Don't follow a cross-origin request, unless we've been explicitly requested to do so with `RENOVATE_X_REBASE_PAGINATION_LINKS` if (firstPageUrl.origin === resolvedUrl.origin) { let pages: HttpResponse[]; if (linkHeader?.last?.page) { logger.debug('Using GitHub offset-based pagination'); let lastPage = parseInt(linkHeader.last.page, 10); // v8 ignore else -- TODO: add test #40625 if (!env.RENOVATE_PAGINATE_ALL && httpOptions.paginate !== 'all') { lastPage = Math.min(pageLimit, lastPage); } const queue = [...range(2, lastPage)].map( (pageNumber) => (): Promise> => { // copy before modifying searchParams const nextUrl = parseUrl(firstPageUrl.toString())!; nextUrl.searchParams.set('page', String(pageNumber)); return super.requestJsonUnsafe(method, { ...opts, url: nextUrl, }); }, ); pages = await p.all(queue); } else { logger.debug('Using GitHub cursor-based pagination'); pages = []; const paginateAll = !!env.RENOVATE_PAGINATE_ALL || httpOptions.paginate === 'all'; const cursorPageLimit = paginateAll ? MAX_PAGINATION_PAGES : pageLimit; let nextUrl: URL | null = firstPageUrl; let pageNumber = 2; for (; nextUrl && pageNumber <= cursorPageLimit; pageNumber += 1) { if (nextUrl.origin !== resolvedUrl.origin) { logger.once.warn( { requestOrigin: resolvedUrl.origin, paginationOrigin: nextUrl.origin, }, 'Ignoring cross-origin GitHub pagination link. Set RENOVATE_X_REBASE_PAGINATION_LINKS if this is a self-hosted instance that returns a different origin in pagination links.', ); break; } const nextPage: HttpResponse = await super.requestJsonUnsafe(method, { ...opts, url: nextUrl, }); pages.push(nextPage); const nextLink = parseLinkHeader(nextPage.headers.link)?.next; nextUrl = nextLink?.url ? resolvePaginationUrl( nextLink.url, baseUrl, rebasePaginationLinks, ) : null; } if (paginateAll && nextUrl && pageNumber > cursorPageLimit) { logger.warn( { maxPages: MAX_PAGINATION_PAGES }, 'GitHub cursor pagination limit reached', ); } } // v8 ignore else -- TODO: add test #40625 if (httpOptions.paginationField && isPlainObject(result.body)) { const paginatedResult = result.body[httpOptions.paginationField]; // v8 ignore else -- TODO: add test #40625 if (isArray(paginatedResult)) { for (const nextPage of pages) { // v8 ignore else -- TODO: add test #40625 if (isPlainObject(nextPage.body)) { const nextPageResults = nextPage.body[httpOptions.paginationField]; // v8 ignore else -- TODO: add test #40625 if (isArray(nextPageResults)) { paginatedResult.push(...nextPageResults); } } } } } else if (isArray(result.body)) { for (const nextPage of pages) { // v8 ignore else -- TODO: add test #40625 if (isArray(nextPage.body)) { result.body.push(...nextPage.body); } } } } else { // make sure that users are aware if there are any (potentially malicious, or misconfigured) pagination links being returned logger.once.warn( { requestOrigin: resolvedUrl.origin, paginationOrigin: firstPageUrl.origin, }, 'Ignoring cross-origin GitHub pagination link. Set RENOVATE_X_REBASE_PAGINATION_LINKS if this is a self-hosted instance that returns a different origin in pagination links.', ); } } } return result; } public async requestGraphql( query: string, options: GraphqlOptions = {}, ): Promise | null> { const path = 'graphql'; const { paginate, count = MAX_GRAPHQL_PAGE_SIZE, cursor = null } = options; let { variables } = options; if (paginate) { variables = { ...variables, count, cursor, }; } const body = variables ? { query, variables } : { query }; const opts: GithubBaseHttpOptions = { baseUrl: baseUrl.replace('/v3/', '/'), // GHE uses unversioned graphql path body, headers: { accept: options?.acceptHeader }, readOnly: options.readOnly, }; if (options.token) { opts.token = options.token; } logger.trace(`Performing Github GraphQL request`); let response: GithubGraphqlResponse | null; try { const res = await this.postJson>(path, opts); response = res.body; } catch (err) { logger.debug({ err, query, options }, 'Unexpected GraphQL Error'); if (err instanceof ExternalHostError && count && count > 10) { logger.info('Reducing pagination count to workaround graphql errors'); return null; } throw handleGotError(err, path, opts); } if (isGraphqlRateLimited(response?.errors)) { throw new Error(PLATFORM_RATE_LIMIT_EXCEEDED); } return response; } async queryRepoField>( query: string, fieldName: string, options: GraphqlOptions = {}, ): Promise { const result: T[] = []; const { paginate = true } = options; let optimalCount: null | number = null; let count = getGraphqlPageSize( fieldName, options.count ?? MAX_GRAPHQL_PAGE_SIZE, ); let limit = options.limit ?? 1000; let cursor: string | null = null; let isIterating = true; while (isIterating) { const res = await this.requestGraphql>(query, { ...options, count: Math.min(count, limit), cursor, paginate, }); const repositoryData = res?.data?.repository; if ( isNonEmptyObject(repositoryData) && !isNullOrUndefined(repositoryData[fieldName]) ) { optimalCount = count; const { nodes = [], edges = [], pageInfo, } = repositoryData[fieldName] as GraphqlPaginatedContent; result.push(...nodes); result.push(...edges); limit = Math.max(0, limit - nodes.length - edges.length); if (limit === 0) { isIterating = false; } else if (paginate && pageInfo) { const { hasNextPage, endCursor } = pageInfo; if (hasNextPage && endCursor) { cursor = endCursor; } else { isIterating = false; } } } else { count = Math.floor(count / 2); if (count === 0) { logger.warn({ query, options, res }, 'Error fetching GraphQL nodes'); isIterating = false; } } if (!paginate) { isIterating = false; } } if (optimalCount && optimalCount < MAX_GRAPHQL_PAGE_SIZE) { setGraphqlPageSize(fieldName, optimalCount); } return result; } /** * Get the raw text file from a URL. * Only use this method to fetch text files. * * @param url Full API URL, contents path or path inside the repository to the file * @param options * * @example url = 'https://api.github.com/repos/renovatebot/renovate/contents/package.json' * @example url = 'renovatebot/renovate/contents/package.json' * @example url = 'package.json' & options.repository = 'renovatebot/renovate' */ public async getRawTextFile( url: string, options: InternalHttpOptions & GithubBaseHttpOptions = {}, ): Promise { const newOptions: InternalHttpOptions & GithubBaseHttpOptions = { ...options, headers: { accept: 'application/vnd.github.raw+json', }, }; let newURL = url; const httpRegex = regEx(/^https?:\/\//); if (options.repository && !httpRegex.test(options.repository)) { newURL = joinUrlParts(options.repository, 'contents', url); } return await this.getText(newURL, newOptions); } }