import { isPlainObject, isUndefined } from '@sindresorhus/is'; import merge from 'deepmerge'; import type { Options, OptionsInit, RetryObject } from 'got'; import type { SetRequired } from 'type-fest'; import type { z } from 'zod/v4'; import { ZodType } from 'zod/v4'; import { GlobalConfig } from '../../config/global.ts'; import { HOST_DISABLED } from '../../constants/error-messages.ts'; import { logger } from '../../logger/index.ts'; import { ExternalHostError } from '../../types/errors/external-host-error.ts'; import type { CombinedHostRule } from '../../types/index.ts'; import { coerceArray } from '../array.ts'; import * as memCache from '../cache/memory/index.ts'; import { getEnv } from '../env.ts'; import { hash } from '../hash.ts'; import { acquireLock } from '../mutex.ts'; import { coerceObject } from '../object.ts'; import { type AsyncResult, Result } from '../result.ts'; import { Toml } from '../schema-utils/index.ts'; import { ObsoleteCacheHitLogger } from '../stats.ts'; import { compile } from '../template/index.ts'; import { isHttpUrl, parseUrl, resolveBaseUrl } from '../url.ts'; import { parseSingleYaml } from '../yaml.ts'; import { applyAuthorization } from './auth.ts'; import type { HttpCacheProvider } from './cache/types.ts'; import { fetch, normalize, stream } from './got.ts'; import { applyHostGuard } from './host-guard.ts'; import { applyHostRule, findMatchingRule } from './host-rules.ts'; import { getQueue } from './queue.ts'; import { getRetryAfter, wrapWithRetry } from './retry-after.ts'; import { getThrottle } from './throttle.ts'; import type { GotOptions, GotTask, HttpMethod, HttpOptions, HttpResponse, SafeJsonError, } from './types.ts'; import { copyResponse } from './util.ts'; export interface InternalJsonUnsafeOptions< Opts extends HttpOptions = HttpOptions, > { url: string | URL; httpOptions?: Opts; } export interface InternalJsonOptions< Opts extends HttpOptions, ResT = unknown, Schema extends ZodType = ZodType, > extends InternalJsonUnsafeOptions { schema?: Schema; } export type InternalGotOptions = SetRequired; export interface InternalHttpOptions extends HttpOptions { json?: HttpOptions['body']; method?: HttpMethod; parseJson?: Options['parseJson']; } export function applyDefaultHeaders(options: OptionsInit): void { const userAgentTemplate = GlobalConfig.get('userAgent'); options.headers = { ...options.headers, 'user-agent': compile(userAgentTemplate, {}), }; } export abstract class HttpBase< JSONOpts extends HttpOptions = HttpOptions, Opts extends HttpOptions = HttpOptions, > { private readonly options: InternalGotOptions; protected get baseUrl(): string | undefined { return undefined; } protected hostType: string; constructor(hostType: string, options: HttpOptions = {}) { this.hostType = hostType; const retryLimit = getEnv().NODE_ENV === 'test' ? 0 : 2; this.options = merge( options, { method: 'get', context: { hostType }, retry: { calculateDelay: (retryObject) => this.calculateRetryDelay(retryObject), limit: retryLimit, maxRetryAfter: 0, // Don't rely on `got` retry-after handling, just let it fail and then we'll handle it enforceRetryRules: false, }, }, { isMergeableObject: isPlainObject }, ); } /** * Builds the final got options for a request to a given `url` * * This merges instance options, applies default headers, host rules and authorization, and rejects requests to disabled hosts. * * Both the `request()` and `stream()` paths must use this single entry point so that policy applied here covers every outbound request. */ private prepareOptions( resolvedUrl: URL, httpOptions: InternalHttpOptions, ): { options: InternalGotOptions & InternalHttpOptions; hostRule: CombinedHostRule; } { const url = resolvedUrl.toString(); let options = merge( { ...this.options, hostType: this.hostType, }, httpOptions, { isMergeableObject: isPlainObject }, ); applyDefaultHeaders(options); if ( isUndefined(options.readOnly) && ['head', 'get'].includes(options.method.toLowerCase()) ) { options.readOnly = true; } const hostRule = findMatchingRule(url, options); options = applyHostRule(url, options, hostRule); if (options.enabled === false) { logger.debug(`Host is disabled - rejecting request. HostUrl: ${url}`); throw new Error(HOST_DISABLED); } options = applyAuthorization(options); // enforced here so it runs before anything else - including cache lookups - can act on the URL, and for the stream path too const guard = applyHostGuard( resolvedUrl, options.hostType, hostRule.internalHostGrant, options.responseBecomesConfig, ); options.dnsLookup = guard.dnsLookup; options.hooks = { ...options.hooks, beforeRedirect: [ ...coerceArray(options.hooks?.beforeRedirect), guard.beforeRedirect, ], }; if (guard.beforeRequest) { options.hooks.beforeRequest = [ ...coerceArray(options.hooks.beforeRequest), guard.beforeRequest, ]; } return { options, hostRule }; } private async request( requestUrl: string | URL, httpOptions: InternalHttpOptions, ): Promise; private async request( requestUrl: string | URL, httpOptions: InternalHttpOptions & { responseType: 'text' }, ): Promise; private async request( requestUrl: string | URL, httpOptions: InternalHttpOptions & { responseType: 'buffer' }, ): Promise>; private async request( requestUrl: string | URL, httpOptions: InternalHttpOptions & { responseType: 'json' }, ): Promise>; private async request( requestUrl: string | URL, httpOptions: InternalHttpOptions, ): Promise> { const resolvedUrl = this.resolveUrl(requestUrl, httpOptions); const url = resolvedUrl.toString(); this.processOptions(resolvedUrl, httpOptions); const { options, hostRule } = this.prepareOptions(resolvedUrl, httpOptions); const method = options.method.toLowerCase(); const isReadMethod = ['head', 'get'].includes(method); logger.trace(`HTTP request: ${method.toUpperCase()} ${url}`); const timeout = options.timeout ?? 60000; options.timeout = timeout; let cacheProvider: HttpCacheProvider | undefined; if (isReadMethod && options.cacheProvider) { cacheProvider = options.cacheProvider; } const memCacheKey = !process.env.RENOVATE_X_DISABLE_HTTP_MEMCACHE && !cacheProvider && options.memCache !== false && isReadMethod ? hash( `got-${JSON.stringify({ url, headers: options.headers, method, })}`, ) : null; let resPromise: Promise> | null = null; // Cache GET requests unless memCache=false if (memCacheKey) { resPromise = memCache.get(memCacheKey); /* v8 ignore next: temporary code */ if (resPromise && !cacheProvider) { ObsoleteCacheHitLogger.write(url); } } // v8 ignore else -- TODO: add test #40625 if (!resPromise) { if (cacheProvider) { await cacheProvider.setCacheHeaders(method, url, options); } const startTime = Date.now(); const httpTask: GotTask = async () => { let releaseLock: undefined | (() => void); if (isReadMethod) { releaseLock = await acquireLock( `${options.method} ${url}`, 'http-mutex', timeout * 2, ); } try { const cachedResponse = await cacheProvider?.bypassServer( options.method, url, ); if (cachedResponse) { return cachedResponse; } const queueMs = Date.now() - startTime; return fetch(url, this._normalizeOptions(options), { queueMs, }); } finally { releaseLock?.(); } }; const throttle = getThrottle(url); const throttledTask = throttle ? () => throttle.add(httpTask) : httpTask; const queue = getQueue(url); const queuedTask = queue ? () => queue.add(throttledTask) : throttledTask; const { maxRetryAfter = 60 } = hostRule; resPromise = wrapWithRetry(queuedTask, url, getRetryAfter, maxRetryAfter); if (memCacheKey) { memCache.set(memCacheKey, resPromise); } } try { const res = await resPromise; const deepCopyNeeded = !!memCacheKey && res.statusCode !== 304; const resCopy = copyResponse(res, deepCopyNeeded); resCopy.authorization = !!options?.headers?.authorization; this.handleResponse(resolvedUrl, resCopy); if (cacheProvider) { return await cacheProvider.wrapServerResponse(method, url, resCopy); } return resCopy; } catch (err) { const { abortOnError, abortIgnoreStatusCodes } = options; if (abortOnError && !abortIgnoreStatusCodes?.includes(err.statusCode)) { throw new ExternalHostError(err); } const staleResponse = await cacheProvider?.bypassServer< string | Uint8Array >(method, url, true); if (staleResponse) { logger.debug( { err }, `Request error: returning stale cache instead for ${url}`, ); return staleResponse; } this.handleError(requestUrl, httpOptions, err); } } private _normalizeOptions(options: T): T { return normalize(options, this.extraOptions()); } /** * Returns Renovate extra options which needs to be removed before passing to got. * @returns extra Renovate options. */ protected extraOptions(): readonly string[] { return [ 'baseUrl', 'cacheProvider', 'readOnly', 'responseBecomesConfig', ] as (keyof HttpOptions)[]; } protected processOptions(_url: URL, _options: InternalHttpOptions): void { // noop } protected handleError( _url: string | URL, _httpOptions: HttpOptions, err: Error, ): never { throw err; } protected handleResponse(_url: URL, _res: HttpResponse): void { // noop } resolveUrl(requestUrl: string | URL, options?: HttpOptions): URL { let url = requestUrl; if (url instanceof URL) { // already a aboslute URL return url; } const baseUrl = options?.baseUrl ?? this.baseUrl; if (baseUrl) { url = resolveBaseUrl(baseUrl, url); } const parsedUrl = parseUrl(url); if (!parsedUrl || !isHttpUrl(parsedUrl)) { logger.error( { url: requestUrl, baseUrl, resolvedUrl: url }, 'Request Error: cannot parse url', ); throw new Error('Invalid URL'); } return parsedUrl; } protected calculateRetryDelay({ computedValue }: RetryObject): number { return computedValue; } get( url: string, options: HttpOptions = {}, ): Promise> { return this.request(url, options); } head(url: string, options: HttpOptions = {}): Promise> { // to complex to validate return this.request(url, { ...options, responseType: 'text', method: 'head', }) as Promise>; } getText(url: string | URL, options: HttpOptions = {}): Promise { return this.request(url, { ...options, responseType: 'text' }); } getBuffer( url: string | URL, options: HttpOptions = {}, ): Promise> { return this.request(url, { ...options, responseType: 'buffer' }); } protected requestJsonUnsafe( method: HttpMethod, { url, httpOptions: requestOptions }: InternalJsonUnsafeOptions, ): Promise> { const { body: json, ...httpOptions } = { ...requestOptions }; const opts: InternalHttpOptions = { ...httpOptions, method, }; // signal that we expect a json response opts.headers = { accept: 'application/json', ...opts.headers, }; if (json) { opts.json = json; } return this.request(url, { ...opts, responseType: 'json' }); } private async requestJson< ResT, Schema extends ZodType = ZodType, >( method: HttpMethod, options: InternalJsonOptions, ): Promise> { const res = await this.requestJsonUnsafe(method, options); if (options.schema) { res.body = await options.schema.parseAsync(res.body); } return res; } private resolveArgs( arg1: string, arg2: JSONOpts | ZodType | undefined, arg3: ZodType | undefined, ): InternalJsonOptions { const res: InternalJsonOptions = { url: arg1 }; if (arg2 instanceof ZodType) { res.schema = arg2; } else if (arg2) { res.httpOptions = arg2; } if (arg3) { res.schema = arg3; } return res; } async getPlain(url: string, options?: Opts): Promise { const opt = coerceObject(options); return await this.getText(url, { headers: { Accept: 'text/plain', }, ...opt, }); } /** * @deprecated use `getYaml` instead */ async getYamlUnchecked( url: string, options?: Opts, ): Promise> { const res = await this.getText(url, options); const body = parseSingleYaml(res.body); return { ...res, body }; } async getYaml>( url: string, schema: Schema, ): Promise>>; async getYaml>( url: string, options: Opts, schema: Schema, ): Promise>>; async getYaml>( arg1: string, arg2?: Opts | Schema, arg3?: Schema, ): Promise>> { const url = arg1; let schema: Schema; let httpOptions: Opts | undefined; if (arg3) { schema = arg3; httpOptions = arg2 as Opts; } else { schema = arg2 as Schema; } const opts: InternalHttpOptions = { ...httpOptions, method: 'get', }; const res = await this.getText(url, opts); const body = await schema.parseAsync(parseSingleYaml(res.body)); return { ...res, body }; } getYamlSafe< ResT extends NonNullable, Schema extends ZodType = ZodType, >(url: string, schema: Schema): AsyncResult, SafeJsonError>; getYamlSafe< ResT extends NonNullable, Schema extends ZodType = ZodType, >( url: string, options: Opts, schema: Schema, ): AsyncResult, SafeJsonError>; getYamlSafe< ResT extends NonNullable, Schema extends ZodType = ZodType, >( arg1: string, arg2: Opts | Schema, arg3?: Schema, ): AsyncResult { const url = arg1; let schema: Schema; let httpOptions: Opts | undefined; if (arg3) { schema = arg3; httpOptions = arg2 as Opts; } else { schema = arg2 as Schema; } let res: AsyncResult, SafeJsonError>; if (httpOptions) { res = Result.wrap(this.getYaml(url, httpOptions, schema)); } else { res = Result.wrap(this.getYaml(url, schema)); } return res.transform((response) => Result.ok(response.body)); } /** * Request JSON and return the response without any validation. * * The usage of this method is discouraged, please use `getJson` instead. * * If you're new to Zod schema validation library: * - consult the [documentation of Zod library](https://github.com/colinhacks/zod?tab=readme-ov-file#basic-usage) * - search the Renovate codebase for 'zod' module usage * - take a look at the `schema-utils.ts` file for Renovate-specific schemas and utilities */ getJsonUnchecked( url: string, options?: JSONOpts, ): Promise> { return this.requestJson('get', { url, httpOptions: options }); } /** * Request JSON with a Zod schema for the response, * throwing an error if the response is not valid. * * @param url * @param schema Zod schema for the response */ getJson>( url: string, schema: Schema, ): Promise>>; getJson>( url: string, options: JSONOpts, schema: Schema, ): Promise>>; getJson>( arg1: string, arg2?: JSONOpts | Schema, arg3?: Schema, ): Promise>> { const args = this.resolveArgs>(arg1, arg2, arg3); return this.requestJson>('get', args); } /** * Request JSON with a Zod schema for the response, * wrapping response data in a `Result` class. * * @param url * @param schema Zod schema for the response */ getJsonSafe, Schema extends ZodType>( url: string, schema: Schema, ): AsyncResult, SafeJsonError>; getJsonSafe, Schema extends ZodType>( url: string, options: JSONOpts, schema: Schema, ): AsyncResult, SafeJsonError>; getJsonSafe, Schema extends ZodType>( arg1: string, arg2?: JSONOpts | Schema, arg3?: Schema, ): AsyncResult { const args = this.resolveArgs(arg1, arg2, arg3); return Result.wrap(this.requestJson('get', args)).transform( (response) => Result.ok(response.body), ); } /** * @deprecated use `head` instead */ headJson(url: string, httpOptions?: JSONOpts): Promise> { return this.requestJson('head', { url, httpOptions }); } postJson(url: string, options?: JSONOpts): Promise>; postJson = ZodType>( url: string, schema: Schema, ): Promise>>; postJson = ZodType>( url: string, options: JSONOpts, schema: Schema, ): Promise>>; postJson = ZodType>( arg1: string, arg2?: JSONOpts | Schema, arg3?: Schema, ): Promise> { const args = this.resolveArgs(arg1, arg2, arg3); return this.requestJson('post', args); } putJson(url: string, options?: JSONOpts): Promise>; putJson = ZodType>( url: string, schema: Schema, ): Promise>>; putJson = ZodType>( url: string, options: JSONOpts, schema: Schema, ): Promise>>; putJson = ZodType>( arg1: string, arg2?: JSONOpts | Schema, arg3?: Schema, ): Promise> { const args = this.resolveArgs(arg1, arg2, arg3); return this.requestJson('put', args); } patchJson(url: string, options?: JSONOpts): Promise>; patchJson = ZodType>( url: string, schema: Schema, ): Promise>>; patchJson = ZodType>( url: string, options: JSONOpts, schema: Schema, ): Promise>>; patchJson = ZodType>( arg1: string, arg2?: JSONOpts | Schema, arg3?: Schema, ): Promise> { const args = this.resolveArgs(arg1, arg2, arg3); return this.requestJson('patch', args); } deleteJson(url: string, options?: JSONOpts): Promise>; deleteJson = ZodType>( url: string, schema: Schema, ): Promise>>; deleteJson = ZodType>( url: string, options: JSONOpts, schema: Schema, ): Promise>>; deleteJson = ZodType>( arg1: string, arg2?: JSONOpts | Schema, arg3?: Schema, ): Promise> { const args = this.resolveArgs(arg1, arg2, arg3); return this.requestJson('delete', args); } stream(url: string, options?: HttpOptions): NodeJS.ReadableStream { const resolvedUrl = this.resolveUrl(url, options); const { options: combinedOptions } = this.prepareOptions(resolvedUrl, { ...options, method: 'get', }); return stream( resolvedUrl.toString(), this._normalizeOptions(combinedOptions), ); } async getToml>( url: string, schema?: Schema, ): Promise>>; async getToml>( url: string, options: JSONOpts, schema: Schema, ): Promise>>; async getToml>( arg1: string, arg2?: JSONOpts | Schema, arg3?: Schema, ): Promise>> { const { url, schema, httpOptions } = this.resolveArgs>( arg1, arg2, arg3, ); const opts: InternalHttpOptions = { ...httpOptions, method: 'get', headers: { 'Content-Type': 'application/toml', ...httpOptions?.headers, }, }; const res = await this.getText(url, opts); if (schema) { res.body = await Toml.pipe(schema).parseAsync(res.body); } else { res.body = (await Toml.parseAsync(res.body)) as z.infer; } return res as HttpResponse>; } }