import { ATTR_CODE_FUNCTION_NAME } from '@opentelemetry/semantic-conventions'; import { isFunction, isNonEmptyArray, isString, isTruthy, } from '@sindresorhus/is'; import { dequal } from 'dequal'; import { GlobalConfig } from '../../config/global.ts'; import { HOST_BLOCKED, HOST_DISABLED } from '../../constants/error-messages.ts'; import { instrument } from '../../instrumentation/index.ts'; import { ATTR_RENOVATE_DATASOURCE, ATTR_RENOVATE_PACKAGE_NAME, ATTR_RENOVATE_REGISTRY_URL, } from '../../instrumentation/types.ts'; import { logger } from '../../logger/index.ts'; import { ExternalHostError } from '../../types/errors/external-host-error.ts'; import { coerceArray } from '../../util/array.ts'; import * as memCache from '../../util/cache/memory/index.ts'; import * as packageCache from '../../util/cache/package/index.ts'; import type { PackageCacheNamespace } from '../../util/cache/package/types.ts'; import { clone } from '../../util/clone.ts'; import { filterMap } from '../../util/filter-map.ts'; import { AsyncResult, Result } from '../../util/result.ts'; import { DatasourceCacheStats } from '../../util/stats.ts'; import { safeStringify } from '../../util/stringify.ts'; import { trimTrailingSlash } from '../../util/url.ts'; import * as versioning from '../versioning/index.ts'; import datasources from './api.ts'; import { applyConstraintsFiltering, applyExtractVersion, applyVersionCompatibility, filterValidVersions, getDatasourceFor, sortAndRemoveDuplicates, } from './common.ts'; import { addMetaData } from './metadata.ts'; import { setNpmrc } from './npm/index.ts'; import { resolveRegistryUrl } from './npm/npmrc.ts'; import type { DatasourceApi, DigestConfig, GetDigestInputConfig, GetPkgReleasesConfig, GetReleasesConfig, ReleaseResult, } from './types.ts'; export { isGetPkgReleasesConfig } from './common.ts'; export * from './types.ts'; export function getDatasources(): Map { return datasources; } export function getDatasourceList(): string[] { return Array.from(datasources.keys()); } type GetReleasesInternalConfig = GetReleasesConfig & GetPkgReleasesConfig; // TODO: fix error Type function logError(datasource: string, packageName: string, err: any): void { const { statusCode, code: errCode, url } = err; if (statusCode === 404) { logger.debug({ datasource, packageName, url }, 'Datasource 404'); } else if (statusCode === 401 || statusCode === 403) { logger.debug({ datasource, packageName, url }, 'Datasource unauthorized'); } else if (errCode) { logger.debug( { datasource, packageName, url, errCode }, 'Datasource connection error', ); } else { logger.debug({ datasource, packageName, err }, 'Datasource unknown error'); } } async function getRegistryReleases( datasource: DatasourceApi, config: GetReleasesConfig, registryUrl: string, ): Promise { const cacheNamespace: PackageCacheNamespace = `datasource-releases-${datasource.id}`; const cacheKey = `${registryUrl}:${config.packageName}`; const cacheEnabled = !!datasource.caching; // tells if `isPrivate` flag is supported in datasource result const cacheForced = GlobalConfig.get('cachePrivatePackages'); // tells if caching is forced via admin config if (cacheEnabled || cacheForced) { const cachedResult = await packageCache.get( cacheNamespace, cacheKey, ); if (cachedResult) { logger.trace({ cacheKey }, 'Returning cached datasource response'); DatasourceCacheStats.hit(datasource.id, registryUrl, config.packageName); return cachedResult; } DatasourceCacheStats.miss(datasource.id, registryUrl, config.packageName); } const res = await instrument( 'getReleases', () => datasource.getReleases({ ...config, registryUrl }), { attributes: { [ATTR_CODE_FUNCTION_NAME]: 'getReleases', [ATTR_RENOVATE_DATASOURCE]: datasource.id, [ATTR_RENOVATE_REGISTRY_URL]: registryUrl, [ATTR_RENOVATE_PACKAGE_NAME]: config.packageName, }, }, ); if (res?.releases.length) { res.registryUrl ??= registryUrl; } if (!res) { return null; } let cache = false; if (cacheForced) { cache = true; } else if (cacheEnabled && !res.isPrivate) { cache = true; } if (cache) { logger.trace({ cacheKey }, 'Caching datasource response'); await packageCache.set(cacheNamespace, cacheKey, res, 15); DatasourceCacheStats.set(datasource.id, registryUrl, config.packageName); } else { DatasourceCacheStats.skip(datasource.id, registryUrl, config.packageName); } return res; } function firstRegistry( config: GetReleasesInternalConfig, datasource: DatasourceApi, registryUrls: string[], ): Promise { if (registryUrls.length > 1) { logger.warn( { datasource: datasource.id, packageName: config.packageName, registryUrls, }, 'Excess registryUrls found for datasource lookup - using first configured only', ); } const registryUrl = registryUrls[0]; return getRegistryReleases(datasource, config, registryUrl); } async function huntRegistries( config: GetReleasesInternalConfig, datasource: DatasourceApi, registryUrls: string[], ): Promise { let res: ReleaseResult | null = null; let caughtError: Error | undefined; for (const registryUrl of registryUrls) { try { res = await getRegistryReleases(datasource, config, registryUrl); if (res) { break; } } catch (err) { if (err instanceof ExternalHostError) { throw err; } // We'll always save the last-thrown error caughtError = err; logger.trace({ err }, 'datasource hunt failure'); } } if (res) { return res; } if (caughtError) { throw caughtError; } return null; } async function mergeRegistries( config: GetReleasesInternalConfig, datasource: DatasourceApi, registryUrls: string[], ): Promise { let combinedRes: ReleaseResult | undefined; let lastErr: Error | undefined; let externalHostError: ExternalHostError | undefined; let singleRegistry = true; const releaseVersioning = versioning.get(config.versioning); for (const registryUrl of registryUrls) { try { // Merging must not mutate responses shared by the package cache. const res = clone( await getRegistryReleases(datasource, config, registryUrl), ); if (!res) { continue; } if (!combinedRes) { // This is the first registry, so we can just use it and continue combinedRes = res; continue; } if (singleRegistry) { // This is the second registry // We need to move the registryUrl from the package level to the release level for (const release of coerceArray(combinedRes.releases)) { release.registryUrl ??= combinedRes.registryUrl; } singleRegistry = false; } const releases = coerceArray(res.releases); for (const release of releases) { // We have more than one registry, so we need to move the registryUrl // from the package level to the release level before merging release.registryUrl ??= res.registryUrl; } combinedRes.releases.push(...releases); // Merge the tags from the two results let tags = combinedRes.tags; if (tags) { // v8 ignore else -- needs merged registries where only one carries tags if (res.tags) { // Both results had tags, so we need to compare them for (const tag of ['release', 'latest']) { const existingTag = combinedRes?.tags?.[tag]; const newTag = res.tags?.[tag]; // v8 ignore else -- needs a merged registry whose tag is not a version if (isString(newTag) && releaseVersioning.isVersion(newTag)) { if ( isString(existingTag) && releaseVersioning.isVersion(existingTag) ) { // We need to compare them if (releaseVersioning.isGreaterThan(newTag, existingTag)) { // New tag is greater than the existing one tags[tag] = newTag; } } else { // Existing tag was not present or not a version // so we can just use the new one tags[tag] = newTag; } } } } } else { // Existing results had no tags, so we can just use the new ones tags = res.tags; } combinedRes = { ...res, ...combinedRes }; if (tags) { combinedRes.tags = tags; } // Remove the registryUrl from the package level when more than one registry delete combinedRes.registryUrl; } catch (err) { if (err instanceof ExternalHostError) { // Don't abort the merge if another registry already returned releases; // a single rate-limited registry shouldn't discard results we have externalHostError = err; logger.debug( { err, registryUrl }, 'datasource merge: external host error from registry; continuing so releases from other registries are not discarded', ); continue; } lastErr = err; logger.trace({ err }, 'datasource merge failure'); } } if (!combinedRes) { if (externalHostError) { throw externalHostError; } if (lastErr) { throw lastErr; } return null; } const seenVersions = new Set(); combinedRes.releases = filterMap(combinedRes.releases, (release) => { if (seenVersions.has(release.version)) { return null; } seenVersions.add(release.version); return release; }); return combinedRes; } function massageRegistryUrls(registryUrls: string[]): string[] { return registryUrls.filter(isTruthy).map(trimTrailingSlash); } function resolveRegistryUrls( datasource: DatasourceApi, defaultRegistryUrls: string[] | undefined, registryUrls: string[] | undefined | null, additionalRegistryUrls: string[] | undefined, ): string[] { if (!datasource.customRegistrySupport) { if ( isNonEmptyArray(registryUrls) || isNonEmptyArray(defaultRegistryUrls) || isNonEmptyArray(additionalRegistryUrls) ) { logger.warn( { datasource: datasource.id, registryUrls, defaultRegistryUrls, additionalRegistryUrls, }, 'Custom registries are not allowed for this datasource and will be ignored', ); } return isFunction(datasource.defaultRegistryUrls) ? datasource.defaultRegistryUrls() : coerceArray(datasource.defaultRegistryUrls); } const customUrls = registryUrls?.filter(isTruthy); let resolvedUrls: string[] = []; if (isNonEmptyArray(customUrls)) { resolvedUrls = [...customUrls]; } else if (isNonEmptyArray(defaultRegistryUrls)) { resolvedUrls = [...defaultRegistryUrls]; resolvedUrls = resolvedUrls.concat(coerceArray(additionalRegistryUrls)); } else if (isFunction(datasource.defaultRegistryUrls)) { resolvedUrls = [...datasource.defaultRegistryUrls()]; resolvedUrls = resolvedUrls.concat(coerceArray(additionalRegistryUrls)); } else if (isNonEmptyArray(datasource.defaultRegistryUrls)) { resolvedUrls = [...datasource.defaultRegistryUrls]; resolvedUrls = resolvedUrls.concat(coerceArray(additionalRegistryUrls)); } return massageRegistryUrls(resolvedUrls); } function applyReplacements( config: GetReleasesInternalConfig, ): Pick | undefined { if (config.replacementName && config.replacementVersion) { return { replacementName: config.replacementName, replacementVersion: config.replacementVersion, }; } return undefined; } async function fetchReleases( config: GetReleasesInternalConfig, ): Promise { const { datasource: datasourceName } = config; let { registryUrls } = config; // istanbul ignore if: need test if (!datasourceName || getDatasourceFor(datasourceName) === undefined) { logger.warn({ datasource: datasourceName }, 'Unknown datasource'); return null; } if (datasourceName === 'npm') { if (isString(config.npmrc)) { setNpmrc(config.npmrc); } // v8 ignore else -- npm lookups here never arrive with explicit registryUrls if (!isNonEmptyArray(registryUrls)) { registryUrls = [resolveRegistryUrl(config.packageName)]; } } const datasource = getDatasourceFor(datasourceName); // istanbul ignore if: needs test if (!datasource) { logger.warn({ datasource: datasourceName }, 'Unknown datasource'); return null; } registryUrls = resolveRegistryUrls( datasource, config.defaultRegistryUrls, registryUrls, config.additionalRegistryUrls, ); let dep: ReleaseResult | null = null; const registryStrategy = config.registryStrategy ?? datasource.registryStrategy ?? 'hunt'; try { if (isNonEmptyArray(registryUrls)) { if (registryStrategy === 'first') { dep = await firstRegistry(config, datasource, registryUrls); } else if (registryStrategy === 'hunt') { dep = await huntRegistries(config, datasource, registryUrls); // NOTE: the strategy is always one of these three, so the implicit // final else is unreachable. A coverage-ignore hint cannot suppress it // on an `else if`, so it stays in the branch count. } else if (registryStrategy === 'merge') { dep = await mergeRegistries(config, datasource, registryUrls); } } else { dep = await instrument( 'getReleases', () => datasource.getReleases(config), { attributes: { [ATTR_CODE_FUNCTION_NAME]: 'getReleases', [ATTR_RENOVATE_DATASOURCE]: datasource.id, [ATTR_RENOVATE_REGISTRY_URL]: config.registryUrl ?? '', [ATTR_RENOVATE_PACKAGE_NAME]: config.packageName, }, }, ); } } catch (err) { if ( [HOST_BLOCKED, HOST_DISABLED].includes(err.message) || [HOST_BLOCKED, HOST_DISABLED].includes(err.err?.message) ) { return null; } if (err instanceof ExternalHostError) { throw err; } logError(datasource.id, config.packageName, err); } if (!dep || dequal(dep, { releases: [] })) { return null; } addMetaData(dep, datasourceName, config.packageName); dep = { ...dep, ...applyReplacements(config) }; return dep; } function fetchCachedReleases( config: GetReleasesInternalConfig, ): Promise { const { datasource, packageName, registryUrls } = config; const cacheKey = `datasource-mem:releases:${datasource}:${packageName}:${config.registryStrategy}:${safeStringify( [registryUrls, config.defaultRegistryUrls, config.additionalRegistryUrls], )}`; // By returning a Promise and reusing it, we should only fetch each package at most once const cachedResult = memCache.get>(cacheKey); // istanbul ignore if if (cachedResult !== undefined) { return cachedResult; } const promisedRes = fetchReleases(config); memCache.set(cacheKey, promisedRes); return promisedRes; } export function getRawPkgReleases( config: GetPkgReleasesConfig, ): AsyncResult< ReleaseResult, Error | 'no-datasource' | 'no-package-name' | 'no-result' > { if (!config.datasource) { logger.warn('No datasource found'); return AsyncResult.err('no-datasource'); } const packageName = config.packageName; if (!packageName) { logger.error({ config }, 'Datasource getReleases without packageName'); return AsyncResult.err('no-package-name'); } return Result.wrapNullable(fetchCachedReleases(config), 'no-result' as const) .catch((e) => { if (e instanceof ExternalHostError) { e.hostType = config.datasource; e.packageName = packageName; } return Result.err(e); }) .transform(clone); } export function applyDatasourceFilters( releaseResult: ReleaseResult, config: GetPkgReleasesConfig, ): ReleaseResult { let res = releaseResult; res = applyExtractVersion(res, config.extractVersion); res = applyVersionCompatibility( res, config.versionCompatibility, config.currentCompatibility, ); res = filterValidVersions(res, config); res = sortAndRemoveDuplicates(res, config); res = applyConstraintsFiltering(res, config); return res; } export async function getPkgReleases( config: GetPkgReleasesConfig, ): Promise { const { val = null, err } = await getRawPkgReleases(config) .transform((res) => applyDatasourceFilters(res, config)) .unwrap(); if (err instanceof Error) { throw err; } return val; } export function supportsDigests(datasource: string | undefined): boolean { const ds = !!datasource && getDatasourceFor(datasource); return !!ds && 'getDigest' in ds; } function getDigestConfig( datasource: DatasourceApi, config: GetDigestInputConfig, ): DigestConfig { const { lookupName, currentValue, currentDigest } = config; const packageName = config.replacementName ?? config.packageName; // Prefer registryUrl from getReleases() lookup if it has been passed const registryUrl = config.registryUrl ?? resolveRegistryUrls( datasource, config.defaultRegistryUrls, config.registryUrls, config.additionalRegistryUrls, )[0]; return { lookupName, packageName, registryUrl, currentValue, currentDigest }; } export function getDigest( config: GetDigestInputConfig, value?: string, ): Promise { const datasource = getDatasourceFor(config.datasource); // istanbul ignore if: need test if (!datasource || !('getDigest' in datasource)) { return Promise.resolve(null); } const digestConfig = getDigestConfig(datasource, config); return datasource.getDigest!(digestConfig, value); } export function getDefaultConfig( datasource: string, ): Promise> { const loadedDatasource = getDatasourceFor(datasource); return Promise.resolve>( loadedDatasource?.defaultConfig ?? Object.create({}), ); }