renovate/lib/modules/datasource/index.ts

import { ATTR_CODE_FUNCTION_NAME } from '@opentelemetry/semantic-conventions';
import { isNonEmptyArray, isString, isTruthy } from '@sindresorhus/is';
import { dequal } from 'dequal';
import { GlobalConfig } from '../../config/global.ts';
import { HOST_BLOCKED, HOST_DISABLED } from '../../constants/error-messages.ts';
import { instrument } from '../../instrumentation/index.ts';
import {
  ATTR_RENOVATE_DATASOURCE,
  ATTR_RENOVATE_PACKAGE_NAME,
  ATTR_RENOVATE_REGISTRY_URL,
} from '../../instrumentation/types.ts';
import { logger } from '../../logger/index.ts';
import { ExternalHostError } from '../../types/errors/external-host-error.ts';
import { coerceArray } from '../../util/array.ts';
import * as memCache from '../../util/cache/memory/index.ts';
import * as packageCache from '../../util/cache/package/index.ts';
import { buildCacheKey } from '../../util/cache/package/key.ts';
import type { PackageCacheNamespace } from '../../util/cache/package/types.ts';
import { clone } from '../../util/clone.ts';
import { filterMap } from '../../util/filter-map.ts';
import { detachUrlCredentials } from '../../util/host-rules.ts';
import { AsyncResult, Result } from '../../util/result.ts';
import { DatasourceCacheStats } from '../../util/stats.ts';
import { safeStringify } from '../../util/stringify.ts';
import { trimTrailingSlash, withoutUrlCredentials } from '../../util/url.ts';
import * as versioning from '../versioning/index.ts';
import datasources from './api.ts';
import {
  applyConstraintsFiltering,
  applyExtractVersion,
  applyVersionCompatibility,
  filterValidVersions,
  getDatasourceFor,
  getDatasourceHostType,
  sortAndRemoveDuplicates,
} from './common.ts';
import { addMetaData, stripMetadataUrlCredentials } from './metadata.ts';
import { setNpmrc } from './npm/index.ts';
import { resolveRegistryUrl } from './npm/npmrc.ts';
import type {
  DatasourceApi,
  DigestConfig,
  GetDigestInputConfig,
  GetPkgReleasesConfig,
  GetReleasesConfig,
  RegistryUrlsConfig,
  ReleaseResult,
} from './types.ts';

export { isGetPkgReleasesConfig } from './common.ts';
export * from './types.ts';

export function getDatasources(): Map<string, DatasourceApi> {
  return datasources;
}
export function getDatasourceList(): string[] {
  return Array.from(datasources.keys());
}

/**
 * Projects the registry-level config onto the fields a datasource
 * implementation is allowed to see.
 */
function toGetReleasesConfig(
  config: GetPkgReleasesConfig,
  registryUrl = config.registryUrl,
): GetReleasesConfig {
  const {
    customDatasources,
    datasource,
    packageName,
    currentValue,
    constraints,
    constraintsVersioning,
    constraintsFiltering,
  } = config;
  return {
    customDatasources,
    datasource,
    packageName,
    registryUrl,
    currentValue,
    constraints,
    constraintsVersioning,
    constraintsFiltering,
  };
}

// TODO: fix error Type
function logError(datasource: string, packageName: string, err: any): void {
  const { statusCode, code: errCode, url } = err;
  if (statusCode === 404) {
    logger.debug({ datasource, packageName, url }, 'Datasource 404');
  } else if (statusCode === 401 || statusCode === 403) {
    logger.debug({ datasource, packageName, url }, 'Datasource unauthorized');
  } else if (errCode) {
    logger.debug(
      { datasource, packageName, url, errCode },
      'Datasource connection error',
    );
  } else {
    logger.debug({ datasource, packageName, err }, 'Datasource unknown error');
  }
}

async function getRegistryReleases(
  datasource: DatasourceApi,
  config: GetPkgReleasesConfig,
  registryUrl: string,
): Promise<ReleaseResult | null> {
  const cacheNamespace: PackageCacheNamespace = `datasource-releases-${datasource.id}`;
  const cacheKey = buildCacheKey(registryUrl, config.packageName);

  const cacheEnabled = !!datasource.caching; // tells if `isPrivate` flag is supported in datasource result
  const cacheForced = GlobalConfig.get('cachePrivatePackages'); // tells if caching is forced via admin config

  if (cacheEnabled || cacheForced) {
    const cachedResult = await packageCache.get<ReleaseResult>(
      cacheNamespace,
      cacheKey,
    );

    if (cachedResult) {
      logger.trace({ cacheKey }, 'Returning cached datasource response');
      DatasourceCacheStats.hit(datasource.id, registryUrl, config.packageName);
      return cachedResult;
    }

    DatasourceCacheStats.miss(datasource.id, registryUrl, config.packageName);
  }

  const res = await instrument(
    'getReleases',
    () => datasource.getReleases(toGetReleasesConfig(config, registryUrl)),
    {
      attributes: {
        [ATTR_CODE_FUNCTION_NAME]: 'getReleases',
        [ATTR_RENOVATE_DATASOURCE]: datasource.id,
        [ATTR_RENOVATE_REGISTRY_URL]: registryUrl,
        [ATTR_RENOVATE_PACKAGE_NAME]: config.packageName,
      },
    },
  );
  if (res?.releases.length) {
    res.registryUrl ??= registryUrl;
  }

  if (!res) {
    return null;
  }

  // for the cached and the returned result alike
  stripMetadataUrlCredentials(res);

  let cache = false;
  if (cacheForced) {
    cache = true;
  } else if (cacheEnabled && !res.isPrivate) {
    cache = true;
  }

  if (cache) {
    logger.trace({ cacheKey }, 'Caching datasource response');
    await packageCache.set(cacheNamespace, cacheKey, res, 15);
    DatasourceCacheStats.set(datasource.id, registryUrl, config.packageName);
  } else {
    DatasourceCacheStats.skip(datasource.id, registryUrl, config.packageName);
  }

  return res;
}

function firstRegistry(
  config: GetPkgReleasesConfig,
  datasource: DatasourceApi,
  registryUrls: string[],
): Promise<ReleaseResult | null> {
  if (registryUrls.length > 1) {
    logger.warn(
      {
        datasource: datasource.id,
        packageName: config.packageName,
        registryUrls,
      },
      'Excess registryUrls found for datasource lookup - using first configured only',
    );
  }
  const registryUrl = registryUrls[0];
  return getRegistryReleases(datasource, config, registryUrl);
}

async function huntRegistries(
  config: GetPkgReleasesConfig,
  datasource: DatasourceApi,
  registryUrls: string[],
): Promise<ReleaseResult | null> {
  let res: ReleaseResult | null = null;
  let caughtError: Error | undefined;
  for (const registryUrl of registryUrls) {
    try {
      res = await getRegistryReleases(datasource, config, registryUrl);
      if (res) {
        break;
      }
    } catch (err) {
      if (err instanceof ExternalHostError) {
        throw err;
      }
      // We'll always save the last-thrown error
      caughtError = err;
      logger.trace({ err }, 'datasource hunt failure');
    }
  }
  if (res) {
    return res;
  }
  if (caughtError) {
    throw caughtError;
  }
  return null;
}

async function mergeRegistries(
  config: GetPkgReleasesConfig,
  datasource: DatasourceApi,
  registryUrls: string[],
): Promise<ReleaseResult | null> {
  let combinedRes: ReleaseResult | undefined;
  let lastErr: Error | undefined;
  let externalHostError: ExternalHostError | undefined;
  let singleRegistry = true;
  const releaseVersioning = versioning.get(config.versioning);
  for (const registryUrl of registryUrls) {
    try {
      // Merging must not mutate responses shared by the package cache.
      const res = clone(
        await getRegistryReleases(datasource, config, registryUrl),
      );
      if (!res) {
        continue;
      }

      if (!combinedRes) {
        // This is the first registry, so we can just use it and continue
        combinedRes = res;
        continue;
      }

      if (singleRegistry) {
        // This is the second registry
        // We need to move the registryUrl from the package level to the release level
        for (const release of coerceArray(combinedRes.releases)) {
          release.registryUrl ??= combinedRes.registryUrl;
        }
        singleRegistry = false;
      }

      const releases = coerceArray(res.releases);
      for (const release of releases) {
        // We have more than one registry, so we need to move the registryUrl
        // from the package level to the release level before merging
        release.registryUrl ??= res.registryUrl;
      }

      combinedRes.releases.push(...releases);

      // Merge the tags from the two results
      let tags = combinedRes.tags;
      if (tags) {
        // v8 ignore else -- needs merged registries where only one carries tags
        if (res.tags) {
          // Both results had tags, so we need to compare them
          for (const tag of ['release', 'latest']) {
            const existingTag = combinedRes?.tags?.[tag];
            const newTag = res.tags?.[tag];
            // v8 ignore else -- needs a merged registry whose tag is not a version
            if (isString(newTag) && releaseVersioning.isVersion(newTag)) {
              if (
                isString(existingTag) &&
                releaseVersioning.isVersion(existingTag)
              ) {
                // We need to compare them
                if (releaseVersioning.isGreaterThan(newTag, existingTag)) {
                  // New tag is greater than the existing one
                  tags[tag] = newTag;
                }
              } else {
                // Existing tag was not present or not a version
                // so we can just use the new one
                tags[tag] = newTag;
              }
            }
          }
        }
      } else {
        // Existing results had no tags, so we can just use the new ones
        tags = res.tags;
      }
      combinedRes = { ...res, ...combinedRes };
      if (tags) {
        combinedRes.tags = tags;
      }
      // Remove the registryUrl from the package level when more than one registry
      delete combinedRes.registryUrl;
    } catch (err) {
      if (err instanceof ExternalHostError) {
        // Don't abort the merge if another registry already returned releases;
        // a single rate-limited registry shouldn't discard results we have
        externalHostError = err;
        logger.debug(
          { err, registryUrl },
          'datasource merge: external host error from registry; continuing so releases from other registries are not discarded',
        );
        continue;
      }

      lastErr = err;
      logger.trace({ err }, 'datasource merge failure');
    }
  }

  if (!combinedRes) {
    if (externalHostError) {
      throw externalHostError;
    }

    if (lastErr) {
      throw lastErr;
    }

    return null;
  }

  const seenVersions = new Set<string>();
  combinedRes.releases = filterMap(combinedRes.releases, (release) => {
    if (seenVersions.has(release.version)) {
      return null;
    }
    seenVersions.add(release.version);
    return release;
  });

  return combinedRes;
}

function massageRegistryUrls(
  datasource: DatasourceApi,
  registryUrls: string[],
): string[] {
  const hostTypes = [getDatasourceHostType(datasource.id)].filter(isTruthy);
  return (
    registryUrls
      .filter(isTruthy)
      // the custom datasource detaches the credentials of a URL template from the URL it compiles, which is the one it requests
      .map((url) =>
        isCustomUrlTemplate(hostTypes, url)
          ? url
          : detachUrlCredentials(url, hostTypes),
      )
      .map(trimTrailingSlash)
  );
}

/**
 * Whether `url` requested with `hostTypes` is a URL template of the custom datasource, such as `https://host/{{packageName}}/versions.json`.
 */
function isCustomUrlTemplate(hostTypes: string[], url: string): boolean {
  return hostTypes.includes('custom') && url.includes('{{');
}

function resolveRegistryUrls(
  datasource: DatasourceApi,
  packageName: string,
  {
    defaultRegistryUrls,
    registryUrls,
    additionalRegistryUrls,
  }: RegistryUrlsConfig,
): string[] {
  const customRegistrySupport = datasource.supportsCustomRegistry(packageName);
  const datasourceDefaultRegistryUrls =
    datasource.getDefaultRegistryUrls(packageName);

  if (!customRegistrySupport) {
    if (
      isNonEmptyArray(registryUrls) ||
      isNonEmptyArray(defaultRegistryUrls) ||
      isNonEmptyArray(additionalRegistryUrls)
    ) {
      // the URLs may carry credentials, which are not logged
      logger.warn(
        {
          datasource: datasource.id,
          registryUrls: registryUrls
            ?.filter(isTruthy)
            .map(withoutUrlCredentials),
          defaultRegistryUrls: defaultRegistryUrls
            ?.filter(isTruthy)
            .map(withoutUrlCredentials),
          additionalRegistryUrls: additionalRegistryUrls
            ?.filter(isTruthy)
            .map(withoutUrlCredentials),
        },
        'Custom registries are not allowed for this datasource and will be ignored',
      );
    }
    return coerceArray(datasourceDefaultRegistryUrls);
  }
  const customUrls = registryUrls?.filter(isTruthy);
  if (isNonEmptyArray(customUrls)) {
    return massageRegistryUrls(datasource, customUrls);
  }
  const defaultUrls = isNonEmptyArray(defaultRegistryUrls)
    ? defaultRegistryUrls
    : datasourceDefaultRegistryUrls;
  if (!isNonEmptyArray(defaultUrls)) {
    return [];
  }
  return massageRegistryUrls(datasource, [
    ...defaultUrls,
    ...coerceArray(additionalRegistryUrls),
  ]);
}

function applyReplacements(
  config: GetPkgReleasesConfig,
): Pick<ReleaseResult, 'replacementName' | 'replacementVersion'> | undefined {
  if (config.replacementName && config.replacementVersion) {
    return {
      replacementName: config.replacementName,
      replacementVersion: config.replacementVersion,
    };
  }
  return undefined;
}

async function fetchReleases(
  config: GetPkgReleasesConfig,
): Promise<ReleaseResult | null> {
  const { datasource: datasourceName } = config;
  let registryUrlsConfig: RegistryUrlsConfig = config;
  // istanbul ignore if: need test
  if (!datasourceName || getDatasourceFor(datasourceName) === undefined) {
    logger.warn({ datasource: datasourceName }, 'Unknown datasource');
    return null;
  }
  if (datasourceName === 'npm') {
    if (isString(config.npmrc)) {
      setNpmrc(config.npmrc);
    }
    // v8 ignore else -- npm lookups here never arrive with explicit registryUrls
    if (!isNonEmptyArray(config.registryUrls)) {
      registryUrlsConfig = {
        ...config,
        registryUrls: [resolveRegistryUrl(config.packageName)],
      };
    }
  }
  const datasource = getDatasourceFor(datasourceName);
  // istanbul ignore if: needs test
  if (!datasource) {
    logger.warn({ datasource: datasourceName }, 'Unknown datasource');
    return null;
  }
  const registryUrls = resolveRegistryUrls(
    datasource,
    config.packageName,
    registryUrlsConfig,
  );
  let dep: ReleaseResult | null = null;
  const registryStrategy =
    config.registryStrategy ?? datasource.registryStrategy;
  try {
    if (isNonEmptyArray(registryUrls)) {
      if (registryStrategy === 'first') {
        dep = await firstRegistry(config, datasource, registryUrls);
      } else if (registryStrategy === 'hunt') {
        dep = await huntRegistries(config, datasource, registryUrls);
      } else {
        // `merge` is the only remaining strategy
        dep = await mergeRegistries(config, datasource, registryUrls);
      }
    } else {
      dep = await instrument(
        'getReleases',
        () => datasource.getReleases(toGetReleasesConfig(config)),
        {
          attributes: {
            [ATTR_CODE_FUNCTION_NAME]: 'getReleases',
            [ATTR_RENOVATE_DATASOURCE]: datasource.id,
            [ATTR_RENOVATE_REGISTRY_URL]: config.registryUrl ?? '',
            [ATTR_RENOVATE_PACKAGE_NAME]: config.packageName,
          },
        },
      );
    }
  } catch (err) {
    if (
      [HOST_BLOCKED, HOST_DISABLED].includes(err.message) ||
      [HOST_BLOCKED, HOST_DISABLED].includes(err.err?.message)
    ) {
      return null;
    }
    if (err instanceof ExternalHostError) {
      throw err;
    }
    logError(datasource.id, config.packageName, err);
  }
  if (!dep || dequal(dep, { releases: [] })) {
    return null;
  }
  addMetaData(dep, datasourceName, config.packageName);
  // also for the results of a lookup without registry URL, which do not pass `getRegistryReleases()`
  stripMetadataUrlCredentials(dep);
  dep = { ...dep, ...applyReplacements(config) };
  return dep;
}

function fetchCachedReleases(
  config: GetPkgReleasesConfig,
): Promise<ReleaseResult | null> {
  const { datasource, packageName, registryUrls } = config;
  const cacheKey = buildCacheKey(
    'datasource-mem',
    'releases',
    datasource,
    packageName,
    config.registryStrategy,
    safeStringify([
      registryUrls,
      config.defaultRegistryUrls,
      config.additionalRegistryUrls,
    ]),
  );
  // By returning a Promise and reusing it, we should only fetch each package at most once
  const cachedResult = memCache.get<Promise<ReleaseResult | null>>(cacheKey);
  // istanbul ignore if
  if (cachedResult !== undefined) {
    return cachedResult;
  }
  const promisedRes = fetchReleases(config);
  memCache.set(cacheKey, promisedRes);
  return promisedRes;
}

export function getRawPkgReleases(
  config: GetPkgReleasesConfig,
): AsyncResult<
  ReleaseResult,
  Error | 'no-datasource' | 'no-package-name' | 'no-result'
> {
  if (!config.datasource) {
    logger.warn('No datasource found');
    return AsyncResult.err('no-datasource');
  }

  const packageName = config.packageName;
  if (!packageName) {
    logger.error({ config }, 'Datasource getReleases without packageName');
    return AsyncResult.err('no-package-name');
  }

  return Result.wrapNullable(fetchCachedReleases(config), 'no-result' as const)
    .catch((e) => {
      if (e instanceof ExternalHostError) {
        e.hostType = config.datasource;
        e.packageName = packageName;
      }
      return Result.err(e);
    })
    .transform(clone);
}

export function applyDatasourceFilters(
  releaseResult: ReleaseResult,
  config: GetPkgReleasesConfig,
): ReleaseResult {
  let res = releaseResult;
  res = applyExtractVersion(res, config.extractVersion);
  res = applyVersionCompatibility(
    res,
    config.versionCompatibility,
    config.currentCompatibility,
  );
  res = filterValidVersions(res, config);
  res = sortAndRemoveDuplicates(res, config);
  res = applyConstraintsFiltering(res, config);
  return res;
}

export async function getPkgReleases(
  config: GetPkgReleasesConfig,
): Promise<ReleaseResult | null> {
  const { val = null, err } = await getRawPkgReleases(config)
    .transform((res) => applyDatasourceFilters(res, config))
    .unwrap();

  if (err instanceof Error) {
    throw err;
  }

  return val;
}

export function supportsDigests(datasource: string | undefined): boolean {
  const ds = !!datasource && getDatasourceFor(datasource);
  return !!ds && 'getDigest' in ds;
}

function getDigestConfig(
  datasource: DatasourceApi,
  config: GetDigestInputConfig,
): DigestConfig {
  const { lookupName, currentValue, currentDigest } = config;
  const packageName = config.replacementName ?? config.packageName;
  // Prefer registryUrl from getReleases() lookup if it has been passed
  const registryUrl =
    config.registryUrl ??
    resolveRegistryUrls(datasource, packageName, config)[0];
  return { lookupName, packageName, registryUrl, currentValue, currentDigest };
}

export function getDigest(
  config: GetDigestInputConfig,
  value?: string,
): Promise<string | null> {
  const datasource = getDatasourceFor(config.datasource);
  // istanbul ignore if: need test
  if (!datasource || !('getDigest' in datasource)) {
    return Promise.resolve(null);
  }
  const digestConfig = getDigestConfig(datasource, config);
  return datasource.getDigest!(digestConfig, value);
}

export function getDefaultConfig(
  datasource: string,
): Promise<Record<string, unknown>> {
  const loadedDatasource = getDatasourceFor(datasource);
  return Promise.resolve<Record<string, unknown>>(
    loadedDatasource?.defaultConfig ?? Object.create({}),
  );
}