renovate/lib/modules/datasource/crate/index.ts

import { isBoolean } from '@sindresorhus/is';
import upath from 'upath';
import { GlobalConfig } from '../../../config/global.ts';
import { logger } from '../../../logger/index.ts';
import * as memCache from '../../../util/cache/memory/index.ts';
import * as packageCache from '../../../util/cache/package/index.ts';
import { withCache } from '../../../util/cache/package/with-cache.ts';
import { privateCacheDir, readCacheFile } from '../../../util/fs/index.ts';
import { createSimpleGit } from '../../../util/git/index.ts';
import { toSha256 } from '../../../util/hash.ts';
import { memCacheProvider } from '../../../util/http/cache/memory-http-cache-provider.ts';
import { HttpError } from '../../../util/http/index.ts';
import { acquireLock } from '../../../util/mutex.ts';
import { newlineRegex, regEx } from '../../../util/regex.ts';
import { Json } from '../../../util/schema-utils/index.ts';
import { asTimestamp } from '../../../util/timestamp.ts';
import { joinUrlParts, parseUrl } from '../../../util/url.ts';
import * as cargoVersioning from '../../versioning/cargo/index.ts';
import { Datasource } from '../datasource.ts';
import type {
  GetReleasesConfig,
  PostprocessReleaseConfig,
  PostprocessReleaseResult,
  Release,
  ReleaseResult,
} from '../types.ts';
import type { CrateMetadata } from './schema.ts';
import {
  CrateMetadataResponse,
  RegistryConfig,
  ReleaseTimestamp,
} from './schema.ts';
import type { CrateRecord, RegistryFlavor, RegistryInfo } from './types.ts';

type CloneResult =
  | {
      err: Error;
      clonePath?: undefined;
    }
  | {
      clonePath: string;
      err?: undefined;
    };

export class CrateDatasource extends Datasource {
  static readonly id = 'crate';

  constructor() {
    super(CrateDatasource.id);
  }

  override getDefaultRegistryUrls(_packageName: string): string[] {
    return ['sparse+https://index.crates.io/'];
  }

  override defaultVersioning = cargoVersioning.id;

  override readonly sourceUrlSupport = 'package';
  override readonly sourceUrlNote =
    'The source URL is determined from the `repository` field in the results.';

  override readonly releaseTimestampSupport = true;
  override readonly releaseTimestampNote =
    'The release timestamp is determined from `pubtime` field from crates.io index if available, or `version.created_at` field from crates.io API otherwise.';

  private async _getReleases({
    packageName,
    registryUrl,
  }: GetReleasesConfig): Promise<ReleaseResult | null> {
    /* v8 ignore if -- should never happen */
    if (!registryUrl) {
      logger.warn(
        'crate datasource: No registryUrl specified, cannot perform getReleases',
      );
      return null;
    }

    const registryInfo = await CrateDatasource.fetchRegistryInfo({
      packageName,
      registryUrl,
    });
    if (!registryInfo) {
      logger.debug(`Could not fetch registry info from ${registryUrl}`);
      return null;
    }

    const dependencyUrl = CrateDatasource.getDependencyUrl(
      registryInfo,
      packageName,
    );

    const payload = await this.fetchCrateRecordsPayload(
      registryInfo,
      packageName,
    );
    const lines = payload
      .split(newlineRegex) // break into lines
      .map((line) => line.trim()) // remove whitespace
      .filter((line) => line.length !== 0) // remove empty lines
      .map((line) => JSON.parse(line) as CrateRecord); // parse

    const metadata = await this.getCrateMetadata(registryInfo, packageName);

    const result: ReleaseResult = {
      dependencyUrl,
      releases: [],
    };

    if (metadata?.homepage) {
      result.homepage = metadata.homepage;
    }

    if (metadata?.repository) {
      result.sourceUrl = metadata.repository;
    }

    result.releases = lines
      .map((line) => {
        const versionOrig = line.vers;
        const version = versionOrig.replace(regEx(/\+.*$/), '');
        const release: Release = { version };

        if (versionOrig !== version) {
          release.versionOrig = versionOrig;
        }

        if (line.yanked) {
          release.isDeprecated = true;
        }

        if (line.rust_version) {
          release.constraints = { rust: [line.rust_version] };
        }

        if (line.pubtime) {
          release.releaseTimestamp = asTimestamp(line.pubtime);
        }

        return release;
      })
      .filter((release) => release.version);
    if (!result.releases.length) {
      return null;
    }

    return result;
  }

  getReleases(config: GetReleasesConfig): Promise<ReleaseResult | null> {
    return withCache(
      {
        namespace: `datasource-${CrateDatasource.id}`,
        // TODO: types (#22198)
        key: `${config.registryUrl}/${config.packageName}`,
        cacheable: CrateDatasource.isCratesIo(config.registryUrl),
        fallback: true,
      },
      () => this._getReleases(config),
    );
  }

  /**
   * Fetches the registry's config.json which provides the API base URL
   * and download URL template.
   * See: https://doc.rust-lang.org/cargo/reference/registry-index.html#index-configuration
   */
  private async fetchRegistryConfig(
    info: RegistryInfo,
  ): Promise<RegistryConfig | null> {
    const cacheKey = `crate-datasource/registry-config/${info.rawUrl}`;
    const cached = memCache.get<RegistryConfig>(cacheKey);
    if (cached) {
      return cached;
    }

    if (info.clonePath) {
      try {
        const configPath = upath.join(info.clonePath, 'config.json');
        const content = await readCacheFile(configPath, 'utf8');
        const parsed = Json.pipe(RegistryConfig).parse(content);
        memCache.set(cacheKey, parsed);
        return parsed;
      } catch {
        logger.debug(
          { registryUrl: info.rawUrl },
          'Could not read config.json from cloned registry',
        );
      }
    } else {
      try {
        const configUrl = joinUrlParts(info.rawUrl, 'config.json');
        const { body } = await this.http.getJson(configUrl, RegistryConfig);
        memCache.set(cacheKey, body);
        return body;
      } catch {
        logger.debug(
          { registryUrl: info.rawUrl },
          'Could not fetch registry config.json',
        );
      }
    }

    return null;
  }

  /**
   * The registry web API specification only defines publish, yank, unyank,
   * owners, search and login endpoints, see
   * https://doc.rust-lang.org/cargo/reference/registry-web-api.html.
   * `GET api/v1/crates/<name>` and `GET api/v1/crates/<name>/<version>` are
   * crates.io extensions. Registries which mirror the crates.io index keep
   * `api` pointing at crates.io and are used without probing. Any other
   * registry is probed once and remembered as unsupported when it answers
   * 404, since private registries (Artifactory, CodeArtifact, ...) advertise
   * an `api` URL for publishing only.
   */
  private static isCratesIoApi(api: string): boolean {
    return parseUrl(api)?.hostname === 'crates.io';
  }

  private static isNotFound(err: unknown): boolean {
    return err instanceof HttpError && err.response?.statusCode === 404;
  }

  /**
   * The marker is keyed by the `api` URL, since the missing endpoints are a
   * property of the API server, not of the index that points to it.
   */
  private async isReadApiUnsupported(api: string): Promise<boolean> {
    if (CrateDatasource.isCratesIoApi(api)) {
      return false;
    }

    const memKey = `crate-datasource/registry-api-unsupported/${api}`;
    const cached = memCache.get<boolean>(memKey);
    if (isBoolean(cached)) {
      return cached;
    }

    const persisted = await packageCache.get<boolean>(
      'datasource-crate-registry-api',
      api,
    );
    const unsupported = persisted === true;
    memCache.set(memKey, unsupported);
    return unsupported;
  }

  private async markReadApiUnsupported(
    registryUrl: string,
    api: string,
  ): Promise<void> {
    logger.debug(
      { registryUrl, api },
      'Registry does not implement the crates.io read API, skipping crate metadata and release timestamp lookups',
    );
    memCache.set(`crate-datasource/registry-api-unsupported/${api}`, true);
    await packageCache.set('datasource-crate-registry-api', api, true, 24 * 60);
  }

  private async _getCrateMetadata(
    info: RegistryInfo,
    packageName: string,
  ): Promise<CrateMetadata | null> {
    const registryConfig = await this.fetchRegistryConfig(info);
    if (!registryConfig?.api) {
      return null;
    }

    const { api } = registryConfig;
    if (await this.isReadApiUnsupported(api)) {
      return null;
    }

    const apiBaseUrl = joinUrlParts(api, 'api/v1/');

    // The `?include=` suffix is required to avoid unnecessary database queries
    // on the crates.io server. This lets us work around the regular request
    // throttling of one request per second.
    const crateUrl = `${apiBaseUrl}crates/${packageName}?include=`;

    logger.trace(
      { crateUrl, packageName, registryUrl: info.rawUrl },
      'downloading crate metadata',
    );

    try {
      const { body } = await this.http.getJson(crateUrl, CrateMetadataResponse);
      return body.crate;
    } catch (err) {
      // The index lookup preceding this call found the crate, so a 404 means
      // the registry does not implement the endpoint at all
      if (
        !CrateDatasource.isCratesIoApi(api) &&
        CrateDatasource.isNotFound(err)
      ) {
        await this.markReadApiUnsupported(info.rawUrl, api);
        return null;
      }
      logger.debug(
        { err, packageName, registryUrl: info.rawUrl },
        'failed to download crate metadata',
      );
    }

    return null;
  }

  public getCrateMetadata(
    info: RegistryInfo,
    packageName: string,
  ): Promise<CrateMetadata | null> {
    return withCache(
      {
        namespace: `datasource-${CrateDatasource.id}-metadata`,
        key: `${info.rawUrl}/${packageName}`,
        cacheable: info.flavor === 'crates.io',
        ttlMinutes: 24 * 60, // 24 hours
      },
      () => this._getCrateMetadata(info, packageName),
    );
  }

  public async fetchCrateRecordsPayload(
    info: RegistryInfo,
    packageName: string,
  ): Promise<string> {
    if (info.clonePath) {
      const path = upath.join(
        info.clonePath,
        ...CrateDatasource.getIndexSuffix(packageName),
      );
      return readCacheFile(path, 'utf8');
    }

    // sparse index
    const packageSuffix = CrateDatasource.getIndexSuffix(
      packageName.toLowerCase(),
    );
    const crateUrl = joinUrlParts(info.rawUrl, ...packageSuffix);
    logger.trace(
      { crateUrl, packageName, registryUrl: info.rawUrl },
      'fetching crate records from sparse index',
    );
    try {
      return (await this.http.getText(crateUrl)).body;
    } catch (err) {
      this.handleGenericErrors(err);
    }
  }

  /**
   * Computes the dependency URL for a crate, given
   * registry information
   */
  private static getDependencyUrl(
    info: RegistryInfo,
    packageName: string,
  ): string {
    switch (info.flavor) {
      case 'crates.io':
        return `https://crates.io/crates/${packageName}`;
      case 'cloudsmith': {
        // input: https://dl.cloudsmith.io/basic/$org/$repo/cargo/index.git
        const tokens = info.url.pathname.split('/');
        const org = tokens[2];
        const repo = tokens[3];
        return `https://cloudsmith.io/~${org}/repos/${repo}/packages/detail/cargo/${packageName}`;
      }
      default:
        return `${info.url}/${packageName}`;
    }
  }

  /**
   * Given a Git URL, computes a semi-human-readable name for a folder in which to
   * clone the repository.
   */
  private static cacheDirFromUrl(url: URL): string {
    const proto = url.protocol.replace(regEx(/:$/), '');
    const host = url.hostname;
    const hash = toSha256(url.pathname).substring(0, 7);

    return `crate-registry-${proto}-${host}-${hash}`;
  }

  private static isSparseRegistry(url: string): boolean {
    const parsed = parseUrl(url);
    if (!parsed) {
      return false;
    }
    return parsed.protocol.startsWith('sparse+');
  }

  /**
   * Fetches information about a registry, by url.
   * If no url is given, assumes crates.io.
   * If an url is given, assumes it's a valid Git repository
   * url and clones it to cache.
   */
  private static async fetchRegistryInfo({
    packageName,
    registryUrl,
  }: GetReleasesConfig): Promise<RegistryInfo | null> {
    /* v8 ignore next -- should never happen */
    if (!registryUrl) {
      return null;
    }

    const isSparseRegistry = CrateDatasource.isSparseRegistry(registryUrl);
    const registryFetchUrl = isSparseRegistry
      ? registryUrl.replace(regEx(/^sparse\+/), '')
      : registryUrl;

    const url = parseUrl(registryFetchUrl);
    if (!url) {
      logger.debug(`Could not parse registry URL ${registryFetchUrl}`);
      return null;
    }

    let flavor: RegistryFlavor;
    if (url.hostname === 'index.crates.io') {
      flavor = 'crates.io';
    } else if (url.hostname === 'dl.cloudsmith.io') {
      flavor = 'cloudsmith';
    } else {
      flavor = 'other';
    }

    const registry: RegistryInfo = {
      flavor,
      rawUrl: registryFetchUrl,
      url,
    };

    if (
      registry.flavor !== 'crates.io' &&
      !GlobalConfig.get('allowCustomCrateRegistries')
    ) {
      logger.warn(
        'crate datasource: allowCustomCrateRegistries=true is required for registries other than crates.io, bailing out',
      );
      return null;
    }
    if (!isSparseRegistry) {
      const cacheKey = `crate-datasource/registry-clone-path/${registryFetchUrl}`;
      const lockKey = registryFetchUrl;

      const executionTimeout = GlobalConfig.get('executionTimeout') * 60 * 1000;
      const gitTimeout = GlobalConfig.get('gitTimeout') || executionTimeout;
      const releaseLock = await acquireLock(
        lockKey,
        'crate-registry',
        gitTimeout,
      );
      try {
        const cached = memCache.get<CloneResult>(cacheKey);

        if (cached?.err) {
          logger.warn(
            { err: cached.err, packageName, registryFetchUrl },
            'Previous git clone failed, bailing out.',
          );
          return null;
        }

        if (cached?.clonePath) {
          registry.clonePath = cached.clonePath;
          return registry;
        }

        const clonePath = upath.join(
          privateCacheDir(),
          CrateDatasource.cacheDirFromUrl(url),
        );

        const result = await CrateDatasource.clone(
          registryFetchUrl,
          clonePath,
          packageName,
        );

        memCache.set(cacheKey, result);

        if (result.err) {
          logger.warn(
            { err: result.err, packageName, registryFetchUrl },
            'Git clone failed, bailing out.',
          );
          return null;
        }

        registry.clonePath = result.clonePath;
      } finally {
        releaseLock();
      }
    }

    return registry;
  }

  private static async clone(
    registryFetchUrl: string,
    clonePath: string,
    packageName: string,
  ): Promise<CloneResult> {
    logger.info(
      { clonePath, registryFetchUrl },
      `Cloning private cargo registry`,
    );

    const git = createSimpleGit({
      config: { maxConcurrentProcesses: 1 },
    });

    try {
      await git.clone(registryFetchUrl, clonePath, {
        '--depth': 1,
      });
      return { clonePath };
    } catch (err) {
      if (
        err.message.includes(
          'fatal: dumb http transport does not support shallow capabilities',
        )
      ) {
        logger.info(
          { packageName, registryFetchUrl },
          'failed to shallow clone git registry, doing full clone',
        );
        try {
          await git.clone(registryFetchUrl, clonePath);
          return { clonePath };
        } catch (err) {
          logger.warn(
            { err, packageName, registryFetchUrl },
            'failed cloning git registry',
          );
          return { err };
        }
      } else {
        logger.warn(
          { err, packageName, registryFetchUrl },
          'failed cloning git registry',
        );
        return { err };
      }
    }
  }

  private static isCratesIo(registryUrl: string | undefined): boolean {
    if (!registryUrl) {
      return false;
    }
    const normalized = registryUrl.replace(regEx(/^sparse\+/), '');
    const parsed = parseUrl(normalized);
    return parsed?.hostname === 'index.crates.io';
  }

  public static getIndexSuffix(packageName: string): string[] {
    const len = packageName.length;

    if (len === 1) {
      return ['1', packageName];
    }
    if (len === 2) {
      return ['2', packageName];
    }
    if (len === 3) {
      return ['3', packageName[0], packageName];
    }

    return [packageName.slice(0, 2), packageName.slice(2, 4), packageName];
  }

  private async _postprocessRelease(
    { packageName, registryUrl }: PostprocessReleaseConfig,
    release: Release,
  ): Promise<PostprocessReleaseResult> {
    if (release.releaseTimestamp) {
      return release;
    }

    // Look up the registry config from cache (populated during getReleases)
    const rawUrl = registryUrl?.replace(regEx(/^sparse\+/), '');
    if (!rawUrl) {
      return release;
    }
    const cacheKey = `crate-datasource/registry-config/${rawUrl}`;
    const config = memCache.get<RegistryConfig>(cacheKey);
    if (!config?.api) {
      return release;
    }

    const { api } = config;
    if (await this.isReadApiUnsupported(api)) {
      return release;
    }

    const apiBaseUrl = joinUrlParts(api, 'api/v1/');
    const url = `${apiBaseUrl}crates/${packageName}/${release.versionOrig ?? release.version}`;
    logger.trace(
      { url, packageName, version: release.version, registryUrl },
      'fetching crate release timestamp',
    );
    // Getting release timestamp could become unnecessary if the manual backfill of `pubtime` mentioned in
    // https://github.com/rust-lang/cargo/issues/15491 is done for all packages.
    try {
      const { body: releaseTimestamp } = await this.http.getJson(
        url,
        { cacheProvider: memCacheProvider },
        ReleaseTimestamp,
      );
      release.releaseTimestamp = releaseTimestamp;
    } catch (err) {
      if (
        CrateDatasource.isCratesIoApi(api) ||
        !CrateDatasource.isNotFound(err)
      ) {
        throw err;
      }
      await this.markReadApiUnsupported(rawUrl, api);
    }
    return release;
  }

  override postprocessRelease(
    config: PostprocessReleaseConfig,
    release: Release,
  ): Promise<PostprocessReleaseResult> {
    return withCache(
      {
        namespace: `datasource-crate`,
        key: `postprocessRelease:${config.registryUrl}:${config.packageName}:${release.version}`,
        ttlMinutes: 7 * 24 * 60,
        cacheable: CrateDatasource.isCratesIo(config.registryUrl ?? undefined),
      },
      () => this._postprocessRelease(config, release),
    );
  }
}