renovate/lib/modules/datasource/nuget/v3.ts

import { isNonEmptyString } from '@sindresorhus/is';
import AdmZip from 'adm-zip';
import semver from 'semver';
import upath from 'upath';
import { XmlDocument } from 'xmldoc';
import { logger } from '../../../logger/index.ts';
import { ExternalHostError } from '../../../types/errors/external-host-error.ts';
import { coerceArray } from '../../../util/array.ts';
import * as packageCache from '../../../util/cache/package/index.ts';
import { withCache } from '../../../util/cache/package/with-cache.ts';
import { getEnv } from '../../../util/env.ts';
import * as fs from '../../../util/fs/index.ts';
import { ensureCacheDir } from '../../../util/fs/index.ts';
import { memCacheProvider } from '../../../util/http/cache/memory-http-cache-provider.ts';
import type { Http } from '../../../util/http/index.ts';
import { HttpError } from '../../../util/http/index.ts';
import * as p from '../../../util/promises.ts';
import { regEx } from '../../../util/regex.ts';
import { asTimestamp } from '../../../util/timestamp.ts';
import { ensureTrailingSlash } from '../../../util/url.ts';
import { api as versioning } from '../../versioning/nuget/index.ts';
import type { Release, ReleaseResult } from '../types.ts';
import { massageUrl, removeBuildMeta, sortNugetVersions } from './common.ts';
import type { CatalogEntry } from './schema.ts';
import {
  CatalogPage,
  PackageRegistration,
  ServicesIndexRaw,
} from './schema.ts';

export class NugetV3Api {
  static readonly cacheNamespace = 'datasource-nuget-v3';

  async getResourceUrl(
    http: Http,
    url: string,
    resourceType = 'RegistrationsBaseUrl',
  ): Promise<string | null> {
    // https://learn.microsoft.com/nuget/api/service-index
    const resultCacheKey = `${url}:${resourceType}`;
    const cachedResult = await packageCache.get<string>(
      NugetV3Api.cacheNamespace,
      resultCacheKey,
    );

    /* v8 ignore next -- TODO: add test */
    if (cachedResult) {
      return cachedResult;
    }
    let servicesIndexRaw: ServicesIndexRaw | undefined;
    try {
      const responseCacheKey = url;
      servicesIndexRaw = await packageCache.get<ServicesIndexRaw>(
        NugetV3Api.cacheNamespace,
        responseCacheKey,
      );
      if (!servicesIndexRaw) {
        servicesIndexRaw = (
          await http.getJson(
            url,
            { cacheProvider: memCacheProvider },
            ServicesIndexRaw,
          )
        ).body;
        await packageCache.set(
          NugetV3Api.cacheNamespace,
          responseCacheKey,
          servicesIndexRaw,
          3 * 24 * 60,
        );
      }

      const services = servicesIndexRaw.resources
        .map(({ '@id': serviceId, '@type': t }) => ({
          serviceId,
          type: t?.split('/')?.shift(),
          version: t?.split('/')?.pop(),
        }))
        .filter(
          ({ type, version }) => type === resourceType && semver.valid(version),
        )
        .sort((x, y) =>
          x.version && y.version
            ? semver.compare(x.version, y.version)
            : /* istanbul ignore next: hard to test */ 0,
        );

      if (services.length === 0) {
        await packageCache.set(
          NugetV3Api.cacheNamespace,
          resultCacheKey,
          null,
          60,
        );
        logger.debug(
          { url, servicesIndexRaw },
          `no ${resourceType} services found`,
        );
        return null;
      }

      const { serviceId, version } = services.pop()!;

      // istanbul ignore if
      if (
        resourceType === 'RegistrationsBaseUrl' &&
        version &&
        !version.startsWith('3.0.0-') &&
        !semver.satisfies(version, '^3.0.0')
      ) {
        logger.warn(
          { url, version },
          `Nuget: Unknown version returned. Only v3 is supported`,
        );
      }

      await packageCache.set(
        NugetV3Api.cacheNamespace,
        resultCacheKey,
        serviceId,
        60,
      );
      return serviceId;
    } catch (err) {
      // istanbul ignore if: not easy testable with nock
      if (err instanceof ExternalHostError) {
        throw err;
      }
      logger.debug(
        { err, url, servicesIndexRaw },
        `nuget registry failure: can't get ${resourceType}`,
      );
      return null;
    }
  }

  async getCatalogEntry(
    http: Http,
    catalogPage: CatalogPage,
  ): Promise<CatalogEntry[]> {
    let items = catalogPage.items;
    if (!items) {
      const url = catalogPage['@id'];
      if (!url) {
        return [];
      }
      const catalogPageFull = await http.getJson(url, CatalogPage);
      items = catalogPageFull.body.items;
    }
    return coerceArray(items).map(({ catalogEntry }) => catalogEntry);
  }

  async getReleases(
    http: Http,
    registryUrl: string,
    feedUrl: string,
    pkgName: string,
  ): Promise<ReleaseResult | null> {
    const baseUrl = feedUrl.replace(regEx(/\/*$/), '');
    const url = `${baseUrl}/${pkgName.toLowerCase()}/index.json`;
    const packageRegistration = await http.getJson(url, PackageRegistration);
    const catalogPages = packageRegistration.body.items;
    const catalogPagesQueue = catalogPages.map(
      (page) => (): Promise<CatalogEntry[]> => this.getCatalogEntry(http, page),
    );
    const catalogEntries = (await p.all(catalogPagesQueue))
      .flat()
      .sort((a, b) => sortNugetVersions(a.version, b.version));

    let homepage: string | null = null;
    let latestStable: string | null = null;
    let nupkgUrl: string | null = null;
    const releases = catalogEntries.map(
      ({
        version,
        published,
        projectUrl,
        listed,
        packageContent,
        deprecation,
      }) => {
        const release: Release = { version: removeBuildMeta(version) };
        const releaseTimestamp = asTimestamp(published);
        if (releaseTimestamp) {
          release.releaseTimestamp = releaseTimestamp;
        }
        if (
          versioning.isValid(version) &&
          versioning.isStable(version) &&
          listed
        ) {
          latestStable = removeBuildMeta(version);
          homepage = projectUrl ? massageUrl(projectUrl) : homepage;
          nupkgUrl = massageUrl(packageContent);
        }

        if (listed === false || deprecation) {
          release.isDeprecated = true;
        }
        return release;
      },
    );

    if (!releases.length) {
      return null;
    }

    // istanbul ignore next: only happens when no stable version exists
    if (latestStable === null && catalogPages.length) {
      const last = catalogEntries.pop()!;
      latestStable = removeBuildMeta(last.version);
      homepage ??= last.projectUrl ?? null;
      nupkgUrl ??= massageUrl(last.packageContent);
    }

    const dep: ReleaseResult = {
      releases,
    };

    if (releases.every((release) => release.isDeprecated === true)) {
      dep.deprecationMessage = this.getDeprecationMessage(pkgName);
    }

    try {
      const packageBaseAddress = await this.getResourceUrl(
        http,
        registryUrl,
        'PackageBaseAddress',
      );
      let shouldTryNupkg = false;
      if (isNonEmptyString(packageBaseAddress)) {
        const nuspecUrl = `${ensureTrailingSlash(
          packageBaseAddress,
        )}${pkgName.toLowerCase()}/${
          latestStable
        }/${pkgName.toLowerCase()}.nuspec`;
        try {
          const metaresult = await http.getText(nuspecUrl, {
            cacheProvider: memCacheProvider,
          });
          const nuspec = new XmlDocument(metaresult.body);
          const releaseNotes = nuspec.valueWithPath('metadata.releaseNotes');
          if (releaseNotes) {
            dep.changelogContent = releaseNotes;
          }
          const sourceUrl = nuspec.valueWithPath('metadata.repository@url');
          if (sourceUrl) {
            dep.sourceUrl = massageUrl(sourceUrl);
          }
        } catch (err) {
          /* v8 ignore else -- not easy testable with nock */
          if (err instanceof HttpError && err.response?.statusCode === 404) {
            shouldTryNupkg = true;
            logger.debug(
              { registryUrl, pkgName, pkgVersion: latestStable },
              `package manifest (.nuspec) not found`,
            );
          } else {
            throw err;
          }
        }
      } else {
        shouldTryNupkg = true;
      }

      if (!dep.sourceUrl && shouldTryNupkg && nupkgUrl) {
        const sourceUrl = await this.getSourceUrlFromNupkg(
          http,
          registryUrl,
          pkgName,
          latestStable,
          nupkgUrl,
        );
        if (sourceUrl) {
          dep.sourceUrl = massageUrl(sourceUrl);
          logger.debug(`Determined sourceUrl ${sourceUrl} from ${nupkgUrl}`);
        }
      }
    } catch (err) {
      // istanbul ignore if: not easy testable with nock
      if (err instanceof ExternalHostError) {
        throw err;
      }
      // ignore / silence 404. Seen on proget, if remote connector is used and package is not yet cached
      if (err instanceof HttpError && err.response?.statusCode === 404) {
        logger.debug(
          { registryUrl, pkgName, pkgVersion: latestStable },
          `package manifest (.nuspec) not found`,
        );
      } else {
        logger.debug(
          { err, registryUrl, pkgName, pkgVersion: latestStable },
          `Cannot obtain sourceUrl`,
        );
      }
    }

    if (homepage) {
      // only assign if not assigned
      dep.sourceUrl ??= homepage;
      dep.homepage ??= homepage;
    }

    return dep;
  }

  private async _getSourceUrlFromNupkg(
    http: Http,
    _registryUrl: string,
    packageName: string,
    packageVersion: string | null,
    nupkgUrl: string,
  ): Promise<string | null> {
    /* v8 ignore if -- specs enable RENOVATE_X_NUGET_DOWNLOAD_NUPKGS, so the disabled path is unexercised */
    if (!getEnv().RENOVATE_X_NUGET_DOWNLOAD_NUPKGS) {
      logger.once.debug('RENOVATE_X_NUGET_DOWNLOAD_NUPKGS is not set');
      return null;
    }
    const cacheDir = await ensureCacheDir('nuget');
    const nupkgFile = upath.join(
      cacheDir,
      `${packageName}.${packageVersion}.nupkg`,
    );
    const nupkgContentsDir = upath.join(
      cacheDir,
      `${packageName}.${packageVersion}`,
    );
    const readStream = http.stream(nupkgUrl);
    try {
      const writeStream = fs.createCacheWriteStream(nupkgFile);
      await fs.pipeline(readStream, writeStream);
      const zip = new AdmZip(nupkgFile);
      zip.extractAllTo(nupkgContentsDir);
      const nuspecFile = upath.join(nupkgContentsDir, `${packageName}.nuspec`);
      const nuspec = new XmlDocument(
        await fs.readCacheFile(nuspecFile, 'utf8'),
      );
      return nuspec.valueWithPath('metadata.repository@url') ?? null;
    } finally {
      await fs.rmCache(nupkgFile);
      await fs.rmCache(nupkgContentsDir);
    }
  }

  getSourceUrlFromNupkg(
    http: Http,
    registryUrl: string,
    packageName: string,
    packageVersion: string | null,
    nupkgUrl: string,
  ): Promise<string | null> {
    return withCache(
      {
        namespace: NugetV3Api.cacheNamespace,
        key: `source-url:${registryUrl}:${packageName}`,
        ttlMinutes: 10080, // 1 week
      },
      () =>
        this._getSourceUrlFromNupkg(
          http,
          registryUrl,
          packageName,
          packageVersion,
          nupkgUrl,
        ),
    );
  }

  getDeprecationMessage(packageName: string): string {
    return `The package \`${packageName}\` is deprecated.`;
  }
}