renovate/lib/modules/datasource/go/base.ts

// TODO: types (#22198)
import { GlobalConfig } from '../../../config/global.ts';
import { logger } from '../../../logger/index.ts';
import { detectPlatform } from '../../../util/common.ts';
import * as hostRules from '../../../util/host-rules.ts';
import { Http } from '../../../util/http/index.ts';
import { regEx } from '../../../util/regex.ts';
import {
  parseUrl,
  trimLeadingSlash,
  trimTrailingSlash,
} from '../../../util/url.ts';
import { BitbucketTagsDatasource } from '../bitbucket-tags/index.ts';
import { ForgejoTagsDatasource } from '../forgejo-tags/index.ts';
import { GitTagsDatasource } from '../git-tags/index.ts';
import { GiteaTagsDatasource } from '../gitea-tags/index.ts';
import { GithubTagsDatasource } from '../github-tags/index.ts';
import { GitlabTagsDatasource } from '../gitlab-tags/index.ts';
import type { DataSource } from './types.ts';

// TODO: figure out class hierarchy (#10532)
export class BaseGoDatasource {
  private static readonly gitlabHttpsRegExp = regEx(
    /^(?<httpsRegExpUrl>https:\/\/[^/]*gitlab\.[^/]*)\/(?<httpsRegExpName>.+?)(?:\/v\d+)?[/]?$/,
  );
  private static readonly gitlabRegExp = regEx(
    /^(?<regExpUrl>gitlab\.[^/]*)\/(?<regExpPath>.+?)(?:\/v\d+)?[/]?$/,
  );
  private static readonly gitVcsRegexp = regEx(
    /^(?:[^/]+)\/(?<module>.*)\.git(?:$|\/)/,
  );
  private static readonly goImportRegexp = regEx(
    /<meta\s+name="?go-import"?\s+content="(?<prefix>[^"\s]+)\s+(?<proto>[^"\s]+)\s+(?<goImportURL>[^"\s]+)/,
  );
  private static readonly goImportAllRegexp = regEx(
    BaseGoDatasource.goImportRegexp.source,
    'g',
  );

  private static readonly id = 'go';
  private static readonly http = new Http(BaseGoDatasource.id);

  static async getDatasource(goModule: string): Promise<DataSource | null> {
    if (goModule.startsWith('gopkg.in/')) {
      const [pkg] = goModule.replace('gopkg.in/', '').split('.');
      const packageName = pkg.includes('/') ? pkg : `go-${pkg}/${pkg}`;
      return {
        datasource: GithubTagsDatasource.id,
        packageName,
        registryUrl: 'https://github.com',
      };
    }

    if (goModule.startsWith('github.com/')) {
      const split = goModule.split('/');
      const packageName = `${split[1]}/${split[2]}`;
      return {
        datasource: GithubTagsDatasource.id,
        packageName,
        registryUrl: 'https://github.com',
      };
    }

    if (goModule.startsWith('bitbucket.org/')) {
      const split = goModule.split('/');
      const packageName = `${split[1]}/${split[2]}`;
      return {
        datasource: BitbucketTagsDatasource.id,
        packageName,
        registryUrl: 'https://bitbucket.org',
      };
    }

    if (goModule.startsWith('code.cloudfoundry.org/')) {
      const packageName = goModule.replace(
        'code.cloudfoundry.org',
        'cloudfoundry',
      );
      return {
        datasource: GithubTagsDatasource.id,
        packageName,
        registryUrl: 'https://github.com',
      };
    }

    if (goModule.startsWith('dev.azure.com/')) {
      const split = goModule.split('/');
      if ((split.length > 4 && split[3] === '_git') || split.length > 3) {
        const packageName = `https://dev.azure.com/${split[1]}/${split[2]}/_git/${(split[3] === '_git' ? split[4] : split[3]).replace(regEx(/\.git$/), '')}`;
        return {
          datasource: GitTagsDatasource.id,
          packageName,
        };
      }
    }

    //#region known gitea compatible hosts
    if (goModule.startsWith('gitea.com/')) {
      const split = goModule.split('/');
      const packageName = `${split[1]}/${split[2]}`;
      return {
        datasource: GiteaTagsDatasource.id,
        packageName,
        registryUrl: 'https://gitea.com',
      };
    }

    if (goModule.startsWith('code.forgejo.org/')) {
      const split = goModule.split('/');
      const packageName = `${split[1]}/${split[2]}`;
      return {
        datasource: ForgejoTagsDatasource.id,
        packageName,
        registryUrl: 'https://code.forgejo.org',
      };
    }

    if (goModule.startsWith('codeberg.org/')) {
      const split = goModule.split('/');
      const packageName = `${split[1]}/${split[2]}`;
      return {
        datasource: ForgejoTagsDatasource.id,
        packageName,
        registryUrl: 'https://codeberg.org',
      };
    }
    //#endregion

    return await BaseGoDatasource.goGetDatasource(goModule);
  }

  /**
   * Returns the module proxy URL that a `go-import` meta tag with the `mod` VCS advertises for the module.
   *
   * @see https://go.dev/ref/mod#serving-from-proxy
   */
  static async getModProxyUrl(goModule: string): Promise<string | null> {
    const html = await BaseGoDatasource.goGet(goModule);
    return BaseGoDatasource.findModProxyUrl(html, goModule);
  }

  /**
   * Finds the first `go-import` meta tag with the `mod` VCS whose prefix is a path prefix of the module.
   *
   * The proxy URL is controlled by the host serving the page, so only a valid `https:` URL is accepted, as with `go` (unless `GOINSECURE` is set).
   */
  private static findModProxyUrl(
    html: string,
    goModule: string,
  ): string | null {
    for (const match of html.matchAll(BaseGoDatasource.goImportAllRegexp)) {
      const { prefix, proto, goImportURL } = match.groups as Record<
        string,
        string
      >;

      if (
        proto !== 'mod' ||
        !(goModule === prefix || goModule.startsWith(`${prefix}/`))
      ) {
        continue;
      }

      if (parseUrl(goImportURL)?.protocol !== 'https:') {
        logger.debug(
          { goModule, goImportURL },
          'Ignoring go-import mod proxy URL which is not a valid https URL',
        );
        continue;
      }

      return goImportURL;
    }

    return null;
  }

  private static async goGet(goModule: string): Promise<string> {
    const goModuleUrl = goModule.replace(regEx(/\.git(?:\/[a-z0-9/]*)?$/), '');
    const pkgUrl = `https://${goModuleUrl}?go-get=1`;
    const { body } = await BaseGoDatasource.http.getText(pkgUrl);
    return body;
  }

  private static async goGetDatasource(
    goModule: string,
  ): Promise<DataSource | null> {
    const html = await BaseGoDatasource.goGet(goModule);

    // `go` prefers a `mod` proxy over a VCS for the same module, so leave it to the caller to look it up
    if (BaseGoDatasource.findModProxyUrl(html, goModule)) {
      logger.debug(
        { goModule },
        'Preferring go-import mod proxy over other go-import and go-source entries',
      );
      return null;
    }

    const goSourceHeader = BaseGoDatasource.goSourceHeader(html, goModule);
    if (goSourceHeader) {
      return goSourceHeader;
    }

    // GitHub Enterprise only returns a go-import meta
    const goImport = BaseGoDatasource.goImportHeader(html, goModule);
    if (goImport) {
      return goImport;
    }

    logger.trace({ goModule }, 'No go-source or go-import header found');
    /* istanbul ignore next */
    return null;
  }

  private static goSourceHeader(
    html: string,
    goModule: string,
  ): DataSource | null {
    const sourceMatchGroups = regEx(
      /<meta\s+name="?go-source"?\s+content="(?<prefix>[^"\s]+)\s+(?<goSourceUrl>[^"\s]+)/,
    ).exec(html)?.groups;
    if (!sourceMatchGroups) {
      return null;
    }
    const { prefix, goSourceUrl } = sourceMatchGroups;

    if (!goModule.startsWith(prefix)) {
      logger.trace({ goModule }, 'go-source header prefix not match');
      return null;
    }

    logger.debug(`Go lookup source url ${goSourceUrl} for module ${goModule}`);
    return this.detectDatasource(goSourceUrl, goModule);
  }

  private static detectDatasource(
    metadataUrl: string,
    goModule: string,
  ): DataSource | null {
    if (metadataUrl.startsWith('https://github.com/')) {
      return {
        datasource: GithubTagsDatasource.id,
        packageName: metadataUrl
          .replace('https://github.com/', '')
          .replace(regEx(/\/$/), ''),
        registryUrl: 'https://github.com',
      };
    }

    const gitlabModuleName =
      BaseGoDatasource.gitlabRegExp.exec(goModule)?.groups?.regExpPath;
    const vcsIndicatedModule =
      BaseGoDatasource.gitVcsRegexp.exec(goModule)?.groups?.module;

    const metadataUrlMatchGroups =
      BaseGoDatasource.gitlabHttpsRegExp.exec(metadataUrl)?.groups;
    if (metadataUrlMatchGroups) {
      const { httpsRegExpUrl, httpsRegExpName } = metadataUrlMatchGroups;

      let packageName = vcsIndicatedModule ?? gitlabModuleName;

      // Detect submodules in monorepos by comparing metadata path and module path
      if (!vcsIndicatedModule && httpsRegExpName && gitlabModuleName) {
        const metadataPath = httpsRegExpName;
        const modulePath = gitlabModuleName;

        if (modulePath.startsWith(`${metadataPath}/`)) {
          packageName = metadataPath;
        }
      }

      // If we still don't have a package name, fall back to the metadata URL path
      packageName = packageName ?? httpsRegExpName;

      return {
        datasource: GitlabTagsDatasource.id,
        registryUrl: httpsRegExpUrl,
        packageName,
      };
    }

    if (hostRules.hostType({ url: metadataUrl }) === 'gitlab') {
      const parsedUrl = parseUrl(metadataUrl);
      if (!parsedUrl) {
        logger.trace({ goModule }, 'Could not parse go-source URL');
        return null;
      }

      const endpoint = GlobalConfig.get('endpoint');
      const endpointPrefix = regEx(
        /https:\/\/[^/]+\/(?<prefix>.*?\/)(?:api\/v4\/?)?/,
      ).exec(endpoint)?.groups?.prefix;

      let packageName =
        // a .git path indicates a concrete git repository, which can be different from metadata returned by gitlab
        vcsIndicatedModule ?? trimLeadingSlash(parsedUrl.pathname);
      if (endpointPrefix && endpointPrefix !== 'api/') {
        packageName = packageName.replace(endpointPrefix, '');
      }

      const registryUrl = endpointPrefix
        ? endpoint.replace(regEx(/\/api\/v4\/?$/), '/')
        : `${parsedUrl.protocol}//${parsedUrl.host}`;

      return {
        datasource: GitlabTagsDatasource.id,
        registryUrl,
        packageName,
      };
    }
    return null;
  }

  private static goImportHeader(
    html: string,
    goModule: string,
  ): DataSource | null {
    const importMatchGroups =
      BaseGoDatasource.goImportRegexp.exec(html)?.groups;
    if (!importMatchGroups) {
      logger.trace({ goModule }, 'No go-source or go-import header found');
      return null;
    }
    const { prefix, proto, goImportURL } = importMatchGroups;

    if (!goModule.startsWith(prefix)) {
      logger.trace({ goModule }, 'go-import header prefix not match');
      return null;
    }

    if (proto !== 'git') {
      logger.trace({ goModule }, 'go-import header proto not git');
      return null;
    }

    // get server base url from import url
    const parsedUrl = parseUrl(goImportURL);
    if (!parsedUrl) {
      logger.trace({ goModule }, 'Could not parse go-import URL');
      return null;
    }

    logger.debug(`Go module: ${goModule} lookup import url ${goImportURL}`);

    const datasource = this.detectDatasource(
      goImportURL.replace(regEx(/\.git$/), ''),
      goModule,
    );
    if (datasource !== null) {
      return datasource;
    }
    // fall back to old behavior if detection did not work

    switch (detectPlatform(goImportURL)) {
      case 'github': {
        // split the go module from the URL: host/go/module -> go/module
        // TODO: `parsedUrl.pathname` can be undefined
        const packageName = trimTrailingSlash(`${parsedUrl.pathname}`)
          .replace(regEx(/\.git$/), '')
          .split('/')
          .slice(-2)
          .join('/');

        return {
          datasource: GithubTagsDatasource.id,
          registryUrl: `${parsedUrl.protocol}//${parsedUrl.host}`,
          packageName,
        };
      }
      case 'azure': {
        return {
          datasource: GitTagsDatasource.id,
          packageName: goImportURL.replace(regEx(/\.git$/), ''),
        };
      }
      default: {
        return {
          datasource: GitTagsDatasource.id,
          packageName: goImportURL,
        };
      }
    }
  }
}