renovate/lib/modules/manager/gomod/artifacts.ts

import { isString } from '@sindresorhus/is';
import semver from 'semver';
import { quote } from 'shlex';
import upath from 'upath';
import { GlobalConfig } from '../../../config/global.ts';
import { TEMPORARY_ERROR } from '../../../constants/error-messages.ts';
import { logger } from '../../../logger/index.ts';
import { getEnv } from '../../../util/env.ts';
import type { ExecOptions } from '../../../util/exec/types.ts';
import { filterMap } from '../../../util/filter-map.ts';
import {
  ensureCacheDir,
  findLocalSiblingOrParent,
  isValidLocalPath,
  readLocalFile,
  writeLocalFile,
} from '../../../util/fs/index.ts';
import { withGitEnvironment } from '../../../util/git/exec.ts';
import { collectFileChanges } from '../../../util/git/file-changes.ts';
import { getRepoStatus } from '../../../util/git/index.ts';
import { regEx } from '../../../util/regex.ts';
import { isValid } from '../../versioning/semver/index.ts';
import type {
  PackageDependency,
  UpdateArtifact,
  UpdateArtifactsConfig,
  UpdateArtifactsResult,
} from '../types.ts';
import {
  artifactErrorResult,
  fileAddition,
  fileChangesToArtifactResults,
  resolveToolConstraint,
} from '../util.ts';
import { getExtraDepsNotice } from './artifacts-extra.ts';
import { getGoModulesInTidyOrder } from './package-tree.ts';

const { major, valid } = semver;
const gitExec = withGitEnvironment(['go']);

async function getUpdateImportPathCmds(
  updatedDeps: PackageDependency[],
  config: UpdateArtifactsConfig,
): Promise<string[]> {
  // Check if we fail to parse any major versions and log that they're skipped
  const invalidMajorDeps = updatedDeps.filter(
    ({ newVersion }) => !valid(newVersion),
  );
  if (invalidMajorDeps.length > 0) {
    invalidMajorDeps.forEach(({ depName }) =>
      logger.warn(
        { depName },
        'Ignoring dependency: Could not get major version',
      ),
    );
  }

  const updateImportCommands = updatedDeps
    .filter(
      ({ newVersion }) =>
        valid(newVersion) && !newVersion!.endsWith('+incompatible'),
    )
    .map(({ depName, newVersion }) => ({
      depName: depName!,
      newMajor: major(newVersion!),
    }))
    // Skip path updates going from v0 to v1
    .filter(
      ({ depName, newMajor }) =>
        depName.startsWith('gopkg.in/') || newMajor > 1,
    )

    .map(
      ({ depName, newMajor }) =>
        `mod upgrade --mod-name=${quote(depName)} -t=${newMajor}`,
    );

  if (updateImportCommands.length > 0) {
    let installMarwanModArgs =
      'install github.com/marwan-at-work/mod/cmd/mod@latest';
    const gomodModCompatibility = await resolveToolConstraint(
      config,
      'gomodMod',
    );
    if (gomodModCompatibility) {
      if (
        gomodModCompatibility.startsWith('v') &&
        isValid(gomodModCompatibility.replace(regEx(/^v/), ''))
      ) {
        installMarwanModArgs = installMarwanModArgs.replace(
          regEx(/@latest$/),
          `@${gomodModCompatibility}`,
        );
      } else {
        logger.debug(
          { gomodModCompatibility },
          'marwan-at-work/mod compatibility range is not valid - skipping',
        );
      }
    } else {
      logger.debug(
        'No marwan-at-work/mod compatibility range found - installing marwan-at-work/mod latest',
      );
    }
    updateImportCommands.unshift(`go ${installMarwanModArgs}`);
  }

  return updateImportCommands;
}

function useModcacherw(goVersion: string | undefined): boolean {
  if (!isString(goVersion)) {
    return true;
  }

  return semver.intersects(goVersion, `>=1.14`);
}

export async function updateArtifacts({
  packageFileName: goModFileName,
  updatedDeps,
  newPackageFileContent: newGoModContent,
  config,
}: UpdateArtifact): Promise<UpdateArtifactsResult[] | null> {
  logger.debug(`gomod.updateArtifacts(${goModFileName})`);

  const sumFileName = goModFileName.replace(regEx(/\.mod$/), '.sum');
  const existingGoSumContent = await readLocalFile(sumFileName);
  if (!existingGoSumContent) {
    logger.debug('No go.sum found');
    return null;
  }
  const goModDir = upath.dirname(goModFileName);
  const goModFileBaseName = upath.basename(goModFileName);
  const modFileFlag =
    goModFileBaseName === 'go.mod'
      ? ''
      : ` -modfile=${quote(goModFileBaseName)}`;

  // The "vendor" directory can be next to the go.mod, but also in the parent directory in case
  // the go workspaces are used.
  const vendorDir = await findLocalSiblingOrParent(goModFileName, 'vendor');
  const vendorModulesFileName = upath.join(vendorDir ?? '', 'modules.txt');
  const useVendor =
    !!config.postUpdateOptions?.includes('gomodVendor') ||
    (!config.postUpdateOptions?.includes('gomodSkipVendor') &&
      vendorDir &&
      (await readLocalFile(vendorModulesFileName)) !== null);
  let massagedGoMod = newGoModContent;
  const useGoGenerate = !!config.postUpdateOptions?.includes('goGenerate');
  const allowedUnsafeExecutions = GlobalConfig.get('allowedUnsafeExecutions');
  const goGenerateAllowed = allowedUnsafeExecutions?.includes('goGenerate');

  if (config.postUpdateOptions?.includes('gomodMassage')) {
    // Regex match inline replace directive, example:
    // replace golang.org/x/net v1.2.3 => example.com/fork/net v1.4.5
    // https://go.dev/ref/mod#go-mod-file-replace

    // replace bracket after comments, so it doesn't break the regex, doing a complex regex causes problems
    // when there's a comment and ")" after it, the regex will read replace block until comment.. and stop.
    massagedGoMod = massagedGoMod
      .split('\n')
      .map((line) => {
        if (line.trim().startsWith('//')) {
          return line.replace(')', 'renovate-replace-bracket');
        }
        return line;
      })
      .join('\n');

    const inlineReplaceRegEx = regEx(
      /(?<newline>\r?\n)(?<directive>replace\s+[^\s]+\s+=>\s+\.\.\/.*)/g,
    );

    // $<newline> will be matched with the (\r?\n) group
    // $<directive> will be matched with the inline replace match, example
    // "// renovate-replace replace golang.org/x/net v1.2.3 => example.com/fork/net v1.4.5"
    const inlineCommentOut = '$<newline>// renovate-replace $<directive>';

    // Regex match replace directive block, example:
    // replace (
    //     golang.org/x/net v1.2.3 => example.com/fork/net v1.4.5
    // )
    const blockReplaceRegEx = regEx(/(?:\r?\n)replace\s*\([^)]+\s*\)/g);

    /**
     * replacerFunction for commenting out replace blocks
     * @param match A string representing a golang replace directive block
     * @returns A commented out block with // renovate-replace
     */
    function blockCommentOut(match: string): string {
      return match.replace(
        regEx(/(?<newline>\r?\n)/g),
        '$<newline>// renovate-replace ',
      );
    }

    // Comment out golang replace directives
    massagedGoMod = massagedGoMod
      .replace(inlineReplaceRegEx, inlineCommentOut)
      .replace(blockReplaceRegEx, blockCommentOut);

    // v8 ignore else -- needs a go.mod the replace massaging leaves unchanged
    if (massagedGoMod !== newGoModContent) {
      logger.debug(
        'Removed some relative replace statements and comments from go.mod',
      );
    }
  }
  const goConstraints = await deriveGoToolchainConstraints(
    config,
    newGoModContent,
  );

  try {
    await writeLocalFile(goModFileName, massagedGoMod);

    const cmd = 'go';
    const env = getEnv();
    const execOptions: ExecOptions = {
      cwdFile: goModFileName,
      extraEnv: {
        GOPATH: await ensureCacheDir('go'),
        GOPROXY: env.GOPROXY,
        GOPRIVATE: env.GOPRIVATE,
        GONOPROXY: env.GONOPROXY,
        GONOSUMDB: env.GONOSUMDB,
        GOSUMDB: env.GOSUMDB,
        GOINSECURE: env.GOINSECURE,
        /* v8 ignore next -- TODO: add test */
        GOFLAGS: useModcacherw(goConstraints) ? '-modcacherw' : null,
        CGO_ENABLED: GlobalConfig.get('binarySource') === 'docker' ? '0' : null,
      },
      docker: {},
      toolConstraints: [
        {
          toolName: 'golang',
          constraint: goConstraints,
        },
      ],
    };

    const execCommands: string[] = [];

    let goGetDirs: string | undefined;
    if (config.goGetDirs) {
      goGetDirs = config.goGetDirs
        .filter((dir) => {
          const isValid = isValidLocalPath(dir);
          if (!isValid) {
            logger.warn({ dir }, 'Invalid path in goGetDirs');
          }
          return isValid;
        })
        .map(quote)
        .join(' ');

      if (goGetDirs === '') {
        throw new Error('Invalid goGetDirs');
      }
    }

    let args = `get${modFileFlag} `;

    if (goConstraints && !semver.intersects(goConstraints, `>=1.18`)) {
      // For Go versions < 1.18, we need to use the -d flag to avoid builds or installs
      // https://go.dev/doc/go1.18#go-get
      args += `-d `;
    }

    args += `-t ${goGetDirs ?? './...'}`;
    logger.trace({ cmd, args }, 'go get command included');
    execCommands.push(`${cmd} ${args}`);

    // Update import paths on major updates
    const isImportPathUpdateRequired =
      config.postUpdateOptions?.includes('gomodUpdateImportPaths') &&
      config.updateType === 'major';

    if (isImportPathUpdateRequired) {
      const updateImportCmds = await getUpdateImportPathCmds(
        updatedDeps,
        config,
      );
      if (updateImportCmds.length > 0) {
        logger.debug(updateImportCmds, 'update import path commands included');
        // The updates
        execCommands.push(...updateImportCmds);
      }
    }

    const mustSkipGoModTidy =
      !config.postUpdateOptions?.includes('gomodUpdateImportPaths') &&
      config.updateType === 'major';
    if (mustSkipGoModTidy) {
      logger.debug('go mod tidy command skipped');
    }

    let tidyOpts = '';
    if (config.postUpdateOptions?.includes('gomodTidy1.17')) {
      tidyOpts += ' -compat=1.17';
    }
    if (config.postUpdateOptions?.includes('gomodTidyE')) {
      tidyOpts += ' -e';
    }

    // dependent modules can only be tidied once the updated module is settled
    const isGoModTidyAllRequired =
      config.postUpdateOptions?.includes('gomodTidyAll') === true;
    const isGoModTidyRequired =
      !mustSkipGoModTidy &&
      (config.postUpdateOptions?.includes('gomodTidy') === true ||
        config.postUpdateOptions?.includes('gomodTidy1.17') === true ||
        config.postUpdateOptions?.includes('gomodTidyE') === true ||
        isGoModTidyAllRequired ||
        (config.updateType === 'major' && isImportPathUpdateRequired));
    if (isGoModTidyRequired) {
      args = `mod tidy${modFileFlag}${tidyOpts}`;
      logger.debug('go mod tidy command included');
      execCommands.push(`${cmd} ${args}`);
    }

    let goWorkSumFileName = upath.join(goModDir, 'go.work.sum');
    if (useVendor) {
      // If we find a go.work, then use go workspace vendoring.
      const goWorkFile = await findLocalSiblingOrParent(
        goModFileName,
        'go.work',
      );

      if (goWorkFile) {
        goWorkSumFileName = upath.join(
          upath.dirname(goWorkFile),
          'go.work.sum',
        );

        args = 'work vendor';
        logger.debug('using go work vendor');
        execCommands.push(`${cmd} ${args}`);

        args = 'work sync';
        logger.debug('using go work sync');
        execCommands.push(`${cmd} ${args}`);
      } else {
        args = `mod vendor${modFileFlag}`;
        logger.debug('using go mod vendor');
        execCommands.push(`${cmd} ${args}`);
      }

      if (isGoModTidyRequired) {
        args = `mod tidy${modFileFlag}${tidyOpts}`;
        logger.debug('go mod tidy command included');
        execCommands.push(`${cmd} ${args}`);
      }
    }

    // We tidy one more time as a solution for #6795
    if (isGoModTidyRequired) {
      args = `mod tidy${modFileFlag}${tidyOpts}`;
      logger.debug('go mod tidy command included');
      execCommands.push(`${cmd} ${args}`);
    }

    let dependentModules: string[] = [];
    if (isGoModTidyAllRequired) {
      try {
        dependentModules = await getGoModulesInTidyOrder(goModFileName);
        for (const dependent of dependentModules) {
          const dir = upath.relative(goModDir, upath.dirname(dependent));
          execCommands.push(`${cmd} -C ${quote(dir)} mod tidy${tidyOpts}`);
        }
        logger.debug({ dependentModules }, 'go mod tidy commands included');
      } catch (err) {
        logger.warn({ err }, 'Failed to find dependent Go modules');
      }
    }

    if (useGoGenerate) {
      if (goGenerateAllowed) {
        logger.debug('go generate command included');
        execCommands.push(`${cmd} generate ./...`);
      } else {
        logger.once.warn(
          `go generate command requested as a post update action, but goGenerate is not permitted in the allowedUnsafeExecutions`,
        );
      }
    }

    await gitExec(execCommands, execOptions);

    const status = await getRepoStatus();
    const dependentFiles = dependentModules.flatMap((f) => [
      f,
      f.replace(regEx(/\.mod$/), '.sum'),
    ]);

    if (
      !status.modified.includes(sumFileName) &&
      !status.modified.includes(goModFileName) &&
      !status.modified.includes(goWorkSumFileName) &&
      !dependentFiles.some((f) => status.modified.includes(f))
    ) {
      return null;
    }

    const res: UpdateArtifactsResult[] = [];
    if (status.modified.includes(sumFileName)) {
      logger.debug('Returning updated go.sum');
      res.push(fileAddition(sumFileName, await readLocalFile(sumFileName)));
    }

    if (status.modified.includes(goWorkSumFileName)) {
      logger.debug('Returning updated go.work.sum');
      res.push(
        fileAddition(goWorkSumFileName, await readLocalFile(goWorkSumFileName)),
      );
    }

    for (const f of dependentFiles) {
      if (status.modified.includes(f)) {
        logger.trace(`Returning updated ${f}`);
        res.push(fileAddition(f, await readLocalFile(f)));
      }
    }

    // Include all the .go file import changes
    if (isImportPathUpdateRequired) {
      logger.debug('Returning updated go source files for import path changes');
      for (const f of status.modified) {
        if (f.endsWith('.go')) {
          res.push(fileAddition(f, await readLocalFile(f)));
        }
      }
    }

    const alreadyAdded = new Set<string>();
    const alreadyDeleted = new Set<string>();
    if (useVendor) {
      const vendorChanges = await collectFileChanges(status, {
        filter: (f) => !!vendorDir && f.startsWith(vendorDir),
      });
      for (const change of vendorChanges) {
        if (change.type === 'addition') {
          alreadyAdded.add(change.path);
        } else {
          alreadyDeleted.add(change.path);
        }
      }
      res.push(...fileChangesToArtifactResults(vendorChanges));
    }

    // TODO: throws in tests (#22198)
    const finalGoModContent = (await readLocalFile(goModFileName, 'utf8'))!
      .replace(regEx(/\/\/ renovate-replace /g), '')
      .replace(regEx(/renovate-replace-bracket/g), ')');
    if (finalGoModContent !== newGoModContent) {
      const artifactResult: UpdateArtifactsResult = {
        file: {
          type: 'addition',
          path: goModFileName,
          contents: finalGoModContent,
        },
      };

      const updatedDepNames = filterMap(updatedDeps, (dep) => dep?.depName);
      const extraDepsNotice = getExtraDepsNotice(
        newGoModContent,
        finalGoModContent,
        updatedDepNames,
        config,
      );

      if (extraDepsNotice) {
        artifactResult.notice = {
          file: goModFileName,
          message: extraDepsNotice,
        };
      }

      logger.debug('Found updated go.mod after go.sum update');
      res.push(artifactResult);
      alreadyAdded.add(goModFileName);
    }

    // add all files added when in `go generate` mode.
    // unfortunately there is not a good way as there is with vendoring or go import path updates to detect this.
    // Do this at the very very end to ensure we only capture files which would have been explicitly
    // modified, added, or deleted from a `go generate` invocation
    if (useGoGenerate && goGenerateAllowed) {
      logger.debug(
        'Updating all modified files since generated files were added',
      );
      res.push(
        ...fileChangesToArtifactResults([
          ...(await collectFileChanges(status, {
            include: ['modified', 'created'],
            filter: (f) => !alreadyAdded.has(f),
          })),
          ...(await collectFileChanges(status, {
            include: ['deleted'],
            filter: (f) => !alreadyDeleted.has(f),
          })),
        ]),
      );
    }
    return res;
  } catch (err) {
    if (err.message === TEMPORARY_ERROR) {
      throw err;
    }
    logger.debug({ err }, 'Failed to update go.sum');
    return artifactErrorResult(sumFileName, err);
  }
}

function getGoConstraints(content: string): string | undefined {
  // prefer toolchain directive when go.mod has one
  const toolchainMatch = regEx(/^toolchain\s*go(?<gover>\d+\.\d+\.\d+)$/m).exec(
    content,
  );
  const toolchainVer = toolchainMatch?.groups?.gover;
  if (toolchainVer) {
    logger.debug(
      `Using go version ${toolchainVer} found in toolchain directive`,
    );
    return toolchainVer;
  }

  // If go.mod doesn't have toolchain directive and has a full go version spec,
  // for example `go 1.23.6`, pick this version, this doesn't match major.minor version spec.
  //
  // This is because when go.mod have same version defined in go directive and toolchain directive,
  // go will remove toolchain directive from go.mod.
  //
  // For example, go will rewrite `go 1.23.5\ntoolchain go1.23.5` to `go 1.23.5` by default,
  // in this case, the go directive is the toolchain directive.
  const goFullVersion = regEx(/^go\s*(?<gover>\d+\.\d+\.\d+)$/m).exec(content)
    ?.groups?.gover;
  if (goFullVersion) {
    return goFullVersion;
  }

  const re = regEx(/^go\s*(?<gover>\d+\.\d+)$/m);
  const match = re.exec(content);
  if (!match?.groups?.gover) {
    return undefined;
  }
  return `^${match.groups.gover}`;
}

/**
 * Derive the version of the Go toolchain needed to run this project.
 *
 * This matches with the `golang` Containerbase tool.
 *
 * In precedence order:
 *
 * 1. config: \`constraints.go\`
 * 1. \`go.mod\`: \`toolchain\` directive
 * 1. \`go.mod\`: \`go\` directive
 * 1. the \`go\` constraint collected during extraction
 *
 * NOTE that the \`constraints.golang\` is not used (TODO #42601)
 */
export async function deriveGoToolchainConstraints(
  config: UpdateArtifactsConfig,
  newGoModContent: string,
): Promise<string | undefined> {
  return await resolveToolConstraint(config, 'go', () =>
    getGoConstraints(newGoModContent),
  );
}