renovate/lib/modules/manager/npm/post-update/npm.ts

// TODO: types (#22198)
import { isNonEmptyString, isNumber, isString } from '@sindresorhus/is';
import ini from 'ini';
import { DateTime } from 'luxon';
import semver from 'semver';
import { quote } from 'shlex';
import upath from 'upath';
import { GlobalConfig } from '../../../../config/global.ts';
import {
  SYSTEM_INSUFFICIENT_DISK_SPACE,
  TEMPORARY_ERROR,
} from '../../../../constants/error-messages.ts';
import { logger } from '../../../../logger/index.ts';
import { exec, getToolSettingsOptions } from '../../../../util/exec/index.ts';
import type {
  ExecOptions,
  ExtraEnv,
  ToolConstraint,
} from '../../../../util/exec/types.ts';
import {
  deleteLocalFile,
  localPathExists,
  readLocalFile,
  renameLocalFile,
} from '../../../../util/fs/index.ts';
import { minimatch } from '../../../../util/minimatch.ts';
import { toMs } from '../../../../util/pretty-time.ts';
import { regEx } from '../../../../util/regex.ts';
import { Result } from '../../../../util/result.ts';
import { trimSlashes } from '../../../../util/url.ts';
import type { PostUpdateConfig, Upgrade } from '../../types.ts';
import { PackageLock } from '../schema.ts';
import { composeLockFile, parseLockFile } from '../utils.ts';
import { getNodeToolConstraint } from './node-version.ts';
import type { GenerateLockFileResult, NpmrcCooldownResult } from './types.ts';
import {
  getNodeOptions,
  getPackageManagerVersion,
  lazyLoadPackageJson,
} from './utils.ts';

export function parseNpmrcCooldownDate(
  npmrcContent: string | null,
): NpmrcCooldownResult | null {
  if (!npmrcContent) {
    return null;
  }

  const parsed = ini.parse(npmrcContent);

  const before = parsed.before;
  if (isNonEmptyString(before)) {
    const dt = DateTime.fromISO(before, { zone: 'utc' });
    if (dt.isValid) {
      return { date: dt, source: 'before' };
    }
    logger.debug(`Invalid before date in .npmrc: ${before}, ignoring`);
  }

  const minReleaseAge = parsed['min-release-age'];
  if (isNonEmptyString(minReleaseAge)) {
    const days = parseInt(minReleaseAge, 10);
    if (isNumber(days) && days >= 0) {
      return {
        date: DateTime.now().minus({ days }).toUTC(),
        source: 'min-release-age',
      };
    }
    logger.debug(
      `Invalid min-release-age in .npmrc: ${minReleaseAge}, ignoring`,
    );
  }

  return null;
}

async function getNpmConstraintFromPackageLock(
  lockFileDir: string,
  filename: string,
): Promise<string | null> {
  const packageLockFileName = upath.join(lockFileDir, filename);
  const packageLockContents = await readLocalFile(packageLockFileName, 'utf8');
  const packageLockJson = Result.parse(
    packageLockContents,
    PackageLock,
  ).unwrapOrNull();
  if (!packageLockJson) {
    logger.debug(`Could not parse ${packageLockFileName}`);
    return null;
  }
  const { lockfileVersion } = packageLockJson;
  if (lockfileVersion === 1) {
    logger.debug(`Using npm constraint <7 for lockfileVersion=1`);
    return `<7`;
  }
  if (lockfileVersion === 2) {
    logger.debug(`Using npm constraint <9 for lockfileVersion=2`);
    return `<9`;
  }
  return null;
}

export async function generateLockFile(
  lockFileDir: string,
  env: NodeJS.ProcessEnv,
  filename: string,
  config: Partial<PostUpdateConfig> = {},
  upgrades: Upgrade[] = [],
  npmrcContent: string | null = null,
): Promise<GenerateLockFileResult> {
  // TODO: don't assume package-lock.json is in the same directory
  const lockFileName = upath.join(lockFileDir, filename);

  logger.debug(`Spawning npm install to create ${lockFileDir}/${filename}`);
  const { skipInstalls, postUpdateOptions } = config;

  let lockFile: string | null = null;
  let beforeFallback = false;
  try {
    const lazyPkgJson = lazyLoadPackageJson(lockFileDir);
    const npmToolConstraint: ToolConstraint = {
      toolName: 'npm',
      constraint:
        config.constraints?.npm ??
        getPackageManagerVersion('npm', await lazyPkgJson.getValue()) ??
        (await getNpmConstraintFromPackageLock(lockFileDir, filename)) ??
        null,
    };
    const supportsPreferDedupeFlag =
      !npmToolConstraint.constraint ||
      semver.intersects('>=7.0.0', npmToolConstraint.constraint);
    let commands: string[] = [];
    let cmdOptions = '';
    if (
      (postUpdateOptions?.includes('npmDedupe') === true &&
        !supportsPreferDedupeFlag) ||
      skipInstalls === false
    ) {
      logger.debug('Performing node_modules install');
      cmdOptions += '--no-audit';
    } else {
      logger.debug('Updating lock file only');
      cmdOptions += '--package-lock-only --no-audit';
    }

    if (postUpdateOptions?.includes('npmDedupe') && supportsPreferDedupeFlag) {
      logger.debug('Deduplicate dependencies on installation');
      cmdOptions += ' --prefer-dedupe';
    }

    if (!GlobalConfig.get('allowScripts') || config.ignoreScripts) {
      cmdOptions += ' --ignore-scripts';
    }

    let beforeFlag = '';
    if (config.minimumReleaseAge) {
      const ms = toMs(config.minimumReleaseAge);
      if (ms === null) {
        logger.debug(
          {
            minimumReleaseAge: config.minimumReleaseAge,
          },
          'Invalid minimumReleaseAge, skipping --before for npm install',
        );
      } else {
        const npmrcCooldown = parseNpmrcCooldownDate(npmrcContent);

        // npm rejects --before when min-release-age is set in .npmrc,
        // so let the .npmrc handle cooldown natively in that case
        if (npmrcCooldown?.source === 'min-release-age') {
          logger.debug(
            {
              npmrcMinReleaseAge: npmrcCooldown.date.toISO(),
              minimumReleaseAge: config.minimumReleaseAge,
            },
            'Skipping --before flag because .npmrc already contains min-release-age',
          );
        } else {
          let beforeDate = DateTime.now().minus(ms).toUTC();

          if (npmrcCooldown && npmrcCooldown.date < beforeDate) {
            logger.debug(
              {
                npmrcDate: npmrcCooldown.date.toISO(),
                beforeDate: beforeDate.toISO(),
              },
              'Using stricter .npmrc cooldown date over minimumReleaseAge date',
            );
            beforeDate = npmrcCooldown.date;
          }

          const beforeISO = beforeDate.toISO();
          logger.debug(
            {
              beforeISO,
              minimumReleaseAge: config.minimumReleaseAge,
            },
            'Setting npm --before based on minimumReleaseAge',
          );
          beforeFlag = ` --before=${beforeISO}`;
        }
      }
    }

    const extraEnv: ExtraEnv = {
      NPM_CONFIG_CACHE: env.NPM_CONFIG_CACHE,
      npm_config_store: env.npm_config_store,
    };

    const { nodeMaxMemory } = getToolSettingsOptions(config.toolSettings);
    if (nodeMaxMemory) {
      extraEnv.NODE_OPTIONS = getNodeOptions(nodeMaxMemory);
    }

    const execOptions: ExecOptions = {
      cwdFile: lockFileName,
      extraEnv,
      toolConstraints: [
        await getNodeToolConstraint(config, upgrades, lockFileDir, lazyPkgJson),
        ...(isNonEmptyString(npmToolConstraint.constraint)
          ? [npmToolConstraint]
          : []),
      ],
      docker: {},
    };
    /* v8 ignore next -- needs test */
    if (GlobalConfig.get('exposeAllEnv')) {
      extraEnv.NPM_AUTH = env.NPM_AUTH;
      extraEnv.NPM_EMAIL = env.NPM_EMAIL;
    }

    if (!upgrades.every((upgrade) => upgrade.isLockfileUpdate)) {
      // This command updates the lock file based on package.json
      commands.push(`npm install ${cmdOptions}${beforeFlag}`.trim());
    }

    // rangeStrategy = update-lockfile
    const lockUpdates = upgrades.filter((upgrade) => upgrade.isLockfileUpdate);

    // divide the deps in two categories: workspace and root
    const { lockRootUpdates, lockWorkspacesUpdates, workspaces, rootDeps } =
      divideWorkspaceAndRootDeps(lockFileDir, lockUpdates);

    if (workspaces.size && lockWorkspacesUpdates.length) {
      logger.debug('Performing lockfileUpdate (npm-workspaces)');
      for (const workspace of workspaces) {
        const currentWorkspaceUpdates = lockWorkspacesUpdates
          .filter((update) => update.workspace === workspace)
          .map((update) => update.managerData?.packageKey)
          .filter((packageKey) => !rootDeps.has(packageKey));

        // v8 ignore else -- TODO: add test #40625
        if (currentWorkspaceUpdates.length) {
          const updateCmd = `npm install ${cmdOptions}${beforeFlag} --workspace=${quote(workspace)} ${currentWorkspaceUpdates
            .map(quote)
            .join(' ')}`;
          commands.push(updateCmd);
        }
      }
    }

    if (lockRootUpdates.length) {
      logger.debug('Performing lockfileUpdate (npm)');
      const updateCmd = `npm install ${cmdOptions}${beforeFlag} ${lockRootUpdates
        .map((update) => update.managerData?.packageKey)
        .map(quote)
        .join(' ')}`;
      commands.push(updateCmd);
    }

    if (upgrades.some((upgrade) => upgrade.isRemediation)) {
      // We need to run twice to get the correct lock file
      commands.push(`npm install ${cmdOptions}${beforeFlag}`.trim());
    }

    // Lock file maintenance recreates the lock file from scratch, and a single
    // `npm install` can generate a lock file which is out of sync with
    // package.json, so we need to run the install a second time (#37531).
    // Skipped if `npmInstallTwice` is configured, as that doubles all install
    // commands already.
    if (
      upgrades.some((upgrade) => upgrade.isLockFileMaintenance) &&
      !postUpdateOptions?.includes('npmInstallTwice')
    ) {
      commands.push(`npm install ${cmdOptions}${beforeFlag}`.trim());
    }

    // postUpdateOptions
    if (
      config.postUpdateOptions?.includes('npmDedupe') &&
      !supportsPreferDedupeFlag
    ) {
      logger.debug('Performing npm dedupe after installation');
      commands.push('npm dedupe');
    }

    if (upgrades.find((upgrade) => upgrade.isLockFileMaintenance)) {
      logger.debug(
        `Removing ${lockFileName} first due to lock file maintenance upgrade`,
      );
      try {
        await deleteLocalFile(lockFileName);
      } catch (err) /* v8 ignore next -- TODO: add test #40625 */ {
        logger.debug(
          { err, lockFileName },
          'Error removing `package-lock.json` for lock file maintenance',
        );
      }
    }

    if (postUpdateOptions?.includes('npmInstallTwice')) {
      logger.debug('Running npm install twice');
      // Run the install command twice to ensure the lock file is up to date
      // iterate through commands and if any command starts with `npm install`, add it again
      const existingCommands = [...commands];
      commands = [];
      for (const command of existingCommands) {
        commands.push(command);
        // v8 ignore else -- TODO: add test #40625
        if (command.startsWith('npm install')) {
          commands.push(command);
        }
      }
    }

    // Run the commands, retrying without --before on ETARGET if needed
    await exec(commands, execOptions).catch(async (err) => {
      if (beforeFlag && err.stderr?.includes('with a date before')) {
        logger.debug('npm --before caused ETARGET, retrying without --before');
        const commandsWithoutBefore = commands.map((cmd) =>
          cmd.replace(beforeFlag, ''),
        );
        beforeFallback = true;
        return exec(commandsWithoutBefore, execOptions);
      }
      throw err;
    });

    // massage to shrinkwrap if necessary
    if (
      filename === 'npm-shrinkwrap.json' &&
      (await localPathExists(upath.join(lockFileDir, 'package-lock.json')))
    ) {
      await renameLocalFile(
        upath.join(lockFileDir, 'package-lock.json'),
        upath.join(lockFileDir, 'npm-shrinkwrap.json'),
      );
    }

    // Read the result
    // TODO #22198
    lockFile = (await readLocalFile(
      upath.join(lockFileDir, filename),
      'utf8',
    ))!;

    // Massage lockfile counterparts of package.json that were modified
    // because npm install was called with an explicit version for rangeStrategy=update-lockfile
    if (lockUpdates.length) {
      const { detectedIndent, lockFileParsed } = parseLockFile(lockFile);
      if (
        lockFileParsed?.lockfileVersion === 2 ||
        lockFileParsed?.lockfileVersion === 3
      ) {
        lockUpdates.forEach((lockUpdate) => {
          const depType = lockUpdate.depType as
            'dependencies' | 'optionalDependencies';

          // TODO #22198
          // v8 ignore else -- TODO: add test #40625
          if (
            lockFileParsed.packages?.['']?.[depType]?.[lockUpdate.packageName!]
          ) {
            lockFileParsed.packages[''][depType][lockUpdate.packageName!] =
              lockUpdate.newValue!;
          }
        });
        lockFile = composeLockFile(lockFileParsed, detectedIndent);
      }
    }
  } catch (err) {
    // v8 ignore if -- TODO: add test #40625
    if (err.message === TEMPORARY_ERROR) {
      throw err;
    }
    logger.debug(
      {
        err,
        type: 'npm',
      },
      'lock file error',
    );
    // v8 ignore if -- TODO: add test #40625
    if (err.stderr?.includes('ENOSPC: no space left on device')) {
      throw new Error(SYSTEM_INSUFFICIENT_DISK_SPACE);
    }
    return { error: true, stderr: err.stderr };
  }
  return { error: !lockFile, lockFile, beforeFallback };
}

export function divideWorkspaceAndRootDeps(
  lockFileDir: string,
  lockUpdates: Upgrade[],
): {
  lockRootUpdates: Upgrade[];
  lockWorkspacesUpdates: Upgrade[];
  workspaces: Set<string>;
  rootDeps: Set<string>;
} {
  const lockRootUpdates: Upgrade[] = []; // stores all upgrades which are present in root package.json
  const lockWorkspacesUpdates: Upgrade[] = []; // stores all upgrades which are present in workspaces package.json
  const workspaces = new Set<string>(); // name of all workspaces
  const rootDeps = new Set<string>(); // packageName of all upgrades in root package.json (makes it check duplicate deps in root)

  // divide the deps in two categories: workspace and root
  for (const upgrade of lockUpdates) {
    upgrade.managerData ??= {};
    upgrade.managerData.packageKey = generatePackageKey(
      upgrade.packageName!,
      upgrade.newVersion!,
    );
    if (
      upgrade.managerData.workspacesPackages?.length &&
      isString(upgrade.packageFile)
    ) {
      const workspacePatterns = upgrade.managerData.workspacesPackages; // glob pattern or directory name/path
      const packageFileDir = trimSlashes(
        upgrade.packageFile.replace('package.json', ''),
      );

      // workspaceDir = packageFileDir - lockFileDir
      const workspaceDir = trimSlashes(
        packageFileDir.startsWith(lockFileDir)
          ? packageFileDir.slice(lockFileDir.length)
          : packageFileDir,
      );

      if (isNonEmptyString(workspaceDir)) {
        let workspaceName: string | undefined;
        // compare workspaceDir to workspace patterns
        // stop when the first match is found and
        // add workspaceDir to workspaces set and upgrade object
        for (const workspacePattern of workspacePatterns) {
          const massagedPattern = (workspacePattern as string).replace(
            regEx(/^\.\//),
            '',
          );
          if (minimatch(massagedPattern).match(workspaceDir)) {
            workspaceName = workspaceDir;
            break;
          }
        }
        if (workspaceName) {
          // v8 ignore else -- TODO: add test #40625
          if (
            !rootDeps.has(upgrade.managerData.packageKey) // prevent same dep from existing in root and workspace
          ) {
            workspaces.add(workspaceName);
            upgrade.workspace = workspaceName;
            lockWorkspacesUpdates.push(upgrade);
          }
        } else {
          logger.warn(
            { workspacePatterns, workspaceDir },
            'workspaceDir not found',
          );
        }
        continue;
      }
    }
    lockRootUpdates.push(upgrade);
    rootDeps.add(upgrade.managerData.packageKey);
  }

  return { lockRootUpdates, lockWorkspacesUpdates, workspaces, rootDeps };
}

function generatePackageKey(packageName: string, version: string): string {
  return `${packageName}@${version}`;
}