import { isString } from '@sindresorhus/is'; import semver from 'semver'; import { quote } from 'shlex'; import upath from 'upath'; import { GlobalConfig } from '../../../../config/global.ts'; import { SYSTEM_INSUFFICIENT_DISK_SPACE, TEMPORARY_ERROR, } from '../../../../constants/error-messages.ts'; import { logger } from '../../../../logger/index.ts'; import { ExternalHostError } from '../../../../types/errors/external-host-error.ts'; import { getEnv } from '../../../../util/env.ts'; import { exec, getToolSettingsOptions } from '../../../../util/exec/index.ts'; import type { CommandWithOptions, ExecOptions, ExtraEnv, ToolConstraint, } from '../../../../util/exec/types.ts'; import { localPathIsFile, readLocalFile, writeLocalFile, } from '../../../../util/fs/index.ts'; import { coerceObject } from '../../../../util/object.ts'; import { newlineRegex, regEx } from '../../../../util/regex.ts'; import { uniqueStrings } from '../../../../util/string.ts'; import { NpmDatasource } from '../../../datasource/npm/index.ts'; import type { PostUpdateConfig, Upgrade } from '../../types.ts'; import { getYarnLock, getYarnVersionFromLock } from '../extract/yarn.ts'; import type { NpmManagerData } from '../types.ts'; import { getNodeToolConstraint } from './node-version.ts'; import type { GenerateLockFileResult } from './types.ts'; import { getNodeOptions, getPackageManagerVersion, lazyLoadPackageJson, } from './utils.ts'; export async function checkYarnrc( lockFileDir: string, ): Promise<{ offlineMirror: boolean; yarnPath: string | null }> { let offlineMirror = false; let yarnPath: string | null = null; try { const yarnrc = await readLocalFile( upath.join(lockFileDir, '.yarnrc'), 'utf8', ); if (isString(yarnrc)) { const mirrorLine = yarnrc .split(newlineRegex) .find((line) => line.startsWith('yarn-offline-mirror ')); offlineMirror = !!mirrorLine; const pathLine = yarnrc .split(newlineRegex) .find((line) => line.startsWith('yarn-path ')); if (pathLine) { yarnPath = pathLine.replace( regEx(/^yarn-path\s+"?(?.+?)"?$/), '$', ); } if (yarnPath) { // resolve binary relative to `yarnrc` yarnPath = upath.join(lockFileDir, yarnPath); } const yarnBinaryExists = yarnPath ? await localPathIsFile(yarnPath) : false; let scrubbedYarnrc = yarnrc .replace('--install.pure-lockfile true', '') .replace('--install.frozen-lockfile true', ''); if (!yarnBinaryExists) { scrubbedYarnrc = scrubbedYarnrc.replace( regEx(/^yarn-path\s+"?.+?"?$/gm), '', ); yarnPath = null; } if (yarnrc !== scrubbedYarnrc) { logger.debug(`Writing scrubbed .yarnrc to ${lockFileDir}`); await writeLocalFile( upath.join(lockFileDir, '.yarnrc'), scrubbedYarnrc, ); } } /* v8 ignore next -- needs test */ } catch { // not found } return { offlineMirror, yarnPath }; } export function getOptimizeCommand(fileName: string): string[] { return ['sed', '-i', 's/ steps,/ steps.slice(0,1),/', fileName]; } export function isYarnUpdate(upgrade: Upgrade): boolean { return upgrade.depType === 'packageManager' && upgrade.depName === 'yarn'; } export async function generateLockFile( lockFileDir: string, env: NodeJS.ProcessEnv, config: Partial> = {}, upgrades: Upgrade[] = [], ): Promise { const lockFileName = upath.join(lockFileDir, 'yarn.lock'); logger.debug(`Spawning yarn install to create ${lockFileName}`); let lockFile: string | null = null; try { const lazyPgkJson = lazyLoadPackageJson(lockFileDir); const toolConstraints: ToolConstraint[] = [ await getNodeToolConstraint(config, upgrades, lockFileDir, lazyPgkJson), ]; const yarnUpdate = upgrades.find(isYarnUpdate); const yarnCompatibility = (yarnUpdate ? yarnUpdate.newValue : config.constraints?.yarn) ?? getPackageManagerVersion('yarn', await lazyPgkJson.getValue()) ?? getYarnVersionFromLock(await getYarnLock(lockFileName)); const minYarnVersion = semver.validRange(yarnCompatibility) && semver.minVersion(yarnCompatibility); const isYarn1 = !minYarnVersion || minYarnVersion.major === 1; const isYarnDedupeAvailable = minYarnVersion && semver.gte(minYarnVersion, '2.2.0'); const isYarnModeAvailable = minYarnVersion && semver.gte(minYarnVersion, '3.0.0'); const yarnTool: ToolConstraint = { toolName: 'yarn', constraint: '^1.22.18', // needs to be a v1 yarn, otherwise v2 will be installed }; // check first upgrade, see #17786 const hasPackageManager = !!config.managerData?.hasPackageManager || !!upgrades[0]?.managerData?.hasPackageManager; if (!isYarn1 && hasPackageManager) { toolConstraints.push({ toolName: 'corepack', constraint: config.constraints?.corepack, }); } else { toolConstraints.push(yarnTool); if (isYarn1 && minYarnVersion) { yarnTool.constraint = yarnCompatibility; } } const extraEnv: ExtraEnv = { NPM_CONFIG_CACHE: env.NPM_CONFIG_CACHE, npm_config_store: env.npm_config_store, }; const commands: (string | CommandWithOptions)[] = []; let cmdOptions = ''; // should have a leading space if (config.skipInstalls !== false) { if (isYarn1) { const { offlineMirror, yarnPath } = await checkYarnrc(lockFileDir); // v8 ignore else -- TODO: add test #40625 if (!offlineMirror) { logger.debug('Updating yarn.lock only - skipping node_modules'); // The following change causes Yarn 1.x to exit gracefully after updating the lock file but without installing node_modules yarnTool.toolName = 'yarn-slim'; if (yarnPath) { commands.push({ command: getOptimizeCommand(yarnPath), ignoreFailure: true, }); } } } else if (isYarnModeAvailable) { // Don't run the link step and only fetch what's necessary to compute an updated lockfile cmdOptions += ' --mode=update-lockfile'; } } if (isYarn1) { cmdOptions += ' --ignore-engines --ignore-platform --network-timeout 100000'; extraEnv.YARN_CACHE_FOLDER = env.YARN_CACHE_FOLDER; } else { extraEnv.YARN_ENABLE_IMMUTABLE_INSTALLS = 'false'; extraEnv.YARN_HTTP_TIMEOUT = '100000'; extraEnv.YARN_GLOBAL_FOLDER = env.YARN_GLOBAL_FOLDER; if (!config.managerData?.yarnZeroInstall) { logger.debug('Enabling global cache as zero-install is not detected'); extraEnv.YARN_ENABLE_GLOBAL_CACHE = '1'; } } if (!GlobalConfig.get('allowScripts') || config.ignoreScripts) { if (isYarn1) { cmdOptions += ' --ignore-scripts'; } else if (isYarnModeAvailable) { if (config.skipInstalls === false) { // Don't run the build scripts cmdOptions += ' --mode=skip-build'; } } else { extraEnv.YARN_ENABLE_SCRIPTS = '0'; } } const { nodeMaxMemory } = getToolSettingsOptions(config.toolSettings); if (nodeMaxMemory) { extraEnv.NODE_OPTIONS = getNodeOptions(nodeMaxMemory); } const execOptions: ExecOptions = { cwdFile: lockFileName, extraEnv, docker: {}, toolConstraints, }; /* v8 ignore next -- needs test */ if (GlobalConfig.get('exposeAllEnv')) { extraEnv.NPM_AUTH = env.NPM_AUTH; extraEnv.NPM_EMAIL = env.NPM_EMAIL; } if (yarnUpdate && !isYarn1) { logger.debug('Updating Yarn binary'); // TODO: types (#22198) commands.push(`yarn set version ${quote(yarnUpdate.newValue!)}`); } const allEnv = getEnv(); if (allEnv.RENOVATE_X_YARN_PROXY) { // v8 ignore else -- TODO: add test #40625 if (allEnv.HTTP_PROXY && !isYarn1) { commands.push('yarn config unset --home httpProxy'); commands.push( `yarn config set --home httpProxy ${quote(allEnv.HTTP_PROXY)}`, ); } // v8 ignore else -- TODO: add test #40625 if (allEnv.HTTPS_PROXY && !isYarn1) { commands.push('yarn config unset --home httpsProxy'); commands.push( `yarn config set --home httpsProxy ${quote(allEnv.HTTPS_PROXY)}`, ); } } // This command updates the lock file based on package.json commands.push(`yarn install${cmdOptions}`); // rangeStrategy = update-lockfile const lockUpdates = upgrades.filter((upgrade) => upgrade.isLockfileUpdate); if (lockUpdates.length) { logger.debug('Performing lockfileUpdate (yarn)'); if (isYarn1) { // `yarn upgrade` updates based on the version range specified in the package file // note - this can hit a yarn bug, see https://github.com/yarnpkg/yarn/issues/8236 commands.push( `yarn upgrade ${lockUpdates .map((update) => update.depName) .filter(isString) .filter(uniqueStrings) .map(quote) .join(' ')}${cmdOptions}`, ); } else { // `yarn up -R` updates to the latest release in each range commands.push( `yarn up -R ${lockUpdates // TODO: types (#22198) .map((update) => `${update.depName!}`) .filter(uniqueStrings) .map(quote) .join(' ')}${cmdOptions}`, ); } } // postUpdateOptions ['fewer', 'highest'].forEach((s) => { if ( config.postUpdateOptions?.includes( `yarnDedupe${s.charAt(0).toUpperCase()}${s.slice(1)}`, ) ) { logger.debug(`Performing yarn dedupe ${s}`); if (isYarn1) { commands.push(`npx yarn-deduplicate --strategy ${s}`); // Run yarn again in case any changes are necessary commands.push(`yarn install${cmdOptions}`); } else if (isYarnDedupeAvailable && s === 'highest') { commands.push(`yarn dedupe --strategy ${s}${cmdOptions}`); } else { logger.debug(`yarn dedupe ${s} not available`); } } }); if (upgrades.find((upgrade) => upgrade.isLockFileMaintenance)) { logger.debug( `Removing ${lockFileName} first due to lock file maintenance upgrade`, ); // Note: Instead of just deleting the `yarn.lock` file, we just wipe it // and keep an empty lock file. Deleting the lock file could result in different // Yarn semantics. e.g. Yarn 2+ will error when `yarn install` is executed in // a subdirectory which is not part of a Yarn workspace. Yarn suggests to create // an empty lock file if a subdirectory should be treated as its own workspace. // https://github.com/yarnpkg/berry/blob/20612e82d26ead5928cc27bf482bb8d62dde87d3/packages/yarnpkg-core/sources/Project.ts#L284. try { await writeLocalFile(lockFileName, ''); } catch (err) { // v8 ignore next -- TODO: add test #40625 logger.debug( { err, lockFileName }, 'Error clearing `yarn.lock` for lock file maintenance', ); } } // Run the commands await exec(commands, execOptions); // Read the result lockFile = await readLocalFile(lockFileName, 'utf8'); } catch (err) { // v8 ignore if -- TODO: add test #40625 if (err.message === TEMPORARY_ERROR) { throw err; } logger.debug( { err, type: 'yarn', }, 'lock file error', ); const stdouterr = String(err.stdout) + String(err.stderr); // v8 ignore if -- TODO: add test #40625 if ( stdouterr.includes('ENOSPC: no space left on device') || stdouterr.includes('Out of diskspace') ) { throw new Error(SYSTEM_INSUFFICIENT_DISK_SPACE); } // v8 ignore if -- TODO: add test #40625 if ( stdouterr.includes('The registry may be down.') || stdouterr.includes('getaddrinfo ENOTFOUND registry.yarnpkg.com') || stdouterr.includes('getaddrinfo ENOTFOUND registry.npmjs.org') ) { throw new ExternalHostError(err, NpmDatasource.id); } return { error: true, stderr: err.stderr, stdout: err.stdout }; } return { lockFile }; } export function fuzzyMatchAdditionalYarnrcYml< T extends { npmRegistries?: Record }, >(additionalYarnRcYml: T, existingYarnrRcYml: T): T { const keys = new Map( Object.keys(coerceObject(existingYarnrRcYml.npmRegistries)).map((x) => [ x.replace(regEx(/\/$/), '').replace(regEx(/^https?:/), ''), x, ]), ); return { ...additionalYarnRcYml, npmRegistries: Object.entries( coerceObject(additionalYarnRcYml.npmRegistries), ) .map(([k, v]) => { const key = keys.get(k.replace(regEx(/\/$/), '')) ?? k; return { [key]: v }; }) .reduce((acc, cur) => ({ ...acc, ...cur }), {}), }; }