import { isNonEmptyArray } from '@sindresorhus/is'; import type { RenovateConfig } from '../../../config/types.ts'; import { instrument } from '../../../instrumentation/index.ts'; import { logger } from '../../../logger/index.ts'; import { hashMap } from '../../../modules/manager/index.ts'; import type { PackageFile } from '../../../modules/manager/types.ts'; import { scm } from '../../../modules/platform/scm.ts'; import { isNotNullOrUndefined } from '../../../util/array.ts'; import * as memCache from '../../../util/cache/memory/index.ts'; import { getCache } from '../../../util/cache/repository/index.ts'; import type { BaseBranchCache } from '../../../util/cache/repository/types.ts'; import { checkGithubToken as ensureGithubToken } from '../../../util/check-token.ts'; import { fingerprint } from '../../../util/fingerprint.ts'; import type { BranchConfig } from '../../types.ts'; import { generateFingerprintConfig } from '../extract/extract-fingerprint-config.ts'; import { extractAllDependencies } from '../extract/index.ts'; import { branchifyUpgrades } from '../updates/branchify.ts'; import { fetchUpdates } from './fetch.ts'; import { calculateLibYears } from './libyear.ts'; import { sortBranches } from './sort.ts'; import { Vulnerabilities } from './vulnerabilities.ts'; import type { WriteUpdateResult } from './write.ts'; import { writeUpdates } from './write.ts'; // Increment this if needing to cache bust ALL extract caches export const EXTRACT_CACHE_REVISION = 1; export interface ExtractResult { branches: BranchConfig[]; branchList: string[]; packageFiles: Record; } export interface StatsResult { fileCount: number; depCount: number; } export interface Stats { managers: Record; total: StatsResult; } // istanbul ignore next function extractStats( packageFiles: Record, ): Stats | null { if (!packageFiles) { return null; } const stats: Stats = { managers: {}, total: { fileCount: 0, depCount: 0, }, }; for (const [manager, managerPackageFiles] of Object.entries(packageFiles)) { const fileCount = managerPackageFiles.length; let depCount = 0; for (const file of managerPackageFiles) { depCount += file.deps.length; } stats.managers[manager] = { fileCount, depCount, }; stats.total.fileCount += fileCount; stats.total.depCount += depCount; } return stats; } export function isCacheExtractValid( baseBranchSha: string, configHash: string, cachedExtract?: BaseBranchCache, ): boolean { if (!cachedExtract) { return false; } if (!cachedExtract.revision) { logger.debug('Cached extract is missing revision, so cannot be used'); return false; } if (cachedExtract.revision !== EXTRACT_CACHE_REVISION) { logger.debug( `Extract cache revision has changed (old=${cachedExtract.revision}, new=${EXTRACT_CACHE_REVISION})`, ); return false; } if (!(cachedExtract.sha && cachedExtract.configHash)) { return false; } if (cachedExtract.sha !== baseBranchSha) { logger.debug( `Cached extract result cannot be used due to base branch SHA change (old=${cachedExtract.sha}, new=${baseBranchSha})`, ); return false; } if (cachedExtract.configHash !== configHash) { logger.debug('Cached extract result cannot be used due to config change'); return false; } if (!cachedExtract.extractionFingerprints) { logger.debug( 'Cached extract is missing extractionFingerprints, so cannot be used', ); return false; } const changedManagers = new Set(); for (const [manager, fingerprint] of Object.entries( cachedExtract.extractionFingerprints, )) { if (fingerprint !== hashMap.get(manager)) { changedManagers.add(manager); } } if (changedManagers.size > 0) { logger.debug( { changedManagers: [...changedManagers] }, 'Manager fingerprint(s) have changed, extract cache cannot be reused', ); return false; } logger.debug( `Cached extract for sha=${baseBranchSha} is valid and can be used`, ); return true; } export async function extract( config: RenovateConfig, overwriteCache = true, ): Promise> { logger.debug('extract()'); const { baseBranch } = config; const baseBranchSha = await scm.getBranchCommit(baseBranch!); let packageFiles: Record; const cache = getCache(); cache.scan ??= {}; const cachedExtract = cache.scan[baseBranch!]; const configHash = instrument('fingerprint', () => fingerprint(generateFingerprintConfig(config)), ); // istanbul ignore if if ( overwriteCache && isCacheExtractValid(baseBranchSha!, configHash, cachedExtract) ) { packageFiles = cachedExtract.packageFiles; try { for (const files of Object.values(packageFiles)) { for (const file of files) { for (const dep of file.deps) { delete dep.updates; } } } logger.debug('Deleted cached dep updates'); } catch (err) { logger.info({ err }, 'Error deleting cached dep updates'); } } else { await instrument( 'checkoutBranch', async () => await scm.checkoutBranch(baseBranch!), ); const extractResult = await instrument( 'extractAllDependencies', async () => (await extractAllDependencies(config)) || {}, ); packageFiles = extractResult.packageFiles; const { extractionFingerprints } = extractResult; if (overwriteCache) { // TODO: fix types (#22198) cache.scan[baseBranch!] = { revision: EXTRACT_CACHE_REVISION, sha: baseBranchSha!, configHash, extractionFingerprints, packageFiles, }; } // Clean up cached branch extracts const baseBranches = isNonEmptyArray(config.baseBranches) ? config.baseBranches : [baseBranch]; Object.keys(cache.scan).forEach((branchName) => { if (!baseBranches.includes(branchName)) { delete cache.scan![branchName]; } }); } const stats = extractStats(packageFiles); logger.info( { baseBranch: config.baseBranch, stats }, `Dependency extraction complete`, ); logger.trace({ config: packageFiles }, 'packageFiles'); ensureGithubToken(packageFiles); return packageFiles; } async function fetchVulnerabilities( config: RenovateConfig, packageFiles: Record, ): Promise { if (config.osvVulnerabilityAlerts) { logger.debug('fetchVulnerabilities() - osvVulnerabilityAlerts=true'); try { const vulnerabilities = await Vulnerabilities.create(); await vulnerabilities.appendVulnerabilityPackageRules( config, packageFiles, ); } catch (err) { logger.warn({ err }, 'Unable to read vulnerability information'); } } } export async function lookup( config: RenovateConfig, packageFiles: Record, ): Promise { await fetchVulnerabilities(config, packageFiles); await fetchUpdates(config, packageFiles); // call this twice, as the second time, the updates will be availalbe for malicious package checks // TODO: this will be refactored as part of #42423 await fetchVulnerabilities(config, packageFiles); memCache.cleanDatasourceKeys(); calculateLibYears(config, packageFiles); const { branches, branchList } = await branchifyUpgrades( config, packageFiles, ); reportMaliciousSkippedDependencies(packageFiles); logger.debug( { baseBranch: config.baseBranch, config: packageFiles }, 'packageFiles with updates', ); sortBranches(branches); return { branches, branchList, packageFiles }; } export function reportMaliciousSkippedDependencies( allPackageFiles: Record, ): void { if (allPackageFiles === undefined) { return; } for (const [manager, packageFiles] of Object.entries(allPackageFiles)) { for (const packageFile of packageFiles) { for (const dep of packageFile.deps) { if (dep.skipReason === 'malicious-version-in-use') { logger.warn( { packageFile: packageFile.packageFile, depName: dep.depName, packageName: dep.packageName, manager: manager, datasource: dep.datasource, }, 'Dependency is currently using a malicious version', ); // and make sure that it then gets updates proposed in the update phase delete dep.skipReason; delete dep.skipStage; } else if (dep.skipReason === 'malicious-update-proposed') { const newVersions = dep.updates ?.map((u) => u.newVersion ?? u.newValue) .filter(isNotNullOrUndefined); logger.warn( { packageFile: packageFile.packageFile, depName: dep.depName, packageName: dep.packageName, manager: manager, datasource: dep.datasource, newVersions, }, 'Dependency has update(s) proposed which would update you to a malicious version - skipping', ); } } } } } export function update( config: RenovateConfig, branches: BranchConfig[], ): Promise { return writeUpdates(config, branches); }