import { lang, query as q } from '@renovatebot/good-enough-parser'; import { logger } from '../../../logger/index.ts'; import { readLocalFile } from '../../../util/fs/index.ts'; import { newlineRegex, regEx } from '../../../util/regex.ts'; import { parseUrl } from '../../../util/url.ts'; import { GithubReleasesDatasource } from '../../datasource/github-releases/index.ts'; import { MAVEN_REPO } from '../../datasource/maven/common.ts'; import { MavenDatasource } from '../../datasource/maven/index.ts'; import { SbtPackageDatasource } from '../../datasource/sbt-package/index.ts'; import { SBT_PLUGINS_REPO, SbtPluginDatasource, } from '../../datasource/sbt-plugin/index.ts'; import { get } from '../../versioning/index.ts'; import * as mavenVersioning from '../../versioning/maven/index.ts'; import * as semverVersioning from '../../versioning/semver/index.ts'; import type { ExtractConfig, PackageDependency, PackageFile, PackageFileContent, } from '../types.ts'; import { normalizeScalaVersion, sortPackageFiles } from './util.ts'; type Vars = Record; interface Ctx { vars: Vars; deps: PackageDependency[]; registryUrls: string[]; scalaVersion?: string; packageFileVersion?: string; groupId?: string; artifactId?: string; currentValue?: string; currentVarName?: string; depType?: string; useScalaVersion?: boolean; variableName?: string; // Use to contain temporary object name // when parsing anonymous object structs. // See objectAssignmentMatch. currentObjectName?: string; } const scala = lang.createLang('scala'); const sbtVersionRegex = regEx( 'sbt\\.version *= *(?\\d+\\.\\d+\\.\\d+)', ); const sbtProxyUrlRegex = regEx( /^\s*(?\S+):\s+(?https?:\/\/[\w./-]+)/, ); const scalaVersionMatch = q .sym('scalaVersion') .op(':=') .alt( // Dotted symbol. Important to have this first in the `alt` as otherwise // the tree walker will eager parse a simple symbol and stop there. q // keep the first part of the path in context .sym((ctx, { value: currentVarName }) => ({ ...ctx, currentVarName, })) .op('.') .sym((ctx, { value: varName }) => { // build the full variable name from what we saved earlier const dottedVariableName = `${ctx.currentVarName}.${varName}`; const currentValue = ctx.vars[dottedVariableName]; if (currentValue) { ctx.scalaVersion = currentValue; } delete ctx.currentVarName; return ctx; }), // Simple symbol defined previously q.sym((ctx, { value: varName }) => { const scalaVersion = ctx.vars[varName]; if (scalaVersion) { ctx.scalaVersion = scalaVersion; } return ctx; }), // Direct string q.str((ctx, { value: scalaVersion }) => ({ ...ctx, scalaVersion })), ) .handler((ctx) => { if (ctx.scalaVersion) { const version = get(mavenVersioning.id); let packageName = 'org.scala-lang:scala-library'; if (version.getMajor(ctx.scalaVersion) === 3) { packageName = 'org.scala-lang:scala3-library_3'; } const dep: PackageDependency = { datasource: MavenDatasource.id, depName: 'scala', packageName, currentValue: ctx.scalaVersion, separateMinorPatch: true, }; ctx.scalaVersion = normalizeScalaVersion(ctx.scalaVersion); ctx.deps.push(dep); } return ctx; }); const packageFileVersionMatch = q .sym('version') .op(':=') .alt( q.str((ctx, { value: packageFileVersion }) => ({ ...ctx, packageFileVersion, })), q.sym((ctx, { value: varName }) => { const packageFileVersion = ctx.vars[varName]; if (packageFileVersion) { ctx.packageFileVersion = packageFileVersion; } return ctx; }), ); const variableNameMatch = q .sym((ctx, { value: varName }) => ({ ...ctx, currentVarName: varName, })) .opt(q.op(':').sym('String')); const variableValueMatch = q.str((ctx, { value }) => { ctx.vars[ctx.currentVarName!] = value; delete ctx.currentVarName; return ctx; }); const assignmentMatch = q.sym('val').join(variableNameMatch).op('='); const variableDefinitionMatch = q .alt( q.sym('lazy').join(assignmentMatch), assignmentMatch, variableNameMatch.op(':='), ) .join(variableValueMatch); // Parses objects fields of the form `{name} = "{version}"`. const objectFieldMatch = q .sym((ctx, { value: fieldName }) => ({ ...ctx, currentVarName: `${ctx.currentObjectName}.${fieldName}`, })) .op('=') .str((ctx, { value }) => { ctx.vars[ctx.currentVarName!] = value; delete ctx.currentVarName; return ctx; }); // Parses anonymous objects of the form // val {objName} = new { // {name} = "{version}" // } // and introduces a `objName.name` variable with value "{version}" // in the context. const objectAssignmentMatch = q .sym('val') .sym((ctx, { value: currentObjectName }) => ({ ...ctx, currentObjectName, })) .op('=') .sym('new') .tree({ search: q.many(objectFieldMatch), }) .handler((ctx) => { delete ctx.currentObjectName; return ctx; }); const groupIdMatch = q.alt( q.sym((ctx, { value: varName }) => { const currentGroupId = ctx.vars[varName]; if (currentGroupId) { ctx.groupId = currentGroupId; } return ctx; }), q.str((ctx, { value: groupId }) => ({ ...ctx, groupId })), ); const artifactIdMatch = q.alt( q.sym((ctx, { value }) => { const artifactId = ctx.vars[value]; if (artifactId) { ctx.artifactId = artifactId; } return ctx; }), q.str((ctx, { value: artifactId }) => ({ ...ctx, artifactId })), ); const versionMatch = q.alt( // Dotted symbol. Important to have this first in the `alt` as otherwise // the tree walker will eager parse a simple symbol and stop there. q .sym((ctx, { value: currentVarName }) => ({ ...ctx, currentVarName })) .op('.') .sym((ctx, { value: varName }) => { // build the full variable name from what we saved earlier const dottedVariableName = `${ctx.currentVarName}.${varName}`; const currentValue = ctx.vars[dottedVariableName]; if (currentValue) { ctx.currentValue = currentValue; ctx.variableName = dottedVariableName; } delete ctx.currentVarName; return ctx; }), // Simple symbol defined earlier q.sym((ctx, { value: varName }) => { const currentValue = ctx.vars[varName]; if (currentValue) { ctx.currentValue = currentValue; ctx.variableName = varName; } return ctx; }), // Direct string q.str((ctx, { value: currentValue }) => ({ ...ctx, currentValue })), ); const simpleDependencyMatch = groupIdMatch .op('%') .join(artifactIdMatch) .op('%') .join(versionMatch); const versionedDependencyMatch = groupIdMatch .op('%%') .join(artifactIdMatch) .handler((ctx) => ({ ...ctx, useScalaVersion: true })) .op('%') .join(versionMatch); const crossDependencyMatch = groupIdMatch .op('%%%') .join(artifactIdMatch) .handler((ctx) => ({ ...ctx, useScalaVersion: true })) .op('%') .join(versionMatch); function depHandler(ctx: Ctx): Ctx { const { scalaVersion, groupId, artifactId, currentValue, useScalaVersion, depType, variableName, } = ctx; delete ctx.groupId; delete ctx.artifactId; delete ctx.currentValue; delete ctx.useScalaVersion; delete ctx.depType; delete ctx.variableName; const depName = `${groupId!}:${artifactId!}`; const dep: PackageDependency = { datasource: SbtPackageDatasource.id, depName, packageName: scalaVersion && useScalaVersion ? `${depName}_${scalaVersion}` : depName, currentValue, }; if (depType) { dep.depType = depType; } if (depType === 'plugin') { dep.datasource = SbtPluginDatasource.id; } if (variableName) { dep.sharedVariableName = variableName; dep.variableName = variableName; } ctx.deps.push(dep); return ctx; } function depTypeHandler(ctx: Ctx, { value: depType }: { value: string }): Ctx { return { ...ctx, depType }; } const sbtPackageMatch = q .opt(q.opt(q.sym('lazy')).sym('val').sym().op('=')) .alt(crossDependencyMatch, simpleDependencyMatch, versionedDependencyMatch) .opt( q.alt( q.sym('classifier').str(depTypeHandler), q.op('%').sym(depTypeHandler), q.op('%').str(depTypeHandler), ), ) .handler(depHandler); const sbtPluginMatch = q .sym(regEx(/^(?:addSbtPlugin|addCompilerPlugin)$/)) .tree({ type: 'wrapped-tree', maxDepth: 1, search: q .begin() .alt(simpleDependencyMatch, versionedDependencyMatch) .end(), }) .handler((ctx) => ({ ...ctx, depType: 'plugin' })) .handler(depHandler); const resolverMatch = q .str() .sym('at') .str((ctx, { value }) => { if (parseUrl(value)) { ctx.registryUrls.push(value); } return ctx; }); const addResolverMatch = q.sym('resolvers').alt( q.op('+=').join(resolverMatch), q.op('++=').sym('Seq').tree({ type: 'wrapped-tree', maxDepth: 1, search: resolverMatch, }), ); function registryUrlHandler(ctx: Ctx): Ctx { for (const dep of ctx.deps) { dep.registryUrls = [...ctx.registryUrls]; } return ctx; } const query = q.tree({ type: 'root-tree', maxDepth: 32, search: q.alt( scalaVersionMatch, packageFileVersionMatch, sbtPackageMatch, sbtPluginMatch, addResolverMatch, variableDefinitionMatch, objectAssignmentMatch, ), postHandler: registryUrlHandler, }); export function extractProxyUrls( content: string, packageFile: string, ): string[] { const extractedProxyUrls: string[] = []; logger.debug(`Parsing proxy repository file ${packageFile}`); for (const line of content.split(newlineRegex)) { const extraction = sbtProxyUrlRegex.exec(line); if (extraction?.groups?.proxy) { extractedProxyUrls.push(extraction.groups.proxy); } else if (line.trim() === 'maven-central') { extractedProxyUrls.push(MAVEN_REPO); } } return extractedProxyUrls; } export function extractPackageFile( content: string, packageFile: string, ): PackageFileContent | null { return extractPackageFileInternal(content, packageFile); } function extractPackageFileInternal( content: string, packageFile: string, ctxScalaVersion?: string, ): PackageFileContent | null { if ( packageFile === 'project/build.properties' || packageFile.endsWith('/project/build.properties') ) { const regexResult = sbtVersionRegex.exec(content); const sbtVersion = regexResult?.groups?.version; const matchString = regexResult?.[0]; if (sbtVersion) { const sbtDependency: PackageDependency = { datasource: GithubReleasesDatasource.id, depName: 'sbt/sbt', packageName: 'sbt/sbt', versioning: semverVersioning.id, currentValue: sbtVersion, replaceString: matchString, extractVersion: '^v(?\\S+)', registryUrls: [], }; return { deps: [sbtDependency], }; } return null; } let parsedResult: Ctx | null = null; try { parsedResult = scala.query(content, query, { vars: {}, deps: [], registryUrls: [], scalaVersion: ctxScalaVersion, }); } catch (err) /* istanbul ignore next */ { logger.debug({ err, packageFile }, 'Sbt parsing error'); } if (!parsedResult) { return null; } const { deps, scalaVersion, packageFileVersion } = parsedResult; if (!deps.length) { return null; } return { deps, packageFileVersion, managerData: { scalaVersion } }; } export async function extractAllPackageFiles( _config: ExtractConfig, packageFiles: string[], ): Promise { const packages: PackageFile[] = []; const proxyUrls: string[] = []; let ctxScalaVersion: string | undefined; const sortedPackageFiles = sortPackageFiles(packageFiles); for (const packageFile of sortedPackageFiles) { const content = await readLocalFile(packageFile, 'utf8'); if (!content) { logger.debug({ packageFile }, 'packageFile has no content'); continue; } if (packageFile === 'repositories') { const urls = extractProxyUrls(content, packageFile); proxyUrls.push(...urls); } else { const pkg = extractPackageFileInternal( content, packageFile, ctxScalaVersion, ); if (pkg) { packages.push({ deps: pkg.deps, packageFile }); if (pkg.managerData?.scalaVersion) { ctxScalaVersion = pkg.managerData.scalaVersion; } } } } for (const pkg of packages) { for (const dep of pkg.deps) { if (dep.datasource !== GithubReleasesDatasource.id) { if (proxyUrls.length > 0) { dep.registryUrls!.unshift(...proxyUrls); } else if (dep.depType === 'plugin') { dep.registryUrls!.unshift(SBT_PLUGINS_REPO, MAVEN_REPO); } else { dep.registryUrls!.unshift(MAVEN_REPO); } } } } return packages; }