import deepmerge from 'deepmerge'; import { z } from 'zod/v4'; import { logger } from '../../../logger/index.ts'; import { coerceArray } from '../../../util/array.ts'; import { getEnv } from '../../../util/env.ts'; import { parseGitUrl } from '../../../util/git/url.ts'; import { LooseArray, LooseRecord, Toml, withDepType, } from '../../../util/schema-utils/index.ts'; import { uniq } from '../../../util/uniq.ts'; import { GitRefsDatasource } from '../../datasource/git-refs/index.ts'; import { GitTagsDatasource } from '../../datasource/git-tags/index.ts'; import { GithubTagsDatasource } from '../../datasource/github-tags/index.ts'; import { GitlabTagsDatasource } from '../../datasource/gitlab-tags/index.ts'; import { normalizePythonDepName } from '../../datasource/pypi/common.ts'; import { PypiDatasource } from '../../datasource/pypi/index.ts'; import * as gitVersioning from '../../versioning/git/index.ts'; import * as pep440Versioning from '../../versioning/pep440/index.ts'; import * as poetryVersioning from '../../versioning/poetry/index.ts'; import { DependencyGroup, ProjectSection } from '../pep621/schema.ts'; import { depTypes, pep508ToPackageDependency } from '../pep621/utils.ts'; import type { PackageDependency, PackageFileContent } from '../types.ts'; const PoetryOptionalDependencyMixin = z .object({ optional: z.boolean().optional().catch(false), }) .transform(({ optional }): PackageDependency => optional ? { depType: 'extras' } : {}, ); const PoetryPathDependency = z .object({ path: z.string(), version: z.string().optional().catch(undefined), }) .transform(({ version }): PackageDependency => { const dep: PackageDependency = { datasource: PypiDatasource.id, skipReason: 'path-dependency', }; if (version) { dep.currentValue = version; } return dep; }) .and(PoetryOptionalDependencyMixin); const PoetryGitDependency = z .object({ git: z.string(), tag: z.string().optional().catch(undefined), version: z.string().optional().catch(undefined), branch: z.string().optional().catch(undefined), rev: z.string().optional().catch(undefined), }) .transform(({ git, tag, version, branch, rev }): PackageDependency => { if (tag) { const { source, owner, name } = parseGitUrl(git); const repo = `${owner}/${name}`; if (source === 'github.com') { return { datasource: GithubTagsDatasource.id, currentValue: tag, packageName: repo, }; } if (source === 'gitlab.com') { return { datasource: GitlabTagsDatasource.id, currentValue: tag, packageName: repo, }; } return { datasource: GitTagsDatasource.id, currentValue: tag, packageName: git, }; } if (rev) { return { datasource: GitRefsDatasource.id, currentValue: branch, currentDigest: rev, replaceString: rev, packageName: git, }; } return { datasource: GitRefsDatasource.id, currentValue: version, packageName: git, skipReason: 'git-dependency', }; }) .and(PoetryOptionalDependencyMixin); const PoetryPypiDependency = z.union([ z .object({ version: z.string().optional(), source: z.string().optional() }) .transform(({ version: currentValue, source }): PackageDependency => { const managerData = source ? { sourceName: source.toLowerCase() } : {}; if (!currentValue) { return { datasource: PypiDatasource.id, managerData }; } return { datasource: PypiDatasource.id, managerData: { ...managerData, nestedVersion: true }, currentValue, }; }) .and(PoetryOptionalDependencyMixin), z.string().transform((version): PackageDependency => ({ datasource: PypiDatasource.id, currentValue: version, managerData: { nestedVersion: false }, })), ]); const PoetryArrayDependency = z .array(z.unknown()) .transform((): PackageDependency => ({ datasource: PypiDatasource.id, skipReason: 'multiple-constraint-dep', })); const PoetryDependency = z.union([ PoetryPathDependency, PoetryGitDependency, PoetryPypiDependency, PoetryArrayDependency, ]); export const PoetryDependencies = LooseRecord( z.string(), PoetryDependency.transform((dep) => { if (dep.skipReason) { return dep; } if (dep.datasource === GitRefsDatasource.id && dep.currentDigest) { dep.versioning = gitVersioning.id; return dep; } // istanbul ignore if: normaly should not happen if (!dep.currentValue) { dep.skipReason = 'unspecified-version'; return dep; } if (pep440Versioning.isValid(dep.currentValue)) { dep.versioning = pep440Versioning.id; return dep; } if (poetryVersioning.isValid(dep.currentValue)) { dep.versioning = poetryVersioning.id; return dep; } dep.skipReason = 'invalid-version'; return dep; }), ).transform((record) => { const deps: PackageDependency[] = []; for (const [depName, dep] of Object.entries(record)) { dep.depName = depName; if (!dep.packageName) { const packageName = normalizePythonDepName(depName); if (depName !== packageName) { dep.packageName = packageName; } } deps.push(dep); } return deps; }); export const PoetryGroupDependencies = LooseRecord( z.string(), z .object({ dependencies: PoetryDependencies }) .transform(({ dependencies }) => dependencies), ).transform((record) => { const deps: PackageDependency[] = []; for (const [name, val] of Object.entries(record)) { for (const dep of Object.values(val)) { dep.depType = name; deps.push(dep); } } return deps; }); const PoetrySourceOrder = [ 'default', 'primary', 'secondary', 'supplemental', 'explicit', ] as const; export const PoetrySource = z.object({ name: z.string().toLowerCase(), url: z.string().optional(), priority: z.enum(PoetrySourceOrder).default('primary'), }); export type PoetrySource = z.infer; export const PoetrySources = LooseArray(PoetrySource, { onError: ({ error: err }) => { logger.debug({ err }, 'Poetry: error parsing sources array'); }, }) .transform((sources) => { const pypiUrl = getEnv().PIP_INDEX_URL ?? 'https://pypi.org/pypi/'; const result: PoetrySource[] = []; let overridesPyPi = false; let hasDefaultSource = false; let hasPrimarySource = false; for (const source of sources) { if (source.name === 'pypi') { source.url = pypiUrl; overridesPyPi = true; } if (!source.url) { continue; } if (source.priority === 'default') { hasDefaultSource = true; } else if (source.priority === 'primary') { hasPrimarySource = true; } result.push(source); } if (sources.length && !hasDefaultSource && !overridesPyPi) { result.push({ name: 'pypi', priority: hasPrimarySource ? 'secondary' : 'default', url: pypiUrl, }); } result.sort( (a, b) => PoetrySourceOrder.indexOf(a.priority) - PoetrySourceOrder.indexOf(b.priority), ); return result; }) .catch([]); export const PoetrySection = z.object({ version: z.string().optional().catch(undefined), dependencies: withDepType( PoetryDependencies, 'dependencies', false, ).optional(), 'dev-dependencies': withDepType( PoetryDependencies, 'dev-dependencies', ).optional(), group: PoetryGroupDependencies.optional(), source: PoetrySources, }); export type PoetrySection = z.infer; const BuildSystemRequires = LooseArray( z .string() .nonempty() .transform((val) => pep508ToPackageDependency(depTypes.buildSystemRequires, val), ) .refine((dep) => dep !== null), ).catch([]); export const PoetryPyProject = Toml.pipe( z .object({ project: ProjectSection.optional().catch(undefined), tool: z.object({ poetry: PoetrySection }).optional().catch(undefined), 'dependency-groups': DependencyGroup(depTypes.dependencyGroups).catch([]), 'build-system': z .object({ 'build-backend': z.string().refine( // https://python-poetry.org/docs/pyproject/#poetry-and-pep-517 (buildBackend) => buildBackend === 'poetry.masonry.api' || buildBackend === 'poetry.core.masonry.api', ), requires: BuildSystemRequires, }) .transform(({ requires }) => { const req = requires.find( ({ depName }) => depName === 'poetry' || depName === 'poetry_core' || depName === 'poetry-core', ); return { poetryRequirement: req?.currentValue, requires, }; }) .optional() .catch(undefined), }) .transform((pyproject) => { const { project, tool, 'build-system': buildSystem, 'dependency-groups': dependencyGroups, } = pyproject; const deps: PackageDependency[] = []; const projectDependencies = coerceArray(project?.dependencies); const projectOptionalDependencies = coerceArray( project?.['optional-dependencies'], ); const projectDepsByName: Record = {}; for (const dep of [ ...projectDependencies, ...dependencyGroups, ...projectOptionalDependencies, ]) { projectDepsByName[dep.depName!] = dep; } if (buildSystem?.requires) { deps.push(...buildSystem.requires); } const poetryDependencies = coerceArray(tool?.poetry?.dependencies); const poetryDevDependencies = coerceArray( tool?.poetry?.['dev-dependencies'], ); const poetryGroupDependencies = coerceArray(tool?.poetry?.group); for (const poetryDep of [ ...poetryDependencies, ...poetryDevDependencies, ...poetryGroupDependencies, ]) { const depName = poetryDep.depName; const projectDep = depName && projectDepsByName[depName]; // When the same dep exists in project.dependencies or dependency-groups, // Poetry just uses the Poetry dep to enrich the project dependency. if (projectDep) { const mergedDep = deepmerge(poetryDep, projectDep); // Poetry supports specifying the version in project.dependencies and // tool.poetry.dependencies *at the same time* and only errors if both // are not compatible - we require a single constraint per dependency. if (projectDep.currentValue && poetryDep.currentValue) { mergedDep.skipReason = 'invalid-dependency-specification'; } // When a skipReason is 'unspecified-version', we defer to the other skipReason, // so that 'unspecified-version' only persists if both deps have no version. if (mergedDep.skipReason === 'unspecified-version') { if (poetryDep.skipReason === 'unspecified-version') { mergedDep.skipReason = projectDep.skipReason; } else { mergedDep.skipReason = poetryDep.skipReason; } } projectDepsByName[depName] = mergedDep; } else { deps.push(poetryDep); } } deps.push(...Object.values(projectDepsByName)); const packageFileVersion = tool?.poetry?.version; const packageFileContent: PackageFileContent = { deps, packageFileVersion, }; const sourceUrls = tool?.poetry?.source; if (sourceUrls) { for (const dep of deps) { if (dep.managerData?.sourceName) { const sourceUrl = sourceUrls.find( ({ name }) => name === dep.managerData?.sourceName, ); if (sourceUrl?.url) { dep.registryUrls = [sourceUrl.url]; } } } const sourceUrlsFiltered = sourceUrls.filter( ({ priority }) => priority !== 'explicit', ); if (sourceUrlsFiltered.length) { packageFileContent.registryUrls = uniq( sourceUrlsFiltered.map(({ url }) => url!), ); } } return { packageFileContent, poetryRequirement: buildSystem?.poetryRequirement, }; }), ); export type PoetryPyProject = z.infer; const poetryConstraint: Record = { '1.0': '<1.1.0', '1.1': '<1.3.0', '2.0': '>=1.3.0 <1.4.0', // 1.4.0 introduced embedding of the poetry version in lock file header }; export const Lockfile = Toml.pipe( z.object({ package: LooseArray( z .object({ name: z.string(), version: z.string(), }) .transform(({ name, version }): [string, string] => [name, version]), ) .transform((entries) => Object.fromEntries(entries)) .catch({}), metadata: z .object({ 'lock-version': z .string() .transform((lockVersion) => poetryConstraint[lockVersion]) .optional() .catch(undefined), 'python-versions': z.string().optional().catch(undefined), }) .transform( ({ 'lock-version': poetryConstraint, 'python-versions': pythonVersions, }) => ({ poetryConstraint, pythonVersions, }), ) .catch({ poetryConstraint: undefined, pythonVersions: undefined, }), }), ).transform( ({ package: lock, metadata: { poetryConstraint, pythonVersions } }) => ({ lock, poetryConstraint, pythonVersions, }), );