import { codeBlock } from 'common-tags'; import upath from 'upath'; import { Fixtures } from '~test/fixtures.ts'; import { fs } from '~test/util.ts'; import { GlobalConfig } from '../../../config/global.ts'; import type { InternalGlobalConfigOptions, RepoGlobalConfig, } from '../../../config/types.ts'; import { logger } from '../../../logger/index.ts'; import { extractAllPackageFiles, extractPackageFile } from './index.ts'; vi.mock('../../../util/fs/index.ts'); const adminConfig: RepoGlobalConfig & InternalGlobalConfigOptions = { // `join` fixes Windows CI localDir: upath.join('/tmp/github/some/repo'), cacheDir: upath.join('/tmp/renovate/cache'), containerbaseDir: upath.join('/tmp/renovate/cache/containerbase'), }; function getSimpleRequirementsFile(command: string, deps: string[] = []) { const header = codeBlock` # # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # ${command} # `; // the deps section is appended separately: `codeBlock` would trim the // trailing blank line away when `deps` is empty return `${header}\n\n${deps.join('\n')}`; } describe('modules/manager/pip-compile/extract', () => { beforeEach(() => { GlobalConfig.set(adminConfig); }); describe('extractPackageFile()', () => { it('returns object for requirements.in', async () => { const packageFile = await extractPackageFile( Fixtures.get('requirementsWithHashes.txt'), 'requirements.in', {}, ); expect(packageFile).toHaveProperty('deps'); expect(packageFile?.deps[0]).toHaveProperty('depName', 'attrs'); }); it('returns object for setup.py', async () => { const packageFile = await extractPackageFile( Fixtures.get('setup.py', '../pip_setup'), 'lib/setup.py', {}, ); expect(packageFile).toHaveProperty('deps'); expect(packageFile?.deps[0]).toHaveProperty('depName', 'celery'); }); it('returns object for pyproject.toml', async () => { const pyproject = codeBlock` [build-system] requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" [project] name = "test-project" requires-python = ">=3.11" version = "1.2.3" description = "Test project for pip-compile with setuptools" readme = "README.md" dependencies = [ "aiohttp", "pydantic>=2.0.0", ] [project.optional-dependencies] dev = [ "black", "flake8", ] `; const packageFile = await extractPackageFile( pyproject, 'pyproject.toml', {}, ); expect(packageFile).toHaveProperty('deps'); expect(packageFile?.deps[0]).toHaveProperty('depType', 'requires-python'); expect(packageFile?.deps[1]).toHaveProperty('depName', 'aiohttp'); }); it.each([ 'random.py', 'app.cfg', 'already_locked.txt', // TODO(not7cd) 'setup.cfg', ])('returns null on not supported package files', async (file: string) => { await expect( extractPackageFile('some content', file, {}), ).resolves.toBeNull(); }); }); describe('extractAllPackageFiles()', () => { it('support package file with multiple lock files', async () => { fs.readLocalFile.mockResolvedValueOnce( getSimpleRequirementsFile( 'pip-compile --output-file=requirements1.txt requirements.in', ['foo==1.0.1'], ), ); // requirements.in is parsed only once fs.readLocalFile.mockResolvedValueOnce('foo>=1.0.0'); fs.readLocalFile.mockResolvedValueOnce( getSimpleRequirementsFile( 'pip-compile --output-file=requirements2.txt requirements.in', ['foo==1.0.2'], ), ); fs.readLocalFile.mockResolvedValueOnce( getSimpleRequirementsFile( 'pip-compile --output-file=requirements3.txt requirements.in', ['foo==1.0.3'], ), ); const lockFiles = [ 'requirements1.txt', 'requirements2.txt', 'requirements3.txt', ]; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(packageFiles).not.toBeNull(); expect(packageFiles!.pop()).toHaveProperty('lockFiles', lockFiles); }); it('no lock files in returned package files', async () => { fs.readLocalFile.mockResolvedValueOnce( getSimpleRequirementsFile('pip-compile --output-file=foo.txt foo.in', [ 'foo==1.0.1', ]), ); fs.readLocalFile.mockResolvedValueOnce('foo>=1.0.0'); fs.readLocalFile.mockResolvedValueOnce( getSimpleRequirementsFile( 'pip-compile --output-file=bar.txt bar.in foo.txt', ['foo==1.0.1', 'bar==2.0.0'], ), ); fs.readLocalFile.mockResolvedValueOnce('bar>=1.0.0'); const lockFiles = ['foo.txt', 'bar.txt']; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(fs.readLocalFile).toHaveBeenCalledTimes(4); expect(packageFiles).not.toBeNull(); packageFiles!.forEach((packageFile) => { expect(packageFile).not.toHaveProperty('packageFile', 'foo.txt'); }); }); it('no override files in returned package files', async () => { fs.readLocalFile.mockResolvedValueOnce( getSimpleRequirementsFile( 'uv pip compile --output-file=requirements.txt --overrides=overrides.txt requirements.in', ['foo==1.0.1'], ), ); fs.readLocalFile.mockResolvedValueOnce('foo>=1.0.0'); const lockFiles = ['requirements.txt']; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(packageFiles).not.toBeNull(); packageFiles!.forEach((packageFile) => { expect(packageFile).not.toHaveProperty('packageFile', 'overrides.txt'); }); }); // TODO(not7cd): update when constraints are supported it('no constraint files in returned package files', async () => { fs.readLocalFile.mockResolvedValueOnce( getSimpleRequirementsFile( 'pip-compile --output-file=requirements.txt --constraint=constraints.txt requirements.in', ['foo==1.0.1'], ), ); fs.readLocalFile.mockResolvedValueOnce('foo>=1.0.0'); const lockFiles = ['requirements.txt']; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(packageFiles).not.toBeNull(); packageFiles!.forEach((packageFile) => { expect(packageFile).not.toHaveProperty( 'packageFile', 'constraints.txt', ); }); }); }); it('return null for malformed files', async () => { // empty.txt fs.readLocalFile.mockResolvedValueOnce(''); // noHeader.txt fs.readLocalFile.mockResolvedValueOnce( Fixtures.get('requirementsNoHeaders.txt'), ); // badSource.txt fs.readLocalFile.mockResolvedValueOnce( getSimpleRequirementsFile( 'pip-compile --output-file=badSource.txt malformed.in empty.in', ['foo==1.0.1'], ), ); // malformed.in fs.readLocalFile.mockResolvedValueOnce('!@#$'); // empty.in fs.readLocalFile.mockResolvedValueOnce(''); // headerOnly.txt fs.readLocalFile.mockResolvedValueOnce( getSimpleRequirementsFile( 'pip-compile --output-file=headerOnly.txt reqs.in', [], ), ); const lockFiles = [ 'empty.txt', 'noHeader.txt', 'badSource.txt', 'headerOnly.txt', ]; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(packageFiles).toBeNull(); expect(fs.readLocalFile).toHaveBeenCalledTimes(6); expect(logger.warn).toHaveBeenCalledTimes(2); // malformed.in, noHeader.txt }); it('return null for bad paths', async () => { // ambigous.txt fs.readLocalFile.mockResolvedValueOnce( getSimpleRequirementsFile( 'pip-compile --output-file=../ambigous.txt reqs.in', ['foo==1.0.1'], ), ); // badSource.txt fs.readLocalFile.mockResolvedValueOnce( getSimpleRequirementsFile( 'pip-compile --output-file=badSource.txt ../outside.in', ['foo==1.0.1'], ), ); const packageFiles = await extractAllPackageFiles({}, [ 'subdir/ambigous.txt', 'badSource.txt', ]); expect(packageFiles).toBeNull(); expect(fs.readLocalFile).toHaveBeenCalledTimes(2); expect(logger.warn).toHaveBeenCalledTimes(2); }); it('return for valid paths', async () => { // reqs.txt fs.readLocalFile.mockResolvedValueOnce( getSimpleRequirementsFile('pip-compile --output-file=reqs.txt reqs.in', [ 'foo==1.0.1', ]), ); fs.readLocalFile.mockResolvedValueOnce('foo>=1.0.0'); // absolute/reqs.txt fs.readLocalFile.mockResolvedValueOnce( getSimpleRequirementsFile( 'pip-compile --output-file=./absolute/reqs.txt ./absolute/reqs.in', ['foo==1.0.1'], ), ); fs.readLocalFile.mockResolvedValueOnce('foo>=1.0.0'); // relative/reqs.txt fs.readLocalFile.mockResolvedValueOnce( getSimpleRequirementsFile( 'pip-compile --output-file=reqs.txt ../outside.in', ['foo==1.0.1'], ), ); fs.readLocalFile.mockResolvedValueOnce('foo>=1.0.0'); const packageFiles = await extractAllPackageFiles({}, [ 'reqs.txt', 'absolute/reqs.txt', 'relative/reqs.txt', ]); expect(packageFiles?.map((p) => p.packageFile).sort()).toEqual( ['reqs.in', 'absolute/reqs.in', 'outside.in'].sort(), ); expect(logger.warn).toHaveBeenCalledTimes(0); }); it('return sorted package files', async () => { fs.readLocalFile.mockImplementation((name): any => { if (name === '1.in') { return 'foo'; } if (name === '2.txt') { return getSimpleRequirementsFile( 'pip-compile --output-file=2.txt 1.in', ['foo==1.0.1'], ); } if (name === '3.in') { return '-r 2.txt\nfoo'; } if (name === '4.txt') { return getSimpleRequirementsFile( 'pip-compile --output-file=4.txt 3.in', ['foo==1.0.1'], ); } return null; }); const lockFiles = ['4.txt', '2.txt']; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(packageFiles).toBeDefined(); expect(packageFiles?.map((p) => p.packageFile)).toEqual(['1.in', '3.in']); expect(packageFiles?.map((p) => p.lockFiles)).toEqual([ ['2.txt', '4.txt'], ['4.txt'], ]); }); it('return sorted package files with constraint in file', async () => { fs.readLocalFile.mockResolvedValueOnce( getSimpleRequirementsFile('pip-compile --output-file=4.txt 3.in', [ 'foo==1.0.1', ]), ); fs.readLocalFile.mockResolvedValueOnce('-c 2.txt\nfoo'); fs.readLocalFile.mockResolvedValueOnce( getSimpleRequirementsFile('pip-compile --output-file=2.txt 1.in', [ 'foo==1.0.1', ]), ); fs.readLocalFile.mockResolvedValueOnce('foo'); const lockFiles = ['4.txt', '2.txt']; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(packageFiles).toBeDefined(); expect(packageFiles?.map((p) => p.packageFile)).toEqual(['1.in', '3.in']); expect(packageFiles?.map((p) => p.lockFiles!.pop())).toEqual([ '2.txt', '4.txt', ]); }); it('return sorted package files with constraint in command', async () => { fs.readLocalFile.mockResolvedValueOnce( getSimpleRequirementsFile( 'pip-compile --constraint=2.txt --output-file=4.txt 3.in', ['foo==1.0.1'], ), ); fs.readLocalFile.mockResolvedValueOnce('foo'); fs.readLocalFile.mockResolvedValueOnce( getSimpleRequirementsFile('pip-compile --output-file=2.txt 1.in', [ 'foo==1.0.1', ]), ); fs.readLocalFile.mockResolvedValueOnce('foo'); const lockFiles = ['4.txt', '2.txt']; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(packageFiles).toBeDefined(); expect(packageFiles?.map((p) => p.packageFile)).toEqual(['1.in', '3.in']); expect(packageFiles?.map((p) => p.lockFiles!.pop())).toEqual([ '2.txt', '4.txt', ]); }); it('adds lockedVersion to deps in package file', async () => { fs.readLocalFile.mockResolvedValueOnce( getSimpleRequirementsFile( 'pip-compile --output-file=requirements.txt requirements.in', ['friendly-bard==1.0.1'], ), ); // also check if normalized name is used fs.readLocalFile.mockResolvedValueOnce('FrIeNdLy-._.-bArD>=1.0.0'); const lockFiles = ['requirements.txt']; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(packageFiles).toBeDefined(); const packageFile = packageFiles!.pop(); expect(packageFile!.deps).toHaveLength(1); expect(packageFile!.deps.pop()).toMatchObject({ currentValue: '>=1.0.0', depName: 'FrIeNdLy-._.-bArD', lockedVersion: '1.0.1', }); }); it('warns if dependency has no locked version', async () => { fs.readLocalFile.mockResolvedValueOnce( getSimpleRequirementsFile( 'pip-compile --output-file=requirements.txt requirements.in', ['foo==1.0.1'], ), ); fs.readLocalFile.mockResolvedValueOnce('foo>=1.0.0\nbar'); const lockFiles = ['requirements.txt']; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(packageFiles).toBeDefined(); const packageFile = packageFiles!.pop(); expect(packageFile!.deps).toHaveLength(2); expect(logger.warn).toHaveBeenCalledWith( { depName: 'bar', lockFile: 'requirements.txt' }, 'pip-compile: dependency not found in lock file', ); }); it('adds transitive dependency to deps in package file', async () => { fs.readLocalFile.mockResolvedValueOnce( getSimpleRequirementsFile( 'pip-compile --output-file=requirements.txt requirements.in', ['friendly-bard==1.0.1', 'bards-friend==1.0.0'], ), ); fs.readLocalFile.mockResolvedValueOnce('FrIeNdLy-._.-bArD>=1.0.0'); const lockFiles = ['requirements.txt']; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(packageFiles).toBeDefined(); const packageFile = packageFiles!.pop(); expect(packageFile!.deps).toHaveLength(2); expect(packageFile!.deps[1]).toEqual({ datasource: 'pypi', depType: 'indirect', depName: 'bards-friend', packageName: 'bards-friend', lockedVersion: '1.0.0', enabled: false, }); }); it('handles -r reference to another input file', async () => { fs.readLocalFile.mockImplementation((name): any => { if (name === '1.in') { return 'foo'; } if (name === '2.txt') { return getSimpleRequirementsFile( 'pip-compile --output-file=2.txt 1.in', ['foo==1.0.1'], ); } if (name === '3.in') { return '-r 1.in\nfoo'; } if (name === '4.txt') { return getSimpleRequirementsFile( 'pip-compile --output-file=4.txt 3.in', ['foo==1.0.1'], ); } return null; }); const lockFiles = ['4.txt', '2.txt']; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(packageFiles?.map((p) => p.lockFiles)).toEqual([ ['2.txt', '4.txt'], ['4.txt'], ]); }); it('handles transitive -r references', async () => { fs.readLocalFile.mockImplementation((name): any => { if (name === '1.in') { return 'foo'; } if (name === '2.txt') { return getSimpleRequirementsFile( 'pip-compile --output-file=2.txt 1.in', ['foo==1.0.1'], ); } if (name === '3.in') { return '-r 1.in\nfoo'; } if (name === '4.txt') { return getSimpleRequirementsFile( 'pip-compile --output-file=4.txt 3.in', ['foo==1.0.1'], ); } if (name === '5.in') { return '-r 4.txt\nfoo'; } if (name === '6.txt') { return getSimpleRequirementsFile( 'pip-compile --output-file=6.txt 5.in', ['foo==1.0.1'], ); } return null; }); const lockFiles = ['4.txt', '2.txt', '6.txt']; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(packageFiles?.map((p) => p.lockFiles)).toEqual([ ['2.txt', '4.txt', '6.txt'], ['4.txt', '6.txt'], ['6.txt'], ]); }); it('warns on -r reference to failed file', async () => { fs.readLocalFile.mockImplementation((name): any => { if (name === 'reqs-no-headers.txt') { return Fixtures.get('requirementsNoHeaders.txt'); } if (name === '1.in') { return '-r reqs-no-headers.txt\nfoo'; } if (name === '2.txt') { return getSimpleRequirementsFile( 'pip-compile --output-file=2.txt 1.in', ['foo==1.0.1'], ); } return null; }); const lockFiles = ['reqs-no-headers.txt', '2.txt']; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(packageFiles?.map((p) => p.lockFiles)).toEqual([['2.txt']]); expect(logger.warn).toHaveBeenCalledWith( { packageFile: '1.in', requirementsFile: 'reqs-no-headers.txt' }, 'pip-compile: Package file references a file which does not appear to be a requirements file managed by pip-compile', ); }); it('warns on -r reference to requirements file not managed by pip-compile', async () => { fs.readLocalFile.mockImplementation((name): any => { if (name === '1.in') { return '-r unmanaged-file.txt\nfoo'; } if (name === '2.txt') { return getSimpleRequirementsFile( 'pip-compile --output-file=2.txt 1.in', ['foo==1.0.1'], ); } return null; }); const lockFiles = ['2.txt']; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(packageFiles?.map((p) => p.lockFiles)).toEqual([['2.txt']]); expect(logger.warn).toHaveBeenCalledWith( { packageFile: '1.in', requirementsFile: 'unmanaged-file.txt' }, 'pip-compile: Package file references a file which does not appear to be a requirements file managed by pip-compile', ); }); it('handles duplicate -r dependencies', async () => { fs.readLocalFile.mockImplementation((name): any => { if (name === '1.in') { return 'foo'; } if (name === '1.txt') { return getSimpleRequirementsFile( 'pip-compile --output-file=1.txt 1.in', ['foo==1.0.1'], ); } if (name === '2.in') { return '-r 1.txt\nbar'; } if (name === '2.txt') { return getSimpleRequirementsFile( 'pip-compile --output-file=2.txt 2.in', ['foo==1.0.1', 'bar==2.0.0'], ); } if (name === '3.in') { return '-r 1.txt\nbaz'; } if (name === '3.txt') { return getSimpleRequirementsFile( 'pip-compile --output-file=3.txt 3.in', ['foo==1.0.1', 'baz==2.0.0'], ); } if (name === '4.in') { return '-r 2.txt\n-r 3.txt\nqux'; } if (name === '4.txt') { return getSimpleRequirementsFile( 'pip-compile --output-file=4.txt 4.in', ['foo==1.0.1', 'bar==2.0.0', 'baz==2.0.0', 'qux==1.2.3'], ); } return null; }); const lockFiles = ['1.txt', '2.txt', '3.txt', '4.txt']; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(packageFiles?.map((p) => p.lockFiles)).toEqual([ ['1.txt', '2.txt', '3.txt', '4.txt'], ['3.txt', '4.txt'], ['2.txt', '4.txt'], ['4.txt'], ]); }); it('handles -r dependency on lock file with multiple input files', async () => { fs.readLocalFile.mockImplementation((name): any => { if (name === '1.in') { return 'foo'; } if (name === '2.in') { return 'bar'; } if (name === 'multi_input.txt') { return getSimpleRequirementsFile( 'pip-compile --output-file=multi_input.txt 1.in 2.in', ['foo==1.0.1', 'bar==2.0.0'], ); } if (name === '3.in') { return '-r multi_input.txt\nbaz'; } if (name === 'dash_r.txt') { return getSimpleRequirementsFile( 'pip-compile --output-file=dash_r.txt 3.in', ['foo==1.0.1', 'bar==2.0.0', 'baz==2.0.0'], ); } return null; }); const lockFiles = ['multi_input.txt', 'dash_r.txt']; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(packageFiles?.map((p) => p.lockFiles)).toEqual([ ['multi_input.txt', 'dash_r.txt'], ['multi_input.txt', 'dash_r.txt'], ['dash_r.txt'], ]); }); it('handles -r dependency on input file that is also used to generate lock file with multiple inputs', async () => { fs.readLocalFile.mockImplementation((name): any => { if (name === '1.in') { return 'foo'; } if (name === '2.in') { return 'bar'; } if (name === 'multi_input.txt') { return getSimpleRequirementsFile( 'pip-compile --output-file=multi_input.txt 1.in 2.in', ['foo==1.0.1', 'bar==2.0.0'], ); } if (name === '3.in') { return '-r 1.in\nbaz'; } if (name === 'dash_r.txt') { return getSimpleRequirementsFile( 'pip-compile --output-file=dash_r.txt 3.in', ['foo==1.0.1', 'baz==2.0.0'], ); } return null; }); const lockFiles = ['multi_input.txt', 'dash_r.txt']; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(packageFiles?.map((p) => p.lockFiles)).toEqual([ ['multi_input.txt'], ['multi_input.txt', 'dash_r.txt'], ['dash_r.txt'], ]); }); it('handles -r dependency on file with relative path same dir', async () => { fs.readLocalFile.mockImplementation((name): any => { if (name === 'dir/1.in') { return 'foo'; } if (name === 'dir/2.in') { return '-r 1.in\nbar'; } if (name === 'dir/1.txt') { return getSimpleRequirementsFile( 'pip-compile --output-file=1.txt 1.in', ['foo==1.0.1'], ); } if (name === 'dir/2.txt') { return getSimpleRequirementsFile( 'pip-compile --output-file=2.txt 2.in', ['foo==1.0.1', 'bar==2.0.0'], ); } return null; }); const lockFiles = ['dir/1.txt', 'dir/2.txt']; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(packageFiles).toMatchObject([ { packageFile: 'dir/1.in', lockFiles: ['dir/1.txt', 'dir/2.txt'] }, { packageFile: 'dir/2.in', lockFiles: ['dir/2.txt'] }, ]); }); it('handles -r dependency on file with relative path above', async () => { fs.readLocalFile.mockImplementation((name): any => { if (name === 'common/1.in') { return 'foo'; } if (name === 'dir/2.in') { return '-r ../common/1.in\nbar'; } if (name === 'common/1.txt') { return getSimpleRequirementsFile( 'pip-compile --output-file=1.txt 1.in', ['foo==1.0.1'], ); } if (name === 'dir/2.txt') { return getSimpleRequirementsFile( 'pip-compile --output-file=2.txt 2.in', ['foo==1.0.1', 'bar==2.0.0'], ); } return null; }); const lockFiles = ['common/1.txt', 'dir/2.txt']; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(packageFiles).toMatchObject([ { packageFile: 'common/1.in', lockFiles: ['common/1.txt', 'dir/2.txt'] }, { packageFile: 'dir/2.in', lockFiles: ['dir/2.txt'] }, ]); }); it('handles -r dependency on file with relative path above with path', async () => { fs.readLocalFile.mockImplementation((name): any => { if (name === 'common/1.in') { return 'foo'; } if (name === 'dir/2.in') { return '-r ../common/1.in\nbar'; } if (name === 'common/1.txt') { return getSimpleRequirementsFile( 'pip-compile --output-file=common/1.txt common/1.in', ['foo==1.0.1'], ); } if (name === 'dir/2.txt') { return getSimpleRequirementsFile( 'pip-compile --output-file=dir/2.txt dir/2.in', ['foo==1.0.1', 'bar==2.0.0'], ); } return null; }); const lockFiles = ['common/1.txt', 'dir/2.txt']; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(packageFiles).toMatchObject([ { packageFile: 'common/1.in', lockFiles: ['common/1.txt', 'dir/2.txt'] }, { packageFile: 'dir/2.in', lockFiles: ['dir/2.txt'] }, ]); }); });