import readline from 'node:readline'; import { logger } from '../../../logger/index.ts'; import { withCache } from '../../../util/cache/package/with-cache.ts'; import * as fs from '../../../util/fs/index.ts'; import { Datasource } from '../datasource.ts'; import type { GetReleasesConfig, ReleaseResult } from '../types.ts'; import { packageKeys, requiredPackageKeys } from './common.ts'; import { downloadAndExtractPackage } from './packages.ts'; import { formatReleaseResult, releaseMetaInformationMatches, } from './release.ts'; import type { PackageDescription } from './types.ts'; import { constructComponentUrls } from './url.ts'; export class DebDatasource extends Datasource { static readonly id = 'deb'; constructor() { super(DebDatasource.id); } /** * Users are able to specify custom Debian repositories as long as they follow * the Debian package repository format as specified here * @see{https://wiki.debian.org/DebianRepository/Format} */ override readonly customRegistrySupport = true; /** * Users can specify multiple upstream repositories and the datasource will aggregate the release * @example * When specifying multiple dependencies both internal and external dependencies from internal/external artifactory */ override readonly registryStrategy = 'merge'; /** * The original apt source list file format is * deb uri distribution [component1] [component2] [...] * @see{https://wiki.debian.org/DebianRepository/Format} * * However, for Renovate, we require the registry URLs to be * valid URLs which is why the parameters are encoded in the URL. * * The following query parameters are required: * - components: comma separated list of components * - suite: stable, oldstable or other alias for a release, either this or release must be given like buster * - binaryArch: e.g. amd64 resolves to http://deb.debian.org/debian/dists/stable/non-free/binary-amd64/ */ override readonly defaultRegistryUrls = [ 'https://deb.debian.org/debian?suite=stable&components=main,contrib,non-free&binaryArch=amd64', ]; override readonly defaultVersioning = 'deb'; /** * Parses the extracted package index file. * * @param extractedFile - The path to the extracted package file. * @param lastTimestamp - The timestamp of the last modification. * @returns a list of packages with minimal Metadata. */ private async _parseExtractedPackageIndex( extractedFile: string, _lastTimestamp: Date, ): Promise> { // read line by line to avoid high memory consumption as the extracted Packages // files can be multiple MBs in size const rl = readline.createInterface({ input: fs.createCacheReadStream(extractedFile), terminal: false, }); let currentPackage: PackageDescription = {}; // A Package Index can contain multiple Versions of the package on private Artifactory (e.g. Jfrog) const allPackages: Record = {}; for await (const line of rl) { if (line === '') { // All information of the package are available, add to the list of packages if (requiredPackageKeys.every((key) => key in currentPackage)) { if (!allPackages[currentPackage.Package!]) { allPackages[currentPackage.Package!] = []; } allPackages[currentPackage.Package!].push(currentPackage); currentPackage = {}; } } else { for (const key of packageKeys) { if (line.startsWith(`${key}:`)) { currentPackage[key] = line.substring(key.length + 1).trim(); break; } } } } // Check the last package after file reading is complete if (requiredPackageKeys.every((key) => key in currentPackage)) { if (!allPackages[currentPackage.Package!]) { allPackages[currentPackage.Package!] = []; } allPackages[currentPackage.Package!].push(currentPackage); } return allPackages; } parseExtractedPackageIndex( extractedFile: string, lastTimestamp: Date, ): Promise> { return withCache( { namespace: `datasource-${DebDatasource.id}`, key: `${extractedFile}:${lastTimestamp.getTime()}`, ttlMinutes: 24 * 60, }, () => this._parseExtractedPackageIndex(extractedFile, lastTimestamp), ); } private async _getPackageIndex( componentUrl: string, ): Promise> { const { extractedFile, lastTimestamp } = await downloadAndExtractPackage( componentUrl, this.http, ); return await this.parseExtractedPackageIndex(extractedFile, lastTimestamp); } getPackageIndex( componentUrl: string, ): Promise> { return withCache( { namespace: `datasource-${DebDatasource.id}`, key: componentUrl, }, () => this._getPackageIndex(componentUrl), ); } /** * Fetches the release information for a given package from the registry URL. * * @param config - Configuration for fetching releases. * @returns The release result if the package is found, otherwise null. */ private async _getReleases({ registryUrl, packageName, }: GetReleasesConfig): Promise { /* v8 ignore next -- should never happen */ if (!registryUrl) { return null; } const componentUrls = constructComponentUrls(registryUrl); let aggregatedRelease: ReleaseResult | null = null; for (const componentUrl of componentUrls) { try { const packageIndex = await this.getPackageIndex(componentUrl); const parsedPackages = packageIndex[packageName]; if (parsedPackages) { const newRelease = formatReleaseResult(parsedPackages); if (aggregatedRelease === null) { aggregatedRelease = newRelease; } else { if (!releaseMetaInformationMatches(aggregatedRelease, newRelease)) { logger.warn( { packageName }, 'Package occurred in more than one repository with different meta information. Aggregating releases anyway.', ); } aggregatedRelease.releases.push(...newRelease.releases); } } } catch (error) { logger.debug( { componentUrl, err: error }, 'Skipping component due to an error', ); } } return aggregatedRelease; } getReleases(config: GetReleasesConfig): Promise { return withCache( { namespace: `datasource-${DebDatasource.id}`, key: `${config.registryUrl}:${config.packageName}`, fallback: true, }, () => this._getReleases(config), ); } }