import { isNonEmptyArray } from '@sindresorhus/is'; import { HOST_BLOCKED, HOST_DISABLED, } from '../../../constants/error-messages.ts'; import { logger } from '../../../logger/index.ts'; import { ExternalHostError } from '../../../types/errors/external-host-error.ts'; import { withCache } from '../../../util/cache/package/with-cache.ts'; import { refusedHostMessage } from '../../../util/http/util.ts'; import { getQueryString, joinUrlParts } from '../../../util/url.ts'; import { Datasource } from '../datasource.ts'; import type { GetReleasesConfig, ReleaseResult } from '../types.ts'; import { type RepologyPackage, RepologyPackages } from './schema.ts'; import type { RepologyPackageType } from './types.ts'; const packageTypes: RepologyPackageType[] = ['binname', 'srcname']; function findPackageInResponse( response: RepologyPackage[], repoName: string, pkgName: string, types: RepologyPackageType[], ): RepologyPackage[] | null { const repoPackages = response.filter((pkg) => pkg.repo === repoName); if (repoPackages.length === 0) { // no packages associated with repoName return null; } if (repoPackages.length === 1) { // repo contains exactly one package, so we can return them safely return repoPackages; } // In some cases Repology bundles multiple packages into a single project, which might result in ambiguous results. // We need to do additional filtering by matching allowed package types passed as params with package description. // Remaining packages are the one we are looking for const packagesWithType = repoPackages.filter((pkg) => { for (const pkgType of types) { if (pkg[pkgType] && pkg[pkgType] === pkgName) { return true; } } return false; }); return packagesWithType.length > 0 ? packagesWithType : null; } export class RepologyDatasource extends Datasource { static readonly id = 'repology'; override readonly defaultRegistryUrls = ['https://repology.org/']; override readonly registryStrategy = 'hunt'; constructor() { super(RepologyDatasource.id); } private async queryPackages(url: string): Promise { try { const res = await this.http.getJson(url, RepologyPackages); return res.body; } catch (err) { if (err.statusCode === 404) { // Return an array here because the api does not return proper http codes // and instead of an 404 error an empty array with code 200 is returned // When querying the resolver 404 is thrown if package could not be resolved // and 403 if the repo is not supported // 403 is handled later because in this case we are trying the API return []; } throw err; } } private async queryPackagesViaResolver( registryUrl: string, repoName: string, packageName: string, packageType: RepologyPackageType, ): Promise { const query = getQueryString({ repo: repoName, name_type: packageType, target_page: 'api_v1_project', noautoresolve: 'on', name: packageName, }); // Retrieve list of packages by looking up Repology project const packages = await this.queryPackages( joinUrlParts(registryUrl, `tools/project-by?${query}`), ); return packages; } private async queryPackagesViaAPI( registryUrl: string, packageName: string, ): Promise { // Directly query the package via the API. This will only work if `packageName` has the // same name as the repology project const packages = await this.queryPackages( joinUrlParts(registryUrl, `api/v1/project`, packageName), ); return packages; } private async _queryPackage( registryUrl: string, repoName: string, pkgName: string, ): Promise { let response: RepologyPackage[]; // Try getting the packages from tools/project-by first for type binname and // afterwards for srcname. This needs to be done first, because some packages // resolve to repology projects which have a different name than the package // e.g. `pulseaudio-utils` resolves to project `pulseaudio`, BUT there is also // a project called `pulseaudio-utils` but it does not contain the package we // are looking for. try { for (const pkgType of packageTypes) { response = await this.queryPackagesViaResolver( registryUrl, repoName, pkgName, pkgType, ); if (response) { const pkg = findPackageInResponse(response, repoName, pkgName, [ pkgType, ]); if (isNonEmptyArray(pkg)) { // exit immediately if package found return pkg; } } } } catch (err) { if (err.statusCode === 403) { logger.debug( { repoName, pkgName }, 'Repology does not support tools/project-by lookups for repository. Will try direct API access now', ); // If the repository is not supported in tools/project-by we try directly accessing the // API. This will support all repositories but requires that the project name is equal to the // package name. This won't be always the case but for a good portion we might be able to resolve // the package this way. response = await this.queryPackagesViaAPI(registryUrl, pkgName); const pkg = findPackageInResponse( response, repoName, pkgName, packageTypes, ); if (isNonEmptyArray(pkg)) { // exit immediately if package found return pkg; } } else if (err.statusCode === 300) { logger.warn( { repoName, pkgName }, 'Ambiguous redirection from package name to project name in Repology. Skipping this package', ); return undefined; } throw err; } logger.debug( { repoName, pkgName }, 'Repository or package not found on Repology', ); return undefined; } queryPackage( registryUrl: string, repoName: string, pkgName: string, ): Promise { return withCache( { ttlMinutes: 60, namespace: `datasource-${RepologyDatasource.id}`, key: joinUrlParts(registryUrl, repoName, pkgName), }, () => this._queryPackage(registryUrl, repoName, pkgName), ); } async getReleases({ packageName, registryUrl, }: GetReleasesConfig): Promise { /* v8 ignore next -- should never happen */ if (!registryUrl) { return null; } // Ensure lookup name contains both repository and package const [repoName, pkgName] = packageName.split('/', 2); if (!repoName || !pkgName) { throw new ExternalHostError( new Error( 'Repology lookup name must contain repository and package separated by slash (/)', ), ); } logger.trace(`repology.getReleases(${repoName}, ${pkgName})`); try { // Try to retrieve (cached) package information from Repology const pkg = await this.queryPackage(registryUrl, repoName, pkgName); if (!pkg) { return null; } // Always prefer origversion if available, otherwise default to version // This is required as source packages usually have no origversion const releases = pkg.map((item) => ({ version: item.origversion ?? item.version, })); return { releases }; } catch (err) { if ([HOST_BLOCKED, HOST_DISABLED].includes(err.message)) { logger.trace({ packageName, err }, refusedHostMessage(err)); } else { logger.once.warn( { packageName, err }, 'Repology lookup failed with unexpected error', ); } this.handleGenericErrors(err); } } }