import { logger } from '../../../logger/index.ts'; import { ExternalHostError } from '../../../types/errors/external-host-error.ts'; import { withCache } from '../../../util/cache/package/with-cache.ts'; import { HttpError } from '../../../util/http/index.ts'; import * as p from '../../../util/promises.ts'; import { regEx } from '../../../util/regex.ts'; import { getQueryString, joinUrlParts } from '../../../util/url.ts'; import * as hashicorpVersioning from '../../versioning/hashicorp/index.ts'; import { TerraformDatasource } from '../terraform-module/base.ts'; import { createSDBackendURL } from '../terraform-module/utils.ts'; import type { GetReleasesConfig, ReleaseResult } from '../types.ts'; import { OpenTofuProviderDocsResponse, OpenTofuProviderPackagesResponse, type TerraformBuild, TerraformProviderReleaseBackend, TerraformProviderV2Response, TerraformProviderVersions, TerraformRegistryBuildResponse, TerraformRegistryVersions, VersionDetailResponse, } from './schema.ts'; export class TerraformProviderDatasource extends TerraformDatasource { static override readonly id = 'terraform-provider'; static readonly hashicorpReleaseUrl = 'https://releases.hashicorp.com'; static readonly defaultRegistryUrls = [ TerraformProviderDatasource.terraformRegistryUrl, TerraformProviderDatasource.hashicorpReleaseUrl, ]; static repositoryRegex = regEx(/^hashicorp\/(?\S+)$/); constructor() { super(TerraformProviderDatasource.id); } override readonly defaultRegistryUrls = TerraformProviderDatasource.defaultRegistryUrls; override readonly defaultVersioning = hashicorpVersioning.id; override readonly registryStrategy = 'hunt'; override readonly releaseTimestampSupport = true; override readonly releaseTimestampNote = 'The release timestamp is only available for `registry.terraform.io` (v2 API) and `registry.opentofu.org` (via `api.opentofu.org`). Other registries using the Provider Registry Protocol do not provide timestamps.'; override readonly sourceUrlSupport = 'package'; override readonly sourceUrlNote = 'For `registry.terraform.io`, the source URL is taken from the `source` field of the v2 API response. For `registry.opentofu.org`, it is derived from the package name following the OpenTofu registry policy of `github.com/NAMESPACE/terraform-provider-NAME`.'; private async _getReleases({ packageName, registryUrl, }: GetReleasesConfig): Promise { /* v8 ignore next -- should never happen */ if (!registryUrl) { return null; } logger.trace( `terraform-provider.getDependencies() packageName: ${packageName}`, ); if (registryUrl === TerraformProviderDatasource.terraformRegistryUrl) { return await this.queryTerraformRegistryV2(registryUrl, packageName); } if ( registryUrl === TerraformProviderDatasource.openTofuRegistryUrl || registryUrl === TerraformProviderDatasource.openTofuApiUrl ) { return await this.queryOpenTofuRegistry(packageName); } if (registryUrl === TerraformProviderDatasource.hashicorpReleaseUrl) { return await this.queryReleaseBackend(packageName, registryUrl); } // Fall back to the standard Provider Registry Protocol for other registries. return await this.queryProviderRegistry(registryUrl, packageName); } getReleases(config: GetReleasesConfig): Promise { const url = config.registryUrl; const repo = TerraformProviderDatasource.getRepository(config); return withCache( { namespace: `datasource-${TerraformProviderDatasource.id}`, key: `getReleases:${url}/${repo}`, fallback: true, }, () => this._getReleases(config), ); } private static getRepository({ packageName }: GetReleasesConfig): string { return packageName.includes('/') ? packageName : `hashicorp/${packageName}`; } /** * Query the Terraform Registry using the undocumented v2 JSON:API. * * Returns release timestamps for all versions, unlike the v1 API * which only exposed the timestamp for the latest version. */ private async queryTerraformRegistryV2( registryUrl: string, packageName: string, ): Promise { const repository = TerraformProviderDatasource.getRepository({ packageName, }); const providerUrl = `${joinUrlParts( registryUrl, 'v2/providers', repository, )}?${getQueryString({ include: 'provider-versions' })}`; const { body: res } = await this.http.getJson( providerUrl, TerraformProviderV2Response, ); res.homepage = `${registryUrl}/providers/${repository}`; return res; } /** * Query the OpenTofu registry docs API. * https://api.opentofu.org/ * * Used when the registry URL is `registry.opentofu.org`. * Queries `api.opentofu.org` for provider versions with release timestamps. */ private async queryOpenTofuRegistry( packageName: string, ): Promise { const repository = TerraformProviderDatasource.getRepository({ packageName, }); const docsUrl = joinUrlParts( TerraformProviderDatasource.openTofuApiUrl, 'registry/docs/providers', repository, 'index.json', ); const { body: res } = await this.http.getJson( docsUrl, OpenTofuProviderDocsResponse, ); res.homepage = `https://search.opentofu.org/provider/${repository}`; // The OpenTofu registry only indexes providers hosted on GitHub under the // `NAMESPACE/terraform-provider-NAME` repository naming convention, so the // source URL can be derived deterministically from the package name. // https://github.com/opentofu/registry/blob/main/PROCEDURES.md const [namespace, name] = repository.split('/'); res.sourceUrl = `https://github.com/${namespace}/terraform-provider-${name}`; return res; } /** * Query a registry using the Provider Registry Protocol that all registries * are required to implement. * https://www.terraform.io/internals/provider-registry-protocol */ private async queryProviderRegistry( registryUrl: string, packageName: string, ): Promise { const repository = TerraformProviderDatasource.getRepository({ packageName, }); const serviceDiscovery = await this.getTerraformServiceDiscoveryResult(registryUrl); const backendURL = createSDBackendURL( registryUrl, 'providers.v1', serviceDiscovery, `${repository}/versions`, ); const res = (await this.http.getJson(backendURL, TerraformProviderVersions)) .body; const dep: ReleaseResult = { releases: res.versions.map(({ version }) => ({ version, })), }; return dep; } private async queryReleaseBackend( packageName: string, registryURL: string, ): Promise { const hashicorpPackage = packageName.replace('hashicorp/', ''); const backendLookUpName = `terraform-provider-${hashicorpPackage}`; const backendURL = joinUrlParts( registryURL, backendLookUpName, `index.json`, ); const res = ( await this.http.getJson(backendURL, TerraformProviderReleaseBackend) ).body; const dep: ReleaseResult = { releases: Object.keys(res.versions).map((version) => ({ version, })), sourceUrl: joinUrlParts( 'https://github.com/terraform-providers', backendLookUpName, ), }; return dep; } private async _getBuilds( registryURL: string, repository: string, version: string, ): Promise { if (registryURL === TerraformProviderDatasource.hashicorpReleaseUrl) { // check if registryURL === secondary backend const repositoryRegexResult = TerraformProviderDatasource.repositoryRegex.exec(repository)?.groups; if (!repositoryRegexResult) { // non hashicorp builds are not supported with releases.hashicorp.com return null; } const packageName = repositoryRegexResult.packageName; const backendLookUpName = `terraform-provider-${packageName}`; let versionReleaseBackend: VersionDetailResponse; try { versionReleaseBackend = await this.getReleaseBackendIndex( backendLookUpName, version, ); } catch (err) { if (err instanceof ExternalHostError) { throw err; } logger.debug( { err, backendLookUpName, version }, `Failed to retrieve builds for ${backendLookUpName} ${version}`, ); // throw an error to disable caching throw new ExternalHostError(err); } return versionReleaseBackend.builds; } // check public or private Terraform registry const serviceDiscovery = await this.getTerraformServiceDiscoveryResult(registryURL); if (!serviceDiscovery) { // throw an error to disable caching throw new ExternalHostError( new Error(`Service discovery not found for ${registryURL}`), ); } const backendURL = createSDBackendURL( registryURL, 'providers.v1', serviceDiscovery, repository, ); const versionsResponse = ( await this.http.getJson( `${backendURL}/versions`, TerraformRegistryVersions, ) ).body; if (!versionsResponse.versions) { // throw an error to disable caching throw new ExternalHostError( new Error(`Failed to retrieve version list for ${backendURL}`), ); } const builds = versionsResponse.versions.find( (value) => value.version === version, ); if (!builds) { // should never happen, but just in case // throw an error to disable caching throw new ExternalHostError( new Error( `No builds found for ${repository}:${version} on ${registryURL}`, ), ); } const result = await p.map( builds.platforms, async (platform) => { const buildURL = `${backendURL}/${version}/download/${platform.os}/${platform.arch}`; try { const res = ( await this.http.getJson(buildURL, TerraformRegistryBuildResponse) ).body; const newBuild: TerraformBuild = { name: repository, url: res.download_url, version, ...res, }; return newBuild; } catch (err) { /* v8 ignore next -- hard to test */ if (err instanceof ExternalHostError) { throw err; } logger.debug({ err, url: buildURL }, 'Failed to retrieve build'); // throw an error to disable caching throw new ExternalHostError(err); } }, { concurrency: 4 }, ); return result; } getBuilds( registryURL: string, repository: string, version: string, ): Promise { return withCache( { namespace: `datasource-${TerraformProviderDatasource.id}`, key: `getBuilds:${registryURL}/${repository}/${version}`, }, () => this._getBuilds(registryURL, repository, version), ); } /** * A single platform's download endpoint returns the hashes for all platforms, * so we query `linux/amd64` and fall back to `/versions` discovery only when a * provider lacks that platform (404). * See https://github.com/opentofu/opentofu/pull/3434 */ private async _getProviderPackages( repository: string, version: string, ): Promise { const baseUrl = joinUrlParts( TerraformProviderDatasource.openTofuRegistryUrl, 'v1/providers', repository, ); try { try { const { body } = await this.http.getJson( `${baseUrl}/${version}/download/linux/amd64`, OpenTofuProviderPackagesResponse, ); return body; } catch (err) { if (!(err instanceof HttpError) || err.response?.statusCode !== 404) { throw err; } } return await this._getProviderPackagesForAvailablePlatform( baseUrl, version, ); } catch (err) { if (err instanceof ExternalHostError) { throw err; } logger.debug( { err, repository, version }, `Failed to retrieve provider packages for ${repository}@${version}`, ); throw new ExternalHostError(err); } } /** * Some providers do not publish a `linux/amd64` build, so discover an * available platform via `/versions` and fetch its download endpoint. */ private async _getProviderPackagesForAvailablePlatform( baseUrl: string, version: string, ): Promise { const { body: versionsResponse } = await this.http.getJson( `${baseUrl}/versions`, TerraformRegistryVersions, ); const platform = versionsResponse.versions?.find( (entry) => entry.version === version, )?.platforms?.[0]; if (!platform) { return null; } const { body: hashes } = await this.http.getJson( `${baseUrl}/${version}/download/${platform.os}/${platform.arch}`, OpenTofuProviderPackagesResponse, ); return hashes; } getProviderPackages( repository: string, version: string, ): Promise { return withCache( { namespace: `datasource-${TerraformProviderDatasource.id}`, key: `getProviderPackages:${repository}/${version}`, }, () => this._getProviderPackages(repository, version), ); } private async _getZipHashes( zipHashUrl: string, ): Promise { // The hashes are formatted as the result of sha256sum in plain text, each line: \t let rawHashData: string; try { rawHashData = (await this.http.getText(zipHashUrl)).body; } catch (err) { /* v8 ignore next -- hard to test */ if (err instanceof ExternalHostError) { throw err; } logger.debug( { err, zipHashUrl }, `Failed to retrieve zip hashes from ${zipHashUrl}`, ); return undefined; } return rawHashData .trimEnd() .split('\n') .map((line) => line.split(regEx(/\s/))[0]); } getZipHashes(zipHashUrl: string): Promise { return withCache( { namespace: `datasource-${TerraformProviderDatasource.id}`, key: `getZipHashes:${zipHashUrl}`, }, () => this._getZipHashes(zipHashUrl), ); } private async _getReleaseBackendIndex( backendLookUpName: string, version: string, ): Promise { return ( await this.http.getJson( `${TerraformProviderDatasource.hashicorpReleaseUrl}/${backendLookUpName}/${version}/index.json`, VersionDetailResponse, ) ).body; } getReleaseBackendIndex( backendLookUpName: string, version: string, ): Promise { return withCache( { namespace: `datasource-${TerraformProviderDatasource.id}`, key: `getReleaseBackendIndex:${backendLookUpName}/${version}`, }, () => this._getReleaseBackendIndex(backendLookUpName, version), ); } }