renovate/lib/modules/datasource/maven/util.ts

import { Readable } from 'node:stream';
import { GetObjectCommand } from '@aws-sdk/client-s3';
import { XmlDocument } from 'xmldoc';
import {
  HOST_BLOCKED,
  HOST_DISABLED,
} from '../../../constants/error-messages.ts';
import { logger } from '../../../logger/index.ts';
import { ExternalHostError } from '../../../types/errors/external-host-error.ts';
import * as packageCache from '../../../util/cache/package/index.ts';
import { PackageHttpCacheProvider } from '../../../util/http/cache/package-http-cache-provider.ts';
import { type Http, HttpError } from '../../../util/http/index.ts';
import type { HttpOptions, HttpResponse } from '../../../util/http/types.ts';
import { refusedHostMessage } from '../../../util/http/util.ts';
import { regEx } from '../../../util/regex.ts';
import { Result } from '../../../util/result.ts';
import { getS3Client, parseS3Url } from '../../../util/s3.ts';
import { streamToString } from '../../../util/streams.ts';
import { asTimestamp } from '../../../util/timestamp.ts';
import { ensureTrailingSlash, isHttpUrl } from '../../../util/url.ts';
import { getGoogleAuthToken } from '../util.ts';
import { isMavenCentral } from './common.ts';
import { CachedMavenXml } from './schema.ts';
import type {
  DependencyInfo,
  MavenDependency,
  MavenFetchResult,
  MavenFetchSuccess,
} from './types.ts';

function isTemporaryError(err: HttpError): boolean {
  if (err.code === 'ECONNRESET') {
    return true;
  }

  if (err.response) {
    const status = err.response.statusCode;
    return status === 429 || (status >= 500 && status < 600);
  }

  return false;
}

function isHostError(err: HttpError): boolean {
  return err.code === 'ETIMEDOUT';
}

function isNotFoundError(err: HttpError): boolean {
  return err.code === 'ENOTFOUND' || err.response?.statusCode === 404;
}

function isPermissionsIssue(err: HttpError): boolean {
  const status = err.response?.statusCode;
  return status === 401 || status === 403;
}

function isConnectionError(err: HttpError): boolean {
  return (
    err.code === 'EAI_AGAIN' ||
    err.code === 'ERR_TLS_CERT_ALTNAME_INVALID' ||
    err.code === 'ECONNREFUSED'
  );
}

function isUnsupportedHostError(err: HttpError): boolean {
  return err.name === 'UnsupportedProtocolError';
}

const cacheProvider = new PackageHttpCacheProvider({
  namespace: 'datasource-maven:cache-provider',
  softTtlMinutes: 15,
  checkAuthorizationHeader: true,
  checkCacheControlHeader: false, // Maven doesn't respond with `cache-control` headers
  writeSchema: CachedMavenXml,
});

// Release POMs and timestamped snapshot POMs are immutable once published, so we can cache them much longer than mutable metadata files.
const pomCacheProvider = new PackageHttpCacheProvider({
  namespace: 'datasource-maven:pom-cache-provider',
  softTtlMinutes: 60 * 24 * 28, // 28 days before we'll give it another check, just in case it's updated
  checkAuthorizationHeader: true,
  checkCacheControlHeader: false,
  writeSchema: CachedMavenXml,
});

function selectCacheProvider(url: string): PackageHttpCacheProvider {
  // Non-timestamped -SNAPSHOT.pom files are mutable; everything else ending in .pom (release POMs and timestamped snapshot POMs) is immutable.
  if (url.endsWith('.pom') && !url.endsWith('-SNAPSHOT.pom')) {
    return pomCacheProvider;
  }
  return cacheProvider;
}

const METADATA_NOT_FOUND_NAMESPACE =
  'datasource-maven:metadata-not-found' as const;
const METADATA_NOT_FOUND_TTL_MINUTES = 60 * 12; // 12 hours

/** introduces jitter to make sure that a given repo's cache expiring doesn't lead to many requests leading to high traffic and/or rate limits */
function getMetadataNotFoundTtl(): number {
  const jitter = Math.floor(Math.random() * 60 * 2); // 0–120 minutes
  return METADATA_NOT_FOUND_TTL_MINUTES + jitter;
}

function isMetadataUrl(url: string): boolean {
  return url.endsWith('/maven-metadata.xml');
}

export async function downloadHttpProtocol(
  http: Http,
  pkgUrl: URL | string,
  opts: HttpOptions = {},
): Promise<MavenFetchResult> {
  const url = pkgUrl.toString();

  if (isMetadataUrl(url)) {
    const cacheHasNotFoundResponse = await packageCache.get<true>(
      METADATA_NOT_FOUND_NAMESPACE,
      url,
    );
    if (cacheHasNotFoundResponse) {
      logger.trace(
        { url },
        'Returning cached 404 response for the metadata-metadata URL request',
      );
      return Result.err({ type: 'not-found' });
    }
  }

  const fetchResult = await Result.wrap<HttpResponse>(
    http.getText(url, { ...opts, cacheProvider: selectCacheProvider(url) }),
  )
    .transform((res): MavenFetchSuccess => {
      const result: MavenFetchSuccess = { data: res.body };

      if (!res.authorization) {
        result.isCacheable = true;
      }

      const lastModified = asTimestamp(res?.headers?.['last-modified']);
      if (lastModified) {
        result.lastModified = lastModified;
      }

      return result;
    })
    .catch(async (err): Promise<MavenFetchResult> => {
      /* v8 ignore next: never happens, needs for type narrowing */
      if (!(err instanceof HttpError)) {
        return Result.err({ type: 'unknown', err });
      }

      const failedUrl = url;
      if ([HOST_BLOCKED, HOST_DISABLED].includes(err.message)) {
        logger.trace({ failedUrl }, refusedHostMessage(err));
        return Result.err({ type: 'host-disabled' });
      }

      if (isNotFoundError(err)) {
        logger.trace({ failedUrl }, `Url not found`);
        if (isMetadataUrl(failedUrl)) {
          await packageCache.set(
            METADATA_NOT_FOUND_NAMESPACE,
            failedUrl,
            true,
            getMetadataNotFoundTtl(),
          );
        }
        return Result.err({ type: 'not-found' });
      }

      if (isHostError(err)) {
        logger.debug(`Cannot connect to host ${failedUrl}`);
        return Result.err({ type: 'host-error' });
      }

      if (isPermissionsIssue(err)) {
        logger.debug(
          `Dependency lookup unauthorized. Please add authentication with a hostRule for ${failedUrl}`,
        );
        return Result.err({ type: 'permission-issue' });
      }

      if (isTemporaryError(err)) {
        logger.debug({ failedUrl, err }, 'Temporary error');

        if (isMavenCentral(url)) {
          const statusCode = err?.response?.statusCode;
          if (statusCode === 429) {
            if (packageCache.getCacheType() === 'redis') {
              logger.once.warn(
                { failedUrl },
                'Maven Central rate limiting detected despite Redis caching.',
              );
            } else {
              logger.once.warn(
                { failedUrl },
                'Maven Central rate limiting detected. Persistent caching required.',
              );
            }
          }
          return Result.err({ type: 'maven-central-temporary-error', err });
        }
        return Result.err({ type: 'temporary-error' });
      }

      if (isConnectionError(err)) {
        logger.debug(`Connection refused to maven registry ${failedUrl}`);
        return Result.err({ type: 'connection-error' });
      }

      if (isUnsupportedHostError(err)) {
        logger.debug(`Unsupported host ${failedUrl}`);
        return Result.err({ type: 'unsupported-host' });
      }

      logger.info({ failedUrl, err }, 'Unknown HTTP download error');
      return Result.err({ type: 'unknown', err });
    });

  const { err } = fetchResult.unwrap();
  if (err?.type === 'maven-central-temporary-error') {
    throw new ExternalHostError(err.err);
  }

  return fetchResult;
}

export async function downloadHttpContent(
  http: Http,
  pkgUrl: URL | string,
  opts: HttpOptions = {},
): Promise<string | null> {
  const fetchResult = await downloadHttpProtocol(http, pkgUrl, opts);
  return fetchResult.transform(({ data }) => data).unwrapOrNull();
}

function isS3NotFound(err: Error): boolean {
  return err.message === 'NotFound' || err.message === 'NoSuchKey';
}

export async function downloadS3Protocol(
  pkgUrl: URL,
): Promise<MavenFetchResult> {
  logger.trace({ url: pkgUrl.toString() }, `Attempting to load S3 dependency`);

  const s3Url = parseS3Url(pkgUrl);
  if (!s3Url) {
    return Result.err({ type: 'invalid-url' });
  }

  return await Result.wrap(() => {
    const command = new GetObjectCommand(s3Url);
    const client = getS3Client();
    return client.send(command);
  })
    .transform(
      async ({
        Body,
        LastModified,
        DeleteMarker,
      }): Promise<MavenFetchResult> => {
        if (DeleteMarker) {
          logger.trace(
            { failedUrl: pkgUrl.toString() },
            'Maven S3 lookup error: DeleteMarker encountered',
          );
          return Result.err({ type: 'not-found' });
        }

        if (!(Body instanceof Readable)) {
          logger.debug(
            { failedUrl: pkgUrl.toString() },
            'Maven S3 lookup error: unsupported Body type',
          );
          return Result.err({ type: 'unsupported-format' });
        }

        const data = await streamToString(Body);
        const result: MavenFetchSuccess = { data };

        const lastModified = asTimestamp(LastModified);
        if (lastModified) {
          result.lastModified = lastModified;
        }

        return Result.ok(result);
      },
    )
    .catch((err): MavenFetchResult => {
      if (!(err instanceof Error)) {
        return Result.err(err);
      }

      const failedUrl = pkgUrl.toString();

      if (err.name === 'CredentialsProviderError') {
        logger.debug(
          { failedUrl },
          'Maven S3 lookup error: credentials provider error, check "AWS_ACCESS_KEY_ID" and "AWS_SECRET_ACCESS_KEY" variables',
        );
        return Result.err({ type: 'credentials-error' });
      }

      if (err.message === 'Region is missing') {
        logger.debug(
          { failedUrl },
          'Maven S3 lookup error: missing region, check "AWS_REGION" variable',
        );
        return Result.err({ type: 'missing-aws-region' });
      }

      if (isS3NotFound(err)) {
        logger.trace({ failedUrl }, 'Maven S3 lookup error: object not found');
        return Result.err({ type: 'not-found' });
      }

      logger.debug({ failedUrl, err }, 'Maven S3 lookup error: unknown error');
      return Result.err({ type: 'unknown', err });
    });
}

export async function downloadArtifactRegistryProtocol(
  http: Http,
  pkgUrl: URL,
): Promise<MavenFetchResult> {
  const opts: HttpOptions = {};
  const host = pkgUrl.host;
  const path = pkgUrl.pathname;

  logger.trace({ host, path }, `Using google auth for Maven repository`);
  const auth = await getGoogleAuthToken();
  if (auth) {
    opts.headers = { authorization: `Basic ${auth}` };
  } else {
    logger.once.debug(
      { host, path },
      'Could not get Google access token, using no auth',
    );
  }

  const url = pkgUrl.toString().replace('artifactregistry:', 'https:');

  return downloadHttpProtocol(http, url, opts);
}

function containsPlaceholder(str: string): boolean {
  return regEx(/\${.*?}/g).test(str);
}

function removeKnownPlaceholders(str: string): string {
  return str.replace(regEx(/\/tree\/\${[^}]+}/), '');
}

export function getMavenUrl(
  dependency: MavenDependency,
  repoUrl: string,
  path: string,
): URL {
  return new URL(
    `${dependency.dependencyUrl}/${path}`,
    ensureTrailingSlash(repoUrl),
  );
}

export async function downloadMaven(
  http: Http,
  url: URL,
): Promise<MavenFetchResult> {
  const protocol = url.protocol;

  let result: MavenFetchResult = Result.err({ type: 'unsupported-protocol' });

  if (isHttpUrl(url)) {
    result = await downloadHttpProtocol(http, url);
  }

  if (protocol === 'artifactregistry:') {
    result = await downloadArtifactRegistryProtocol(http, url);
  }

  if (protocol === 's3:') {
    result = await downloadS3Protocol(url);
  }

  return result.onError((err) => {
    if (err.type === 'unsupported-protocol') {
      logger.debug(
        { url: url.toString() },
        `Maven lookup error: unsupported protocol (${protocol})`,
      );
    }
  });
}

export async function downloadMavenXml(
  http: Http,
  url: URL,
): Promise<MavenFetchResult<XmlDocument>> {
  const rawResult = await downloadMaven(http, url);
  return rawResult.transform((result): MavenFetchResult<XmlDocument> => {
    try {
      return Result.ok({
        ...result,
        data: new XmlDocument(result.data),
      });
    } catch (err) {
      return Result.err({ type: 'xml-parse-error', err });
    }
  });
}

export function getDependencyParts(packageName: string): MavenDependency {
  const [group, name] = packageName.split(':');
  const dependencyUrl = `${group.replace(regEx(/\./g), '/')}/${name}`;
  return {
    display: packageName,
    group,
    name,
    dependencyUrl,
  };
}

function extractSnapshotVersion(metadata: XmlDocument): string | null {
  // Parse the maven-metadata.xml for the snapshot version and determine
  // the fixed version of the latest deployed snapshot.
  // The metadata descriptor can be found at
  // https://maven.apache.org/ref/3.3.3/maven-repository-metadata/repository-metadata.html
  //
  // Basically, we need to replace -SNAPSHOT with the artifact timestanp & build number,
  // so for example 1.0.0-SNAPSHOT will become 1.0.0-<timestamp>-<buildNumber>
  const version = metadata
    .descendantWithPath('version')
    ?.val?.replace('-SNAPSHOT', '');

  const snapshot = metadata.descendantWithPath('versioning.snapshot');
  const timestamp = snapshot?.childNamed('timestamp')?.val;
  const build = snapshot?.childNamed('buildNumber')?.val;

  // If we weren't able to parse out the required 3 version elements,
  // return null because we can't determine the fixed version of the latest snapshot.
  if (!version || !timestamp || !build) {
    return null;
  }
  return `${version}-${timestamp}-${build}`;
}

async function getSnapshotFullVersion(
  http: Http,
  version: string,
  dependency: MavenDependency,
  repoUrl: string,
): Promise<string | null> {
  // To determine what actual files are available for the snapshot, first we have to fetch and parse
  // the metadata located at http://<repo>/<group>/<artifact>/<version-SNAPSHOT>/maven-metadata.xml
  const metadataUrl = getMavenUrl(
    dependency,
    repoUrl,
    `${version}/maven-metadata.xml`,
  );

  const metadataXmlResult = await downloadMavenXml(http, metadataUrl);

  return metadataXmlResult
    .transform(({ data }) =>
      Result.wrapNullable(extractSnapshotVersion(data), {
        type: 'snapshot-extract-error',
      }),
    )
    .unwrapOrNull();
}

function isSnapshotVersion(version: string): boolean {
  if (version.endsWith('-SNAPSHOT')) {
    return true;
  }
  return false;
}

export async function createUrlForDependencyPom(
  http: Http,
  version: string,
  dependency: MavenDependency,
  repoUrl: string,
): Promise<string> {
  if (isSnapshotVersion(version)) {
    // By default, Maven snapshots are deployed to the repository with fixed file names.
    // Resolve the full, actual pom file name for the version.
    const fullVersion = await getSnapshotFullVersion(
      http,
      version,
      dependency,
      repoUrl,
    );

    // If we were able to resolve the version, use that, otherwise fall back to using -SNAPSHOT
    if (fullVersion !== null) {
      // TODO: types (#22198)
      return `${version}/${dependency.name}-${fullVersion}.pom`;
    }
  }

  // TODO: types (#22198)
  return `${version}/${dependency.name}-${version}.pom`;
}

export async function getDependencyInfo(
  http: Http,
  dependency: MavenDependency,
  repoUrl: string,
  version: string,
  recursionLimit = 5,
): Promise<DependencyInfo> {
  const path = await createUrlForDependencyPom(
    http,
    version,
    dependency,
    repoUrl,
  );

  const pomUrl = getMavenUrl(dependency, repoUrl, path);
  const pomXmlResult = await downloadMavenXml(http, pomUrl);
  const dependencyInfoResult = await pomXmlResult.transform(
    async ({ data: pomContent }) => {
      const result: DependencyInfo = {};

      const homepage = pomContent.valueWithPath('url');
      if (homepage && !containsPlaceholder(homepage)) {
        result.homepage = homepage;
      }

      const sourceUrl = pomContent.valueWithPath('scm.url');
      if (
        sourceUrl &&
        !containsPlaceholder(removeKnownPlaceholders(sourceUrl))
      ) {
        result.sourceUrl = sourceUrl
          .replace(regEx(/^scm:/), '')
          .replace(regEx(/^git:/), '')
          .replace(regEx(/^git@github.com:/), 'https://github.com/')
          .replace(regEx(/^git@github.com\//), 'https://github.com/');

        if (result.sourceUrl.startsWith('//')) {
          // most likely the result of us stripping scm:, git: etc
          // going with prepending https: here which should result in potential information retrival
          result.sourceUrl = `https:${result.sourceUrl}`;
        }
      }

      const relocation = pomContent.descendantWithPath(
        'distributionManagement.relocation',
      );
      if (relocation) {
        const relocationGroup =
          relocation.valueWithPath('groupId') ?? dependency.group;
        const relocationName =
          relocation.valueWithPath('artifactId') ?? dependency.name;
        result.replacementName = `${relocationGroup}:${relocationName}`;
        const relocationVersion = relocation.valueWithPath('version');
        result.replacementVersion = relocationVersion ?? version;
        const relocationMessage = relocation.valueWithPath('message');
        if (relocationMessage) {
          result.deprecationMessage = relocationMessage;
        }
      }

      const groupId = pomContent.valueWithPath('groupId');
      if (groupId) {
        result.packageScope = groupId;
      }

      const parent = pomContent.childNamed('parent');
      if (
        recursionLimit > 0 &&
        parent &&
        (!result.sourceUrl || !result.homepage)
      ) {
        // if we found a parent and are missing some information
        // trying to get the scm/homepage information from it
        const [parentGroupId, parentArtifactId, parentVersion] = [
          'groupId',
          'artifactId',
          'version',
        ].map((k) => parent.valueWithPath(k)?.replace(regEx(/\s+/g), ''));
        if (parentGroupId && parentArtifactId && parentVersion) {
          const parentDisplayId = `${parentGroupId}:${parentArtifactId}`;
          const parentDependency = getDependencyParts(parentDisplayId);
          const parentInformation = await getDependencyInfo(
            http,
            parentDependency,
            repoUrl,
            parentVersion,
            recursionLimit - 1,
          );
          if (!result.sourceUrl && parentInformation.sourceUrl) {
            result.sourceUrl = parentInformation.sourceUrl;
          }
          if (!result.homepage && parentInformation.homepage) {
            result.homepage = parentInformation.homepage;
          }
        }
      }

      return result;
    },
  );

  return dependencyInfoResult.unwrapOr({});
}