renovate/lib/workers/global/index.ts

import {
  ATTR_VCS_OWNER_NAME,
  ATTR_VCS_PROVIDER_NAME,
  ATTR_VCS_REPOSITORY_NAME,
} from '@opentelemetry/semantic-conventions/incubating';
import {
  isNonEmptyObject,
  isNonEmptyString,
  isNonEmptyStringAndNotWhitespace,
  isString,
} from '@sindresorhus/is';
import { ERROR } from 'bunyan';
import fs from 'fs-extra';
import semver from 'semver';
import upath from 'upath';
import { GlobalConfig } from '../../config/global.ts';
import * as configParser from '../../config/index.ts';
import { resolveConfigPresets } from '../../config/presets/index.ts';
import { validateConfigSecretsAndVariables } from '../../config/secrets.ts';
import type {
  AllConfig,
  InternalGlobalConfigOptions,
  RenovateConfig,
  RenovateRepository,
  RepoGlobalConfig,
} from '../../config/types.ts';
import { CONFIG_PRESETS_INVALID } from '../../constants/error-messages.ts';
import { pkg } from '../../expose.ts';
import { instrument } from '../../instrumentation/index.ts';
import {
  exportStats,
  finalizeReport,
} from '../../instrumentation/reporting.ts';
import { getProblems, logLevel, logger, setMeta } from '../../logger/index.ts';
import { setGlobalLogLevelRemaps } from '../../logger/remap.ts';
import type { HostRule } from '../../types/index.ts';
import { getEnv } from '../../util/env.ts';
import * as hostRules from '../../util/host-rules.ts';
import * as queue from '../../util/http/queue.ts';
import * as throttle from '../../util/http/throttle.ts';
import { regexEngineStatus } from '../../util/regex.ts';
import { addSecretForSanitizing } from '../../util/sanitize.ts';
import { coerceString } from '../../util/string.ts';
import * as repositoryWorker from '../repository/index.ts';
import type { RepositoryWorkerConfig } from '../repository/init/types.ts';
import { autodiscoverRepositories } from './autodiscover.ts';
import { parseConfigs } from './config/parse/index.ts';
import { globalFinalize, globalInitialize } from './initialize.ts';
import { isLimitReached } from './limits.ts';

function applyGlobalOption<
  K extends keyof RepoGlobalConfig | keyof InternalGlobalConfigOptions,
>(
  target: RepoGlobalConfig & InternalGlobalConfigOptions,
  source: RepoGlobalConfig & InternalGlobalConfigOptions,
  key: K,
): void {
  target[key] = source[key];
  delete source[key];
}

export async function getRepositoryConfig(
  globalConfig: RenovateConfig,
  repository: RenovateRepository,
): Promise<RepositoryWorkerConfig> {
  const repoIsString = isString(repository);
  const repoName = repoIsString ? repository : repository.repository;

  const repoConfig: RepositoryWorkerConfig = {
    ...globalConfig,
    repository: repoName,
  };

  if (!repoIsString) {
    const { repository: _repository, ...repositoryEntryConfig } = repository;

    // Promote GlobalConfig.OPTIONS keys into repoConfig directly so that
    // GlobalConfig.set(repoConfig) in the repository worker picks them up
    // with per-repo overrides before onboarding checks run.
    for (const option of GlobalConfig.OPTIONS) {
      if (option in repositoryEntryConfig) {
        applyGlobalOption(repoConfig, repositoryEntryConfig, option);
      }
    }

    if (isNonEmptyObject(repositoryEntryConfig)) {
      // mergeRenovateConfig later resolves this repositories[] object-entry
      // config in the correct order
      repoConfig.repositoryEntryConfig = repositoryEntryConfig;
    }
  }

  const repoParts = repoName.split('/');
  repoParts.pop();
  repoConfig.parentOrg = repoParts.join('/');
  repoConfig.topLevelOrg = repoParts.shift();
  const platform = GlobalConfig.get('platform');
  repoConfig.localDir =
    platform === 'local'
      ? process.cwd()
      : // TODO: types (#22198)
        upath.join(repoConfig.baseDir!, `./repos/${platform}/${repoName}`);
  await fs.ensureDir(repoConfig.localDir);
  delete repoConfig.baseDir;
  return configParser.filterConfig(repoConfig, 'repository');
}

function getGlobalConfig(): Promise<RenovateConfig> {
  return parseConfigs(getEnv(), process.argv);
}

function haveReachedLimits(): boolean {
  if (isLimitReached('Commits')) {
    logger.info('Max commits created for this run.');
    return true;
  }
  return false;
}

/* istanbul ignore next */
function checkEnv(): void {
  const range = pkg.engines.node;
  if (process.release?.name !== 'node' || !process.versions?.node) {
    logger.warn(
      { release: process.release, versions: process.versions },
      'Unknown node environment detected.',
    );
  } else if (!semver.satisfies(process.versions?.node, range)) {
    logger.error(
      { versions: process.versions, range },
      'Unsupported node environment detected. Please update your node version.',
    );
  }
}

export async function validatePresets(config: AllConfig): Promise<void> {
  logger.debug('validatePresets()');
  try {
    await resolveConfigPresets(config);
  } catch (err) /* istanbul ignore next */ {
    logger.error({ err }, CONFIG_PRESETS_INVALID);
    throw new Error(CONFIG_PRESETS_INVALID);
  }
}

export async function start(): Promise<number> {
  logger.info({ renovateVersion: pkg.version }, 'Renovate started');
  // istanbul ignore next
  if (regexEngineStatus.type === 'available') {
    logger.debug('Using RE2 regex engine');
  } else if (regexEngineStatus.type === 'unavailable') {
    logger.warn(
      { err: regexEngineStatus.err },
      'RE2 not usable, falling back to RegExp',
    );
  } else if (regexEngineStatus.type === 'ignored') {
    logger.debug('RE2 regex engine is ignored via RENOVATE_X_IGNORE_RE2');
  }

  let config: AllConfig;
  let repoExitCode = 0;
  const env = getEnv();
  try {
    if (isNonEmptyStringAndNotWhitespace(env.AWS_SECRET_ACCESS_KEY)) {
      addSecretForSanitizing(env.AWS_SECRET_ACCESS_KEY, 'global');
    }
    if (isNonEmptyStringAndNotWhitespace(env.AWS_SESSION_TOKEN)) {
      addSecretForSanitizing(env.AWS_SESSION_TOKEN, 'global');
    }
    if (isNonEmptyStringAndNotWhitespace(env.COREPACK_NPM_TOKEN)) {
      addSecretForSanitizing(env.COREPACK_NPM_TOKEN, 'global');
    }
    if (isNonEmptyStringAndNotWhitespace(env.COREPACK_NPM_PASSWORD)) {
      addSecretForSanitizing(env.COREPACK_NPM_PASSWORD, 'global');
    }

    await instrument('config', async () => {
      // read global config from file, env and cli args
      config = await getGlobalConfig();

      // Set allowedHeaders and userAgent in case hostRules headers are configured in file config
      GlobalConfig.set({
        allowedHeaders: config.allowedHeaders,
        userAgent: config.userAgent,
      });
      // initialize all submodules
      config = await globalInitialize(config);

      // Set platform, endpoint, allowedHeaders and userAgent in case local presets are used
      GlobalConfig.set({
        allowedHeaders: config.allowedHeaders,
        platform: config.platform,
        endpoint: config.endpoint,
        userAgent: config.userAgent,
      });

      await validatePresets(config);

      setGlobalLogLevelRemaps(config.logLevelRemap);

      checkEnv();

      // validate secrets and variables. Will throw and abort if invalid
      validateConfigSecretsAndVariables(config);
    });

    // autodiscover repositories (needs to come after platform initialization)
    config = await instrument('discover', () =>
      autodiscoverRepositories(config),
    );

    if (isNonEmptyString(config.writeDiscoveredRepos)) {
      const content = JSON.stringify(config.repositories);
      await fs.writeFile(config.writeDiscoveredRepos, content);
      logger.info(
        `Written discovered repositories to ${config.writeDiscoveredRepos}`,
      );
      return 0;
    }

    // the self-hosted admin's own headers get no exemption from `allowedHeaders` either, as `applyHostRule` filters the rule it matches by header name alone whoever set it - so we drop them here too, with a WARN, rather than leave them to be silently discarded at request time
    // filtered once, outside the loop, rather than for every repository it processes
    let filteredGlobalHostRules: HostRule[] | undefined;

    // Iterate through repositories sequentially
    for (const repository of config.repositories!) {
      if (haveReachedLimits()) {
        break;
      }

      const { owner, repo } = repositoryToOwnerAndRepo(
        isString(repository) ? repository : repository.repository,
      );

      await instrument(
        'repository',
        async () => {
          const repoConfig = await getRepositoryConfig(config, repository);
          if (repoConfig.hostRules) {
            logger.debug('Reinitializing hostRules for repo');
            hostRules.clear();
            // `GlobalConfig` still reflects the previous repository at this point, so filter with this repository's own `allowedHeaders`: usually the global allowlist (filtered once, above), re-filtered only for a `repositories[]` entry carrying an override of its own
            const rules =
              // reuse the memo only for the exact global inputs it was computed from - today `repoConfig.hostRules` is always the global array, but nothing should break if that ever changes
              repoConfig.hostRules === config.hostRules &&
              repoConfig.allowedHeaders === config.allowedHeaders
                ? (filteredGlobalHostRules ??= hostRules.filterAllowedHeaders(
                    repoConfig.hostRules,
                    config.allowedHeaders,
                    // `globalInitialize` already registered these very rules against this very allowlist, and warned about whatever it dropped
                    false,
                  ))
                : hostRules.filterAllowedHeaders(
                    repoConfig.hostRules,
                    repoConfig.allowedHeaders,
                  );
            for (const rule of rules) {
              // already filtered: pass the same allowlist through so `add()` does not re-filter against a stale `GlobalConfig`
              // the self-hosted admin's own rules: `trusted`, so that their `headers` are applied over any a repository or preset sets for the same host
              hostRules.add(rule, {
                // we haven't yet set `GlobalConfig`, so we need to explicitly pass these in
                allowedHeaders: repoConfig.allowedHeaders,
                trusted: true,
              });
            }
            repoConfig.hostRules = [];
          }

          // host rules can change concurrency
          queue.clear();
          throttle.clear();

          const repoResult =
            await repositoryWorker.renovateRepository(repoConfig);
          if (config.exitCodeForErrors && !repoExitCode) {
            repoExitCode = repoResult?.exitCode ?? 0;
          }
          setMeta({});
        },
        {
          attributes: {
            [ATTR_VCS_PROVIDER_NAME]: config.platform,
            [ATTR_VCS_OWNER_NAME]: owner,
            [ATTR_VCS_REPOSITORY_NAME]: repo,
            /** @deprecated TODO remove */
            repository: isString(repository)
              ? repository
              : repository.repository,
          },
        },
      );
    }

    finalizeReport();
    await exportStats(config);
  } catch (err) /* istanbul ignore next */ {
    if (err.message.startsWith('Init: ')) {
      logger.fatal(
        { errorMessage: err.message.substring(6) },
        'Initialization error',
      );
    } else {
      logger.fatal({ err }, 'Unknown error');
    }
    if (!config!) {
      // return early if we can't parse config options
      logger.debug(`Missing config`);
      return 2;
    }
  } finally {
    await globalFinalize(config!);
    if (logLevel() === 'info') {
      logger.info(
        `Renovate was run at log level "${logLevel()}". Set LOG_LEVEL=debug in environment variables to see extended debug logs.`,
      );
    }
  }
  if (repoExitCode) {
    logger.info(
      { exitCode: repoExitCode },
      'Renovate is exiting with an error-specific code due to a repository error',
    );
    return repoExitCode;
  }
  const loggerErrors = getProblems().filter((p) => p.level >= ERROR);
  if (loggerErrors.length) {
    logger.info(
      { loggerErrors },
      'Renovate is exiting with a non-zero code due to the following logged errors',
    );
    return 1;
  }
  return 0;
}

function repositoryToOwnerAndRepo(fullName: string): {
  owner: string;
  repo: string;
} {
  const parts = fullName.split('/');
  const repo = coerceString(parts.pop());
  const owner = parts.join('/');
  return { owner, repo };
}