renovate/lib/modules/datasource/crate/index.spec.ts

import { setTimeout } from 'node:timers/promises';
import { isUndefined } from '@sindresorhus/is';
import fs from 'fs-extra';
import type { SimpleGit } from 'simple-git';
import type { DirectoryResult } from 'tmp-promise';
import { dir } from 'tmp-promise';
import upath from 'upath';
import type { MockedFunction } from 'vitest';
import { Fixtures } from '~test/fixtures.ts';
import * as httpMock from '~test/http-mock.ts';
import { partial } from '~test/util.ts';
import { GlobalConfig } from '../../../config/global.ts';
import type {
  InternalGlobalConfigOptions,
  RepoGlobalConfig,
} from '../../../config/types.ts';
import { EXTERNAL_HOST_ERROR } from '../../../constants/error-messages.ts';
import * as memCache from '../../../util/cache/memory/index.ts';
import * as git from '../../../util/git/index.ts';
import type { Timestamp } from '../../../util/timestamp.ts';
import { getPkgReleases } from '../index.ts';
import { CrateDatasource } from './index.ts';
import type { RegistryConfig } from './schema.ts';

vi.unmock('../../../util/mutex.ts');
const createSimpleGit = vi.mocked(git.createSimpleGit);

const API_BASE_URL = 'https://crates.io';
const DL_BASE_URL = 'https://static.crates.io/crates';
const CRATES_IO_REGISTRY_URL = 'sparse+https://index.crates.io/';
const CRATES_IO_REGISTRY_URL_PARSED = 'https://index.crates.io/';

const datasource = CrateDatasource.id;

const cratesIoConfig: RegistryConfig = {
  dl: DL_BASE_URL,
  api: API_BASE_URL,
};

function mockCratesIoConfig(): void {
  httpMock
    .scope(CRATES_IO_REGISTRY_URL_PARSED)
    .get('/config.json')
    .reply(200, cratesIoConfig);
}

function setupGitMocks(delayMs?: number): {
  mockClone: MockedFunction<SimpleGit['clone']>;
} {
  const mockClone = vi
    .fn()
    .mockName('clone')
    .mockImplementation(
      async (_registryUrl: string, clonePath: string, _opts) => {
        if (delayMs && delayMs > 0) {
          await setTimeout(delayMs);
        }

        const path = `${clonePath}/my/pk/mypkg`;
        await fs.mkdir(upath.dirname(path), { recursive: true });
        await fs.writeFile(path, Fixtures.get('mypkg'), { encoding: 'utf8' });
        await fs.writeFile(
          `${clonePath}/config.json`,
          JSON.stringify({ dl: 'https://example.com/crates' }),
          { encoding: 'utf8' },
        );
      },
    );

  const gitMock = partial<SimpleGit>({ clone: mockClone });
  createSimpleGit.mockReturnValue(gitMock);
  return { mockClone };
}

function setupErrorGitMock(): {
  mockClone: MockedFunction<SimpleGit['clone']>;
} {
  const mockClone = vi
    .fn()
    .mockName('clone')
    .mockImplementation((_registryUrl: string, _clonePath: string, _opts) =>
      Promise.reject(new Error('mocked error')),
    );

  const gitMock = partial<SimpleGit>({
    clone: mockClone,
  });
  createSimpleGit.mockReturnValue(gitMock);

  return { mockClone };
}

function mockCratesApiCallFor(crateName: string, response?: httpMock.Body) {
  httpMock
    .scope(API_BASE_URL)
    .get(`/api/v1/crates/${crateName}?include=`)
    .reply(response ? 200 : 404, response);
}

describe('modules/datasource/crate/index', () => {
  describe('getIndexSuffix', () => {
    it('returns correct suffixes', () => {
      expect(CrateDatasource.getIndexSuffix('a')).toStrictEqual(['1', 'a']);
      expect(CrateDatasource.getIndexSuffix('1')).toStrictEqual(['1', '1']);
      expect(CrateDatasource.getIndexSuffix('1234567')).toStrictEqual([
        '12',
        '34',
        '1234567',
      ]);
      expect(CrateDatasource.getIndexSuffix('ab')).toStrictEqual(['2', 'ab']);
      expect(CrateDatasource.getIndexSuffix('abc')).toStrictEqual([
        '3',
        'a',
        'abc',
      ]);
      expect(CrateDatasource.getIndexSuffix('abcd')).toStrictEqual([
        'ab',
        'cd',
        'abcd',
      ]);
      expect(CrateDatasource.getIndexSuffix('abcde')).toStrictEqual([
        'ab',
        'cd',
        'abcde',
      ]);
    });
  });

  describe('getReleases', () => {
    let tmpDir: DirectoryResult | null;
    let adminConfig: RepoGlobalConfig & InternalGlobalConfigOptions;

    beforeEach(async () => {
      tmpDir = await dir({ unsafeCleanup: true });

      adminConfig = {
        localDir: upath.join(tmpDir.path, 'local'),
        cacheDir: upath.join(tmpDir.path, 'cache'),
      };
      GlobalConfig.set(adminConfig);

      memCache.init();
    });

    afterEach(async () => {
      await tmpDir?.cleanup();
      tmpDir = null;
      GlobalConfig.reset();
    });

    it('returns null for missing registry url', async () => {
      // FIXME: should not call default registry?
      httpMock
        .scope(CRATES_IO_REGISTRY_URL_PARSED)
        .get('/no/n_/non_existent_crate')
        .reply(404, {});
      await expect(
        getPkgReleases({
          datasource,
          packageName: 'non_existent_crate',
          registryUrls: [],
        }),
      ).resolves.toBeNull();
    });

    it('returns null for invalid registry url', async () => {
      await expect(
        getPkgReleases({
          datasource,
          packageName: 'non_existent_crate',
          registryUrls: ['3'],
        }),
      ).resolves.toBeNull();
    });

    it('returns null for empty result', async () => {
      mockCratesIoConfig();
      mockCratesApiCallFor('non_existent_crate');
      httpMock
        .scope(CRATES_IO_REGISTRY_URL_PARSED)
        .get('/no/n_/non_existent_crate')
        .reply(200, {});
      await expect(
        getPkgReleases({
          datasource,
          packageName: 'non_existent_crate',
          registryUrls: [CRATES_IO_REGISTRY_URL],
        }),
      ).resolves.toBeNull();
    });

    it('returns null for missing fields', async () => {
      mockCratesIoConfig();
      mockCratesApiCallFor('non_existent_crate');
      httpMock
        .scope(CRATES_IO_REGISTRY_URL_PARSED)
        .get('/no/n_/non_existent_crate')
        .reply(200, undefined);
      await expect(
        getPkgReleases({
          datasource,
          packageName: 'non_existent_crate',
          registryUrls: [CRATES_IO_REGISTRY_URL],
        }),
      ).resolves.toBeNull();
    });

    it('returns null for empty list', async () => {
      mockCratesIoConfig();
      mockCratesApiCallFor('non_existent_crate');
      httpMock
        .scope(CRATES_IO_REGISTRY_URL_PARSED)
        .get('/no/n_/non_existent_crate')
        .reply(200, '\n');
      await expect(
        getPkgReleases({
          datasource,
          packageName: 'non_existent_crate',
          registryUrls: [CRATES_IO_REGISTRY_URL],
        }),
      ).resolves.toBeNull();
    });

    it('returns null for 404', async () => {
      httpMock
        .scope(CRATES_IO_REGISTRY_URL_PARSED)
        .get('/so/me/some_crate')
        .reply(404);
      await expect(
        getPkgReleases({
          datasource,
          packageName: 'some_crate',
          registryUrls: [CRATES_IO_REGISTRY_URL],
        }),
      ).resolves.toBeNull();
    });

    it('throws for 5xx', async () => {
      httpMock
        .scope(CRATES_IO_REGISTRY_URL_PARSED)
        .get('/so/me/some_crate')
        .reply(502);
      await expect(
        getPkgReleases({
          datasource,
          packageName: 'some_crate',
          registryUrls: [CRATES_IO_REGISTRY_URL],
        }),
      ).rejects.toThrow(EXTERNAL_HOST_ERROR);
    });

    it('returns null for unknown error', async () => {
      httpMock
        .scope(CRATES_IO_REGISTRY_URL_PARSED)
        .get('/so/me/some_crate')
        .replyWithError('');
      await expect(
        getPkgReleases({
          datasource,
          packageName: 'some_crate',
          registryUrls: [CRATES_IO_REGISTRY_URL],
        }),
      ).resolves.toBeNull();
    });

    it('processes real data: libc', async () => {
      mockCratesIoConfig();
      mockCratesApiCallFor('libc', Fixtures.get('libc.json'));

      httpMock
        .scope(CRATES_IO_REGISTRY_URL_PARSED)
        .get('/li/bc/libc')
        .reply(200, Fixtures.get('libc'));
      const res = await getPkgReleases({
        datasource,
        packageName: 'libc',
        registryUrls: [CRATES_IO_REGISTRY_URL],
      });
      expect(res).toMatchObject({
        sourceUrl: 'https://github.com/rust-lang/libc',
        releases: [
          { version: '0.1.0' },
          { version: '0.1.1' },
          { version: '0.1.2' },
          { version: '0.1.3' },
          { version: '0.1.4' },
          { version: '0.1.5' },
          { version: '0.1.6' },
          { version: '0.1.7' },
          { version: '0.1.8' },
          { version: '0.1.9', isDeprecated: true },
          { version: '0.1.10' },
          { version: '0.1.11', isDeprecated: true },
          { version: '0.1.12' },
          { version: '0.2.0' },
          { version: '0.2.1' },
          { version: '0.2.2' },
          { version: '0.2.3' },
          { version: '0.2.4' },
          { version: '0.2.5' },
          { version: '0.2.6' },
          { version: '0.2.7' },
          { version: '0.2.8' },
          { version: '0.2.9' },
          { version: '0.2.10' },
          { version: '0.2.11' },
          { version: '0.2.12' },
          { version: '0.2.13' },
          { version: '0.2.14' },
          { version: '0.2.15' },
          { version: '0.2.16' },
          { version: '0.2.17' },
          { version: '0.2.18' },
          { version: '0.2.19' },
          { version: '0.2.20' },
          { version: '0.2.21' },
          { version: '0.2.22' },
          { version: '0.2.23' },
          { version: '0.2.24' },
          { version: '0.2.25' },
          { version: '0.2.26' },
          { version: '0.2.27' },
          { version: '0.2.28' },
          { version: '0.2.29' },
          { version: '0.2.30' },
          { version: '0.2.31' },
          { version: '0.2.32' },
          { version: '0.2.33' },
          { version: '0.2.34' },
          { version: '0.2.35' },
          { version: '0.2.36' },
          { version: '0.2.37' },
          { version: '0.2.38' },
          { version: '0.2.39' },
          { version: '0.2.40' },
          { version: '0.2.41' },
          { version: '0.2.42' },
          { version: '0.2.43' },
          { version: '0.2.44' },
          { version: '0.2.45' },
          { version: '0.2.46' },
          { version: '0.2.47' },
          { version: '0.2.48' },
          { version: '0.2.49' },
          { version: '0.2.50', releaseTimestamp: '2019-03-05T11:06:03.000Z' },
          { version: '0.2.51', versionOrig: '0.2.51+metadata' },
        ],
      });
    });

    it('processes real data: amethyst', async () => {
      mockCratesIoConfig();
      mockCratesApiCallFor('amethyst', Fixtures.get('amethyst.json'));

      httpMock
        .scope(CRATES_IO_REGISTRY_URL_PARSED)
        .get('/am/et/amethyst')
        .reply(200, Fixtures.get('amethyst'));
      const res = await getPkgReleases({
        datasource,
        packageName: 'amethyst',
        registryUrls: [CRATES_IO_REGISTRY_URL],
      });
      expect(res).toMatchObject({
        homepage: 'https://amethyst.rs/',
        sourceUrl: 'https://github.com/amethyst/amethyst',
        releases: [
          { version: '0.1.0' },
          { version: '0.1.1' },
          { version: '0.1.3' },
          { version: '0.1.4' },
          { version: '0.2.1' },
          { version: '0.3.0' },
          { version: '0.3.1' },
          { version: '0.4.0' },
          { version: '0.4.1' },
          { version: '0.4.2' },
          { version: '0.4.3' },
          { version: '0.5.0' },
          { version: '0.5.1' },
          { version: '0.6.0' },
          { version: '0.7.0' },
          { version: '0.8.0' },
          { version: '0.9.0' },
          { version: '0.10.0' },
          { version: '0.10.1', isDeprecated: true },
        ],
      });
    });

    it('processes real data: sentry', async () => {
      mockCratesIoConfig();
      mockCratesApiCallFor('sentry', Fixtures.get('sentry.json'));

      httpMock
        .scope(CRATES_IO_REGISTRY_URL_PARSED)
        .get('/se/nt/sentry')
        .reply(200, Fixtures.get('sentry'));
      const res = await getPkgReleases({
        datasource,
        packageName: 'sentry',
        registryUrls: [CRATES_IO_REGISTRY_URL],
      });
      expect(res).toMatchObject({
        homepage: 'https://sentry.io/welcome/',
        sourceUrl: 'https://github.com/getsentry/sentry-rust',
        releases: [
          { version: '0.1.0', releaseTimestamp: '2016-05-20T17:24:17.000Z' },
          { version: '0.1.1', releaseTimestamp: '2016-05-22T17:14:04.000Z' },
          { version: '0.1.2', releaseTimestamp: '2016-05-29T18:29:40.000Z' },
        ],
      });
    });

    it('uses cached registry config for subsequent packages', async () => {
      mockCratesIoConfig();
      mockCratesApiCallFor('libc', Fixtures.get('libc.json'));
      httpMock
        .scope(CRATES_IO_REGISTRY_URL_PARSED)
        .get('/li/bc/libc')
        .reply(200, Fixtures.get('libc'));

      const res1 = await getPkgReleases({
        datasource,
        packageName: 'libc',
        registryUrls: [CRATES_IO_REGISTRY_URL],
      });
      expect(res1).not.toBeNull();

      // Second package: config.json should come from memCache (no extra HTTP mock needed)
      mockCratesApiCallFor('amethyst', Fixtures.get('amethyst.json'));
      httpMock
        .scope(CRATES_IO_REGISTRY_URL_PARSED)
        .get('/am/et/amethyst')
        .reply(200, Fixtures.get('amethyst'));

      const res2 = await getPkgReleases({
        datasource,
        packageName: 'amethyst',
        registryUrls: [CRATES_IO_REGISTRY_URL],
      });
      expect(res2).not.toBeNull();
    });

    it('refuses to clone if allowCustomCrateRegistries is not true', async () => {
      const { mockClone } = setupGitMocks();

      const url = 'https://dl.cloudsmith.io/basic/myorg/myrepo/cargo/index.git';
      const res = await getPkgReleases({
        datasource,
        packageName: 'mypkg',
        registryUrls: [url],
      });
      expect(mockClone).toHaveBeenCalledTimes(0);
      expect(res).toBeNull();
    });

    it('clones cloudsmith private registry', async () => {
      const { mockClone } = setupGitMocks();
      GlobalConfig.set({ ...adminConfig, allowCustomCrateRegistries: true });
      const url = 'https://dl.cloudsmith.io/basic/myorg/myrepo/cargo/index.git';
      const res = await getPkgReleases({
        datasource,
        packageName: 'mypkg',
        registryUrls: [url],
      });
      expect(mockClone).toHaveBeenCalled();
      expect(res).toMatchObject({
        dependencyUrl:
          'https://cloudsmith.io/~myorg/repos/myrepo/packages/detail/cargo/mypkg',
        releases: [{ version: '0.1.0' }, { version: '0.1.1' }],
      });
    });

    it('clones other private registry with explicit gitTimeout', async () => {
      const { mockClone } = setupGitMocks();
      GlobalConfig.set({
        ...adminConfig,
        allowCustomCrateRegistries: true,
        gitTimeout: 30000,
      });
      const url = 'https://github.com/mcorbin/testregistry';
      const res = await getPkgReleases({
        datasource,
        packageName: 'mypkg',
        registryUrls: [url],
      });
      expect(mockClone).toHaveBeenCalled();
      expect(res).not.toBeNull();
    });

    it('clones other private registry', async () => {
      const { mockClone } = setupGitMocks();
      GlobalConfig.set({ ...adminConfig, allowCustomCrateRegistries: true });
      const url = 'https://github.com/mcorbin/testregistry';
      const res = await getPkgReleases({
        datasource,
        packageName: 'mypkg',
        registryUrls: [url],
      });
      expect(mockClone).toHaveBeenCalled();
      expect(res).toMatchObject({
        dependencyUrl: 'https://github.com/mcorbin/testregistry/mypkg',
        releases: [{ version: '0.1.0' }, { version: '0.1.1' }],
      });
    });

    it('clones once then reuses the cache', async () => {
      const { mockClone } = setupGitMocks();
      GlobalConfig.set({ ...adminConfig, allowCustomCrateRegistries: true });
      const url = 'https://github.com/mcorbin/othertestregistry';
      await getPkgReleases({
        datasource,
        packageName: 'mypkg',
        registryUrls: [url],
      });
      await getPkgReleases({
        datasource,
        packageName: 'mypkg',
        registryUrls: [url],
      });
      expect(mockClone).toHaveBeenCalledTimes(1);
    });

    it('reads config.json from cloned registry', async () => {
      const { mockClone } = setupGitMocks();
      GlobalConfig.set({ ...adminConfig, allowCustomCrateRegistries: true });
      const url = 'https://github.com/mcorbin/testregistry';
      const res = await getPkgReleases({
        datasource,
        packageName: 'mypkg',
        registryUrls: [url],
      });
      expect(mockClone).toHaveBeenCalled();
      expect(res).not.toBeNull();
    });

    it('guards against race conditions while cloning', async () => {
      const { mockClone } = setupGitMocks(250);
      GlobalConfig.set({ ...adminConfig, allowCustomCrateRegistries: true });
      const url = 'https://github.com/mcorbin/othertestregistry';

      await Promise.all([
        getPkgReleases({
          datasource,
          packageName: 'mypkg',
          registryUrls: [url],
        }),
        getPkgReleases({
          datasource,
          packageName: 'mypkg-2',
          registryUrls: [url],
        }),
      ]);

      await getPkgReleases({
        datasource,
        packageName: 'mypkg-3',
        registryUrls: [url],
      });

      expect(mockClone).toHaveBeenCalledTimes(1);
    });

    it('returns null when git clone fails', async () => {
      setupErrorGitMock();
      GlobalConfig.set({ ...adminConfig, allowCustomCrateRegistries: true });
      const url = 'https://github.com/mcorbin/othertestregistry';

      const result = await getPkgReleases({
        datasource,
        packageName: 'mypkg',
        registryUrls: [url],
      });
      const result2 = await getPkgReleases({
        datasource,
        packageName: 'mypkg-2',
        registryUrls: [url],
      });

      expect(result).toBeNull();
      expect(result2).toBeNull();
    });

    it('does not clone for sparse registries', async () => {
      GlobalConfig.set({ ...adminConfig, allowCustomCrateRegistries: true });
      const { mockClone } = setupGitMocks();

      const url = 'https://github.com/mcorbin/othertestregistry';
      const sparseUrl = `sparse+${url}`;
      httpMock.scope(url).get('/config.json').reply(404);
      httpMock.scope(url).get('/my/pk/mypkg').reply(200, {});

      const res = await getPkgReleases({
        datasource,
        packageName: 'mypkg',
        registryUrls: [sparseUrl],
      });
      expect(mockClone).toHaveBeenCalledTimes(0);
      expect(res).toBeNull();
    });

    it('retries if shallow fails because of dumb http git repo', async () => {
      const mockClone = vi
        .fn()
        .mockName('clone')
        .mockImplementation(
          async (_registryUrl: string, clonePath: string, opts) => {
            if (!isUndefined(opts) && Object.hasOwn(opts, '--depth')) {
              throw new Error(
                'fatal: dumb http transport does not support shallow capabilities',
              );
            }
            const path = `${clonePath}/my/pk/mypkg`;
            await fs.mkdir(upath.dirname(path), { recursive: true });
            await fs.writeFile(path, Fixtures.get('mypkg'), {
              encoding: 'utf8',
            });
          },
        );

      const gitMock = partial<SimpleGit>({
        clone: mockClone,
      });
      createSimpleGit.mockReturnValue(gitMock);
      GlobalConfig.set({ ...adminConfig, allowCustomCrateRegistries: true });
      const url = 'https://github.com/mcorbin/testregistry';
      const res = await getPkgReleases({
        datasource,
        packageName: 'mypkg',
        registryUrls: [url],
      });
      expect(mockClone).toHaveBeenCalledTimes(2);
      expect(res).not.toBeNull();
      expect(res).toBeDefined();
      expect(res).toMatchObject({
        dependencyUrl: 'https://github.com/mcorbin/testregistry/mypkg',
        registryUrl: 'https://github.com/mcorbin/testregistry',
        releases: [
          {
            version: '0.1.0',
          },
          {
            version: '0.1.1',
          },
        ],
      });
    });

    it('retries if shallow fails but retry can also fail', async () => {
      const mockClone = vi
        .fn()
        .mockName('clone')
        .mockImplementation((_registryUrl: string, clonePath: string, opts) => {
          if (!isUndefined(opts) && Object.hasOwn(opts, '--depth')) {
            return Promise.reject(
              new Error(
                'fatal: dumb http transport does not support shallow capabilities',
              ),
            );
          }
          return Promise.reject(new Error('mocked error'));
        });

      const gitMock = partial<SimpleGit>({
        clone: mockClone,
      });
      createSimpleGit.mockReturnValue(gitMock);
      GlobalConfig.set({ ...adminConfig, allowCustomCrateRegistries: true });
      const url = 'https://github.com/mcorbin/testregistry';
      const res = await getPkgReleases({
        datasource,
        packageName: 'mypkg',
        registryUrls: [url],
      });
      expect(mockClone).toHaveBeenCalledTimes(2);
      expect(res).toBeNull();
    });
  });

  describe('postprocessRelease', () => {
    const datasource = new CrateDatasource();

    beforeEach(() => {
      memCache.init();
    });

    it('no-op for registries without cached config', async () => {
      const releaseOrig = { version: '4.5.17' };

      const res = await datasource.postprocessRelease(
        {
          packageName: 'clap',
          registryUrl: 'https://example.com',
        },
        releaseOrig,
      );

      expect(res).toBe(releaseOrig);
    });

    it('no-op when registryUrl is null', async () => {
      const releaseOrig = { version: '4.5.17' };

      const res = await datasource.postprocessRelease(
        {
          packageName: 'clap',
          registryUrl: null,
        },
        releaseOrig,
      );

      expect(res).toBe(releaseOrig);
    });

    it('no-op for release with timestamp', async () => {
      const releaseOrig = {
        version: '4.5.17',
        releaseTimestamp: '2024-09-04T19:16:41.355Z' as Timestamp,
      };

      const res = await datasource.postprocessRelease(
        {
          packageName: 'clap',
          registryUrl: CRATES_IO_REGISTRY_URL,
        },
        releaseOrig,
      );

      expect(res).toBe(releaseOrig);
    });

    it('fetches releaseTimestamp', async () => {
      memCache.set(
        `crate-datasource/registry-config/${CRATES_IO_REGISTRY_URL_PARSED}`,
        cratesIoConfig,
      );

      httpMock
        .scope(API_BASE_URL)
        .get('/api/v1/crates/clap/4.5.17')
        .reply(200, {
          version: {
            created_at: '2024-09-04T19:16:41.355243+00:00',
          },
        });

      const res = await datasource.postprocessRelease(
        {
          packageName: 'clap',
          registryUrl: CRATES_IO_REGISTRY_URL,
        },
        { version: '4.5.17' },
      );

      expect(res).toEqual({
        version: '4.5.17',
        releaseTimestamp: '2024-09-04T19:16:41.355Z',
      });
    });
  });
});