import PQueue from 'p-queue'; import { logger } from '../../logger/index.ts'; import { parseUrl } from '../url.ts'; import { getConcurrentRequestsLimit } from './rate-limits.ts'; const hostQueues = new Map(); export function getQueue(url: string): PQueue | null { const host = parseUrl(url)?.host; if (!host) { // should never happen logger.debug(`No host on ${url}`); return null; } let queue = hostQueues.get(host); if (queue === undefined) { queue = null; // null represents "no queue", as opposed to undefined const concurrency = getConcurrentRequestsLimit(url); if (concurrency) { logger.debug(`Using queue: host=${host}, concurrency=${concurrency}`); queue = new PQueue({ concurrency }); } else { logger.trace({ host }, 'No concurrency limits'); } } hostQueues.set(host, queue); return queue; } export function clear(): void { hostQueues.clear(); }