Firelink/scripts/engine-download.js

import fs from 'node:fs';
import { Readable, Transform } from 'node:stream';
import { pipeline } from 'node:stream/promises';

import { sha256 } from './engine-payload-integrity.js';

const DEFAULT_ATTEMPTS = 3;
const DEFAULT_IDLE_TIMEOUT_MS = 120_000;
const DEFAULT_RETRY_DELAYS_MS = [2_000, 5_000];
const FILE_RESET_RETRY_DELAYS_MS = [100, 250, 500, 1_000, 2_000];

function parseContentRange(value) {
  const match = /^bytes (\d+)-(\d+)\/(\d+|\*)$/.exec(value || '');
  if (!match) return undefined;
  const start = Number(match[1]);
  const end = Number(match[2]);
  const total = match[3] === '*' ? undefined : Number(match[3]);
  if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || end < start) {
    return undefined;
  }
  if (total !== undefined && (!Number.isSafeInteger(total) || end >= total)) {
    return undefined;
  }
  return { start, end, total };
}

function responseLength(response) {
  const value = response.headers.get('content-length');
  if (!value || !/^\d+$/.test(value)) return undefined;
  const length = Number(value);
  return Number.isSafeInteger(length) ? length : undefined;
}

function createDownloadTimeout(idleTimeoutMs) {
  const controller = new AbortController();
  let timer;
  const refresh = () => {
    clearTimeout(timer);
    timer = setTimeout(() => {
      controller.abort(new Error(`Download idle for ${idleTimeoutMs}ms`));
    }, idleTimeoutMs);
  };
  const dispose = () => clearTimeout(timer);
  refresh();
  return { signal: controller.signal, refresh, dispose };
}

function archiveSize(archive) {
  try {
    return fs.statSync(archive).size;
  } catch (error) {
    if (error?.code === 'ENOENT') return 0;
    throw error;
  }
}

function checksumMismatchError(name, expected, actual) {
  const error = new Error(`Archive checksum mismatch for ${name}. Expected ${expected}, got ${actual}`);
  error.code = 'ARCHIVE_CHECKSUM_MISMATCH';
  return error;
}

function sleep(milliseconds) {
  return new Promise(resolve => setTimeout(resolve, milliseconds));
}

async function resetArchive(archive) {
  for (let attempt = 0; ; attempt += 1) {
    try {
      fs.rmSync(archive, { force: true });
      return;
    } catch (error) {
      const retryable = process.platform === 'win32'
        && ['EACCES', 'EBUSY', 'EPERM'].includes(error?.code);
      if (!retryable || attempt >= FILE_RESET_RETRY_DELAYS_MS.length) throw error;
      await sleep(FILE_RESET_RETRY_DELAYS_MS[attempt]);
    }
  }
}

/**
 * Download and checksum an engine archive, resuming an interrupted response
 * when the provider honors HTTP range requests. A provider that ignores the
 * range is handled safely by replacing the partial file instead of appending
 * a second full archive to it.
 */
export async function downloadEngineArchive({
  name,
  url,
  archive,
  expectedSha256,
  attempts = DEFAULT_ATTEMPTS,
  idleTimeoutMs = DEFAULT_IDLE_TIMEOUT_MS,
  retryDelaysMs = DEFAULT_RETRY_DELAYS_MS,
}) {
  let lastError;

  for (let attempt = 1; attempt <= attempts; attempt += 1) {
    const partialSize = archiveSize(archive);
    if (partialSize > 0 && sha256(archive) === expectedSha256) return archive;
    const downloadTimeout = createDownloadTimeout(idleTimeoutMs);
    let resetForRetry = false;

    try {
      const response = await fetch(url, {
        headers: partialSize > 0 ? { Range: `bytes=${partialSize}-` } : undefined,
        redirect: 'follow',
        signal: downloadTimeout.signal,
      });

      if (response.status === 416 && partialSize > 0) {
        await response.body?.cancel();
        resetForRetry = true;
        throw new Error(`Retained partial archive range is not satisfiable for ${name}`);
      }

      if (!response.ok || !response.body) {
        await response.body?.cancel();
        throw new Error(`Failed to download ${name}: HTTP ${response.status}`);
      }

      const contentRange = response.status === 206
        ? parseContentRange(response.headers.get('content-range'))
        : undefined;
      if (response.status === 206 && (!contentRange || contentRange.start !== partialSize)) {
        await response.body.cancel();
        throw new Error(`Invalid Content-Range while downloading ${name}`);
      }

      const append = response.status === 206 && partialSize > 0;
      const expectedResponseLength = responseLength(response);
      if (!append && partialSize > 0) {
        // The provider ignored Range and returned the complete archive.
        await resetArchive(archive);
      }

      await pipeline(
        Readable.fromWeb(response.body),
        new Transform({
          transform(chunk, encoding, callback) {
            downloadTimeout.refresh();
            callback(null, chunk, encoding);
          },
        }),
        fs.createWriteStream(archive, { flags: append ? 'a' : 'w' }),
        { signal: downloadTimeout.signal },
      );

      const finalSize = archiveSize(archive);
      const expectedFinalSize = contentRange?.total
        ?? (expectedResponseLength === undefined
          ? undefined
          : (append ? partialSize + expectedResponseLength : expectedResponseLength));
      if (expectedFinalSize !== undefined && finalSize !== expectedFinalSize) {
        throw new Error(
          `Incomplete archive for ${name}: expected ${expectedFinalSize} bytes, got ${finalSize}`,
        );
      }

      const actual = sha256(archive);
      if (actual === expectedSha256) return archive;

      resetForRetry = true;
      throw checksumMismatchError(name, expectedSha256, actual);
    } catch (error) {
      lastError = error;
      if (resetForRetry || error?.code === 'ARCHIVE_CHECKSUM_MISMATCH') {
        await resetArchive(archive);
      }
      if (attempt === attempts) {
        throw new Error(
          `Failed to download ${name} after ${attempts} attempts: ${
            error instanceof Error ? error.message : String(error)
          }`,
          { cause: error },
        );
      }
      await new Promise(resolve => setTimeout(resolve, retryDelaysMs[attempt - 1] ?? 0));
    } finally {
      downloadTimeout.dispose();
    }
  }

  throw lastError;
}