pulse/tests/integration/scripts/check-e2e-tiering.mjs

#!/usr/bin/env node

import fs from 'node:fs';
import path from 'node:path';
import { spawnSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';

import {
  PROBATION_SPECS,
  QUARANTINED_SPECS,
} from '../e2e-tiering.mjs';
import { parsePlaywrightTestIdentity } from './e2e-tier-identity.mjs';

const integrationRoot = path.resolve(fileURLToPath(new URL('..', import.meta.url)));
const testsRoot = path.join(integrationRoot, 'tests');
const playwrightBin = path.join(integrationRoot, 'node_modules', '.bin', 'playwright');
const MIN_STABLE_SPEC_FILES = 10;

function listSpecFiles() {
  const specs = [];
  function walk(directory, relativeDirectory = '') {
    for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
      if (entry.isDirectory()) {
        walk(path.join(directory, entry.name), `${relativeDirectory}${entry.name}/`);
      } else if (entry.name.endsWith('.spec.ts')) {
        specs.push(`${relativeDirectory}${entry.name}`);
      }
    }
  }
  walk(testsRoot);
  return specs.sort();
}

function duplicates(values) {
  const seen = new Set();
  return values.filter((value) => {
    if (seen.has(value)) return true;
    seen.add(value);
    return false;
  });
}

function ledgerFiles(values, label, allFiles, errors) {
  const badFormat = values.filter((value) => !/^\*\*\/.+\.spec\.ts$/.test(value));
  if (badFormat.length) errors.push(`${label} has invalid entries: ${badFormat.join(', ')}`);

  const duplicateValues = duplicates(values);
  if (duplicateValues.length) errors.push(`${label} has duplicates: ${duplicateValues.join(', ')}`);

  const sortedValues = [...values].sort();
  if (values.some((value, index) => value !== sortedValues[index])) {
    errors.push(`${label} must be sorted`);
  }

  const files = new Set(values.map((value) => value.replace(/^\*\*\//, '')));
  const unknown = [...files].filter((value) => !allFiles.has(value));
  if (unknown.length) errors.push(`${label} has unknown specs: ${unknown.join(', ')}`);
  return files;
}

function listPlaywrightTests(tier) {
  if (!fs.existsSync(playwrightBin)) {
    throw new Error('Playwright is not installed; run npm ci before this check');
  }
  const env = { ...process.env };
  if (tier) env.PULSE_E2E_TIER = tier;
  else delete env.PULSE_E2E_TIER;

  const result = spawnSync(playwrightBin, ['test', '--list'], {
    cwd: integrationRoot,
    env,
    encoding: 'utf8',
    maxBuffer: 64 * 1024 * 1024,
  });
  if (result.status !== 0) {
    throw new Error(
      `Playwright --list failed for ${tier || 'all'} tier:\n${result.stderr || result.stdout}`,
    );
  }

  const tests = new Map();
  for (const rawLine of result.stdout.split('\n')) {
    const identity = parsePlaywrightTestIdentity(rawLine);
    if (!identity) continue;
    if (tests.has(identity.key)) {
      throw new Error(`Playwright listed a duplicate test: ${identity.key}`);
    }
    tests.set(identity.key, identity.specFile);
  }
  return tests;
}

function difference(left, right) {
  return new Set([...left].filter((value) => !right.has(value)));
}

function sameSet(actual, expected, label, errors) {
  const missing = difference(expected, actual);
  const extra = difference(actual, expected);
  if (missing.size) errors.push(`${label} is missing: ${[...missing].join(', ')}`);
  if (extra.size) errors.push(`${label} has unexpected entries: ${[...extra].join(', ')}`);
}

const errors = [];
const allFiles = new Set(listSpecFiles());
const probationFiles = ledgerFiles(PROBATION_SPECS, 'PROBATION_SPECS', allFiles, errors);
const quarantineFiles = ledgerFiles(QUARANTINED_SPECS, 'QUARANTINED_SPECS', allFiles, errors);
const overlap = [...probationFiles].filter((value) => quarantineFiles.has(value));
if (overlap.length) errors.push(`Probation/quarantine overlap: ${overlap.join(', ')}`);

const allTests = listPlaywrightTests(null);
const stableTests = listPlaywrightTests('stable');
const probationTests = listPlaywrightTests('probation');
const allKeys = new Set(allTests.keys());
const stableKeys = new Set(stableTests.keys());
const probationKeys = new Set(probationTests.keys());

const instanceOverlap = [...stableKeys].filter((key) => probationKeys.has(key));
if (instanceOverlap.length) errors.push(`Stable/probation test overlap: ${instanceOverlap.join(', ')}`);
sameSet(new Set([...stableKeys, ...probationKeys]), allKeys, 'Stable + probation tests', errors);

const listedAllFiles = new Set(allTests.values());
const listedStableFiles = new Set(stableTests.values());
const listedProbationFiles = new Set(probationTests.values());
const expectedAllFiles = difference(allFiles, quarantineFiles);
const expectedStableFiles = difference(expectedAllFiles, probationFiles);
sameSet(listedAllFiles, expectedAllFiles, 'All-tier spec files', errors);
sameSet(listedStableFiles, expectedStableFiles, 'Stable-tier spec files', errors);
sameSet(listedProbationFiles, probationFiles, 'Probation-tier spec files', errors);

if (listedStableFiles.size < MIN_STABLE_SPEC_FILES) {
  errors.push(
    `Stable tier has ${listedStableFiles.size} spec files; the signal floor is ${MIN_STABLE_SPEC_FILES}`,
  );
}

if (errors.length) {
  console.error('E2E tier selection validation failed:');
  for (const error of errors) console.error(`- ${error}`);
  process.exit(1);
}

console.log('E2E tier selection validation passed');
console.log(`  spec_files: ${allFiles.size}`);
console.log(`  stable_spec_files: ${listedStableFiles.size}`);
console.log(`  probation_spec_files: ${listedProbationFiles.size}`);
console.log(`  quarantined_spec_files: ${quarantineFiles.size}`);
console.log(`  test_project_instances: ${allTests.size}`);
console.log(`  stable_test_project_instances: ${stableTests.size}`);
console.log(`  probation_test_project_instances: ${probationTests.size}`);