test: harden release and browser gates

scripts/README.md

@@ -10,6 +10,7 @@ npm run verify
 npm run lint
 npm run test:unit
 npm run test:coverage
+npm run prepare:release -- 3.1.5
 ```
 
 - `npm run build:extension` runs `scripts/build-extension.cjs`.
@@ -17,6 +18,7 @@ npm run test:coverage
 - `npm run lint` runs ESLint across the repository.
 - `npm run test:unit` runs Vitest tests.
 - `npm run test:coverage` runs the same tests with the enforced coverage floor.
+- `npm run prepare:release -- MAJOR.MINOR.PATCH` updates every release-version source consistently before the release PR.
 
 ## build-extension.cjs
 
@@ -89,10 +91,17 @@ both global and risk-specific per-module floors. Browser entry points
 (`background.js`, `content.js`, and `popup.js`) and server process startup are
 deliberately measured by extension E2E and integration tests instead of being
 reported as zero-coverage unit code.
+`scripts/check-coverage-inventory.mjs` additionally requires every JavaScript
+source file to be classified as V8-covered or assigned to a named external
+integration gate. New unclassified files fail `npm run verify`.
 
 ## Published Release Verification
 
-After a GitHub Release is created, the release workflow runs:
+Before publication, the release workflow validates the exact annotated SemVer
+tag, requires it to point at current `origin/main`, requires successful
+`verify`, `node20`, and `e2e` checks, and runs the complete gates again. It then
+creates a draft release, publishes and smoke-tests the relay image, and only
+afterwards makes the GitHub Release public. The published-asset gate runs:
 
 ```bash
 node scripts/verify-published-release.mjs v3.1.4 --repo Shik3i/KoalaSync

scripts/check-coverage-inventory.mjs

@@ -0,0 +1,62 @@
+#!/usr/bin/env node
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { EXTERNALLY_GATED_SOURCES, VITEST_COVERAGE_INCLUDE } from './coverage-plan.mjs';
+
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const SOURCE_ROOTS = Object.freeze(['extension', 'scripts', 'server', 'shared', 'website']);
+const SOURCE_EXTENSION = /\.(?:cjs|js|mjs)$/u;
+const TEST_FILE = /\.test\.(?:cjs|js|mjs)$/u;
+const GENERATED_OR_DEPENDENCY_DIRECTORIES = new Set(['extension/shared', 'server/node_modules', 'website/www']);
+
+export function validateCoverageInventory(discoveredSources, coveredSources, externallyGatedSources) {
+    const discovered = new Set(discoveredSources);
+    const assignments = [...coveredSources, ...externallyGatedSources];
+    const assigned = new Set();
+    const duplicates = new Set();
+    for (const source of assignments) {
+        if (assigned.has(source)) duplicates.add(source);
+        assigned.add(source);
+    }
+    const unclassified = [...discovered].filter(source => !assigned.has(source)).sort();
+    const stale = [...assigned].filter(source => !discovered.has(source)).sort();
+    if (duplicates.size || unclassified.length || stale.length) {
+        const details = [];
+        if (duplicates.size) details.push(`assigned more than once: ${[...duplicates].sort().join(', ')}`);
+        if (unclassified.length) details.push(`unclassified sources: ${unclassified.join(', ')}`);
+        if (stale.length) details.push(`stale assignments: ${stale.join(', ')}`);
+        throw new Error(details.join('; '));
+    }
+}
+
+function collectSources(directory, output = []) {
+    for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
+        const absolutePath = path.join(directory, entry.name);
+        if (entry.isDirectory()) {
+            const relativeDirectory = path.relative(repoRoot, absolutePath).split(path.sep).join('/');
+            if (!GENERATED_OR_DEPENDENCY_DIRECTORIES.has(relativeDirectory)) collectSources(absolutePath, output);
+        } else if (SOURCE_EXTENSION.test(entry.name) && !TEST_FILE.test(entry.name)) {
+            output.push(path.relative(repoRoot, absolutePath).split(path.sep).join('/'));
+        }
+    }
+    return output;
+}
+
+function main() {
+    const discoveredSources = SOURCE_ROOTS.flatMap(root => collectSources(path.join(repoRoot, root))).sort();
+    const externallyGatedSources = Object.values(EXTERNALLY_GATED_SOURCES).flat();
+    validateCoverageInventory(discoveredSources, VITEST_COVERAGE_INCLUDE, externallyGatedSources);
+    console.log(`Coverage inventory passed: ${VITEST_COVERAGE_INCLUDE.length} V8-covered, ${externallyGatedSources.length} externally gated`);
+}
+
+const isMainModule = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
+if (isMainModule) {
+    try {
+        main();
+    } catch (error) {
+        console.error(`Coverage inventory failed: ${error.message}`);
+        process.exitCode = 1;
+    }
+}

scripts/coverage-plan.mjs

@@ -0,0 +1,68 @@
+export const VITEST_COVERAGE_INCLUDE = Object.freeze([
+    'server/chat.js',
+    'server/ops.js',
+    'server/rate-limiter.js',
+    'shared/blacklist.js',
+    'shared/invite-links.js',
+    'shared/names.js',
+    'extension/chat-activity.js',
+    'extension/chat-crypto.js',
+    'extension/chat-format.js',
+    'extension/chat-session.js',
+    'extension/chat-wire.js',
+    'extension/episode-utils.js',
+    'extension/host-access.js',
+    'extension/media-frame-target.js',
+    'extension/title-privacy.js',
+    'scripts/release-artifact-checks.mjs'
+]);
+
+// Exact list by design: adding a runtime/tooling module requires choosing its
+// automated gate instead of silently leaving it unmeasured.
+export const EXTERNALLY_GATED_SOURCES = Object.freeze({
+    'packed extension E2E': Object.freeze([
+        'extension/audio-options.js',
+        'extension/background.js',
+        'extension/bridge.js',
+        'extension/chat-overlay.js',
+        'extension/content.js',
+        'extension/i18n.js',
+        'extension/media-frame-monitor.js',
+        'extension/modules/tab-manager.js',
+        'extension/page-api-seek-overrides.js',
+        'extension/popup.js',
+        'extension/theme-init.js',
+        'shared/constants.js'
+    ]),
+    'relay integration': Object.freeze([
+        'server/index.js'
+    ]),
+    'release and repository integration': Object.freeze([
+        'scripts/build-extension.cjs',
+        'scripts/check-coverage-inventory.mjs',
+        'scripts/coverage-plan.mjs',
+        'scripts/prepare-release.mjs',
+        'scripts/release-preflight.mjs',
+        'scripts/test-audio-settings.mjs',
+        'scripts/test-chat-settings.mjs',
+        'scripts/test-content-video-finder.cjs',
+        'scripts/test-locales.cjs',
+        'scripts/test-popup-refresh-cooldown.mjs',
+        'scripts/test-server-routes.mjs',
+        'scripts/test-server-ws.mjs',
+        'scripts/test-website-locales.mjs',
+        'scripts/test-website-theme.mjs',
+        'scripts/translate-locales-tool.cjs',
+        'scripts/validate-brand-names.cjs',
+        'scripts/verify-published-release.mjs',
+        'scripts/verify-release.mjs'
+    ]),
+    'website build and contract checks': Object.freeze([
+        'website/app.js',
+        'website/build.cjs',
+        'website/flag-font-utils.cjs',
+        'website/lang-init.js',
+        'website/submit-indexnow.cjs',
+        'website/tools/subset-flag-font.mjs'
+    ])
+});

scripts/coverage-plan.test.mjs

@@ -0,0 +1,21 @@
+import { describe, expect, it } from 'vitest';
+import { validateCoverageInventory } from './check-coverage-inventory.mjs';
+
+describe('coverage inventory', () => {
+    it('accepts an exact, unique classification', () => {
+        expect(() => validateCoverageInventory(
+            ['covered.js', 'browser.js'],
+            ['covered.js'],
+            ['browser.js']
+        )).not.toThrow();
+    });
+
+    it('rejects unclassified, stale, and duplicate assignments', () => {
+        expect(() => validateCoverageInventory(['new.js'], [], []))
+            .toThrow('unclassified sources: new.js');
+        expect(() => validateCoverageInventory([], ['deleted.js'], []))
+            .toThrow('stale assignments: deleted.js');
+        expect(() => validateCoverageInventory(['same.js'], ['same.js'], ['same.js']))
+            .toThrow('assigned more than once: same.js');
+    });
+});

scripts/prepare-release.mjs

@@ -0,0 +1,80 @@
+#!/usr/bin/env node
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { versionFromTag } from './release-artifact-checks.mjs';
+
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+
+export function replaceExactly(text, pattern, replacement, label) {
+    const matches = String(text).match(pattern);
+    if (!matches || matches.length !== 1) {
+        throw new Error(`${label} must contain exactly one release-version marker`);
+    }
+    return text.replace(pattern, replacement);
+}
+
+function writeJson(relativePath, update) {
+    const absolutePath = path.join(repoRoot, relativePath);
+    const value = JSON.parse(fs.readFileSync(absolutePath, 'utf8'));
+    update(value);
+    fs.writeFileSync(absolutePath, `${JSON.stringify(value, null, 2)}\n`, 'utf8');
+}
+
+function updateText(relativePath, pattern, replacement, label) {
+    const absolutePath = path.join(repoRoot, relativePath);
+    const current = fs.readFileSync(absolutePath, 'utf8');
+    fs.writeFileSync(absolutePath, replaceExactly(current, pattern, replacement, label), 'utf8');
+}
+
+export function prepareRelease(version, date = new Date()) {
+    versionFromTag(`v${version}`);
+    const timestamp = date.toISOString().replace(/\.\d{3}Z$/u, 'Z');
+    writeJson('package.json', value => { value.version = version; });
+    writeJson('package-lock.json', value => {
+        value.version = version;
+        value.packages[''].version = version;
+    });
+    writeJson('extension/manifest.base.json', value => { value.version = version; });
+    writeJson('website/version.json', value => {
+        value.version = version;
+        value.date = timestamp;
+    });
+    updateText(
+        'shared/constants.js',
+        /export const APP_VERSION = ["'][^"']+["'];/gu,
+        `export const APP_VERSION = "${version}";`,
+        'shared/constants.js'
+    );
+    updateText(
+        'website/template.html',
+        /"softwareVersion": "[^"]+"/gu,
+        `"softwareVersion": "${version}"`,
+        'website/template.html'
+    );
+    updateText(
+        'website/llms.txt',
+        /Current website release: .+/gu,
+        `Current website release: ${version}`,
+        'website/llms.txt'
+    );
+    updateText(
+        'README.md',
+        /Release-v\d+\.\d+\.\d+-blue/gu,
+        `Release-v${version}-blue`,
+        'README.md release badge'
+    );
+    console.log(`Prepared release v${version} at ${timestamp}`);
+}
+
+const isMainModule = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
+if (isMainModule) {
+    try {
+        if (process.argv.length !== 3) throw new Error('Usage: npm run prepare:release -- MAJOR.MINOR.PATCH');
+        prepareRelease(process.argv[2]);
+    } catch (error) {
+        console.error(`Release preparation failed: ${error.message}`);
+        process.exitCode = 1;
+    }
+}

scripts/prepare-release.test.mjs

@@ -0,0 +1,13 @@
+import { describe, expect, it } from 'vitest';
+import { replaceExactly } from './prepare-release.mjs';
+
+describe('release preparation helpers', () => {
+    it('replaces one and only one version marker', () => {
+        expect(replaceExactly('version=3.1.4', /version=\d+\.\d+\.\d+/gu, 'version=3.1.5', 'fixture'))
+            .toBe('version=3.1.5');
+        expect(() => replaceExactly('none', /version=\d+/gu, 'version=4', 'fixture'))
+            .toThrow('fixture must contain exactly one release-version marker');
+        expect(() => replaceExactly('version=1 version=2', /version=\d+/gu, 'version=3', 'fixture'))
+            .toThrow('fixture must contain exactly one release-version marker');
+    });
+});

scripts/release-preflight.mjs

@@ -0,0 +1,111 @@
+#!/usr/bin/env node
+
+import { execFileSync } from 'node:child_process';
+import fs from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import path from 'node:path';
+import { versionFromTag } from './release-artifact-checks.mjs';
+
+export const REQUIRED_RELEASE_CHECKS = Object.freeze(['verify', 'node20', 'e2e']);
+
+export function parseCheckRuns(text) {
+    return String(text).split(/\r?\n/u).filter(Boolean).map(line => {
+        const [name, conclusion, url = ''] = line.split('\t');
+        if (!name || !conclusion) throw new Error(`Invalid check-run record: ${line}`);
+        return { name, conclusion, url };
+    });
+}
+
+export function validateRequiredChecks(checkRuns, required = REQUIRED_RELEASE_CHECKS) {
+    for (const name of required) {
+        const matches = checkRuns.filter(check => check.name === name);
+        if (matches.length === 0) throw new Error(`Required check is missing for the release commit: ${name}`);
+        if (matches.some(check => check.conclusion !== 'success')) {
+            const conclusions = matches.map(check => check.conclusion).join(', ');
+            throw new Error(`Required check ${name} did not succeed: ${conclusions}`);
+        }
+    }
+}
+
+function run(command, args) {
+    return execFileSync(command, args, {
+        cwd: process.cwd(),
+        encoding: 'utf8',
+        stdio: ['ignore', 'pipe', 'pipe']
+    }).trim();
+}
+
+export function validateRepositoryName(repo) {
+    if (!/^[^/\s]+\/[^/\s]+$/u.test(repo || '')) {
+        throw new Error(`Invalid GitHub repository: ${repo || '<empty>'}`);
+    }
+    return repo;
+}
+
+export function validateVersionSnapshot(expectedVersion, snapshot) {
+    for (const [label, actualVersion] of Object.entries(snapshot)) {
+        if (actualVersion !== expectedVersion) {
+            throw new Error(`${label} version ${actualVersion || '<missing>'} does not match tag version ${expectedVersion}`);
+        }
+    }
+}
+
+export function validateReleaseSourceVersion(expectedVersion, repoRoot = process.cwd()) {
+    const readJson = relativePath => JSON.parse(fs.readFileSync(path.join(repoRoot, relativePath), 'utf8'));
+    const packageJson = readJson('package.json');
+    const packageLock = readJson('package-lock.json');
+    const manifest = readJson('extension/manifest.base.json');
+    const websiteVersion = readJson('website/version.json');
+    const constants = fs.readFileSync(path.join(repoRoot, 'shared/constants.js'), 'utf8');
+    const appVersion = /export const APP_VERSION = ["']([^"']+)["']/u.exec(constants)?.[1] || '';
+    validateVersionSnapshot(expectedVersion, {
+        'package.json': packageJson.version,
+        'package-lock.json': packageLock.version,
+        'package-lock root package': packageLock.packages?.['']?.version,
+        'extension manifest': manifest.version,
+        'shared constants': appVersion,
+        'website/version.json': websiteVersion.version
+    });
+}
+
+export function verifyReleaseRef({ tag, repo }) {
+    const version = versionFromTag(tag);
+    validateRepositoryName(repo);
+    validateReleaseSourceVersion(version);
+    const tagRef = `refs/tags/${tag}`;
+    if (run('git', ['cat-file', '-t', tagRef]) !== 'tag') {
+        throw new Error(`${tag} must be an annotated tag`);
+    }
+    const tagCommit = run('git', ['rev-list', '-n', '1', tagRef]);
+    const mainCommit = run('git', ['rev-parse', 'origin/main']);
+    if (tagCommit !== mainCommit) {
+        throw new Error(`Release tag ${tag} points to ${tagCommit}, but origin/main is ${mainCommit}`);
+    }
+
+    const checks = parseCheckRuns(run('gh', [
+        'api', `repos/${repo}/commits/${tagCommit}/check-runs`,
+        '--jq', '.check_runs[] | [.name, .conclusion, .html_url] | @tsv'
+    ]));
+    validateRequiredChecks(checks);
+    return { version, tagCommit };
+}
+
+function main() {
+    const tag = process.env.GITHUB_REF_NAME || '';
+    const repo = process.env.GITHUB_REPOSITORY || '';
+    const outputPath = process.env.GITHUB_OUTPUT || '';
+    const result = verifyReleaseRef({ tag, repo });
+    if (!outputPath) throw new Error('GITHUB_OUTPUT is required');
+    fs.appendFileSync(outputPath, `version=${result.version}\ntag_commit=${result.tagCommit}\n`, 'utf8');
+    console.log(`Release preflight accepted ${tag} at ${result.tagCommit}`);
+}
+
+const isMainModule = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
+if (isMainModule) {
+    try {
+        main();
+    } catch (error) {
+        console.error(`Release preflight failed: ${error.message}`);
+        process.exitCode = 1;
+    }
+}

scripts/release-preflight.test.mjs

@@ -0,0 +1,59 @@
+import { describe, expect, it } from 'vitest';
+import {
+    parseCheckRuns,
+    validateRepositoryName,
+    validateRequiredChecks,
+    validateVersionSnapshot
+} from './release-preflight.mjs';
+
+describe('release preflight helpers', () => {
+    it('parses successful GitHub check runs', () => {
+        const checks = parseCheckRuns('verify\tsuccess\thttps://example.test/1\nnode20\tsuccess\thttps://example.test/2\ne2e\tsuccess\thttps://example.test/3');
+        expect(checks).toEqual([
+            { name: 'verify', conclusion: 'success', url: 'https://example.test/1' },
+            { name: 'node20', conclusion: 'success', url: 'https://example.test/2' },
+            { name: 'e2e', conclusion: 'success', url: 'https://example.test/3' }
+        ]);
+        expect(() => validateRequiredChecks(checks)).not.toThrow();
+    });
+
+    it('rejects missing, pending, and failed release checks', () => {
+        expect(() => validateRequiredChecks([{ name: 'verify', conclusion: 'success' }]))
+            .toThrow('Required check is missing for the release commit: node20');
+        expect(() => validateRequiredChecks([
+            { name: 'verify', conclusion: 'success' },
+            { name: 'node20', conclusion: 'success' },
+            { name: 'e2e', conclusion: 'in_progress' }
+        ])).toThrow('Required check e2e did not succeed: in_progress');
+        expect(() => validateRequiredChecks([
+            { name: 'verify', conclusion: 'failure' },
+            { name: 'node20', conclusion: 'success' },
+            { name: 'e2e', conclusion: 'success' }
+        ])).toThrow('Required check verify did not succeed: failure');
+        expect(() => validateRequiredChecks([
+            { name: 'verify', conclusion: 'success' },
+            { name: 'verify', conclusion: 'failure' },
+            { name: 'node20', conclusion: 'success' },
+            { name: 'e2e', conclusion: 'success' }
+        ])).toThrow('Required check verify did not succeed: success, failure');
+    });
+
+    it('validates repository names and malformed check output', () => {
+        expect(validateRepositoryName('Shik3i/KoalaSync')).toBe('Shik3i/KoalaSync');
+        for (const invalid of ['', 'KoalaSync', 'owner/repo/extra', 'owner /repo']) {
+            expect(() => validateRepositoryName(invalid)).toThrow('Invalid GitHub repository');
+        }
+        expect(() => parseCheckRuns('verify')).toThrow('Invalid check-run record');
+    });
+
+    it('requires every release source to already match the tag version', () => {
+        expect(() => validateVersionSnapshot('3.1.5', {
+            package: '3.1.5',
+            manifest: '3.1.5'
+        })).not.toThrow();
+        expect(() => validateVersionSnapshot('3.1.5', {
+            package: '3.1.5',
+            manifest: '3.1.4'
+        })).toThrow('manifest version 3.1.4 does not match tag version 3.1.5');
+    });
+});

scripts/verify-published-release.mjs

@@ -85,7 +85,8 @@ async function verify() {
     if (run('git', ['cat-file', '-t', tagRef]).trim() !== 'tag') {
         throw new Error(`${options.tag} must be an annotated tag`);
     }
-    run('git', ['merge-base', '--is-ancestor', tagRef, 'HEAD']);
+    run('git', ['merge-base', '--is-ancestor', tagRef, 'origin/main']);
+    const tagCommit = run('git', ['rev-list', '-n', '1', tagRef]).trim();
 
     const temporaryDirectory = options.assetDir
         ? null
@@ -132,7 +133,14 @@ async function verify() {
             validateManifest(browserName, manifest, version);
             assertRuntimeBuild(browserName, archivePath, version);
             if (!options.skipAttestation && !options.assetDir) {
-                run('gh', ['attestation', 'verify', archivePath, '--repo', repo], { capture: false });
+                run('gh', [
+                    'attestation', 'verify', archivePath,
+                    '--repo', repo,
+                    '--signer-workflow', `${repo}/.github/workflows/release.yml`,
+                    '--source-ref', tagRef,
+                    '--source-digest', tagCommit,
+                    '--deny-self-hosted-runners'
+                ], { capture: false });
             }
         }
         validateArchiveParity(archiveEntries.chrome, archiveEntries.firefox);

scripts/verify-release.mjs

@@ -7,6 +7,7 @@ import path from 'node:path';
 const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
 
 const checks = [
+  ['coverage source inventory', 'node', ['scripts/check-coverage-inventory.mjs']],
   ['vitest unit tests and coverage', 'npm', ['run', 'test:coverage']],
   ['server routes', 'node', ['scripts/test-server-routes.mjs'], {
     env: { ADMIN_METRICS_TOKEN: 'verify-admin-token-with-more-than-32-chars' }