feat(release): add cross-platform packaging

Add target-aware engine provisioning, platform package configs, and CI/release verification for macOS arm64, Windows x64, and Linux AppImage.
This commit is contained in:
NimBold
2026-06-23 21:26:51 +03:30
parent 0e3118ec2c
commit f603b74a99
41 changed files with 1889 additions and 382 deletions
+165
View File
@@ -0,0 +1,165 @@
#!/usr/bin/env node
import crypto from 'node:crypto';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { execFileSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const repoRoot = path.resolve(__dirname, '..');
const sourceLock = JSON.parse(
fs.readFileSync(path.join(repoRoot, 'engine-sources.lock.json'), 'utf8')
);
function argValue(name) {
const index = process.argv.indexOf(name);
return index >= 0 ? process.argv[index + 1] : undefined;
}
const target = argValue('--target')
|| process.env.FIRELINK_TARGET_TRIPLE
|| process.env.TAURI_ENV_TARGET_TRIPLE;
if (!target) {
console.error('Pass --target <Rust target triple>.');
process.exit(1);
}
const targetSources = sourceLock.targets?.[target];
if (!targetSources) {
console.error(`No source lock exists for ${target}.`);
process.exit(1);
}
const destination = path.join(repoRoot, 'src-tauri', 'provisioned-engines', target);
const temporary = fs.mkdtempSync(path.join(os.tmpdir(), `firelink-engines-${target}-`));
const isWindows = target.includes('windows');
const executableSuffix = isWindows ? '.exe' : '';
function sha256(file) {
return crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex');
}
async function download(name, source) {
const sourcePath = new URL(source.url).pathname;
const archive = path.join(
temporary,
`${name}${sourcePath.endsWith('.tar.xz') ? '.tar.xz' : '.zip'}`
);
const response = await fetch(source.url, { redirect: 'follow' });
if (!response.ok || !response.body) {
throw new Error(`Failed to download ${name}: HTTP ${response.status}`);
}
const output = fs.createWriteStream(archive);
const reader = response.body.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (!output.write(Buffer.from(value))) {
await new Promise(resolve => output.once('drain', resolve));
}
}
await new Promise(resolve => output.end(resolve));
const actual = sha256(archive);
if (actual !== source.sha256) {
throw new Error(`Archive checksum mismatch for ${name}. Expected ${source.sha256}, got ${actual}`);
}
const extracted = path.join(temporary, `${name}-extracted`);
fs.mkdirSync(extracted);
if (archive.endsWith('.zip') && process.platform !== 'win32') {
execFileSync('unzip', ['-q', archive, '-d', extracted], { stdio: 'inherit' });
} else {
execFileSync('tar', ['-xf', archive, '-C', extracted], { stdio: 'inherit' });
}
return extracted;
}
function findFile(root, names) {
const wanted = new Set(names.map(name => name.toLowerCase()));
const matches = [];
const walk = directory => {
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
const file = path.join(directory, entry.name);
if (entry.isDirectory()) walk(file);
else if (entry.isFile() && wanted.has(entry.name.toLowerCase())) matches.push(file);
}
};
walk(root);
if (matches.length !== 1) {
throw new Error(`Expected one of [${names.join(', ')}] under ${root}, found ${matches.length}`);
}
return matches[0];
}
function copyExecutable(source, engine) {
const output = path.join(destination, `${engine}-${target}${executableSuffix}`);
fs.copyFileSync(source, output);
if (!isWindows) fs.chmodSync(output, 0o755);
}
function writePayloadManifest() {
const files = [];
const walk = directory => {
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
const file = path.join(directory, entry.name);
if (entry.isDirectory()) walk(file);
else if (entry.isFile() && entry.name !== 'payload-manifest.json') files.push(file);
}
};
walk(destination);
files.sort((left, right) => left.localeCompare(right));
const manifest = {
schemaVersion: 1,
target,
generatedFrom: Object.fromEntries(
Object.entries(targetSources).map(([name, source]) => [
name,
{
version: source.version,
url: source.url || source.sourceUrl,
sha256: source.sha256 || source.sourceSha256
}
])
),
files: Object.fromEntries(
files.map(file => [
path.relative(destination, file).split(path.sep).join('/'),
sha256(file)
])
)
};
fs.writeFileSync(
path.join(destination, 'payload-manifest.json'),
`${JSON.stringify(manifest, null, 2)}\n`
);
}
try {
fs.rmSync(destination, { recursive: true, force: true });
fs.mkdirSync(destination, { recursive: true });
const ytdlp = await download('yt-dlp', targetSources['yt-dlp']);
copyExecutable(
findFile(ytdlp, isWindows ? ['yt-dlp.exe'] : ['yt-dlp_linux']),
'yt-dlp'
);
fs.cpSync(path.join(ytdlp, '_internal'), path.join(destination, '_internal'), {
recursive: true,
preserveTimestamps: true
});
const deno = await download('deno', targetSources.deno);
copyExecutable(findFile(deno, isWindows ? ['deno.exe'] : ['deno']), 'deno');
const ffmpeg = await download('ffmpeg', targetSources.ffmpeg);
copyExecutable(findFile(ffmpeg, isWindows ? ['ffmpeg.exe'] : ['ffmpeg']), 'ffmpeg');
const aria2 = await download('aria2c', targetSources.aria2c);
copyExecutable(findFile(aria2, isWindows ? ['aria2c.exe'] : ['aria2c']), 'aria2c');
writePayloadManifest();
console.log(`Provisioned locked engine payload at ${destination}`);
} finally {
fs.rmSync(temporary, { recursive: true, force: true });
}
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env node
import { spawn } from 'node:child_process';
import path from 'node:path';
function argValue(name) {
const index = process.argv.indexOf(name);
return index >= 0 ? process.argv[index + 1] : undefined;
}
const executableArg = argValue('--executable');
if (!executableArg) {
console.error('Pass --executable <path>.');
process.exit(1);
}
const executable = path.resolve(executableArg);
const child = spawn(executable, [], {
cwd: process.env.RUNNER_TEMP || process.env.TMPDIR || process.cwd(),
detached: process.platform !== 'win32',
stdio: ['ignore', 'pipe', 'pipe']
});
let stderr = '';
let spawnError = null;
child.on('error', error => {
spawnError = error;
});
child.stderr.on('data', data => {
stderr += data.toString();
});
let readyPort = null;
for (let attempt = 0; attempt < 40 && readyPort === null; attempt += 1) {
for (let port = 6412; port <= 6422; port += 1) {
try {
const response = await fetch(`http://127.0.0.1:${port}/ping`);
if (response.headers.get('x-firelink-server') === '1') {
readyPort = port;
break;
}
} catch {}
}
if (readyPort === null) {
await new Promise(resolve => setTimeout(resolve, 250));
}
}
if (process.platform === 'win32') {
spawn('taskkill', ['/pid', String(child.pid), '/t', '/f'], { stdio: 'ignore' });
} else {
try {
process.kill(-child.pid, 'SIGTERM');
} catch {
child.kill('SIGTERM');
}
}
if (readyPort === null) {
const detail = spawnError?.message || stderr.slice(-1000);
console.error(`Packaged Firelink did not expose extension server. ${detail}`);
process.exit(1);
}
console.log(`Packaged Firelink smoke passed on 127.0.0.1:${readyPort}`);
+163
View File
@@ -0,0 +1,163 @@
#!/usr/bin/env node
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import crypto from 'node:crypto';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const repoRoot = path.resolve(__dirname, '..');
const binariesRoot = path.join(repoRoot, 'src-tauri', 'binaries');
const outputRoot = path.join(repoRoot, 'src-tauri', 'engine-dist');
const lock = JSON.parse(fs.readFileSync(path.join(repoRoot, 'engines.lock.json'), 'utf8'));
const archMap = { x64: 'x86_64', arm64: 'aarch64' };
const platformMap = {
darwin: 'apple-darwin',
win32: 'pc-windows-msvc',
linux: 'unknown-linux-gnu',
};
function argValue(name) {
const index = process.argv.indexOf(name);
return index >= 0 ? process.argv[index + 1] : undefined;
}
const hostTarget = `${archMap[os.arch()]}-${platformMap[os.platform()]}`;
const target = argValue('--target')
|| process.env.TAURI_ENV_TARGET_TRIPLE
|| process.env.FIRELINK_TARGET_TRIPLE
|| hostTarget;
const isWindowsTarget = target.includes('windows');
const suffix = isWindowsTarget ? '.exe' : '';
const engines = ['yt-dlp', 'aria2c', 'ffmpeg', 'deno'];
const expectedNames = engines.map(engine => `${engine}-${target}${suffix}`);
const targetLock = lock.targets?.[target];
const configuredSource = process.env.FIRELINK_ENGINE_SOURCE_DIR
? path.resolve(process.env.FIRELINK_ENGINE_SOURCE_DIR)
: null;
const canonicalSource = path.join(binariesRoot, target);
const provisionedSource = path.join(repoRoot, 'src-tauri', 'provisioned-engines', target);
const legacyMacSource = target.endsWith('apple-darwin') ? binariesRoot : null;
const source = [configuredSource, canonicalSource, provisionedSource, legacyMacSource]
.filter(Boolean)
.find(candidate => expectedNames.every(name => fs.existsSync(path.join(candidate, name))));
if (!source) {
console.error(`No complete engine payload found for ${target}.`);
console.error(`Expected source directory: ${canonicalSource}`);
console.error(`Expected files: ${expectedNames.join(', ')}`);
process.exit(1);
}
function sha256(file) {
return crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex');
}
function treeDigest(root) {
const files = [];
const walk = directory => {
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
const file = path.join(directory, entry.name);
if (entry.isDirectory()) walk(file);
else if (entry.isFile()) files.push(file);
}
};
walk(root);
files.sort((left, right) => left.localeCompare(right));
const digest = crypto.createHash('sha256');
for (const file of files) {
const relative = path.relative(root, file).split(path.sep).join('/');
digest.update(`${relative}\0${sha256(file)}\n`);
}
return { files: files.length, sha256: digest.digest('hex') };
}
if (targetLock) {
for (const engine of engines) {
const name = `${engine}-${target}${suffix}`;
const expected = targetLock.engines?.[engine]?.sha256;
const actual = sha256(path.join(source, name));
if (!expected || actual !== expected) {
console.error(`Checksum mismatch for ${name}. Expected ${expected || 'missing lock'}, got ${actual}.`);
process.exit(1);
}
}
for (const [runtimeDir, expected] of Object.entries(targetLock.runtimeTrees || {})) {
const sourceDir = path.join(source, runtimeDir);
if (!fs.existsSync(sourceDir)) {
console.error(`Missing locked runtime directory ${runtimeDir} for ${target}.`);
process.exit(1);
}
const actual = treeDigest(sourceDir);
if (actual.files !== expected.files || actual.sha256 !== expected.sha256) {
console.error(`Runtime checksum mismatch for ${runtimeDir}.`);
process.exit(1);
}
}
} else {
const manifestPath = path.join(source, 'payload-manifest.json');
if (!fs.existsSync(manifestPath)) {
console.error(`No committed lock or payload manifest exists for ${target}.`);
process.exit(1);
}
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
if (manifest.target !== target) {
console.error(`Payload manifest target mismatch: ${manifest.target}`);
process.exit(1);
}
for (const [relative, expected] of Object.entries(manifest.files || {})) {
const file = path.join(source, relative);
if (!fs.existsSync(file) || sha256(file) !== expected) {
console.error(`Payload manifest mismatch: ${relative}`);
process.exit(1);
}
}
const actualFiles = [];
const walk = directory => {
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
const file = path.join(directory, entry.name);
if (entry.isDirectory()) walk(file);
else if (entry.isFile() && entry.name !== 'payload-manifest.json') {
actualFiles.push(path.relative(source, file).split(path.sep).join('/'));
}
}
};
walk(source);
const expectedFiles = Object.keys(manifest.files || {}).sort();
actualFiles.sort();
if (JSON.stringify(actualFiles) !== JSON.stringify(expectedFiles)) {
console.error(`Payload contains files not covered by manifest for ${target}.`);
process.exit(1);
}
}
const destination = path.join(outputRoot, target);
fs.rmSync(outputRoot, { recursive: true, force: true });
fs.mkdirSync(destination, { recursive: true });
for (const name of expectedNames) {
fs.copyFileSync(path.join(source, name), path.join(destination, name));
if (!isWindowsTarget) {
fs.chmodSync(path.join(destination, name), 0o755);
}
}
for (const runtimeDir of ['_internal', 'aria2-libs']) {
const sourceDir = path.join(source, runtimeDir);
if (fs.existsSync(sourceDir)) {
fs.cpSync(sourceDir, path.join(destination, runtimeDir), {
recursive: true,
dereference: false,
preserveTimestamps: true,
});
}
}
const payloadManifest = path.join(source, 'payload-manifest.json');
if (fs.existsSync(payloadManifest)) {
fs.copyFileSync(payloadManifest, path.join(destination, 'payload-manifest.json'));
}
console.log(`Staged Firelink engines for ${target} from ${source}`);
+88 -12
View File
@@ -8,6 +8,11 @@ import { fileURLToPath } from 'node:url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
function argValue(name) {
const index = process.argv.indexOf(name);
return index >= 0 ? process.argv[index + 1] : undefined;
}
const archMap = { x64: 'x86_64', arm64: 'aarch64' };
const platformMap = {
darwin: 'apple-darwin',
@@ -23,14 +28,47 @@ if (!currentArch || !currentPlatform) {
process.exit(1);
}
const targetTriple = `${currentArch}-${currentPlatform}`;
const isWindows = os.platform() === 'win32';
const isMacOS = os.platform() === 'darwin';
const targetTriple = argValue('--target')
|| process.env.FIRELINK_TARGET_TRIPLE
|| `${currentArch}-${currentPlatform}`;
const hostTriple = `${currentArch}-${currentPlatform}`;
const canExecuteTarget = targetTriple === hostTriple;
const isWindows = targetTriple.includes('windows');
const isMacOS = targetTriple.includes('apple-darwin');
const isLinux = targetTriple.includes('linux');
const ext = isWindows ? '.exe' : '';
const suffix = `-${targetTriple}${ext}`;
const scriptsDir = __dirname;
const binariesDir = path.join(scriptsDir, '..', 'src-tauri', 'binaries');
const searchRoot = argValue('--search-root');
function findEngineRoot(root) {
const expected = `yt-dlp-${targetTriple}${ext}`;
const matches = [];
const walk = directory => {
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
const candidate = path.join(directory, entry.name);
if (entry.isDirectory()) {
if (fs.existsSync(path.join(candidate, expected))) matches.push(candidate);
walk(candidate);
}
}
};
walk(path.resolve(root));
if (matches.length !== 1) {
throw new Error(`Expected exactly one packaged engine root under ${root}, found ${matches.length}`);
}
return matches[0];
}
const configuredRoot = argValue('--root')
|| (process.argv.includes('--staged')
? path.join(scriptsDir, '..', 'src-tauri', 'engine-dist', targetTriple)
: searchRoot
? findEngineRoot(searchRoot)
: null);
const binariesDir = configuredRoot
? path.resolve(configuredRoot)
: path.join(scriptsDir, '..', 'src-tauri', 'binaries');
const requiredEngines = ['yt-dlp', 'aria2c', 'ffmpeg', 'deno'];
const FORBIDDEN_OTOOL_PATHS = ['/opt/homebrew', '/usr/local/Cellar'];
@@ -79,6 +117,10 @@ if (exitCode !== 0) {
// ───── Check 2: Executable permission ─────
console.log('\n─── 2. Executable permission ───');
for (const eng of requiredEngines) {
if (isWindows) {
ok(`Executable permission is represented by PE format on Windows: ${binName(eng)}`);
continue;
}
try {
fs.accessSync(binPath(eng), fs.constants.X_OK);
ok(`Executable ${binName(eng)}`);
@@ -90,6 +132,19 @@ for (const eng of requiredEngines) {
// ───── Check 3: file(1) identification ─────
console.log('\n─── 3. file(1) identification ───');
for (const eng of requiredEngines) {
if (os.platform() === 'win32') {
try {
const header = fs.readFileSync(binPath(eng)).subarray(0, 2).toString('ascii');
if (header === 'MZ') {
ok(`${binName(eng)}: Windows PE executable`);
} else {
fail(`${binName(eng)}: missing Windows PE MZ header`);
}
} catch (e) {
fail(`identify ${binName(eng)}: ${e.message}`);
}
continue;
}
try {
const out = execFileSync('file', ['--brief', binPath(eng)], {
encoding: 'utf-8',
@@ -172,7 +227,6 @@ console.log('\n─── 6. yt-dlp packaging ───');
}
const requiredRuntimeFiles = [
path.join(internalDir, 'Python'),
path.join(internalDir, 'yt_dlp_ejs', 'yt', 'solver', 'core.min.js'),
path.join(internalDir, 'yt_dlp_ejs', 'yt', 'solver', 'lib.min.js'),
];
@@ -183,6 +237,16 @@ console.log('\n─── 6. yt-dlp packaging ───');
fail(`Missing yt-dlp runtime component: ${path.relative(binariesDir, required)}`);
}
}
const runtimeCandidates = isWindows
? fs.readdirSync(internalDir).filter(name => /^python.*\.dll$/i.test(name))
: isLinux
? fs.readdirSync(internalDir).filter(name => /^libpython.*\.so/i.test(name) || name === 'Python')
: ['Python', 'Python.framework'].filter(name => fs.existsSync(path.join(internalDir, name)));
if (runtimeCandidates.length > 0) {
ok(`yt-dlp embedded Python runtime: ${runtimeCandidates[0]}`);
} else {
fail(`Missing embedded Python runtime in ${internalDir}`);
}
} else {
fail('yt-dlp must use the self-contained onedir distribution; onefile adds ~17 seconds to every launch');
}
@@ -222,14 +286,26 @@ function runEngine(label, engine, args, timeout = 30000) {
}
}
runEngine('yt-dlp cold start', 'yt-dlp', ['--version'], 20000);
runEngine('yt-dlp warm start', 'yt-dlp', ['--version'], 8000);
runEngine('ffmpeg', 'ffmpeg', ['-version']);
runEngine('deno', 'deno', ['--version']);
runEngine('aria2c', 'aria2c', ['--version']);
const coldStartTimeout = isMacOS ? 120000 : 30000;
if (canExecuteTarget) {
runEngine('yt-dlp cold start', 'yt-dlp', ['--version'], coldStartTimeout);
runEngine('yt-dlp warm start', 'yt-dlp', ['--version'], 8000);
runEngine('ffmpeg cold start', 'ffmpeg', ['-version'], coldStartTimeout);
runEngine('deno cold start', 'deno', ['--version'], coldStartTimeout);
runEngine('aria2c cold start', 'aria2c', ['--version'], coldStartTimeout);
if (isMacOS) {
// Unsigned binaries can incur a one-time macOS provenance scan after copying.
// Warm checks enforce engine startup performance after that OS validation.
runEngine('ffmpeg warm start', 'ffmpeg', ['-version'], 8000);
runEngine('deno warm start', 'deno', ['--version'], 8000);
runEngine('aria2c warm start', 'aria2c', ['--version'], 8000);
}
} else {
console.log(` Runtime tests skipped on ${hostTriple}; native ${targetTriple} CI runs them.`);
}
// ───── aria2 RPC smoke test (macOS only) ─────
if (isMacOS) {
// ───── aria2 RPC smoke test (native target only) ─────
if (canExecuteTarget) {
console.log('\n─── aria2 RPC smoke test ───');
await (async function testAria2Rpc() {
const p = binPath('aria2c');