mirror of
https://github.com/nimbold/Firelink.git
synced 2026-09-09 09:16:57 +00:00
fix(ci): optimize engine build times and add granular Aria2 caching
- replace hardcoded make -j2 with dynamic CPU core detection in aria2 build.sh - isolate Aria2 source build cache from prebuilt engine downloads - add atomic cache validation, restoration, and promotion in scripts/engine-aria2-cache.js - add Rust dependency and target caching with Swatinem/rust-cache@v2 - raise Desktop checks timeout to 45 minutes to prevent job aborts - harden temporary directory cleanup on Windows process abort
This commit is contained in:
@@ -1 +1,2 @@
|
||||
scripts/aria2/firelink.patch text eol=lf
|
||||
scripts/aria2/build.sh text eol=lf
|
||||
|
||||
@@ -39,7 +39,7 @@ jobs:
|
||||
|
||||
desktop:
|
||||
name: Desktop checks (${{ matrix.target }})
|
||||
timeout-minutes: 30
|
||||
timeout-minutes: 45
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -62,6 +62,10 @@ jobs:
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
- name: Cache Rust dependencies and build targets
|
||||
uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
workspaces: src-tauri -> target
|
||||
- name: Install Linux dependencies
|
||||
if: runner.os == 'Linux'
|
||||
run: |
|
||||
@@ -169,6 +173,16 @@ jobs:
|
||||
run: |
|
||||
node scripts/stage-engines.js
|
||||
node scripts/verify-binaries.js --staged
|
||||
- name: Restore verified Aria2 build cache
|
||||
id: aria2-cache
|
||||
if: >-
|
||||
runner.os != 'macOS' &&
|
||||
(steps.engine-cache.outputs.cache-hit != 'true' ||
|
||||
steps.engine-cache-validation.outcome != 'success')
|
||||
uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57
|
||||
with:
|
||||
path: src-tauri/provisioned-engines/.aria2-cache/${{ matrix.target }}
|
||||
key: firelink-aria2-build-v1-${{ matrix.target }}-${{ steps.engine-toolchain.outputs.aria2-fingerprint }}
|
||||
- name: Provision locked engines
|
||||
if: >-
|
||||
runner.os != 'macOS' &&
|
||||
@@ -176,6 +190,7 @@ jobs:
|
||||
steps.engine-cache-validation.outcome != 'success')
|
||||
env:
|
||||
FIRELINK_MSYS2_ROOT: ${{ steps.aria2-msys.outputs.msys2-location }}
|
||||
FIRELINK_TOOLCHAIN_FINGERPRINT: ${{ steps.engine-toolchain.outputs.fingerprint }}
|
||||
run: node scripts/provision-engines.js --target ${{ matrix.target }}
|
||||
- name: Stage and verify engines
|
||||
env:
|
||||
@@ -195,6 +210,16 @@ jobs:
|
||||
env:
|
||||
FIRELINK_ENGINE_OUTPUT_ROOT: ${{ runner.temp }}/firelink-engine-workspace/${{ matrix.target }}/engine-dist
|
||||
run: node scripts/smoke-aria2-transfers.js
|
||||
- name: Save verified Aria2 build cache
|
||||
if: >-
|
||||
runner.os != 'macOS' &&
|
||||
github.event_name == 'push' &&
|
||||
github.ref == 'refs/heads/main' &&
|
||||
steps.aria2-cache.outputs.cache-hit != 'true'
|
||||
uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57
|
||||
with:
|
||||
path: src-tauri/provisioned-engines/.aria2-cache/${{ matrix.target }}
|
||||
key: firelink-aria2-build-v1-${{ matrix.target }}-${{ steps.engine-toolchain.outputs.aria2-fingerprint }}
|
||||
- name: Save verified engine payload cache
|
||||
if: >-
|
||||
runner.os != 'macOS' &&
|
||||
|
||||
@@ -141,6 +141,16 @@ jobs:
|
||||
run: |
|
||||
node scripts/stage-engines.js
|
||||
node scripts/verify-binaries.js --staged
|
||||
- name: Restore verified Aria2 build cache
|
||||
id: aria2-cache
|
||||
if: >-
|
||||
runner.os != 'macOS' &&
|
||||
(steps.engine-cache.outputs.cache-hit != 'true' ||
|
||||
steps.engine-cache-validation.outcome != 'success')
|
||||
uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57
|
||||
with:
|
||||
path: src-tauri/provisioned-engines/.aria2-cache/${{ matrix.target }}
|
||||
key: firelink-aria2-build-v1-${{ matrix.target }}-${{ steps.engine-toolchain.outputs.aria2-fingerprint }}
|
||||
- name: Provision locked engines
|
||||
if: >-
|
||||
runner.os != 'macOS' &&
|
||||
@@ -148,6 +158,7 @@ jobs:
|
||||
steps.engine-cache-validation.outcome != 'success')
|
||||
env:
|
||||
FIRELINK_MSYS2_ROOT: ${{ steps.aria2-msys.outputs.msys2-location }}
|
||||
FIRELINK_TOOLCHAIN_FINGERPRINT: ${{ steps.engine-toolchain.outputs.fingerprint }}
|
||||
run: node scripts/provision-engines.js --target ${{ matrix.target }}
|
||||
- name: Build package
|
||||
if: runner.os != 'Linux'
|
||||
|
||||
+15
-1
@@ -45,7 +45,21 @@ export PKG_CONFIG="pkg-config --static"
|
||||
--without-gnutls --with-openssl --without-libxml2 --with-libexpat \
|
||||
--without-libgmp --without-libnettle --without-libgcrypt \
|
||||
--with-libssh2 --with-libcares
|
||||
make -j2
|
||||
if command -v nproc >/dev/null 2>&1; then
|
||||
JOBS="$(nproc 2>/dev/null || echo 4)"
|
||||
elif [[ -n "${NUMBER_OF_PROCESSORS:-}" ]]; then
|
||||
JOBS="$NUMBER_OF_PROCESSORS"
|
||||
elif command -v sysctl >/dev/null 2>&1; then
|
||||
JOBS="$(sysctl -n hw.ncpu 2>/dev/null || echo 4)"
|
||||
else
|
||||
JOBS=4
|
||||
fi
|
||||
JOBS="${JOBS//$'\r'/}"
|
||||
JOBS="${JOBS// /}"
|
||||
if ! [[ "$JOBS" =~ ^[1-9][0-9]*$ ]]; then
|
||||
JOBS=4
|
||||
fi
|
||||
make -j"$JOBS"
|
||||
|
||||
if command -v cygpath >/dev/null 2>&1; then
|
||||
command -v objdump >/dev/null 2>&1 || {
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
import crypto from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { collectRegularFiles, sha256 } from './engine-payload-integrity.js';
|
||||
import { promoteDirectory, removePathWithRetry } from './engine-payload-promotion.js';
|
||||
|
||||
export function getAria2BuildScriptSha256(repoRoot) {
|
||||
const buildScriptPath = path.join(repoRoot, 'scripts/aria2/build.sh');
|
||||
return crypto.createHash('sha256')
|
||||
.update(fs.readFileSync(buildScriptPath, 'utf8').replaceAll('\r\n', '\n'))
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
export function validateAria2Cache({
|
||||
aria2CacheRoot,
|
||||
target,
|
||||
aria2Source,
|
||||
buildScriptSha256,
|
||||
toolchainFingerprint = null,
|
||||
executableSuffix = '',
|
||||
}) {
|
||||
const manifestPath = path.join(aria2CacheRoot, 'aria2-build-manifest.json');
|
||||
if (!fs.existsSync(manifestPath)) {
|
||||
return { valid: false, reason: 'manifest-not-found' };
|
||||
}
|
||||
|
||||
try {
|
||||
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
|
||||
if (
|
||||
manifest.schemaVersion !== 1
|
||||
|| manifest.target !== target
|
||||
|| manifest.sourceSha256 !== aria2Source.sha256
|
||||
|| manifest.patchSha256 !== aria2Source.patchSha256
|
||||
|| manifest.buildScriptSha256 !== buildScriptSha256
|
||||
) {
|
||||
return { valid: false, reason: 'manifest-metadata-mismatch' };
|
||||
}
|
||||
|
||||
if (
|
||||
toolchainFingerprint !== null
|
||||
&& manifest.toolchainFingerprint !== toolchainFingerprint
|
||||
) {
|
||||
return { valid: false, reason: 'toolchain-fingerprint-mismatch' };
|
||||
}
|
||||
|
||||
const exeName = `aria2c-${target}${executableSuffix}`;
|
||||
const cachedExe = path.join(aria2CacheRoot, exeName);
|
||||
if (!fs.existsSync(cachedExe) || sha256(cachedExe) !== manifest.files?.[exeName]) {
|
||||
return { valid: false, reason: 'executable-mismatch' };
|
||||
}
|
||||
|
||||
if (manifest.files) {
|
||||
for (const [rel, expectedSha] of Object.entries(manifest.files)) {
|
||||
if (rel === exeName) continue;
|
||||
const libFile = path.join(aria2CacheRoot, rel);
|
||||
if (!fs.existsSync(libFile) || sha256(libFile) !== expectedSha) {
|
||||
return { valid: false, reason: 'library-mismatch' };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const actualFiles = collectRegularFiles(aria2CacheRoot, {
|
||||
ignoredNames: ['aria2-build-manifest.json'],
|
||||
}).map(f => path.relative(aria2CacheRoot, f).split(path.sep).join('/'));
|
||||
const expectedFiles = Object.keys(manifest.files || {}).sort();
|
||||
actualFiles.sort();
|
||||
if (JSON.stringify(actualFiles) !== JSON.stringify(expectedFiles)) {
|
||||
return { valid: false, reason: 'file-list-mismatch' };
|
||||
}
|
||||
|
||||
return { valid: true, manifest };
|
||||
} catch {
|
||||
return { valid: false, reason: 'manifest-corrupted' };
|
||||
}
|
||||
}
|
||||
|
||||
export function restoreAria2Cache({
|
||||
aria2CacheRoot,
|
||||
payloadDestination,
|
||||
target,
|
||||
executableSuffix = '',
|
||||
isWindows = false,
|
||||
}) {
|
||||
const exeName = `aria2c-${target}${executableSuffix}`;
|
||||
const cachedExe = path.join(aria2CacheRoot, exeName);
|
||||
const targetExe = path.join(payloadDestination, exeName);
|
||||
fs.copyFileSync(cachedExe, targetExe);
|
||||
if (!isWindows) fs.chmodSync(targetExe, 0o755);
|
||||
|
||||
const cachedLibs = path.join(aria2CacheRoot, 'aria2-libs');
|
||||
if (fs.existsSync(cachedLibs)) {
|
||||
fs.cpSync(cachedLibs, path.join(payloadDestination, 'aria2-libs'), {
|
||||
recursive: true,
|
||||
preserveTimestamps: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function saveAria2Cache({
|
||||
aria2CacheRoot,
|
||||
payloadDestination,
|
||||
target,
|
||||
aria2Source,
|
||||
buildScriptSha256,
|
||||
toolchainFingerprint = null,
|
||||
executableSuffix = '',
|
||||
aria2Runtime = null,
|
||||
isWindows = false,
|
||||
}) {
|
||||
const cacheParent = path.dirname(aria2CacheRoot);
|
||||
fs.mkdirSync(cacheParent, { recursive: true });
|
||||
|
||||
const stagingDir = fs.mkdtempSync(
|
||||
path.join(cacheParent, `.${path.basename(aria2CacheRoot)}-staging-${process.pid}-`)
|
||||
);
|
||||
|
||||
try {
|
||||
const targetExeName = `aria2c-${target}${executableSuffix}`;
|
||||
const cachedExeDest = path.join(stagingDir, targetExeName);
|
||||
fs.copyFileSync(path.join(payloadDestination, targetExeName), cachedExeDest);
|
||||
if (!isWindows) fs.chmodSync(cachedExeDest, 0o755);
|
||||
|
||||
const manifestFiles = {
|
||||
[targetExeName]: sha256(cachedExeDest),
|
||||
};
|
||||
|
||||
if (aria2Runtime && fs.existsSync(aria2Runtime)) {
|
||||
const cachedLibsDest = path.join(stagingDir, 'aria2-libs');
|
||||
fs.cpSync(aria2Runtime, cachedLibsDest, { recursive: true, preserveTimestamps: true });
|
||||
const libFiles = collectRegularFiles(cachedLibsDest);
|
||||
for (const lib of libFiles) {
|
||||
const rel = path.relative(stagingDir, lib).split(path.sep).join('/');
|
||||
manifestFiles[rel] = sha256(lib);
|
||||
}
|
||||
}
|
||||
|
||||
const cacheManifest = {
|
||||
schemaVersion: 1,
|
||||
target,
|
||||
sourceSha256: aria2Source.sha256,
|
||||
patchSha256: aria2Source.patchSha256,
|
||||
buildScriptSha256,
|
||||
toolchainFingerprint: toolchainFingerprint || null,
|
||||
files: manifestFiles,
|
||||
};
|
||||
|
||||
fs.writeFileSync(
|
||||
path.join(stagingDir, 'aria2-build-manifest.json'),
|
||||
`${JSON.stringify(cacheManifest, null, 2)}\n`
|
||||
);
|
||||
|
||||
await promoteDirectory(stagingDir, aria2CacheRoot);
|
||||
} catch (error) {
|
||||
try {
|
||||
await removePathWithRetry(stagingDir);
|
||||
} catch {}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
|
||||
import {
|
||||
getAria2BuildScriptSha256,
|
||||
restoreAria2Cache,
|
||||
saveAria2Cache,
|
||||
validateAria2Cache,
|
||||
} from './engine-aria2-cache.js';
|
||||
import { sha256 } from './engine-payload-integrity.js';
|
||||
|
||||
const TARGET = 'x86_64-pc-windows-msvc';
|
||||
const ARIA2_SOURCE = {
|
||||
version: '1.37.0-firelink-native-dns-v1',
|
||||
url: 'https://example.invalid/aria2.tar.xz',
|
||||
sha256: 'a'.repeat(64),
|
||||
buildFromSource: true,
|
||||
patch: 'scripts/aria2/firelink.patch',
|
||||
patchSha256: 'b'.repeat(64),
|
||||
allocationTelemetry: true,
|
||||
};
|
||||
|
||||
function createTestWorkspace() {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'firelink-aria2-cache-test-'));
|
||||
const payloadDir = path.join(root, 'payload');
|
||||
const cacheRoot = path.join(root, 'cache', TARGET);
|
||||
fs.mkdirSync(payloadDir, { recursive: true });
|
||||
|
||||
const exeName = `aria2c-${TARGET}.exe`;
|
||||
const exePath = path.join(payloadDir, exeName);
|
||||
fs.writeFileSync(exePath, 'binary-content-for-testing');
|
||||
|
||||
const libsDir = path.join(payloadDir, 'aria2-libs');
|
||||
fs.mkdirSync(libsDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(libsDir, 'test.dll'), 'dll-content');
|
||||
|
||||
return { root, payloadDir, cacheRoot, exeName, exePath, libsDir };
|
||||
}
|
||||
|
||||
test('validateAria2Cache fails closed when cache manifest is missing or invalid', () => {
|
||||
const { root, cacheRoot } = createTestWorkspace();
|
||||
try {
|
||||
const missing = validateAria2Cache({
|
||||
aria2CacheRoot: cacheRoot,
|
||||
target: TARGET,
|
||||
aria2Source: ARIA2_SOURCE,
|
||||
buildScriptSha256: 'c'.repeat(64),
|
||||
executableSuffix: '.exe',
|
||||
});
|
||||
assert.equal(missing.valid, false);
|
||||
assert.equal(missing.reason, 'manifest-not-found');
|
||||
|
||||
fs.mkdirSync(cacheRoot, { recursive: true });
|
||||
fs.writeFileSync(path.join(cacheRoot, 'aria2-build-manifest.json'), 'not json');
|
||||
const corrupt = validateAria2Cache({
|
||||
aria2CacheRoot: cacheRoot,
|
||||
target: TARGET,
|
||||
aria2Source: ARIA2_SOURCE,
|
||||
buildScriptSha256: 'c'.repeat(64),
|
||||
executableSuffix: '.exe',
|
||||
});
|
||||
assert.equal(corrupt.valid, false);
|
||||
assert.equal(corrupt.reason, 'manifest-corrupted');
|
||||
} finally {
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('saveAria2Cache, validateAria2Cache, and restoreAria2Cache work end-to-end', async () => {
|
||||
const { root, payloadDir, cacheRoot, libsDir } = createTestWorkspace();
|
||||
try {
|
||||
const buildScriptSha = 'c'.repeat(64);
|
||||
const toolchainFingerprint = 'toolchain-v1';
|
||||
|
||||
await saveAria2Cache({
|
||||
aria2CacheRoot: cacheRoot,
|
||||
payloadDestination: payloadDir,
|
||||
target: TARGET,
|
||||
aria2Source: ARIA2_SOURCE,
|
||||
buildScriptSha256: buildScriptSha,
|
||||
toolchainFingerprint,
|
||||
executableSuffix: '.exe',
|
||||
aria2Runtime: libsDir,
|
||||
isWindows: true,
|
||||
});
|
||||
|
||||
const valid = validateAria2Cache({
|
||||
aria2CacheRoot: cacheRoot,
|
||||
target: TARGET,
|
||||
aria2Source: ARIA2_SOURCE,
|
||||
buildScriptSha256: buildScriptSha,
|
||||
toolchainFingerprint,
|
||||
executableSuffix: '.exe',
|
||||
});
|
||||
assert.equal(valid.valid, true);
|
||||
|
||||
const wrongFingerprint = validateAria2Cache({
|
||||
aria2CacheRoot: cacheRoot,
|
||||
target: TARGET,
|
||||
aria2Source: ARIA2_SOURCE,
|
||||
buildScriptSha256: buildScriptSha,
|
||||
toolchainFingerprint: 'toolchain-v2',
|
||||
executableSuffix: '.exe',
|
||||
});
|
||||
assert.equal(wrongFingerprint.valid, false);
|
||||
assert.equal(wrongFingerprint.reason, 'toolchain-fingerprint-mismatch');
|
||||
|
||||
const restoreDir = path.join(root, 'restored');
|
||||
fs.mkdirSync(restoreDir, { recursive: true });
|
||||
restoreAria2Cache({
|
||||
aria2CacheRoot: cacheRoot,
|
||||
payloadDestination: restoreDir,
|
||||
target: TARGET,
|
||||
executableSuffix: '.exe',
|
||||
isWindows: true,
|
||||
});
|
||||
|
||||
assert.equal(
|
||||
sha256(path.join(restoreDir, `aria2c-${TARGET}.exe`)),
|
||||
sha256(path.join(payloadDir, `aria2c-${TARGET}.exe`))
|
||||
);
|
||||
assert.equal(
|
||||
sha256(path.join(restoreDir, 'aria2-libs', 'test.dll')),
|
||||
sha256(path.join(payloadDir, 'aria2-libs', 'test.dll'))
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('validateAria2Cache rejects tampered files or rogue untracked files', async () => {
|
||||
const { root, payloadDir, cacheRoot, libsDir } = createTestWorkspace();
|
||||
try {
|
||||
const buildScriptSha = 'c'.repeat(64);
|
||||
await saveAria2Cache({
|
||||
aria2CacheRoot: cacheRoot,
|
||||
payloadDestination: payloadDir,
|
||||
target: TARGET,
|
||||
aria2Source: ARIA2_SOURCE,
|
||||
buildScriptSha256: buildScriptSha,
|
||||
executableSuffix: '.exe',
|
||||
aria2Runtime: libsDir,
|
||||
isWindows: true,
|
||||
});
|
||||
|
||||
// Tamper with cached exe
|
||||
const exePath = path.join(cacheRoot, `aria2c-${TARGET}.exe`);
|
||||
fs.appendFileSync(exePath, 'tampered');
|
||||
const tamperedExe = validateAria2Cache({
|
||||
aria2CacheRoot: cacheRoot,
|
||||
target: TARGET,
|
||||
aria2Source: ARIA2_SOURCE,
|
||||
buildScriptSha256: buildScriptSha,
|
||||
executableSuffix: '.exe',
|
||||
});
|
||||
assert.equal(tamperedExe.valid, false);
|
||||
assert.equal(tamperedExe.reason, 'executable-mismatch');
|
||||
|
||||
// Restore exe, tamper with library
|
||||
fs.writeFileSync(exePath, 'binary-content-for-testing');
|
||||
const libPath = path.join(cacheRoot, 'aria2-libs', 'test.dll');
|
||||
fs.appendFileSync(libPath, 'tampered');
|
||||
const tamperedLib = validateAria2Cache({
|
||||
aria2CacheRoot: cacheRoot,
|
||||
target: TARGET,
|
||||
aria2Source: ARIA2_SOURCE,
|
||||
buildScriptSha256: buildScriptSha,
|
||||
executableSuffix: '.exe',
|
||||
});
|
||||
assert.equal(tamperedLib.valid, false);
|
||||
assert.equal(tamperedLib.reason, 'library-mismatch');
|
||||
|
||||
// Restore library, add rogue file
|
||||
fs.writeFileSync(libPath, 'dll-content');
|
||||
fs.writeFileSync(path.join(cacheRoot, 'rogue.txt'), 'rogue');
|
||||
const rogueFile = validateAria2Cache({
|
||||
aria2CacheRoot: cacheRoot,
|
||||
target: TARGET,
|
||||
aria2Source: ARIA2_SOURCE,
|
||||
buildScriptSha256: buildScriptSha,
|
||||
executableSuffix: '.exe',
|
||||
});
|
||||
assert.equal(rogueFile.valid, false);
|
||||
assert.equal(rogueFile.reason, 'file-list-mismatch');
|
||||
} finally {
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('getAria2BuildScriptSha256 reads and hashes build.sh with normalized line endings', () => {
|
||||
const repoRoot = path.resolve(import.meta.dirname, '..');
|
||||
const hash = getAria2BuildScriptSha256(repoRoot);
|
||||
assert.equal(typeof hash, 'string');
|
||||
assert.equal(hash.length, 64);
|
||||
});
|
||||
@@ -43,3 +43,13 @@ test('only trusted main pushes save the shared engine cache', () => {
|
||||
assert.match(save, /github\.ref == 'refs\/heads\/main'/);
|
||||
assert.doesNotMatch(releaseWorkflow, /actions\/cache\/save@/);
|
||||
});
|
||||
|
||||
test('CI and release use granular Aria2 build caching and safe timeouts', () => {
|
||||
assert.match(ciWorkflow, /timeout-minutes: (?:4[5-9]|[5-9][0-9])/);
|
||||
assert.match(ciWorkflow, /uses: Swatinem\/rust-cache@v2/);
|
||||
assert.match(ciWorkflow, /key: firelink-aria2-build-v1-\$\{\{ matrix\.target \}\}-\$\{\{ steps\.engine-toolchain\.outputs\.aria2-fingerprint \}\}/);
|
||||
assert.match(releaseWorkflow, /key: firelink-aria2-build-v1-\$\{\{ matrix\.target \}\}-\$\{\{ steps\.engine-toolchain\.outputs\.aria2-fingerprint \}\}/);
|
||||
const saveAria2 = ciWorkflow.slice(ciWorkflow.indexOf('- name: Save verified Aria2 build cache'));
|
||||
assert.match(saveAria2, /github\.event_name == 'push'/);
|
||||
assert.match(saveAria2, /github\.ref == 'refs\/heads\/main'/);
|
||||
});
|
||||
|
||||
@@ -4,6 +4,8 @@ import crypto from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { getAria2BuildScriptSha256 } from './engine-aria2-cache.js';
|
||||
|
||||
const WINDOWS_PACKAGES = [
|
||||
'autoconf',
|
||||
@@ -99,6 +101,42 @@ function main() {
|
||||
|
||||
const fingerprint = crypto.createHash('sha256').update(records.join('\n')).digest('hex');
|
||||
writeOutput('fingerprint', fingerprint);
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const sourceLockPath = path.join(repoRoot, 'engine-sources.lock.json');
|
||||
if (fs.existsSync(sourceLockPath)) {
|
||||
const sourceLock = JSON.parse(fs.readFileSync(sourceLockPath, 'utf8'));
|
||||
const aria2Source = sourceLock.targets?.[target]?.aria2c;
|
||||
if (aria2Source) {
|
||||
const canonicalize = val => {
|
||||
if (Array.isArray(val)) return val.map(canonicalize);
|
||||
if (val && typeof val === 'object') {
|
||||
return Object.fromEntries(
|
||||
Object.keys(val).sort().map(k => [k, canonicalize(val[k])])
|
||||
);
|
||||
}
|
||||
return val;
|
||||
};
|
||||
|
||||
const patchPath = path.join(repoRoot, aria2Source.patch || 'scripts/aria2/firelink.patch');
|
||||
const buildShPath = path.join(repoRoot, 'scripts/aria2/build.sh');
|
||||
const patchSha = fs.existsSync(patchPath)
|
||||
? crypto.createHash('sha256').update(fs.readFileSync(patchPath, 'utf8').replaceAll('\r\n', '\n')).digest('hex')
|
||||
: (aria2Source.patchSha256 || '');
|
||||
const buildShSha = fs.existsSync(buildShPath)
|
||||
? getAria2BuildScriptSha256(repoRoot)
|
||||
: '';
|
||||
|
||||
const aria2Records = [
|
||||
...records,
|
||||
`aria2-source=${JSON.stringify(canonicalize(aria2Source))}`,
|
||||
`aria2-patch-sha256=${patchSha}`,
|
||||
`aria2-build-sh-sha256=${buildShSha}`,
|
||||
];
|
||||
const aria2Fingerprint = crypto.createHash('sha256').update(aria2Records.join('\n')).digest('hex');
|
||||
writeOutput('aria2-fingerprint', aria2Fingerprint);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
|
||||
@@ -14,6 +14,12 @@ import {
|
||||
removePathWithRetry,
|
||||
} from './engine-payload-promotion.js';
|
||||
import { assertAria2RouteSource } from './aria2-route-contract.js';
|
||||
import {
|
||||
getAria2BuildScriptSha256,
|
||||
restoreAria2Cache,
|
||||
saveAria2Cache,
|
||||
validateAria2Cache,
|
||||
} from './engine-aria2-cache.js';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const repoRoot = path.resolve(__dirname, '..');
|
||||
@@ -182,32 +188,75 @@ try {
|
||||
const ffmpeg = await download('ffmpeg', targetSources.ffmpeg);
|
||||
copyExecutable(findFile(ffmpeg, isWindows ? ['ffmpeg.exe'] : ['ffmpeg']), 'ffmpeg');
|
||||
|
||||
const aria2 = await download('aria2c', targetSources.aria2c);
|
||||
const aria2Source = targetSources.aria2c;
|
||||
if (aria2Source.buildFromSource !== true || aria2Source.allocationTelemetry !== true) {
|
||||
throw new Error('Aria2 provisioning requires the allocation telemetry source build.');
|
||||
}
|
||||
const patchFile = path.join(repoRoot, aria2Source.patch);
|
||||
if (sha256(patchFile) !== aria2Source.patchSha256) throw new Error('Aria2 source patch checksum mismatch');
|
||||
const sourceRoots = fs.readdirSync(aria2, { withFileTypes: true })
|
||||
.filter(entry => entry.isDirectory() && fs.existsSync(path.join(aria2, entry.name, 'configure.ac')))
|
||||
.map(entry => path.join(aria2, entry.name));
|
||||
if (sourceRoots.length !== 1) throw new Error('Aria2 archive must contain exactly one source root');
|
||||
const [sourceRoot] = sourceRoots;
|
||||
const bash = isWindows ? path.join(process.env.FIRELINK_MSYS2_ROOT || 'C:/msys64', 'usr/bin/bash.exe') : 'bash';
|
||||
await execFileAsync(bash, [path.join(repoRoot, 'scripts/aria2/build.sh').replaceAll('\\', '/'), sourceRoot, patchFile], {
|
||||
signal: provisioningAbortController.signal,
|
||||
env: { ...process.env, ...(isWindows ? { MSYSTEM: 'MINGW64' } : {}) },
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
timeout: 30 * 60 * 1000,
|
||||
|
||||
const aria2CacheRoot = process.env.FIRELINK_ARIA2_CACHE_DIR
|
||||
|| path.join(destinationParent, '.aria2-cache', target);
|
||||
const buildScriptSha256 = getAria2BuildScriptSha256(repoRoot);
|
||||
const toolchainFingerprint = process.env.FIRELINK_TOOLCHAIN_FINGERPRINT || null;
|
||||
|
||||
const cacheValidation = validateAria2Cache({
|
||||
aria2CacheRoot,
|
||||
target,
|
||||
aria2Source,
|
||||
buildScriptSha256,
|
||||
toolchainFingerprint,
|
||||
executableSuffix,
|
||||
});
|
||||
copyExecutable(path.join(sourceRoot, 'firelink-build', 'src', `aria2c${executableSuffix}`), 'aria2c');
|
||||
const aria2Runtime = path.join(sourceRoot, 'aria2-libs');
|
||||
if (fs.existsSync(aria2Runtime)) {
|
||||
fs.cpSync(aria2Runtime, path.join(payloadDestination, 'aria2-libs'), {
|
||||
recursive: true,
|
||||
preserveTimestamps: true,
|
||||
|
||||
if (cacheValidation.valid) {
|
||||
restoreAria2Cache({
|
||||
aria2CacheRoot,
|
||||
payloadDestination,
|
||||
target,
|
||||
executableSuffix,
|
||||
isWindows,
|
||||
});
|
||||
console.log(`Reused cached Aria2 build from ${aria2CacheRoot}`);
|
||||
} else {
|
||||
const aria2 = await download('aria2c', targetSources.aria2c);
|
||||
const sourceRoots = fs.readdirSync(aria2, { withFileTypes: true })
|
||||
.filter(entry => entry.isDirectory() && fs.existsSync(path.join(aria2, entry.name, 'configure.ac')))
|
||||
.map(entry => path.join(aria2, entry.name));
|
||||
if (sourceRoots.length !== 1) throw new Error('Aria2 archive must contain exactly one source root');
|
||||
const [sourceRoot] = sourceRoots;
|
||||
const bash = isWindows ? path.join(process.env.FIRELINK_MSYS2_ROOT || 'C:/msys64', 'usr/bin/bash.exe') : 'bash';
|
||||
await execFileAsync(bash, [path.join(repoRoot, 'scripts/aria2/build.sh').replaceAll('\\', '/'), sourceRoot, patchFile], {
|
||||
signal: provisioningAbortController.signal,
|
||||
env: { ...process.env, ...(isWindows ? { MSYSTEM: 'MINGW64' } : {}) },
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
timeout: 30 * 60 * 1000,
|
||||
});
|
||||
copyExecutable(path.join(sourceRoot, 'firelink-build', 'src', `aria2c${executableSuffix}`), 'aria2c');
|
||||
const aria2Runtime = path.join(sourceRoot, 'aria2-libs');
|
||||
if (fs.existsSync(aria2Runtime)) {
|
||||
fs.cpSync(aria2Runtime, path.join(payloadDestination, 'aria2-libs'), {
|
||||
recursive: true,
|
||||
preserveTimestamps: true,
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
await saveAria2Cache({
|
||||
aria2CacheRoot,
|
||||
payloadDestination,
|
||||
target,
|
||||
aria2Source,
|
||||
buildScriptSha256,
|
||||
toolchainFingerprint,
|
||||
executableSuffix,
|
||||
aria2Runtime,
|
||||
isWindows,
|
||||
});
|
||||
console.log(`Saved built Aria2 cache to ${aria2CacheRoot}`);
|
||||
} catch (cacheError) {
|
||||
console.warn(`Could not save Aria2 build cache: ${cacheError.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
writePayloadManifest();
|
||||
@@ -215,7 +264,17 @@ try {
|
||||
await promoteDirectory(payloadDestination, destination);
|
||||
console.log(`Provisioned locked engine payload at ${destination}`);
|
||||
} finally {
|
||||
if (temporary) await removePathWithRetry(temporary);
|
||||
if (temporary) {
|
||||
try {
|
||||
await removePathWithRetry(temporary);
|
||||
} catch (cleanupError) {
|
||||
if (provisioningAbortController.signal.aborted) {
|
||||
console.warn(`Could not remove temporary directory during abort: ${cleanupError.message}`);
|
||||
} else {
|
||||
throw cleanupError;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const [signalName, handler] of signalHandlers) {
|
||||
process.removeListener(signalName, handler);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user