refactor(yt-dlp): modernize bot evasion and progress parsing

- Fix config location typo to enable authentication
- Wire up bundled Deno to handle YouTube PoW challenges
- Implement iOS/TV player spoofing to bypass web blocks
- Replace fragile regex progress parsing with structured templates
- Add auto-reconnect logic to Aria2 WebSocket
- Deduplicate media domains and fetch dynamically on frontend
This commit is contained in:
NimBold
2026-06-15 19:02:12 +03:30
parent 44c089fadf
commit f4b74898fd
4 changed files with 86 additions and 48 deletions
+52 -35
View File
@@ -180,6 +180,7 @@ async fn fetch_media_metadata(app_handle: tauri::AppHandle, url: String, cookie_
let resource_dir = app_handle.path().resource_dir().map_err(|e| e.to_string())?; let resource_dir = app_handle.path().resource_dir().map_err(|e| e.to_string())?;
let ytdlp_path = resource_dir.join("binaries").join(get_binary_name("yt-dlp")); let ytdlp_path = resource_dir.join("binaries").join(get_binary_name("yt-dlp"));
let deno_path = resource_dir.join("binaries").join(get_binary_name("deno"));
let mut cmd = AsyncCommand::new(&ytdlp_path); let mut cmd = AsyncCommand::new(&ytdlp_path);
cmd.arg("-J") cmd.arg("-J")
.arg("--no-warnings") .arg("--no-warnings")
@@ -189,7 +190,9 @@ async fn fetch_media_metadata(app_handle: tauri::AppHandle, url: String, cookie_
.arg("--retries").arg("3") .arg("--retries").arg("3")
.arg("--extractor-retries").arg("3") .arg("--extractor-retries").arg("3")
.arg("--compat-options").arg("no-youtube-unavailable-videos") .arg("--compat-options").arg("no-youtube-unavailable-videos")
.arg("--js-runtimes").arg("deno,node"); .arg("--js-runtimes").arg("deno,node")
.arg("--js-runtime-path").arg(format!("deno:{}", deno_path.display()))
.arg("--extractor-args").arg("youtube:player_client=ios,tv");
if let Some(browser) = cookie_browser { if let Some(browser) = cookie_browser {
if !browser.is_empty() { if !browser.is_empty() {
@@ -213,7 +216,7 @@ async fn fetch_media_metadata(app_handle: tauri::AppHandle, url: String, cookie_
config_file.write_all(config_content.as_bytes()).map_err(|e| e.to_string())?; config_file.write_all(config_content.as_bytes()).map_err(|e| e.to_string())?;
let config_path = config_file.into_temp_path(); let config_path = config_file.into_temp_path();
if !config_content.is_empty() { if !config_content.is_empty() {
cmd.arg("--config-locations").arg(&config_path); cmd.arg("--config-location").arg(&config_path);
} }
cmd.arg("--").arg(&url); cmd.arg("--").arg(&url);
@@ -310,9 +313,8 @@ async fn test_deno(app_handle: tauri::AppHandle) -> Result<String, String> {
println!("deno execution finished with status: {}", output.status); println!("deno execution finished with status: {}", output.status);
if output.status.success() { if output.status.success() {
let text = String::from_utf8_lossy(&output.stdout).trim().to_string(); let text = String::from_utf8_lossy(&output.stdout).trim().to_string();
let first_line = text.lines().next().unwrap_or("").to_string(); let re = regex::Regex::new(r"deno\s+(\d+\.\d+\.\d+)").unwrap();
let parts: Vec<&str> = first_line.split_whitespace().collect(); let clean = re.captures(&text).and_then(|c| c.get(1)).map(|m| m.as_str()).unwrap_or(&text).to_string();
let clean = parts.get(1).unwrap_or(&first_line.as_str()).to_string();
println!("deno output: {}", clean); println!("deno output: {}", clean);
Ok(clean) Ok(clean)
} else { } else {
@@ -748,8 +750,8 @@ async fn start_media_download(
} }
}; };
let _ = start_media_download_internal( if let Err(e) = start_media_download_internal(
app_handle, app_handle.clone(),
&id, url, &id, url,
destination, destination,
filename, filename,
@@ -763,7 +765,11 @@ async fn start_media_download(
user_agent, user_agent,
max_tries, max_tries,
&mut cancel_rx, &mut cancel_rx,
).await; ).await {
eprintln!("Media download {} failed: {}", id, e);
use tauri::Emitter;
let _ = app_handle.emit("download-failed", id.clone());
}
drop(permit); drop(permit);
coordinator.finish_media(id).await; coordinator.finish_media(id).await;
@@ -818,6 +824,7 @@ pub(crate) async fn start_media_download_internal(
1.0 1.0
}; };
let deno_path = resource_dir.join("binaries").join(get_binary_name("deno"));
let mut cmd = AsyncCommand::new(&ytdlp_path); let mut cmd = AsyncCommand::new(&ytdlp_path);
cmd.arg("--newline") cmd.arg("--newline")
.arg("--ffmpeg-location") .arg("--ffmpeg-location")
@@ -829,6 +836,12 @@ pub(crate) async fn start_media_download_internal(
.arg("--downloader").arg("aria2c") .arg("--downloader").arg("aria2c")
.arg("--downloader-args").arg("aria2c:-x 16 -s 16 -k 1M") .arg("--downloader-args").arg("aria2c:-x 16 -s 16 -k 1M")
.arg("--concurrent-fragments").arg("4") .arg("--concurrent-fragments").arg("4")
.arg("--no-warnings")
.arg("--compat-options").arg("no-youtube-unavailable-videos")
.arg("--js-runtimes").arg("deno,node")
.arg("--js-runtime-path").arg(format!("deno:{}", deno_path.display()))
.arg("--extractor-args").arg("youtube:player_client=ios,tv")
.arg("--progress-template").arg("download:[%(progress.downloaded_bytes)s/%(progress.total_bytes)s]")
.arg("-o").arg(out_path.to_string_lossy().to_string()); .arg("-o").arg(out_path.to_string_lossy().to_string());
if let Some(limit) = speed_limit { if let Some(limit) = speed_limit {
@@ -881,7 +894,7 @@ pub(crate) async fn start_media_download_internal(
config_file.write_all(config_content.as_bytes()).map_err(|e| e.to_string())?; config_file.write_all(config_content.as_bytes()).map_err(|e| e.to_string())?;
let config_path = config_file.into_temp_path(); let config_path = config_file.into_temp_path();
if !config_content.is_empty() { if !config_content.is_empty() {
cmd.arg("--config-locations").arg(&config_path); cmd.arg("--config-location").arg(&config_path);
} }
if let Some(format) = format_selector { if let Some(format) = format_selector {
@@ -1463,32 +1476,34 @@ pub fn run() {
let app_handle_ws = app.handle().clone(); let app_handle_ws = app.handle().clone();
let ws_port = aria2_port; let ws_port = aria2_port;
tauri::async_runtime::spawn(async move { tauri::async_runtime::spawn(async move {
tokio::time::sleep(std::time::Duration::from_secs(1)).await; loop {
let ws_url = format!("ws://127.0.0.1:{}/jsonrpc", ws_port); tokio::time::sleep(std::time::Duration::from_secs(1)).await;
if let Ok((ws_stream, _)) = tokio_tungstenite::connect_async(&ws_url).await { let ws_url = format!("ws://127.0.0.1:{}/jsonrpc", ws_port);
use futures_util::StreamExt; if let Ok((ws_stream, _)) = tokio_tungstenite::connect_async(&ws_url).await {
let (_, mut read) = ws_stream.split(); use futures_util::StreamExt;
while let Some(msg) = read.next().await { let (_, mut read) = ws_stream.split();
if let Ok(tokio_tungstenite::tungstenite::Message::Text(text)) = msg { while let Some(msg) = read.next().await {
if let Ok(json) = serde_json::from_str::<serde_json::Value>(&text) { if let Ok(tokio_tungstenite::tungstenite::Message::Text(text)) = msg {
if let Some(method) = json.get("method").and_then(|m| m.as_str()) { if let Ok(json) = serde_json::from_str::<serde_json::Value>(&text) {
if let Some(params) = json.get("params").and_then(|p| p.as_array()) { if let Some(method) = json.get("method").and_then(|m| m.as_str()) {
if let Some(event) = params.first().and_then(|p| p.as_object()) { if let Some(params) = json.get("params").and_then(|p| p.as_array()) {
if let Some(gid) = event.get("gid").and_then(|g| g.as_str()) { if let Some(event) = params.first().and_then(|p| p.as_object()) {
let id = { if let Some(gid) = event.get("gid").and_then(|g| g.as_str()) {
let map = aria2_gids_clone1.read().unwrap(); let id = {
map.iter().find(|(_, g)| *g == gid).map(|(i, _)| i.clone()) let map = aria2_gids_clone1.read().unwrap();
}; map.iter().find(|(_, g)| *g == gid).map(|(i, _)| i.clone())
if let Some(id) = id { };
use tauri::Emitter; if let Some(id) = id {
match method { use tauri::Emitter;
"aria2.onDownloadComplete" => { match method {
let _ = app_handle_ws.emit("download-complete", id); "aria2.onDownloadComplete" => {
let _ = app_handle_ws.emit("download-complete", id);
}
"aria2.onDownloadError" => {
let _ = app_handle_ws.emit("download-failed", id);
}
_ => {}
} }
"aria2.onDownloadError" => {
let _ = app_handle_ws.emit("download-failed", id);
}
_ => {}
} }
} }
} }
@@ -1498,6 +1513,8 @@ pub fn run() {
} }
} }
} }
// Connection lost, loop and reconnect
tokio::time::sleep(std::time::Duration::from_secs(4)).await;
} }
}); });
@@ -1584,7 +1601,7 @@ pub fn run() {
set_keychain_password, get_keychain_password, delete_keychain_password, set_keychain_password, get_keychain_password, delete_keychain_password,
check_file_exists, delete_file, toggle_tray_icon, set_extension_pairing_token, check_file_exists, delete_file, toggle_tray_icon, set_extension_pairing_token,
set_extension_frontend_ready, set_concurrent_limit, set_global_speed_limit, remove_download, set_extension_frontend_ready, set_concurrent_limit, set_global_speed_limit, remove_download,
parity::get_system_proxy, parity::get_file_category, parity::check_for_updates, parity::is_supported_media, parity::get_system_proxy, parity::get_file_category, parity::check_for_updates, parity::is_supported_media, parity::get_supported_media_domains,
db_save_settings, db_load_settings, db_get_all_downloads, db_save_download, db_delete_download, db_get_all_queues, db_save_queue, db_delete_queue, db_save_settings, db_load_settings, db_get_all_downloads, db_save_download, db_delete_download, db_get_all_queues, db_save_queue, db_delete_queue,
parity::create_category_directories parity::create_category_directories
]) ])
+18 -12
View File
@@ -159,23 +159,29 @@ pub async fn create_category_directories(app_handle: tauri::AppHandle, paths: Ve
Ok(()) Ok(())
} }
pub static SUPPORTED_DOMAINS: &[&str] = &[
"youtube.com", "youtu.be",
"twitter.com", "x.com",
"vimeo.com",
"twitch.tv",
"instagram.com",
"tiktok.com",
"facebook.com", "fb.watch",
"reddit.com", "v.redd.it",
"soundcloud.com",
];
#[tauri::command]
pub fn get_supported_media_domains() -> Vec<String> {
SUPPORTED_DOMAINS.iter().map(|s| s.to_string()).collect()
}
#[tauri::command] #[tauri::command]
pub fn is_supported_media(url: String) -> bool { pub fn is_supported_media(url: String) -> bool {
if let Ok(parsed_url) = reqwest::Url::parse(&url) { if let Ok(parsed_url) = reqwest::Url::parse(&url) {
if let Some(host) = parsed_url.host_str() { if let Some(host) = parsed_url.host_str() {
let host_lower = host.to_lowercase(); let host_lower = host.to_lowercase();
let supported_domains = [ for domain in SUPPORTED_DOMAINS.iter() {
"youtube.com", "youtu.be",
"twitter.com", "x.com",
"vimeo.com",
"twitch.tv",
"instagram.com",
"tiktok.com",
"facebook.com", "fb.watch",
"reddit.com", "v.redd.it",
"soundcloud.com"
];
for domain in supported_domains.iter() {
if host_lower == *domain || host_lower.ends_with(&format!(".{}", domain)) { if host_lower == *domain || host_lower.ends_with(&format!(".{}", domain)) {
return true; return true;
} }
+2
View File
@@ -1,3 +1,4 @@
import { initMediaDomains } from './utils/downloads';
import { useEffect, useRef, useState } from "react"; import { useEffect, useRef, useState } from "react";
import { Sidebar, SidebarFilter } from "./components/Sidebar"; import { Sidebar, SidebarFilter } from "./components/Sidebar";
import { DownloadTable } from "./components/DownloadTable"; import { DownloadTable } from "./components/DownloadTable";
@@ -57,6 +58,7 @@ function App() {
}; };
useEffect(() => { useEffect(() => {
initMediaDomains();
window.localStorage.setItem('firelink-sidebar-width', String(sidebarWidth)); window.localStorage.setItem('firelink-sidebar-width', String(sidebarWidth));
}, [sidebarWidth]); }, [sidebarWidth]);
+14 -1
View File
@@ -1,7 +1,9 @@
import type { DownloadCategory } from '../bindings/DownloadCategory'; import type { DownloadCategory } from '../bindings/DownloadCategory';
export type { DownloadCategory } from '../bindings/DownloadCategory'; export type { DownloadCategory } from '../bindings/DownloadCategory';
const MEDIA_DOMAINS = [ import { invoke } from '@tauri-apps/api/core';
let MEDIA_DOMAINS = [
'youtube.com', 'youtube.com',
'youtu.be', 'youtu.be',
'twitter.com', 'twitter.com',
@@ -17,6 +19,17 @@ const MEDIA_DOMAINS = [
'fb.watch' 'fb.watch'
]; ];
export const initMediaDomains = async () => {
try {
const domains = await invoke<string[]>('get_supported_media_domains');
if (domains && domains.length > 0) {
MEDIA_DOMAINS = domains;
}
} catch (e) {
console.error('Failed to init media domains:', e);
}
};
export const categoryForFileName = (fileName: string): DownloadCategory => { export const categoryForFileName = (fileName: string): DownloadCategory => {
const ext = fileName.split('.').pop()?.toLowerCase() || ''; const ext = fileName.split('.').pop()?.toLowerCase() || '';
if (['mp4', 'mkv', 'avi', 'mov', 'wmv', 'flv', 'webm', 'm4v', 'mpeg', 'mpg', '3gp', 'ts', 'vob'].includes(ext)) return 'Movies'; if (['mp4', 'mkv', 'avi', 'mov', 'wmv', 'flv', 'webm', 'm4v', 'mpeg', 'mpg', '3gp', 'ts', 'vob'].includes(ext)) return 'Movies';