mirror of
https://github.com/nimbold/Firelink.git
synced 2026-08-05 08:59:05 +00:00
feat(retry): add connection-aware exponential backoff across all download backends
Transient network drops and Wi-Fi timeouts no longer promote a download straight to a hard Failed state. A shared retry engine classifies the error, emits a transient `Retrying` state, and runs a 3-strike exponential backoff (2s / 5s / 10s) while preserving the active download allocation (semaphore permit / worker slot). Resumability primitives are reused on every retry so no downloaded bytes are discarded. Engine core (src-tauri/src/retry.rs, new): - BACKOFF_SCHEDULE = [2s, 5s, 10s], MAX_RETRIES = 3 - backoff_for(strike): schedule index with graceful clamp beyond range - is_transient_network_error(msg): string classifier covering reqwest, yt-dlp, and aria2c phrasing; permanent conditions (HTTP 401/403/404/ 410/451, not-found, permission denied, out-of-disk) checked first so they always fail fast - backoff_and_emit / backoff_and_emit_cancel: cancel-safe sleep helpers driving a Retrying state emit before each delay - 9 unit tests covering schedule math, clamping, transient vs permanent classification, and permanent-wins-over-transient precedence State model (src-tauri/src/ipc.rs): - DownloadStatus::Retrying variant (serialized "retrying") - DownloadStateEvent::retrying(id, reason) constructor - Regenerated TypeScript binding: src/bindings/DownloadStatus.ts Native reqwest backend (src-tauri/src/download.rs): - DownloadEvent::Retrying variant (headless mirror) - CoordinatorEventSink::emit_retrying() drives the production download-state channel with status "retrying" - download_file retry core rewritten: transient -> emit_retrying -> backoff_for sleep inside a control_rx select (pause/cancel honored mid-backoff) -> re-issue Range header; permanent errors or strike exhaustion advance to the next URL then hard Failed yt-dlp media backend (src-tauri/src/lib.rs): - child spawn+stream loop wrapped in a 3-strike re-spawn loop; stderr tail classified and, if transient, the process is re-spawned after backoff (--continue resumes); --retry-wait=2 added to aria2c downloader aria2c backend (src-tauri/src/queue.rs): - retry-wait=2 option so aria2's internal retries are not rapid-fire - handle_aria2_download_error: intercepts transient onDownloadError, backs off, re-issues addUri, and rotates the stale gid -> id mapping via rotate_aria2_gid (fresh addUri mints a new gid; not rotating would detach subsequent WS events and leak the semaphore permit permanently) - aria2_payloads / aria2_retry_strikes tracking with cleanup on terminal outcomes Verification: cargo build clean; cargo test --lib 37 passed / 0 failed.
This commit is contained in:
+143
-9
@@ -1,8 +1,10 @@
|
||||
use crate::ipc::{DownloadStateEvent, DownloadStatus, QueueDirection};
|
||||
use crate::retry::{BackoffOutcome, MAX_RETRIES, backoff_and_emit, is_transient_network_error};
|
||||
use serde::Deserialize;
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tauri::{AppHandle, Manager};
|
||||
use tokio::sync::{Mutex, Notify, OwnedSemaphorePermit, Semaphore};
|
||||
use serde_json;
|
||||
@@ -93,6 +95,12 @@ pub struct QueueManager<R: tauri::Runtime = tauri::Wry> {
|
||||
/// before the gid was stored. Drained by `remember_gid`.
|
||||
pub pending_completion: Arc<Mutex<HashMap<String, (String, PendingOutcome)>>>,
|
||||
|
||||
/// download id -> spawn payload for aria2 transient-error re-addUri retries.
|
||||
aria2_payloads: Mutex<HashMap<String, SpawnPayload>>,
|
||||
|
||||
/// 0-based transient-error strike counter per aria2 download id.
|
||||
aria2_retry_strikes: Mutex<HashMap<String, usize>>,
|
||||
|
||||
spawner: Arc<dyn SidecarSpawner>,
|
||||
app_handle: AppHandle<R>,
|
||||
}
|
||||
@@ -123,6 +131,8 @@ impl<R: tauri::Runtime> QueueManager<R> {
|
||||
notify: Notify::new(),
|
||||
aria2_gids: Arc::new(std::sync::RwLock::new(HashMap::new())),
|
||||
pending_completion: Arc::new(Mutex::new(HashMap::new())),
|
||||
aria2_payloads: Mutex::new(HashMap::new()),
|
||||
aria2_retry_strikes: Mutex::new(HashMap::new()),
|
||||
spawner,
|
||||
app_handle,
|
||||
}
|
||||
@@ -292,9 +302,15 @@ impl<R: tauri::Runtime> QueueManager<R> {
|
||||
|
||||
match task.kind {
|
||||
TaskKind::Aria2 => {
|
||||
self.aria2_payloads
|
||||
.lock()
|
||||
.await
|
||||
.insert(id.clone(), task.payload.clone());
|
||||
self.aria2_retry_strikes.lock().await.remove(&id);
|
||||
match self.spawner.add_uri(&id, &task.payload).await {
|
||||
Ok(gid) => self.remember_gid(id.clone(), gid).await,
|
||||
Err(error) => {
|
||||
self.clear_aria2_retry_state(&id).await;
|
||||
self.emit_failed(&id, error);
|
||||
self.release_permit(&id).await;
|
||||
}
|
||||
@@ -364,31 +380,148 @@ impl<R: tauri::Runtime> QueueManager<R> {
|
||||
pub async fn apply_completion(&self, id: &str, outcome: PendingOutcome) {
|
||||
match outcome {
|
||||
PendingOutcome::Complete => {
|
||||
self.clear_aria2_retry_state(id).await;
|
||||
self.emit_state(id, DownloadStatus::Completed);
|
||||
}
|
||||
PendingOutcome::Error(error) => {
|
||||
self.clear_aria2_retry_state(id).await;
|
||||
self.emit_failed(id, error);
|
||||
}
|
||||
}
|
||||
self.release_permit(id).await;
|
||||
}
|
||||
|
||||
/// Entry point for the aria2 WS poller. Resolves gid -> id; if not yet
|
||||
/// stored, buffers the outcome for reconciliation by remember_gid.
|
||||
pub async fn handle_aria2_event(&self, gid: &str, outcome: PendingOutcome) {
|
||||
let id_opt = {
|
||||
async fn clear_aria2_retry_state(&self, id: &str) {
|
||||
self.aria2_payloads.lock().await.remove(id);
|
||||
self.aria2_retry_strikes.lock().await.remove(id);
|
||||
}
|
||||
|
||||
/// Overwrite a stale aria2 gid with the fresh gid minted by a retry
|
||||
/// `addUri`. Failing to call this after re-add leaks the semaphore permit.
|
||||
pub fn rotate_aria2_gid(&self, id: &str, stale_gid: &str, new_gid: &str) {
|
||||
let mut gids = self.aria2_gids.write().unwrap();
|
||||
gids.remove(stale_gid);
|
||||
gids.insert(new_gid.to_string(), id.to_string());
|
||||
}
|
||||
|
||||
async fn wait_permit_released(self: &Arc<Self>, id: &str) {
|
||||
loop {
|
||||
if !self.active_permits.lock().await.contains_key(id) {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Intercept transient `onDownloadError` events: backoff, re-issue
|
||||
/// `addUri`, and rotate the gid mapping. Permanent errors and exhausted
|
||||
/// strikes fall through to a hard `Failed` state.
|
||||
async fn handle_aria2_download_error(self: &Arc<Self>, gid: &str, error: String) {
|
||||
let id = {
|
||||
let gids = self.aria2_gids.read().unwrap();
|
||||
gids.get(gid).cloned()
|
||||
};
|
||||
match id_opt {
|
||||
Some(id) => {
|
||||
self.apply_completion(&id, outcome).await;
|
||||
}
|
||||
let id = match id {
|
||||
Some(id) => id,
|
||||
None => {
|
||||
self.pending_completion
|
||||
.lock()
|
||||
.await
|
||||
.insert(gid.to_string(), (String::new(), outcome));
|
||||
.insert(gid.to_string(), (String::new(), PendingOutcome::Error(error)));
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let strike = {
|
||||
let mut strikes = self.aria2_retry_strikes.lock().await;
|
||||
let entry = strikes.entry(id.clone()).or_insert(0);
|
||||
*entry
|
||||
};
|
||||
|
||||
let transient = is_transient_network_error(&error);
|
||||
let strikes_left = strike < MAX_RETRIES;
|
||||
if !(transient && strikes_left) {
|
||||
self.apply_completion(&id, PendingOutcome::Error(error)).await;
|
||||
return;
|
||||
}
|
||||
|
||||
let payload = self.aria2_payloads.lock().await.get(&id).cloned();
|
||||
if payload.is_none() {
|
||||
self.apply_completion(&id, PendingOutcome::Error(error)).await;
|
||||
return;
|
||||
}
|
||||
let payload = payload.unwrap();
|
||||
|
||||
let this = Arc::clone(&self);
|
||||
let stale_gid = gid.to_string();
|
||||
let id_for_task = id.clone();
|
||||
let error_for_emit = error.clone();
|
||||
tauri::async_runtime::spawn(async move {
|
||||
let outcome = backoff_and_emit(
|
||||
strike,
|
||||
error_for_emit,
|
||||
this.wait_permit_released(&id_for_task),
|
||||
|reason| {
|
||||
use tauri::Emitter;
|
||||
let _ = this.app_handle.emit(
|
||||
"download-state",
|
||||
DownloadStateEvent::retrying(&id_for_task, reason),
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
if outcome == BackoffOutcome::Aborted {
|
||||
return;
|
||||
}
|
||||
|
||||
if !this.active_permits.lock().await.contains_key(&id_for_task) {
|
||||
return;
|
||||
}
|
||||
|
||||
match this.spawner.add_uri(&id_for_task, &payload).await {
|
||||
Ok(new_gid) => {
|
||||
this.aria2_retry_strikes
|
||||
.lock()
|
||||
.await
|
||||
.insert(id_for_task.clone(), strike + 1);
|
||||
this.rotate_aria2_gid(&id_for_task, &stale_gid, &new_gid);
|
||||
this.emit_state(&id_for_task, DownloadStatus::Downloading);
|
||||
}
|
||||
Err(retry_error) => {
|
||||
this.apply_completion(
|
||||
&id_for_task,
|
||||
PendingOutcome::Error(retry_error),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Entry point for the aria2 WS poller. Resolves gid -> id; if not yet
|
||||
/// stored, buffers the outcome for reconciliation by remember_gid.
|
||||
pub async fn handle_aria2_event(self: &Arc<Self>, gid: &str, outcome: PendingOutcome) {
|
||||
match outcome {
|
||||
PendingOutcome::Error(error) => {
|
||||
self.handle_aria2_download_error(gid, error).await;
|
||||
}
|
||||
other => {
|
||||
let id_opt = {
|
||||
let gids = self.aria2_gids.read().unwrap();
|
||||
gids.get(gid).cloned()
|
||||
};
|
||||
match id_opt {
|
||||
Some(id) => {
|
||||
self.apply_completion(&id, other).await;
|
||||
}
|
||||
None => {
|
||||
self.pending_completion
|
||||
.lock()
|
||||
.await
|
||||
.insert(gid.to_string(), (String::new(), other));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -487,6 +620,7 @@ impl SidecarSpawner for ProductionSpawner {
|
||||
);
|
||||
let mt = payload.max_tries.unwrap_or(1).max(1) as u32;
|
||||
options.insert("max-tries".to_string(), serde_json::json!(mt.to_string()));
|
||||
options.insert("retry-wait".to_string(), serde_json::json!("2"));
|
||||
options.insert("continue".to_string(), serde_json::json!("true"));
|
||||
if let Some(speed) = &payload.speed_limit {
|
||||
options.insert("max-download-limit".to_string(), serde_json::json!(speed));
|
||||
|
||||
Reference in New Issue
Block a user