feat: retry the restore backup download

Extracts the download body to download_once and makes download_backup a
retry wrapper around it. This path had no retry at all before, so a
single dropped connection failed the whole restore job.

Retrying is safe because File::create truncates and the target filename
is derived from Content-Disposition or the URL, so it is stable across
attempts. There is no Range resume: a download that fails at 90% starts
over.
This commit is contained in:
charles-gauthereau
2026-08-27 19:01:10 +02:00
parent 5298d82576
commit b6a120fcbf
3 changed files with 88 additions and 0 deletions
+23
View File
@@ -8,6 +8,7 @@ use std::sync::Arc;
use std::time::Instant;
use tokio::io::AsyncWriteExt;
use crate::services::backup::logger::JobLogger;
use crate::utils::retry::{RetryPolicy, retry};
fn human_size(bytes: u64) -> String {
if bytes >= 1024 * 1024 {
@@ -26,6 +27,28 @@ impl RestoreService {
tmp_path: &Path,
logger: Arc<JobLogger>,
expected_size: Option<String>,
) -> Result<PathBuf> {
let policy = RetryPolicy::default();
let logger_ref = &logger;
retry("Backup download", &logger, &policy, move |_| {
let expected = expected_size.clone();
async move {
self.download_once(file_url, tmp_path, Arc::clone(logger_ref), expected)
.await
}
})
.await
}
pub async fn download_once(
&self,
file_url: &str,
tmp_path: &Path,
logger: Arc<JobLogger>,
expected_size: Option<String>,
) -> Result<PathBuf> {
logger.log("info", "Start downloading backup archive".to_string());
+1
View File
@@ -3,3 +3,4 @@ mod backup_runner_tests;
mod backup_uploader_tests;
mod config_tests;
mod dashboard_config_tests;
mod restore_downloader_tests;
@@ -0,0 +1,64 @@
use crate::core::context::Context;
use crate::services::api::ApiClient;
use crate::services::backup::logger::JobLogger;
use crate::services::restore::RestoreService;
use crate::tests::init_tracing_for_test;
use crate::utils::edge_key::EdgeKey;
use std::sync::Arc;
use tempfile::TempDir;
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
#[tokio::test]
async fn a_failing_download_is_retried_until_it_succeeds() {
init_tracing_for_test();
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/backups/archive.tar.gz"))
.respond_with(ResponseTemplate::new(503))
.up_to_n_times(2)
.with_priority(1)
.expect(2)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/backups/archive.tar.gz"))
.respond_with(ResponseTemplate::new(200).set_body_bytes(b"portabase-archive".to_vec()))
.with_priority(2)
.expect(1)
.mount(&server)
.await;
let ctx = Context {
edge_key: EdgeKey {
server_url: server.uri(),
agent_id: "agent-1".to_string(),
master_key_b64: String::new(),
},
api: ApiClient::new(server.uri()),
};
let service = RestoreService::new(Arc::new(ctx));
let temp_dir = TempDir::new().unwrap();
let logger = Arc::new(JobLogger::new());
let url = format!("{}/backups/archive.tar.gz", server.uri());
let downloaded = service
.download_backup(&url, temp_dir.path(), Arc::clone(&logger), None)
.await
.unwrap();
assert_eq!(std::fs::read(&downloaded).unwrap(), b"portabase-archive");
let entries = Arc::try_unwrap(logger).unwrap().into_entries();
assert_eq!(entries.iter().filter(|e| e.level == "warn").count(), 2);
assert!(
entries
.iter()
.any(|e| e.message == "Backup download succeeded on attempt 3/3")
);
}