chore: release 1.20.0

feat: retry-system

.github/workflows/release.yml

@@ -118,12 +118,24 @@ jobs:
     secrets:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+  build-windows:
+    needs: create-release
+    if: ${{ needs.create-release.result == 'success' }}
+    uses: ./.github/workflows/windows-release.yml
+    with:
+      version: ${{ needs.create-release.outputs.version }}
+      ref: ${{ needs.create-release.outputs.version }}
+      draft_tag: ${{ needs.create-release.outputs.draft_tag }}
+    secrets:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
   finalize-release:
     needs:
       - create-release
       - publish-docker
       - publish-docker-ghcr
       - publish-helm
+      - build-windows
     runs-on: ubuntu-latest
     outputs:
       release_tag: ${{ steps.publish_release_step.outputs.release_tag }}

.github/workflows/windows-release.yml

@@ -1,14 +1,30 @@
 name: Build Windows release
 
-
 on:
+  workflow_call:
+    inputs:
+      version:
+        description: 'Release version (git tag), e.g. 1.18.4'
+        type: string
+        required: false
+      ref:
+        description: 'Git ref to check out and build'
+        type: string
+        required: false
+      draft_tag:
+        description: 'Draft GitHub release tag to attach the asset to (e.g. untagged-xxxx). Empty = skip upload.'
+        type: string
+        required: false
+    secrets:
+      GH_TOKEN:
+        required: false
+
   workflow_dispatch:
-  push:
-    tags:
-      - '[0-9]+.[0-9]+.[0-9]+'
-    branches:
-      - main
-      - master
+    inputs:
+      ref:
+        description: 'Git ref to check out and build'
+        type: string
+        required: false
 
 jobs:
   build-windows:
@@ -17,80 +33,68 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.ref }}
 
       - name: Set up Rust toolchain (MSVC)
-        uses: actions-rs/toolchain@v1
+        uses: dtolnay/rust-toolchain@stable
         with:
-          toolchain: stable-x86_64-pc-windows-msvc
-          profile: minimal
-          override: true
+          targets: x86_64-pc-windows-msvc
 
-      - name: Install vcpkg and OpenSSL (x64)
+      - name: Cache cargo build
+        uses: Swatinem/rust-cache@v2
+
+      - name: Cache vcpkg installed packages
+        uses: actions/cache@v4
+        with:
+          path: C:\vcpkg\installed
+          key: vcpkg-openssl-x64-windows-v1
+
+      - name: Install OpenSSL (x64) via vcpkg
         shell: pwsh
         run: |
-          # Install vcpkg and the prebuilt OpenSSL package
-          git clone https://github.com/microsoft/vcpkg C:\vcpkg
-          C:\vcpkg\bootstrap-vcpkg.bat
-          C:\vcpkg\vcpkg install openssl:x64-windows
-          # Export variables for subsequent steps
+          # windows-latest ships vcpkg preinstalled; the install is a no-op when the
+          # package is restored from cache.
+          & "$env:VCPKG_INSTALLATION_ROOT\vcpkg.exe" install openssl:x64-windows
           'VCPKG_ROOT=C:\vcpkg' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
           'OPENSSL_DIR=C:\vcpkg\installed\x64-windows' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
 
       - name: Build (cargo release)
         shell: pwsh
-        env:
-          # Cargo / openssl-sys will pick up OPENSSL_DIR from the environment
-          OPENSSL_DIR: ${{ env.OPENSSL_DIR }}
-        run: |
-          # Ensure the environment variable is present for this step
-          if (-Not $env:OPENSSL_DIR) { Write-Host "OPENSSL_DIR not set, printing env for debugging"; Get-ChildItem Env: | ForEach-Object { Write-Host $_ } }
-          # Build the declared bin target explicitly (Cargo.toml [[bin]] name = "app")
-          cargo build --release --bin app
+        run: cargo build --release --bin app
 
       - name: Prepare artifact zip
         id: prepare_artifact
         shell: pwsh
         env:
-          RELEASE_TAG: ${{ github.ref_name }}
+          RELEASE_VERSION: ${{ inputs.version }}
         run: |
-          $tag = $env:RELEASE_TAG
+          $tag = $env:RELEASE_VERSION
           if (-not $tag) { $tag = $env:GITHUB_SHA }
 
-          # Package the declared bin target deterministically (Cargo.toml [[bin]] name = "app")
           $exe = "target\release\app.exe"
           if (-not (Test-Path $exe)) { Write-Error "Built binary $exe not found in target/release"; exit 1 }
 
           $outDir = "artifact"
           New-Item -ItemType Directory -Path $outDir -Force | Out-Null
-          # Ship under the package name, not the internal bin name "app"
+          # Ship under the package name, not the internal bin name "app".
           Copy-Item -Path $exe -Destination "$outDir\portabase-agent.exe"
 
           $zipName = "windows-release-$tag.zip"
           if (Test-Path $zipName) { Remove-Item $zipName }
           Compress-Archive -Path "$outDir\*" -DestinationPath $zipName -Force
-          Write-Host "ZIP=$zipName"
-          Write-Output "zip=$zipName" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
+          "zip=$zipName" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
 
       - name: Upload build artifact
         uses: actions/upload-artifact@v4
         with:
           name: windows-release
-          path: windows-release-*.zip
+          path: ${{ steps.prepare_artifact.outputs.zip }}
 
-      - name: Create GitHub Release
-        if: startsWith(github.ref, 'refs/tags/')
-        id: create_release
-        uses: softprops/action-gh-release@v1
-        with:
-          tag_name: ${{ github.ref_name }}
+      - name: Attach asset to draft release
+        if: ${{ inputs.draft_tag != '' }}
+        shell: pwsh
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Upload release asset
-        if: startsWith(github.ref, 'refs/tags/')
-        uses: actions/upload-release-asset@v1
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: windows-release-${{ github.ref_name }}.zip
-          asset_name: windows-release-${{ github.ref_name }}.zip
-          asset_content_type: application/zip
+          GH_TOKEN: ${{ secrets.GH_TOKEN }}
+        run: |
+          gh release upload "${{ inputs.draft_tag }}" "${{ steps.prepare_artifact.outputs.zip }}" --clobber

CITATION.cff

@@ -27,5 +27,5 @@ keywords:
   - self-hosted
   - portabase
 license: Apache-2.0
-version: 1.18.3
+version: 1.20.0
 date-released: '2026-02-24'

Cargo.lock

@@ -3503,7 +3503,7 @@ dependencies = [
 
 [[package]]
 name = "portabase-agent"
-version = "1.18.3"
+version = "1.20.0"
 dependencies = [
  "aes",
  "aes-gcm",
@@ -3535,6 +3535,7 @@ dependencies = [
  "oauth2",
  "once_cell",
  "openssl",
+ "percent-encoding",
  "postgres",
  "rand 0.9.2",
  "redis",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "portabase-agent"
-version = "1.18.3"
+version = "1.20.0"
 edition = "2024"
 
 [dependencies]
@@ -57,6 +57,7 @@ testcontainers = "0.27.1"
 testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "valkey", "mysql", "mariadb", "mongo"] }
 postgres = "0.19.12"
 url = "2.5.8"
+percent-encoding = "2.3.2"
 bollard = "0.20.0"
 
 [dev-dependencies]

docker-compose.prod.yml

@@ -1,13 +1,14 @@
 services:
   rust-app:
-#    build:
-#      context: .
-#      dockerfile: docker/Dockerfile
-#      target: prod
-    image: portabase/agent:latest
+    build:
+      context: .
+      dockerfile: docker/Dockerfile
+      target: prod
+#    image: portabase/agent:latest
     container_name: rust-prod
     volumes:
       - ./databases.json:/config/config.json
+      - /var/run/docker.sock:/var/run/docker.sock
     environment:
       LOG: info
       TZ: "Europe/Paris"
@@ -18,48 +19,6 @@ services:
     networks:
       - portabase
 
-
-  db-mongodb-auth:
-    container_name: db-mongodb-auth
-    image: mongo:latest
-    ports:
-      - "27082:27017"
-    environment:
-      MONGO_INITDB_ROOT_USERNAME: root
-      MONGO_INITDB_ROOT_PASSWORD: rootpassword
-      MONGO_INITDB_DATABASE: testdbauth
-    command: mongod --auth
-    networks:
-      - portabase
-    volumes:
-      - mongodb-data-auth:/data/db
-    healthcheck:
-      test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ]
-      interval: 5s
-      timeout: 5s
-      retries: 10
-
-  db-mongodb:
-    container_name: db-mongodb
-    image: mongo:latest
-    ports:
-      - "27083:27017"
-    volumes:
-      - mongodb-data:/data/db
-    healthcheck:
-      test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ]
-      interval: 5s
-      timeout: 5s
-      retries: 10
-    environment:
-      MONGO_INITDB_DATABASE: testdb
-    networks:
-      - portabase
-
-volumes:
-  mongodb-data:
-  mongodb-data-auth:
-
 networks:
   portabase:
     name: portabase_network

docker-compose.yml

@@ -21,9 +21,11 @@ services:
       LOG: debug
       TZ: "Europe/Paris"
 #      TMPDIR: /scratch
-      EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiNmM4NWE3ODQtODRkMi00YzUyLTgzYmUtZTc2MDZkZjg2YjM5IiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ=="
+      EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiY2UxNjRiZDItZGZkMy00YzY4LThlZGItNmQ3OTczODAzZWEyIiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ=="
       #CHUNK_SIZE_MB: "1"
       #POOLING: 1
+      #RETRY_ATTEMPTS: 3
+      #RETRY_BACKOFF_MS: 1000
       #DATABASES_CONFIG_FILE: "config.toml"
     extra_hosts:
       - "localhost:host-gateway"

docker/Dockerfile

@@ -142,6 +142,8 @@ RUN curl -sSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh \
 
 WORKDIR /app
 
+RUN mkdir -p /config
+
 COPY --from=builder /app/target/release/app /usr/local/bin/app
 COPY --from=builder /app/version.env /app/version.env
 COPY entrypoint.sh /entrypoint.sh

helm/templates/env-configmap.yaml

@@ -7,4 +7,6 @@ data:
   TZ: {{ .Values.env.TZ | quote }}
   POLLING: {{ .Values.env.POLLING | quote }}
   APP_ENV: {{ .Values.env.APP_ENV | quote }}
-  LOG: {{ .Values.env.LOG | quote }}
+  LOG: {{ .Values.env.LOG | quote }}
+  RETRY_ATTEMPTS: {{ .Values.env.RETRY_ATTEMPTS | quote }}
+  RETRY_BACKOFF_MS: {{ .Values.env.RETRY_BACKOFF_MS | quote }}

helm/values.yaml

@@ -11,6 +11,8 @@ env:
   POLLING: "5"
   APP_ENV: "production"
   LOG: "info"
+  RETRY_ATTEMPTS: "3"
+  RETRY_BACKOFF_MS: "1000"
 
 resources:
   limits:

src/core/agent.rs

@@ -2,13 +2,16 @@
 
 use crate::core::context::Context;
 use crate::services::backup::BackupService;
-use crate::services::config::ConfigService;
+use crate::services::config::{ConfigService, DatabaseConfig};
 use crate::services::cron::CronService;
+use crate::services::dashboard_config::{collect_configs, load_cache, merge, persist_cache};
 use crate::services::restore::RestoreService;
 use crate::services::status::StatusService;
+use crate::settings::CONFIG;
 use crate::utils::common::BackupMethod;
+use std::path::PathBuf;
 use std::sync::Arc;
-use tracing::info;
+use tracing::{error, info, warn};
 
 pub struct Agent {
     ctx: Arc<Context>,
@@ -17,6 +20,8 @@ pub struct Agent {
     cron_service: CronService,
     backup_service: BackupService,
     restore_service: RestoreService,
+    dashboard_cache: Vec<DatabaseConfig>,
+    cache_path: PathBuf,
 }
 
 impl Agent {
@@ -28,6 +33,9 @@ impl Agent {
         let backup_service = BackupService::new(ctx.clone());
         let restore_service = RestoreService::new(ctx.clone());
 
+        let cache_path = PathBuf::from(&CONFIG.data_path).join("dashboard_databases.json");
+        let dashboard_cache = load_cache(&cache_path);
+
         Agent {
             ctx,
             config_service,
@@ -35,19 +43,33 @@ impl Agent {
             cron_service,
             backup_service,
             restore_service,
+            dashboard_cache,
+            cache_path,
         }
     }
 
     pub async fn run(&mut self, method: BackupMethod) -> Result<(), Box<dyn std::error::Error>> {
-        let config = self.config_service.load(None)?;
-        let ping_result = self.status_service.ping(&config.databases).await?;
+        let local = self.config_service.load_optional(None);
+
+        let merged_in = merge(&local.databases, &self.dashboard_cache);
+        let ping_result = self.status_service.ping(&merged_in.databases).await?;
+
+        self.dashboard_cache = collect_configs(&ping_result);
+        if let Err(e) = persist_cache(&self.cache_path, &self.dashboard_cache) {
+            error!("Failed to persist dashboard cache: {e}");
+        }
+
+        let merged = merge(&local.databases, &self.dashboard_cache);
 
         for db in ping_result.databases.iter() {
-            let database = config
+            let Some(database) = merged
                 .databases
                 .iter()
                 .find(|cfg_db| cfg_db.generated_id == db.generated_id)
-                .unwrap();
+            else {
+                warn!("No config for returned database {}; skipping", db.generated_id);
+                continue;
+            };
             info!(
                 "Generated Id: {} | backup action: {} | restore action: {} | Database Name: {}",
                 db.generated_id, db.data.backup.action, db.data.restore.action, database.name,
@@ -59,14 +81,14 @@ impl Agent {
                     .backup_service
                     .dispatch(
                         &db.generated_id,
-                        &config,
+                        &merged,
                         method.clone(),
                         &db.storages,
                         db.encrypt,
                     )
                     .await;
             } else if db.data.restore.action {
-                let _ = self.restore_service.dispatch(db, &config).await;
+                let _ = self.restore_service.dispatch(db, &merged).await;
             }
         }
 

src/domain/docker_volume/docker.rs

@@ -15,7 +15,7 @@ pub const EPHEMERAL_LABEL: &str = "io.portabase.ephemeral";
 const HELPER_MOUNT: &str = "/vol";
 
 pub fn client() -> Result<Docker> {
-    Docker::connect_with_unix_defaults().context("Failed to connect to Docker daemon socket")
+    Docker::connect_with_defaults().context("Failed to connect to Docker daemon socket")
 }
 
 pub fn parse_container_id(mountinfo: &str, cgroup: &str) -> Option<String> {

src/domain/firebird/ping.rs

@@ -20,6 +20,7 @@ pub async fn run(cfg: DatabaseConfig) -> anyhow::Result<bool> {
         .stdin(Stdio::piped())
         .stdout(Stdio::piped())
         .stderr(Stdio::piped())
+        .kill_on_drop(true)
         .spawn()?;
 
     let query = b"SELECT 1 FROM RDB$DATABASE;\nQUIT;\n";

src/domain/mariadb/ping.rs

@@ -12,7 +12,8 @@ pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::R
         .arg("--user")
         .arg(cfg.username)
         .arg("ping")
-        .envs(env);
+        .envs(env)
+        .kill_on_drop(true);
 
     let result = timeout(Duration::from_secs(10), cmd.output()).await;
 

src/domain/mongodb/connection.rs

@@ -1,6 +1,13 @@
 use crate::services::config::DatabaseConfig;
 use anyhow::Result;
 use mongodb::Client;
+use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
+
+const USERINFO_ENCODE: &AsciiSet = &NON_ALPHANUMERIC
+    .remove(b'-')
+    .remove(b'_')
+    .remove(b'.')
+    .remove(b'~');
 
 pub async fn connect(cfg: DatabaseConfig) -> Result<Client> {
     let uri = get_mongo_uri(cfg)?;
@@ -16,19 +23,40 @@ pub fn select_mongo_path() -> std::path::PathBuf {
 }
 
 pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result<String> {
-    if cfg.username.is_empty() || cfg.password.is_empty() {
-        Ok(format!(
-            "mongodb://{}:{}/{}",
-            cfg.host, cfg.port, cfg.database
-        ))
-    } else {
-        Ok(format!(
-            "mongodb://{}:{}@{}:{}/{}?authSource=admin",
-            cfg.username, cfg.password, cfg.host, cfg.port, cfg.database
-        ))
-    }
+    Ok(build_mongo_uri(&cfg, true))
 }
 
+pub fn build_mongo_uri(cfg: &DatabaseConfig, include_db: bool) -> String {
+    let is_srv = cfg.port == 0;
+    let scheme = if is_srv { "mongodb+srv" } else { "mongodb" };
+    let has_auth = !cfg.username.is_empty() && !cfg.password.is_empty();
+
+    let credentials = if has_auth {
+        format!(
+            "{}:{}@",
+            utf8_percent_encode(&cfg.username, USERINFO_ENCODE),
+            utf8_percent_encode(&cfg.password, USERINFO_ENCODE)
+        )
+    } else {
+        String::new()
+    };
+
+    let authority = if is_srv {
+        cfg.host.clone()
+    } else {
+        format!("{}:{}", cfg.host, cfg.port)
+    };
+
+    let path = if include_db {
+        format!("/{}", cfg.database)
+    } else {
+        "/".to_string()
+    };
+
+    let query = if has_auth { "?authSource=admin" } else { "" };
+
+    format!("{}://{}{}{}{}", scheme, credentials, authority, path, query)
+}
 
 pub fn extract_db_name(dry_output: &str) -> Option<String> {
     let mut dbs = std::collections::HashSet::new();
@@ -43,3 +71,70 @@ pub fn extract_db_name(dry_output: &str) -> Option<String> {
     }
     dbs.into_iter().next()
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::services::config::{DatabaseConfig, DbType};
+    use std::collections::HashMap;
+
+    fn cfg(host: &str, port: u16, user: &str, pass: &str) -> DatabaseConfig {
+        DatabaseConfig {
+            name: "t".into(),
+            database: "mydb".into(),
+            db_type: DbType::MongoDB,
+            username: user.into(),
+            password: pass.into(),
+            port,
+            host: host.into(),
+            generated_id: "id".into(),
+            path: String::new(),
+            max_packet_size: String::new(),
+            volume_name: String::new(),
+            container_name: None,
+            options: HashMap::new(),
+        }
+    }
+
+    #[test]
+    fn standard_with_auth() {
+        let c = cfg("localhost", 27017, "user", "pass");
+        assert_eq!(
+            build_mongo_uri(&c, true),
+            "mongodb://user:pass@localhost:27017/mydb?authSource=admin"
+        );
+    }
+
+    #[test]
+    fn standard_no_auth() {
+        let c = cfg("localhost", 27017, "", "");
+        assert_eq!(build_mongo_uri(&c, true), "mongodb://localhost:27017/mydb");
+    }
+
+    #[test]
+    fn srv_with_auth() {
+        let c = cfg("cluster.example.mongodb.net", 0, "user", "pass");
+        assert_eq!(
+            build_mongo_uri(&c, true),
+            "mongodb+srv://user:pass@cluster.example.mongodb.net/mydb?authSource=admin"
+        );
+    }
+
+    #[test]
+    fn srv_no_db_for_dryrun() {
+        let c = cfg("cluster.example.mongodb.net", 0, "user", "pass");
+        assert_eq!(
+            build_mongo_uri(&c, false),
+            "mongodb+srv://user:pass@cluster.example.mongodb.net/?authSource=admin"
+        );
+    }
+
+    #[test]
+    fn encodes_special_chars_in_credentials() {
+        let c = cfg("cluster.example.mongodb.net", 0, "user", "p@ss:w/rd?");
+        assert_eq!(
+            build_mongo_uri(&c, true),
+            "mongodb+srv://user:p%40ss%3Aw%2Frd%3F@cluster.example.mongodb.net/mydb?authSource=admin"
+        );
+    }
+}

src/domain/mongodb/ping.rs

@@ -19,7 +19,11 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
         Ok(_) => Ok(true),
         Err(e) => {
             error!("--- MongoDB Connection Error Details ---");
-            error!("Target Host: {}:{}", cfg.host, cfg.port);
+            if cfg.port == 0 {
+                error!("Target Host: {} (srv)", cfg.host);
+            } else {
+                error!("Target Host: {}:{}", cfg.host, cfg.port);
+            }
             error!("Error Kind: {:?}", e.kind);
             error!("Full Error: {}", e);
             error!("Check you database network connectivity");

src/domain/mongodb/restore.rs

@@ -1,4 +1,6 @@
-use crate::domain::mongodb::connection::{extract_db_name, get_mongo_uri, select_mongo_path};
+use crate::domain::mongodb::connection::{
+    build_mongo_uri, extract_db_name, get_mongo_uri, select_mongo_path,
+};
 use crate::services::backup::logger::JobLogger;
 use crate::services::config::DatabaseConfig;
 use anyhow::{Context, Result};
@@ -16,13 +18,7 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogg
 
         let dry_start = Instant::now();
         let dry_run = Command::new(&mongorestore)
-            .arg(format!(
-                "--uri={}",
-                format!(
-                    "mongodb://{}:{}@{}:{}/?authSource=admin",
-                    cfg.username, cfg.password, cfg.host, cfg.port
-                )
-            ))
+            .arg(format!("--uri={}", build_mongo_uri(&cfg, false)))
             .arg(format!("--archive={}", restore_file.display()))
             .arg("--gzip")
             .arg("--dryRun")

src/domain/mysql/ping.rs

@@ -12,7 +12,8 @@ pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::R
         .arg("--user")
         .arg(cfg.username)
         .arg("ping")
-        .envs(env);
+        .envs(env)
+        .kill_on_drop(true);
 
     let result = timeout(Duration::from_secs(10), cmd.output()).await;
 

src/domain/redis/ping.rs

@@ -21,6 +21,8 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
 
     cmd.arg("PING");
 
+    cmd.kill_on_drop(true);
+
     debug!("Command Ping Redis: {:?}", cmd);
 
     let result = timeout(Duration::from_secs(10), cmd.output()).await;

src/domain/valkey/ping.rs

@@ -20,6 +20,7 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
     }
 
     cmd.arg("PING");
+    cmd.kill_on_drop(true);
 
     debug!("Command Ping Valkey: {:?}", cmd);
 

src/main.rs

@@ -22,7 +22,6 @@ async fn main() {
         eprintln!("Failed to clean locks on startup: {:?}", e);
     }
 
-    // Best-effort cleanup of ephemeral helper containers orphaned by a crash.
     match crate::domain::docker_volume::docker::client() {
         Ok(docker) => match crate::domain::docker_volume::docker::sweep_ephemeral(&docker).await {
             Ok(n) if n > 0 => tracing::info!("Removed {n} orphaned ephemeral helper container(s)"),

src/services/api/models/agent/status.rs

@@ -1,5 +1,6 @@
 #![allow(dead_code)]
 
+use crate::services::config::DatabaseConfig;
 use crate::utils::deserializer::{deserialize_snake_case, string_or_number_to_string};
 use serde::{Deserialize, Serialize};
 use toml::Value;
@@ -39,6 +40,13 @@ pub struct DatabaseStatus {
     pub storages_encrypted: Option<bool>,
     #[serde(default)]
     pub storages_ciphertext: Option<String>,
+    #[serde(default)]
+    pub config_encrypted: Option<bool>,
+    #[serde(default)]
+    pub config_ciphertext: Option<String>,
+    /// Filled in memory after decrypting `config_ciphertext`; never on the wire.
+    #[serde(skip)]
+    pub resolved_config: Option<DatabaseConfig>,
     pub encrypt: bool,
     pub data: DatabaseData,
 }

src/services/backup/models.rs

@@ -1,6 +1,7 @@
 #![allow(dead_code)]
 
 use crate::services::config::DbType;
+use std::fmt::{self, Display, Formatter};
 use std::path::PathBuf;
 
 #[derive(Debug, Clone)]
@@ -20,3 +21,9 @@ pub struct UploadResult {
     pub remote_file_path: Option<String>,
     pub total_size: Option<u64>,
 }
+
+impl Display for UploadResult {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.error.as_deref().unwrap_or("unknown error"))
+    }
+}

src/services/backup/runner.rs

@@ -4,6 +4,7 @@ use super::service::BackupService;
 
 use crate::domain::factory::DatabaseFactory;
 use crate::services::config::DatabaseConfig;
+use crate::utils::retry::{RetryPolicy, retry};
 
 use anyhow::Result;
 use std::path::Path;
@@ -39,7 +40,31 @@ impl BackupService {
             });
         }
 
-        match db.backup(tmp_path, Arc::clone(&logger)).await {
+        let policy = RetryPolicy::default();
+
+        let db_ref = &db;
+        let logger_ref = &logger;
+
+        let outcome = retry("Database backup", &logger, &policy, move |attempt| {
+            let dir = tmp_path.join(format!("attempt-{attempt}"));
+
+            async move {
+                if let Err(e) = tokio::fs::create_dir_all(&dir).await {
+                    return Err(anyhow::Error::from(e));
+                }
+
+                match db_ref.backup(&dir, Arc::clone(logger_ref)).await {
+                    Ok(f) => Ok(f),
+                    Err(e) => {
+                        let _ = tokio::fs::remove_dir_all(&dir).await;
+                        Err(e)
+                    }
+                }
+            }
+        })
+        .await;
+
+        match outcome {
             Ok(file) => Ok(BackupResult {
                 generated_id,
                 db_type,

src/services/backup/uploader.rs

@@ -4,6 +4,7 @@ use super::service::BackupService;
 use crate::services::api::models::agent::status::DatabaseStorage;
 use crate::services::storage;
 use crate::utils::common::BackupMethod;
+use crate::utils::retry::{RetryPolicy, retry};
 use anyhow::{Result, bail};
 use futures::future::join_all;
 use std::sync::Arc;
@@ -97,16 +98,44 @@ impl BackupService {
                 /*
                  STORAGE UPLOAD
                 */
-                let upload_result = provider
-                    .upload(
-                        ctx_clone.clone(),
-                        result_clone,
-                        method,
-                        &storage,
-                        Some(encrypt),
-                        &backup_storage_id,
+                let policy = RetryPolicy::default();
+
+                let attempt_result = if result_clone.backup_file.is_none() {
+                    logger_clone.log("error", format!("Missing backup file for storage {}", storage_id));
+
+                    Err(UploadResult {
+                        storage_id: storage_id.clone(),
+                        success: false,
+                        error: Some("Missing backup file path".into()),
+                        remote_file_path: None,
+                        total_size: None,
+                    })
+                } else {
+                    retry(
+                        &format!("Upload to storage {storage_id}"),
+                        &logger_clone,
+                        &policy,
+                        |_| async {
+                            let r = provider
+                                .upload(
+                                    ctx_clone.clone(),
+                                    result_clone.clone(),
+                                    method,
+                                    &storage,
+                                    Some(encrypt),
+                                    &backup_storage_id,
+                                )
+                                .await;
+
+                            if r.success { Ok(r) } else { Err(r) }
+                        },
                     )
-                    .await;
+                    .await
+                };
+
+                let upload_result = match attempt_result {
+                    Ok(r) | Err(r) => r,
+                };
 
                 let status = if upload_result.success { "success" } else { "failed" };
 
@@ -117,8 +146,6 @@ impl BackupService {
                         upload_result.error.as_deref().unwrap_or("unknown error")
                     ));
 
-                    // `backup_upload_init` opened a per-storage record; close it as "failed"
-                    // so the server is notified of the failure (no path/size on this path).
                     if let Err(err) = ctx_clone
                         .api
                         .backup_upload_status(

src/services/config.rs

@@ -1,7 +1,7 @@
 #![allow(dead_code)]
 
 use crate::core::context::Context;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use serde_json;
 use std::collections::HashMap;
 use std::fs::File;
@@ -12,7 +12,7 @@ use toml;
 use tracing::info;
 use uuid::Uuid;
 
-#[derive(Debug, Deserialize, Clone)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 #[serde(rename_all = "lowercase")]
 pub enum DbType {
     Mysql,
@@ -49,7 +49,7 @@ impl DbType {
 }
 
 #[allow(dead_code)]
-#[derive(Debug, Deserialize, Clone)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct DatabaseConfig {
     pub name: String,
     pub database: String,
@@ -68,7 +68,7 @@ pub struct DatabaseConfig {
 }
 
 #[allow(dead_code)]
-#[derive(Debug, Deserialize, Clone)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct DatabasesConfig {
     pub databases: Vec<DatabaseConfig>,
 }
@@ -98,6 +98,106 @@ pub struct InputDatabasesConfig {
     pub databases: Vec<InputDatabaseConfig>,
 }
 
+fn required<T: Clone>(opt: &Option<T>, db_name: &str, field_name: &str) -> Result<T, String> {
+    match opt {
+        Some(v) => Ok(v.clone()),
+        None => Err(format!(
+            "Missing required field '{}' for database '{}'",
+            field_name, db_name
+        )),
+    }
+}
+
+fn optional<T: Clone + Default>(opt: &Option<T>) -> T {
+    opt.clone().unwrap_or_default()
+}
+
+pub fn build_config(db: InputDatabaseConfig) -> Result<DatabaseConfig, String> {
+    if Uuid::parse_str(&db.generated_id).is_err() {
+        return Err(format!("Invalid UUID for database '{}'", db.name));
+    }
+
+    let username = match db.db_type {
+        DbType::Postgresql
+        | DbType::PostgresqlCluster
+        | DbType::Mysql
+        | DbType::Mariadb
+        | DbType::Mssql => required(&db.username, &db.name, "username")?,
+        _ => optional(&db.username),
+    };
+    let password = match db.db_type {
+        DbType::Postgresql
+        | DbType::PostgresqlCluster
+        | DbType::Mysql
+        | DbType::Mariadb
+        | DbType::Mssql => required(&db.password, &db.name, "password")?,
+        _ => optional(&db.password),
+    };
+    let host = match db.db_type {
+        DbType::Postgresql
+        | DbType::PostgresqlCluster
+        | DbType::Mysql
+        | DbType::Mariadb
+        | DbType::MongoDB
+        | DbType::Redis
+        | DbType::Firebird
+        | DbType::Valkey
+        | DbType::Mssql => required(&db.host, &db.name, "host")?,
+        DbType::Sqlite | DbType::DockerVolume => optional(&db.host),
+    };
+
+    let port = match db.db_type {
+        DbType::Postgresql
+        | DbType::PostgresqlCluster
+        | DbType::Mysql
+        | DbType::Mariadb
+        | DbType::Redis
+        | DbType::Firebird
+        | DbType::Valkey
+        | DbType::Mssql => required(&db.port, &db.name, "port")?,
+        DbType::MongoDB | DbType::Sqlite | DbType::DockerVolume => db.port.unwrap_or(0),
+    };
+
+    let database_name = match db.db_type {
+        DbType::Sqlite | DbType::Redis | DbType::Valkey | DbType::DockerVolume => {
+            optional(&db.database)
+        }
+        DbType::PostgresqlCluster => db
+            .database
+            .clone()
+            .unwrap_or_else(|| "postgres".to_string()),
+        _ => required(&db.database, &db.name, "database")?,
+    };
+    let path_val = match db.db_type {
+        DbType::Sqlite => required(&db.path, &db.name, "path")?,
+        _ => optional(&db.path),
+    };
+    let max_packet_size = match db.db_type {
+        DbType::Mysql | DbType::Mariadb => db.max_packet_size.unwrap_or_else(|| "512M".to_string()),
+        _ => String::new(),
+    };
+    let volume_name = match db.db_type {
+        DbType::DockerVolume => required(&db.volume_name, &db.name, "volume_name")?,
+        _ => optional(&db.volume_name),
+    };
+
+    Ok(DatabaseConfig {
+        name: db.name,
+        database: database_name,
+        db_type: db.db_type,
+        username,
+        password,
+        host,
+        port,
+        generated_id: db.generated_id,
+        path: path_val,
+        max_packet_size,
+        volume_name,
+        container_name: db.container_name.clone(),
+        options: db.options.unwrap_or_default(),
+    })
+}
+
 pub struct ConfigService {
     ctx: Arc<Context>,
 }
@@ -107,16 +207,19 @@ impl ConfigService {
         ConfigService { ctx }
     }
 
-    pub fn load(&self, file_path: Option<&str>) -> Result<DatabasesConfig, String> {
-        let path: String = if let Some(fp) = file_path {
-            fp.to_string()
-        } else {
-            format!(
+    fn resolve_path(file_path: Option<&str>) -> String {
+        match file_path {
+            Some(fp) => fp.to_string(),
+            None => format!(
                 "{}/{}",
                 crate::settings::CONFIG.data_path,
                 crate::settings::CONFIG.databases_config_file
-            )
-        };
+            ),
+        }
+    }
+
+    pub fn load(&self, file_path: Option<&str>) -> Result<DatabasesConfig, String> {
+        let path = Self::resolve_path(file_path);
 
         info!("Loading databases config from: {}", path);
 
@@ -150,128 +253,33 @@ impl ConfigService {
             _ => return Err("Unsupported config file format. Use .json or .toml".to_string()),
         };
 
-        fn required<T: Clone>(
-            opt: &Option<T>,
-            db_name: &str,
-            field_name: &str,
-        ) -> Result<T, String> {
-            match opt {
-                Some(v) => Ok(v.clone()),
-                None => {
-                    let msg = format!(
-                        "Missing required field '{}' for database '{}'",
-                        field_name, db_name
-                    );
-                    Err(msg)
-                }
-            }
-        }
-
-        fn optional<T: Clone>(opt: &Option<T>) -> T
-        where
-            T: Default,
-        {
-            opt.clone().unwrap_or_default()
-        }
-
         let mut databases = Vec::with_capacity(input_config.databases.len());
-
         for db in input_config.databases {
-            if Uuid::parse_str(&db.generated_id).is_err() {
-                return Err(format!("Invalid UUID for database '{}'", db.name));
-            }
-
-            let username = match db.db_type {
-                DbType::Postgresql
-                | DbType::PostgresqlCluster
-                | DbType::Mysql
-                | DbType::Mariadb
-                | DbType::Mssql => required(&db.username, &db.name, "username")?,
-                _ => optional(&db.username),
-            };
-
-            let password = match db.db_type {
-                DbType::Postgresql
-                | DbType::PostgresqlCluster
-                | DbType::Mysql
-                | DbType::Mariadb
-                | DbType::Mssql => required(&db.password, &db.name, "password")?,
-                _ => optional(&db.password),
-            };
-
-            let host = match db.db_type {
-                DbType::Postgresql
-                | DbType::PostgresqlCluster
-                | DbType::Mysql
-                | DbType::Mariadb
-                | DbType::MongoDB
-                | DbType::Redis
-                | DbType::Firebird
-                | DbType::Valkey
-                | DbType::Mssql => required(&db.host, &db.name, "host")?,
-                DbType::Sqlite | DbType::DockerVolume => optional(&db.host),
-            };
-
-            let port = match db.db_type {
-                DbType::Postgresql
-                | DbType::PostgresqlCluster
-                | DbType::Mysql
-                | DbType::Mariadb
-                | DbType::MongoDB
-                | DbType::Redis
-                | DbType::Firebird
-                | DbType::Valkey
-                | DbType::Mssql => required(&db.port, &db.name, "port")?,
-                DbType::Sqlite | DbType::DockerVolume => db.port.unwrap_or(0),
-            };
-
-            let database_name = match db.db_type {
-                DbType::Sqlite | DbType::Redis | DbType::Valkey | DbType::DockerVolume => {
-                    optional(&db.database)
-                }
-                DbType::PostgresqlCluster => db
-                    .database
-                    .clone()
-                    .unwrap_or_else(|| "postgres".to_string()),
-                _ => required(&db.database, &db.name, "database")?,
-            };
-
-            let path_val = match db.db_type {
-                DbType::Sqlite => required(&db.path, &db.name, "path")?,
-                _ => optional(&db.path),
-            };
-
-            let max_packet_size = match db.db_type {
-                DbType::Mysql | DbType::Mariadb => {
-                    db.max_packet_size.unwrap_or_else(|| "512M".to_string())
-                }
-                _ => String::new(),
-            };
-
-            let volume_name = match db.db_type {
-                DbType::DockerVolume => required(&db.volume_name, &db.name, "volume_name")?,
-                _ => optional(&db.volume_name),
-            };
-            let container_name = db.container_name.clone();
-
-            databases.push(DatabaseConfig {
-                name: db.name,
-                database: database_name,
-                db_type: db.db_type,
-                username,
-                password,
-                host,
-                port,
-                generated_id: db.generated_id,
-                path: path_val,
-                max_packet_size,
-                volume_name,
-                container_name,
-                options: db.options.unwrap_or_default(),
-            });
+            databases.push(build_config(db)?);
         }
 
         info!("Databases: {} instances loaded", databases.len());
         Ok(DatabasesConfig { databases })
     }
+
+    pub fn load_optional(&self, file_path: Option<&str>) -> DatabasesConfig {
+        let path = Self::resolve_path(file_path);
+
+        if !Path::new(&path).exists() {
+            info!(
+                "No local databases config at {}; using dashboard-defined databases only",
+                path
+            );
+            return DatabasesConfig {
+                databases: Vec::new(),
+            };
+        }
+
+        self.load(file_path).unwrap_or_else(|e| {
+            tracing::warn!(
+                    "Local databases config unavailable ({e}); continuing with dashboard-defined databases only"
+                );
+            DatabasesConfig { databases: Vec::new() }
+        })
+    }
 }

src/services/dashboard_config.rs

@@ -0,0 +1,56 @@
+#![allow(dead_code)]
+
+use crate::services::api::models::agent::status::PingResult;
+use crate::services::config::{DatabaseConfig, DatabasesConfig};
+use std::path::Path;
+
+pub fn merge(local: &[DatabaseConfig], dashboard: &[DatabaseConfig]) -> DatabasesConfig {
+    let mut databases: Vec<DatabaseConfig> = local.to_vec();
+    for d in dashboard {
+        if let Some(slot) = databases
+            .iter_mut()
+            .find(|c| c.generated_id == d.generated_id)
+        {
+            *slot = d.clone();
+        } else {
+            databases.push(d.clone());
+        }
+    }
+    DatabasesConfig { databases }
+}
+
+pub fn collect_configs(ping: &PingResult) -> Vec<DatabaseConfig> {
+    ping.databases
+        .iter()
+        .filter_map(|db| db.resolved_config.clone())
+        .collect()
+}
+
+pub fn load_cache(path: &Path) -> Vec<DatabaseConfig> {
+    let contents = match std::fs::read_to_string(path) {
+        Ok(c) => c,
+        Err(_) => return Vec::new(),
+    };
+    match serde_json::from_str::<DatabasesConfig>(&contents) {
+        Ok(cfg) => cfg.databases,
+        Err(e) => {
+            tracing::warn!("Dashboard cache at {:?} is corrupt ({e}); ignoring", path);
+            Vec::new()
+        }
+    }
+}
+
+pub fn persist_cache(path: &Path, databases: &[DatabaseConfig]) -> std::io::Result<()> {
+    let wrapper = DatabasesConfig {
+        databases: databases.to_vec(),
+    };
+    let json = serde_json::to_string_pretty(&wrapper)
+        .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
+    if let Some(parent) = path.parent() {
+        std::fs::create_dir_all(parent)?;
+    }
+    let tmp = path.with_extension("json.tmp");
+    std::fs::write(&tmp, json)?;
+    std::fs::rename(&tmp, path)?;
+    Ok(())
+}

src/services/mod.rs

@@ -2,6 +2,7 @@ pub mod api;
 pub mod backup;
 pub mod config;
 pub mod cron;
+pub mod dashboard_config;
 pub mod restore;
 pub mod status;
 pub mod storage;

src/services/restore/downloader.rs

@@ -8,6 +8,7 @@ use std::sync::Arc;
 use std::time::Instant;
 use tokio::io::AsyncWriteExt;
 use crate::services::backup::logger::JobLogger;
+use crate::utils::retry::{RetryPolicy, retry};
 
 fn human_size(bytes: u64) -> String {
     if bytes >= 1024 * 1024 {
@@ -26,6 +27,34 @@ impl RestoreService {
         tmp_path: &Path,
         logger: Arc<JobLogger>,
         expected_size: Option<String>,
+    ) -> Result<PathBuf> {
+        let policy = RetryPolicy::default();
+
+        let logger_ref = &logger;
+
+        let outcome = retry("Backup download", &logger, &policy, move |_| {
+            let expected = expected_size.clone();
+
+            async move {
+                self.download_once(file_url, tmp_path, Arc::clone(logger_ref), expected)
+                    .await
+            }
+        })
+        .await;
+
+        if let Err(e) = &outcome {
+            logger.log("error", format!("Download failed: {e}"));
+        }
+
+        outcome
+    }
+
+    pub async fn download_once(
+        &self,
+        file_url: &str,
+        tmp_path: &Path,
+        logger: Arc<JobLogger>,
+        expected_size: Option<String>,
     ) -> Result<PathBuf> {
         logger.log("info", "Start downloading backup archive".to_string());
 

src/services/status.rs

@@ -3,9 +3,10 @@
 use crate::core::context::Context;
 use crate::domain::factory::DatabaseFactory;
 use crate::services::api::endpoints::status::DatabasePayload;
+use crate::services::api::models::agent::status::DatabaseStatus;
 use crate::services::api::models::agent::status::DatabaseStorage;
 use crate::services::api::models::agent::status::PingResult;
-use crate::services::config::DatabaseConfig;
+use crate::services::config::{build_config, DatabaseConfig, InputDatabaseConfig};
 use crate::settings::CONFIG;
 use crate::utils::file::decrypt_json_gcm;
 use futures_util::future::try_join_all;
@@ -14,6 +15,26 @@ use std::error::Error;
 use std::sync::Arc;
 use tracing::info;
 
+pub fn resolve_dashboard_config(
+    status: &mut DatabaseStatus,
+    master_key_b64: &str,
+) -> Result<(), String> {
+    if status.config_encrypted != Some(true) {
+        return Ok(());
+    }
+    let ciphertext = status
+        .config_ciphertext
+        .as_deref()
+        .ok_or("config_encrypted set but config_ciphertext missing")?;
+
+    let plaintext = decrypt_json_gcm(ciphertext, master_key_b64)
+        .map_err(|e| format!("Failed to decrypt config: {e}"))?;
+    let input: InputDatabaseConfig = serde_json::from_slice(&plaintext)
+        .map_err(|e| format!("Failed to parse decrypted config: {e}"))?;
+    status.resolved_config = Some(build_config(input)?);
+    Ok(())
+}
+
 pub struct StatusService {
     ctx: Arc<Context>,
     client: Client,
@@ -67,6 +88,10 @@ impl StatusService {
                 db.storages = serde_json::from_slice::<Vec<DatabaseStorage>>(&plaintext)
                     .map_err(|e| format!("Failed to parse decrypted storages: {e}"))?;
             }
+
+            if let Err(e) = resolve_dashboard_config(db, &edge_key.master_key_b64) {
+                tracing::warn!("Skipping dashboard config for {}: {e}", db.generated_id);
+            }
         }
         Ok(result)
     }

src/services/storage/providers/s3/mod.rs

@@ -13,7 +13,9 @@ use aws_config::retry::RetryConfig;
 use aws_sdk_s3 as s3;
 use aws_sdk_s3::config::BehaviorVersion;
 use aws_sdk_s3::config::Region;
+use aws_sdk_s3::config::RequestChecksumCalculation;
 use aws_sdk_s3::config::retry::ReconnectMode;
+use aws_sdk_s3::error::DisplayErrorContext;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
 use futures::StreamExt;
@@ -135,6 +137,7 @@ impl StorageProvider for S3Provider {
             .credentials_provider(credentials)
             .region(region)
             .force_path_style(true)
+            .request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
             .endpoint_url(endpoint)
             .behavior_version(BehaviorVersion::latest())
             .build();
@@ -164,11 +167,12 @@ impl StorageProvider for S3Provider {
         {
             Ok(r) => r,
             Err(e) => {
-                error!("Failed to create multipart upload: {}", e);
+                let detail = DisplayErrorContext(&e).to_string();
+                error!("Failed to create multipart upload: {}", detail);
                 return UploadResult {
                     storage_id: storage.id.clone(),
                     success: false,
-                    error: Some(e.to_string()),
+                    error: Some(detail),
                     remote_file_path: None,
                     total_size: None,
                 };
@@ -251,7 +255,8 @@ impl StorageProvider for S3Provider {
                         }
                     }
                     Err(e) => {
-                        error!("Failed to upload part {}: {}", part_number, e);
+                        let detail = DisplayErrorContext(&e).to_string();
+                        error!("Failed to upload part {}: {}", part_number, detail);
                         let _ = client
                             .abort_multipart_upload()
                             .bucket(bucket)
@@ -262,7 +267,7 @@ impl StorageProvider for S3Provider {
                         return UploadResult {
                             storage_id: storage.id.clone(),
                             success: false,
-                            error: Some(e.to_string()),
+                            error: Some(detail),
                             remote_file_path: None,
                             total_size: None,
                         };
@@ -317,7 +322,8 @@ impl StorageProvider for S3Provider {
                 }
             }
             Err(e) => {
-                error!("Failed to complete multipart upload: {}", e);
+                let detail = DisplayErrorContext(&e).to_string();
+                error!("Failed to complete multipart upload: {}", detail);
                 let _ = client
                     .abort_multipart_upload()
                     .bucket(bucket)
@@ -328,7 +334,7 @@ impl StorageProvider for S3Provider {
                 UploadResult {
                     storage_id: storage.id.clone(),
                     success: false,
-                    error: Some(e.to_string()),
+                    error: Some(detail),
                     remote_file_path: None,
                     total_size: None,
                 }

src/settings.rs

@@ -16,6 +16,8 @@ pub struct Settings {
     pub timezone: String,
     pub log: String,
     pub chunk_size: usize, // bytes
+    pub retry_attempts: u32,
+    pub retry_backoff_ms: u64,
 }
 
 impl Settings {
@@ -49,6 +51,24 @@ impl Settings {
 
         let chunk_size = chunk_size_mb * 1024 * 1024;
 
+        let retry_attempts = env::var("RETRY_ATTEMPTS")
+            .unwrap_or_else(|_| "3".to_string())
+            .parse::<u32>()
+            .expect("RETRY_ATTEMPTS must be a valid positive integer");
+
+        if retry_attempts < 3 || retry_attempts > 5 {
+            panic!("RETRY_ATTEMPTS must be between 3 and 5");
+        }
+
+        let retry_backoff_ms = env::var("RETRY_BACKOFF_MS")
+            .unwrap_or_else(|_| "1000".to_string())
+            .parse::<u64>()
+            .expect("RETRY_BACKOFF_MS must be a valid positive integer");
+
+        if retry_backoff_ms < 100 || retry_backoff_ms > 30_000 {
+            panic!("RETRY_BACKOFF_MS must be between 100 and 30000 milliseconds");
+        }
+
         let tz = env::var("TZ").unwrap_or_else(|_| "UTC".to_string());
 
         Self {
@@ -64,7 +84,9 @@ impl Settings {
             pooling: pooling_seconds,
             timezone: tz,
             log: env::var("LOG").unwrap_or_else(|_| "info".into()),
-            chunk_size
+            chunk_size,
+            retry_attempts,
+            retry_backoff_ms,
         }
     }
 }

src/tests/services/api_models_tests.rs

@@ -148,3 +148,98 @@ fn database_status_encrypted_envelope() {
     assert_eq!(status.storages_encrypted, Some(true));
     assert_eq!(status.storages_ciphertext.as_deref(), Some("AQIDBA=="));
 }
+
+#[test]
+fn database_status_defaults_config_fields_absent() {
+    let json = r#"{
+        "dbms": "postgresql",
+        "generatedId": "16678159-ff7e-4c97-8c83-0adeff214681",
+        "encrypt": false,
+        "data": { "backup": { "action": false, "cron": null },
+                  "restore": { "action": false, "file": null, "metaFile": null, "size": null } }
+    }"#;
+    let status: crate::services::api::models::agent::status::DatabaseStatus =
+        serde_json::from_str(json).unwrap();
+    assert_eq!(status.config_encrypted, None);
+    assert!(status.config_ciphertext.is_none());
+    assert!(status.resolved_config.is_none());
+}
+
+#[test]
+fn resolve_dashboard_config_decrypts_full_entry() {
+    use crate::services::status::resolve_dashboard_config;
+    use base64::{engine::general_purpose, Engine};
+
+    // 32-byte master key, base64 STANDARD (matches decrypt_json_gcm).
+    let master_key_b64 = general_purpose::STANDARD.encode([7u8; 32]);
+
+    // Full agent-entry shape the dashboard encrypts.
+    let entry = r#"{
+        "name": "Dashboard PG",
+        "type": "postgresql",
+        "database": "app",
+        "username": "postgres",
+        "password": "s3cret",
+        "port": 5432,
+        "host": "10.0.0.10",
+        "generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"
+    }"#;
+    let ciphertext = encrypt_json_gcm(entry.as_bytes(), &master_key_b64);
+
+    let mut status: crate::services::api::models::agent::status::DatabaseStatus =
+        serde_json::from_str(
+            r#"{
+                "dbms": "postgresql",
+                "generatedId": "16678159-ff7e-4c97-8c83-0adeff214681",
+                "encrypt": false,
+                "config_encrypted": true,
+                "config_ciphertext": "PLACEHOLDER",
+                "data": { "backup": { "action": false, "cron": null },
+                          "restore": { "action": false, "file": null, "metaFile": null, "size": null } }
+            }"#,
+        )
+        .unwrap();
+    status.config_ciphertext = Some(ciphertext);
+
+    resolve_dashboard_config(&mut status, &master_key_b64).unwrap();
+
+    let cfg = status.resolved_config.expect("resolved");
+    assert_eq!(cfg.name, "Dashboard PG");
+    assert_eq!(cfg.password, "s3cret");
+    assert_eq!(cfg.host, "10.0.0.10");
+    assert_eq!(cfg.db_type.as_str(), "postgresql");
+}
+
+#[test]
+fn resolve_dashboard_config_noop_when_not_encrypted() {
+    use crate::services::status::resolve_dashboard_config;
+    let mut status: crate::services::api::models::agent::status::DatabaseStatus =
+        serde_json::from_str(
+            r#"{
+                "dbms": "postgresql",
+                "generatedId": "16678159-ff7e-4c97-8c83-0adeff214681",
+                "encrypt": false,
+                "data": { "backup": { "action": false, "cron": null },
+                          "restore": { "action": false, "file": null, "metaFile": null, "size": null } }
+            }"#,
+        )
+        .unwrap();
+    resolve_dashboard_config(&mut status, "unused").unwrap();
+    assert!(status.resolved_config.is_none());
+}
+
+fn encrypt_json_gcm(plaintext: &[u8], master_key_b64: &str) -> String {
+    use aes_gcm::aead::{Aead, KeyInit};
+    use aes_gcm::{Aes256Gcm, Key, Nonce};
+    use base64::{engine::general_purpose, Engine};
+
+    let key_bytes = general_purpose::STANDARD.decode(master_key_b64).unwrap();
+    let key = Key::<Aes256Gcm>::try_from(key_bytes.as_slice()).unwrap();
+    let cipher = Aes256Gcm::new(&key);
+    let nonce_bytes = [0u8; 12];
+    let nonce = Nonce::try_from(&nonce_bytes[..]).unwrap();
+    let ct = cipher.encrypt(&nonce, plaintext).unwrap();
+    let mut data = nonce_bytes.to_vec();
+    data.extend_from_slice(&ct);
+    general_purpose::STANDARD.encode(data)
+}

src/tests/services/backup_runner_tests.rs

@@ -0,0 +1,68 @@
+use crate::services::backup::BackupService;
+use crate::services::backup::logger::JobLogger;
+use crate::services::config::{DatabaseConfig, DbType};
+use crate::tests::init_tracing_for_test;
+
+use std::collections::HashMap;
+use std::sync::Arc;
+use tempfile::TempDir;
+
+fn sqlite_config(path: &str) -> DatabaseConfig {
+    DatabaseConfig {
+        name: "retry-test".to_string(),
+        database: String::new(),
+        db_type: DbType::Sqlite,
+        username: String::new(),
+        password: String::new(),
+        port: 0,
+        host: String::new(),
+        generated_id: "retry-test-gen".to_string(),
+        path: path.to_string(),
+        max_packet_size: String::new(),
+        volume_name: String::new(),
+        container_name: None,
+        options: HashMap::new(),
+    }
+}
+
+#[tokio::test]
+async fn a_failing_backup_is_retried_and_leaves_no_attempt_directory() {
+    init_tracing_for_test();
+
+    let temp_dir = TempDir::new().unwrap();
+    let tmp_path = temp_dir.path();
+    let logger = Arc::new(JobLogger::new());
+
+    let cfg = sqlite_config("/nonexistent/definitely-not-here.sqlite");
+
+    let result = BackupService::run(cfg, tmp_path, Arc::clone(&logger))
+        .await
+        .unwrap();
+
+    assert_eq!(result.status, "failed");
+    assert!(result.backup_file.is_none());
+
+    let entries = Arc::try_unwrap(logger).unwrap().into_entries();
+    assert_eq!(
+        entries.iter().filter(|e| e.level == "warn").count(),
+        2,
+        "expected one warn per non-final failed attempt"
+    );
+    assert!(
+        entries
+            .iter()
+            .any(|e| e.level == "error" && e.message.starts_with("Backup failed:")),
+        "expected a single terminal error from the runner"
+    );
+
+    let leftovers: Vec<_> = std::fs::read_dir(tmp_path)
+        .unwrap()
+        .filter_map(|e| e.ok())
+        .filter(|e| e.file_name().to_string_lossy().starts_with("attempt-"))
+        .collect();
+    assert!(
+        leftovers.is_empty(),
+        "failed attempt directories must be cleaned up, found {:?}",
+        leftovers.iter().map(|e| e.file_name()).collect::<Vec<_>>()
+    );
+}

src/tests/services/backup_uploader_tests.rs

@@ -14,7 +14,9 @@ use crate::utils::common::BackupMethod;
 use crate::utils::edge_key::EdgeKey;
 
 use serde_json::json;
+use std::io::Write;
 use std::sync::Arc;
+use tempfile::NamedTempFile;
 use wiremock::matchers::{body_partial_json, method, path};
 use wiremock::{Mock, MockServer, ResponseTemplate};
 
@@ -93,3 +95,111 @@ async fn failed_upload_reports_failed_status_to_server() {
 
     // MockServer drop verifies both `.expect(1)` mounts were hit — including the "failed" PATCH.
 }
+
+#[tokio::test]
+async fn a_failing_upload_is_retried_until_it_succeeds() {
+    init_tracing_for_test();
+    let server = MockServer::start().await;
+
+    Mock::given(method("POST"))
+        .and(path("/agent/agent-1/backup/upload/init"))
+        .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+            "message": "ok",
+            "backupStorage": { "id": "bs-1" }
+        })))
+        .expect(1)
+        .mount(&server)
+        .await;
+
+    Mock::given(method("POST"))
+        .and(path("/tus/files"))
+        .respond_with(ResponseTemplate::new(500))
+        .up_to_n_times(2)
+        .with_priority(1)
+        .expect(2)
+        .mount(&server)
+        .await;
+
+    Mock::given(method("POST"))
+        .and(path("/tus/files"))
+        .respond_with(
+            ResponseTemplate::new(201)
+                .insert_header("Location", format!("{}/tus/files/upload-1", server.uri()).as_str()),
+        )
+        .with_priority(2)
+        .expect(1)
+        .mount(&server)
+        .await;
+
+    Mock::given(method("PATCH"))
+        .and(path("/tus/files/upload-1"))
+        .respond_with(ResponseTemplate::new(204))
+        .mount(&server)
+        .await;
+
+    Mock::given(method("PATCH"))
+        .and(path("/agent/agent-1/backup/upload/status"))
+        .and(body_partial_json(json!({ "status": "success" })))
+        .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+            "message": "ok",
+            "backupStorage": { "id": "bs-1" }
+        })))
+        .expect(1)
+        .mount(&server)
+        .await;
+
+    let mut backup_file = NamedTempFile::new().unwrap();
+    backup_file.write_all(b"portabase-retry-test-payload").unwrap();
+    backup_file.flush().unwrap();
+
+    let ctx = Context {
+        edge_key: EdgeKey {
+            server_url: server.uri(),
+            agent_id: "agent-1".to_string(),
+            master_key_b64: String::new(),
+        },
+        api: ApiClient::new(server.uri()),
+    };
+
+    let service = BackupService::new(Arc::new(ctx));
+
+    let result = BackupResult {
+        generated_id: "gen-1".to_string(),
+        db_type: DbType::Postgresql,
+        status: "success".to_string(),
+        backup_file: Some(backup_file.path().to_path_buf()),
+        code: None,
+    };
+
+    let storage: DatabaseStorage = serde_json::from_value(json!({
+        "id": "storage-1",
+        "provider": "local",
+        "config": {}
+    }))
+    .unwrap();
+
+    let backup_id = "backup-1".to_string();
+    let logger = Arc::new(JobLogger::new());
+
+    let results = service
+        .upload(
+            result,
+            BackupMethod::Manual,
+            vec![storage],
+            false,
+            &backup_id,
+            Arc::clone(&logger),
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(results.len(), 1);
+    assert!(results[0].success);
+
+    let entries = Arc::try_unwrap(logger).unwrap().into_entries();
+    assert_eq!(entries.iter().filter(|e| e.level == "warn").count(), 2);
+    assert!(
+        entries.iter().any(|e| e.message
+            == "Upload to storage storage-1 succeeded on attempt 3/3")
+    );
+}

src/tests/services/config_tests.rs

@@ -1,15 +1,12 @@
 use crate::core::context::Context;
 use crate::services::api::ApiClient;
 use crate::services::config::ConfigService;
+use crate::services::config::{build_config, DatabasesConfig, InputDatabaseConfig};
 use crate::utils::edge_key::EdgeKey;
 use std::io::Write;
 use std::sync::Arc;
 use tempfile::NamedTempFile;
 
-// `ConfigService::load` never touches `self.ctx` on the `Some(file_path)` path,
-// so the values here don't matter — but `Context::new()` panics without an
-// `EDGE_KEY` env var, so build the struct directly (mirrors
-// backup_uploader_tests.rs's `ctx_pointing_at`).
 fn test_context() -> Arc<Context> {
     Arc::new(Context {
         edge_key: EdgeKey {
@@ -264,3 +261,73 @@ fn docker_volume_requires_volume_name() {
     let err = service.load(Some(file.path().to_str().unwrap())).unwrap_err();
     assert!(err.contains("volume_name"), "error was: {err}");
 }
+
+#[test]
+fn build_config_applies_type_defaults() {
+    let input: InputDatabaseConfig = serde_json::from_str(
+        r#"{
+            "name": "cluster1",
+            "type": "postgresql-cluster",
+            "username": "postgres",
+            "password": "p",
+            "port": 5432,
+            "host": "localhost",
+            "generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"
+        }"#,
+    )
+    .unwrap();
+
+    let cfg = build_config(input).unwrap();
+    assert_eq!(cfg.db_type.as_str(), "postgresql-cluster");
+    assert_eq!(cfg.database, "postgres"); // cluster default
+}
+
+#[test]
+fn build_config_rejects_missing_required_field() {
+    let input: InputDatabaseConfig = serde_json::from_str(
+        r#"{
+            "name": "pg",
+            "type": "postgresql",
+            "username": "postgres",
+            "port": 5432,
+            "host": "localhost",
+            "generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"
+        }"#,
+    )
+    .unwrap();
+
+    let err = build_config(input).unwrap_err();
+    assert!(err.contains("password"), "unexpected error: {err}");
+}
+
+#[test]
+fn load_optional_returns_empty_when_file_missing() {
+    let service = ConfigService::new(test_context());
+    let cfg = service.load_optional(Some("/nonexistent/path/does-not-exist.json"));
+    assert!(cfg.databases.is_empty());
+}
+
+#[test]
+fn databases_config_roundtrips_through_serde() {
+    let input: InputDatabaseConfig = serde_json::from_str(
+        r#"{
+            "name": "pg",
+            "type": "postgresql",
+            "database": "app",
+            "username": "postgres",
+            "password": "secret",
+            "port": 5432,
+            "host": "localhost",
+            "generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"
+        }"#,
+    )
+    .unwrap();
+    let cfg = build_config(input).unwrap();
+    let wrapped = DatabasesConfig { databases: vec![cfg] };
+
+    let json = serde_json::to_string(&wrapped).unwrap();
+    let back: DatabasesConfig = serde_json::from_str(&json).unwrap();
+    assert_eq!(back.databases[0].name, "pg");
+    assert_eq!(back.databases[0].db_type.as_str(), "postgresql");
+    assert_eq!(back.databases[0].password, "secret");
+}

src/tests/services/dashboard_config_tests.rs

@@ -0,0 +1,84 @@
+use crate::services::config::{build_config, DatabaseConfig, InputDatabaseConfig};
+use crate::services::dashboard_config::merge;
+use crate::services::dashboard_config::{load_cache, persist_cache};
+
+fn cfg(name: &str, gen_id: &str, host: &str) -> DatabaseConfig {
+    let json = format!(
+        r#"{{ "name": "{name}", "type": "postgresql", "database": "app",
+               "username": "u", "password": "p", "port": 5432,
+               "host": "{host}", "generated_id": "{gen_id}" }}"#
+    );
+    let input: InputDatabaseConfig = serde_json::from_str(&json).unwrap();
+    build_config(input).unwrap()
+}
+
+const ID_A: &str = "16678159-ff7e-4c97-8c83-0adeff214681";
+const ID_B: &str = "16678124-ff7e-4c97-8c83-0adeff214681";
+
+#[test]
+fn merge_keeps_local_only_databases() {
+    let local = vec![cfg("local-a", ID_A, "local-host")];
+    let merged = merge(&local, &[]);
+    assert_eq!(merged.databases.len(), 1);
+    assert_eq!(merged.databases[0].host, "local-host");
+}
+
+#[test]
+fn merge_appends_dashboard_only_databases() {
+    let local = vec![cfg("local-a", ID_A, "local-host")];
+    let dashboard = vec![cfg("dash-b", ID_B, "dash-host")];
+    let merged = merge(&local, &dashboard);
+    assert_eq!(merged.databases.len(), 2);
+    assert!(merged.databases.iter().any(|d| d.generated_id == ID_B));
+}
+
+#[test]
+fn merge_dashboard_wins_on_id_collision() {
+    let local = vec![cfg("local-a", ID_A, "local-host")];
+    let dashboard = vec![cfg("dash-a", ID_A, "dash-host")];
+    let merged = merge(&local, &dashboard);
+    assert_eq!(merged.databases.len(), 1);
+    assert_eq!(merged.databases[0].host, "dash-host"); // dashboard wins
+    assert_eq!(merged.databases[0].name, "dash-a");
+}
+
+#[test]
+fn cache_roundtrips() {
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("dashboard_databases.json");
+
+    let dbs = vec![cfg("dash-a", ID_A, "dash-host")];
+    persist_cache(&path, &dbs).unwrap();
+
+    let loaded = load_cache(&path);
+    assert_eq!(loaded.len(), 1);
+    assert_eq!(loaded[0].generated_id, ID_A);
+    assert_eq!(loaded[0].host, "dash-host");
+}
+
+#[test]
+fn load_cache_missing_file_is_empty() {
+    let loaded = load_cache(std::path::Path::new("/nonexistent/dashboard_databases.json"));
+    assert!(loaded.is_empty());
+}
+
+#[test]
+fn load_cache_corrupt_file_is_empty() {
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("dashboard_databases.json");
+    std::fs::write(&path, b"{ this is not valid json").unwrap();
+
+    let loaded = load_cache(&path);
+    assert!(loaded.is_empty());
+}
+
+#[test]
+fn persist_cache_leaves_no_tmp_file() {
+    let dir = tempfile::tempdir().unwrap();
+    let path = dir.path().join("dashboard_databases.json");
+    persist_cache(&path, &[cfg("dash-a", ID_A, "h")]).unwrap();
+
+    let tmp = path.with_extension("json.tmp");
+    assert!(!tmp.exists(), "temp file should have been renamed away");
+    assert!(path.exists());
+}

src/tests/services/mod.rs

@@ -1,3 +1,6 @@
 mod api_models_tests;
+mod backup_runner_tests;
 mod backup_uploader_tests;
 mod config_tests;
+mod dashboard_config_tests;
+mod restore_downloader_tests;

src/tests/services/restore_downloader_tests.rs

@@ -0,0 +1,64 @@
+use crate::core::context::Context;
+use crate::services::api::ApiClient;
+use crate::services::backup::logger::JobLogger;
+use crate::services::restore::RestoreService;
+use crate::tests::init_tracing_for_test;
+use crate::utils::edge_key::EdgeKey;
+
+use std::sync::Arc;
+use tempfile::TempDir;
+use wiremock::matchers::{method, path};
+use wiremock::{Mock, MockServer, ResponseTemplate};
+
+#[tokio::test]
+async fn a_failing_download_is_retried_until_it_succeeds() {
+    init_tracing_for_test();
+    let server = MockServer::start().await;
+
+    Mock::given(method("GET"))
+        .and(path("/backups/archive.tar.gz"))
+        .respond_with(ResponseTemplate::new(503))
+        .up_to_n_times(2)
+        .with_priority(1)
+        .expect(2)
+        .mount(&server)
+        .await;
+
+    Mock::given(method("GET"))
+        .and(path("/backups/archive.tar.gz"))
+        .respond_with(ResponseTemplate::new(200).set_body_bytes(b"portabase-archive".to_vec()))
+        .with_priority(2)
+        .expect(1)
+        .mount(&server)
+        .await;
+
+    let ctx = Context {
+        edge_key: EdgeKey {
+            server_url: server.uri(),
+            agent_id: "agent-1".to_string(),
+            master_key_b64: String::new(),
+        },
+        api: ApiClient::new(server.uri()),
+    };
+
+    let service = RestoreService::new(Arc::new(ctx));
+
+    let temp_dir = TempDir::new().unwrap();
+    let logger = Arc::new(JobLogger::new());
+    let url = format!("{}/backups/archive.tar.gz", server.uri());
+
+    let downloaded = service
+        .download_backup(&url, temp_dir.path(), Arc::clone(&logger), None)
+        .await
+        .unwrap();
+
+    assert_eq!(std::fs::read(&downloaded).unwrap(), b"portabase-archive");
+
+    let entries = Arc::try_unwrap(logger).unwrap().into_entries();
+    assert_eq!(entries.iter().filter(|e| e.level == "warn").count(), 2);
+    assert!(
+        entries
+            .iter()
+            .any(|e| e.message == "Backup download succeeded on attempt 3/3")
+    );
+}

src/tests/utils/mod.rs

@@ -4,4 +4,5 @@ mod deserializer;
 mod edge_key_tests;
 mod file_tests;
 mod normalize_cron_tests;
+mod retry_tests;
 mod stream_tests;

src/tests/utils/retry_tests.rs

@@ -0,0 +1,135 @@
+use crate::services::backup::logger::JobLogger;
+use crate::tests::init_tracing_for_test;
+use crate::utils::retry::{RetryPolicy, retry};
+
+use std::sync::Mutex;
+use std::sync::atomic::{AtomicU32, Ordering};
+use std::time::Duration;
+
+fn fast_policy(attempts: u32) -> RetryPolicy {
+    RetryPolicy {
+        attempts,
+        base_backoff: Duration::from_millis(1),
+        max_backoff: Duration::from_millis(4),
+    }
+}
+
+#[test]
+fn delay_grows_with_the_attempt_number() {
+    let policy = RetryPolicy {
+        attempts: 5,
+        base_backoff: Duration::from_millis(100),
+        max_backoff: Duration::from_secs(30),
+    };
+
+    assert!(policy.delay(1) >= Duration::from_millis(50));
+    assert!(policy.delay(1) <= Duration::from_millis(100));
+    assert!(policy.delay(2) >= Duration::from_millis(100));
+    assert!(policy.delay(2) <= Duration::from_millis(200));
+    assert!(policy.delay(3) >= Duration::from_millis(200));
+    assert!(policy.delay(3) <= Duration::from_millis(400));
+}
+
+#[test]
+fn delay_never_exceeds_max_backoff() {
+    let policy = RetryPolicy {
+        attempts: 5,
+        base_backoff: Duration::from_millis(1000),
+        max_backoff: Duration::from_millis(2000),
+    };
+
+    for attempt in 1..=5 {
+        assert!(policy.delay(attempt) <= Duration::from_millis(2000));
+    }
+}
+
+#[tokio::test]
+async fn first_attempt_success_logs_nothing() {
+    init_tracing_for_test();
+    let logger = JobLogger::new();
+
+    let result: Result<u32, anyhow::Error> =
+        retry("Test op", &logger, &fast_policy(3), |_| async { Ok(7) }).await;
+
+    assert_eq!(result.unwrap(), 7);
+    assert!(logger.into_entries().is_empty());
+}
+
+#[tokio::test]
+async fn retries_until_success_and_logs_each_attempt() {
+    init_tracing_for_test();
+    let logger = JobLogger::new();
+    let calls = AtomicU32::new(0);
+
+    let result: Result<u32, anyhow::Error> =
+        retry("Test op", &logger, &fast_policy(3), |_| async {
+            let n = calls.fetch_add(1, Ordering::SeqCst) + 1;
+            if n < 3 {
+                Err(anyhow::anyhow!("boom {n}"))
+            } else {
+                Ok(n)
+            }
+        })
+        .await;
+
+    assert_eq!(result.unwrap(), 3);
+    assert_eq!(calls.load(Ordering::SeqCst), 3);
+
+    let entries = logger.into_entries();
+
+    let warns: Vec<_> = entries.iter().filter(|e| e.level == "warn").collect();
+    assert_eq!(warns.len(), 2);
+    assert!(warns[0].message.starts_with("Test op attempt 1/3 failed: boom 1"));
+    assert!(warns[1].message.starts_with("Test op attempt 2/3 failed: boom 2"));
+
+    let infos: Vec<_> = entries.iter().filter(|e| e.level == "info").collect();
+    assert_eq!(infos.len(), 1);
+    assert_eq!(infos[0].message, "Test op succeeded on attempt 3/3");
+}
+
+#[tokio::test]
+async fn exhausts_attempts_and_logs_no_terminal_error() {
+    init_tracing_for_test();
+    let logger = JobLogger::new();
+    let calls = AtomicU32::new(0);
+
+    let result: Result<(), anyhow::Error> =
+        retry("Test op", &logger, &fast_policy(3), |_| async {
+            calls.fetch_add(1, Ordering::SeqCst);
+            Err(anyhow::anyhow!("always"))
+        })
+        .await;
+
+    assert!(result.is_err());
+    assert_eq!(calls.load(Ordering::SeqCst), 3);
+
+    let entries = logger.into_entries();
+    assert_eq!(entries.iter().filter(|e| e.level == "warn").count(), 2);
+    assert_eq!(entries.iter().filter(|e| e.level == "error").count(), 0);
+}
+
+#[tokio::test]
+async fn closure_receives_the_attempt_number() {
+    init_tracing_for_test();
+    let logger = JobLogger::new();
+    let seen = Mutex::new(Vec::new());
+    let seen_ref = &seen;
+
+    let result: Result<(), anyhow::Error> =
+        retry("Test op", &logger, &fast_policy(3), move |attempt| async move {
+            seen_ref.lock().unwrap().push(attempt);
+            Err(anyhow::anyhow!("nope"))
+        })
+        .await;
+
+    assert!(result.is_err());
+    assert_eq!(*seen.lock().unwrap(), vec![1, 2, 3]);
+}
+
+#[test]
+fn config_defaults_are_within_the_documented_range() {
+    let policy = RetryPolicy::default();
+    assert!(policy.attempts >= 3 && policy.attempts <= 5);
+    assert!(policy.base_backoff >= Duration::from_millis(100));
+    assert!(policy.base_backoff <= Duration::from_millis(30_000));
+}

src/utils/mod.rs

@@ -6,6 +6,7 @@ pub mod file;
 pub mod locks;
 pub mod logging;
 pub mod redis_client;
+pub mod retry;
 pub mod stream;
 pub mod task_manager;
 pub mod text;

src/utils/retry.rs

@@ -0,0 +1,75 @@
+use crate::services::backup::logger::JobLogger;
+use crate::settings::CONFIG;
+use rand::Rng;
+use std::fmt::Display;
+use std::future::Future;
+use std::time::Duration;
+
+pub struct RetryPolicy {
+    pub attempts: u32,
+    pub base_backoff: Duration,
+    pub max_backoff: Duration,
+}
+
+impl Default for RetryPolicy {
+    fn default() -> Self {
+        Self {
+            attempts: CONFIG.retry_attempts,
+            base_backoff: Duration::from_millis(CONFIG.retry_backoff_ms),
+            max_backoff: Duration::from_secs(30),
+        }
+    }
+}
+
+impl RetryPolicy {
+    pub(crate) fn delay(&self, attempt: u32) -> Duration {
+        let exp = self.base_backoff.saturating_mul(1u32 << (attempt - 1).min(16));
+        let capped = exp.min(self.max_backoff);
+        let half = capped / 2;
+        let jitter = rand::rng().random_range(0..=half.as_millis() as u64);
+
+        half + Duration::from_millis(jitter)
+    }
+}
+
+pub async fn retry<T, E, F, Fut>(
+    op: &str,
+    logger: &JobLogger,
+    policy: &RetryPolicy,
+    mut f: F,
+) -> Result<T, E>
+where
+    F: FnMut(u32) -> Fut,
+    Fut: Future<Output = Result<T, E>> + Send,
+    T: Send,
+    E: Display + Send,
+{
+    let total = policy.attempts;
+    let mut attempt = 1;
+
+    loop {
+        match f(attempt).await {
+            Ok(v) => {
+                if attempt > 1 {
+                    logger.log("info", format!("{op} succeeded on attempt {attempt}/{total}"));
+                }
+                return Ok(v);
+            }
+            Err(e) if attempt < total => {
+                let delay = policy.delay(attempt);
+                logger.log(
+                    "warn",
+                    format!(
+                        "{op} attempt {attempt}/{total} failed: {e} - retrying in {}ms",
+                        delay.as_millis()
+                    ),
+                );
+                tokio::time::sleep(delay).await;
+                attempt += 1;
+            }
+            Err(e) => {
+                return Err(e);
+            }
+        }
+    }
+}