Compare commits

..

13 Commits

Author SHA1 Message Date
Charles GTE 9bb830327e fix: refactoring 2026-09-10 22:55:47 +02:00
Charles GTE 31e55a7b4f fix(storage): block virtual and non-viable rclone backends 2026-09-10 22:38:08 +02:00
Charles GTE 576c055092 fix(storage): bound rclone timeouts, detect restore truncation, track sdd workspace 2026-09-10 22:38:08 +02:00
Charles GTE ba83b6b2b7 feat(storage): add RcloneProvider upload path 2026-09-10 22:38:07 +02:00
Charles GTE ea356ed54f fix(storage): abort truncated rclone uploads and strengthen stderr test 2026-09-10 22:37:37 +02:00
Charles GTE b222b13934 feat(storage): stream backups into rclone rcat 2026-09-10 22:37:09 +02:00
Charles GTE 9409f0ff25 feat(storage): add rclone config model and validation helpers 2026-09-10 22:37:00 +02:00
Charles GTE 59312215e9 build: exclude target/ and local artifacts from the docker context 2026-09-10 22:36:46 +02:00
Charles GTE 3ff360a971 build: install rclone 1.75.1 in agent images 2026-09-10 22:36:38 +02:00
github-actions[bot] 5f92297108 chore: release 1.20.1 2026-09-03 15:41:20 +00:00
Charles GTE d1821f7045 Merge pull request #102 from Portabase/fix/cron-crash
fix: cron crash for databases setup from dashboard
2026-09-03 17:38:53 +02:00
charles-gauthereau 0fe4d50cbd fix: docker-compose.yml 2026-09-03 17:38:35 +02:00
charles-gauthereau 1f29466a28 fix: cron crash for databases setup from dashboard 2026-09-03 17:22:03 +02:00
16 changed files with 770 additions and 15 deletions
+3 -5
View File
@@ -1,11 +1,9 @@
# Git
.git
.gitignore
# MD files
CHANGELOG.md
README.md
RELEASE.md
#IDE configurations
.idea
target
dump.rdb
.superpowers
+1
View File
@@ -8,3 +8,4 @@
.claude
/docs
.superpowers
+1 -1
View File
@@ -27,5 +27,5 @@ keywords:
- self-hosted
- portabase
license: Apache-2.0
version: 1.20.0
version: 1.20.1
date-released: '2026-02-24'
Generated
+1 -1
View File
@@ -3503,7 +3503,7 @@ dependencies = [
[[package]]
name = "portabase-agent"
version = "1.20.0"
version = "1.20.1"
dependencies = [
"aes",
"aes-gcm",
+2 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "portabase-agent"
version = "1.20.0"
version = "1.20.1"
edition = "2024"
[dependencies]
@@ -19,7 +19,7 @@ log = "0.4.29"
toml = "0.9.10"
reqwest = { version = "0.13.1", features = ["json", "blocking", "multipart", "stream", "query"] }
anyhow = "1.0.100"
tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "fs"] }
tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "fs", "process", "io-util"] }
async-trait = "0.1.89"
tempfile = "3.24.0"
openssl = "0.10.75"
+1 -1
View File
@@ -21,7 +21,7 @@ services:
LOG: debug
TZ: "Europe/Paris"
# TMPDIR: /scratch
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiY2UxNjRiZDItZGZkMy00YzY4LThlZGItNmQ3OTczODAzZWEyIiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ=="
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiMWNhYTY2ZjEtMWJjNi00MzQzLThiMmItNGEwZDFmM2UzMWI5IiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ=="
#CHUNK_SIZE_MB: "1"
#POOLING: 1
#RETRY_ATTEMPTS: 3
+15
View File
@@ -44,6 +44,15 @@ RUN ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') \
| tar -xjf - -C /usr/local/bin sqlcmd \
&& chmod +x /usr/local/bin/sqlcmd
# =========================
# rclone (all storage backends)
# =========================
ARG RCLONE_VERSION=1.75.1
RUN ARCH=$(dpkg --print-architecture) \
&& curl -fsSL -o /tmp/rclone.deb "https://downloads.rclone.org/v${RCLONE_VERSION}/rclone-v${RCLONE_VERSION}-linux-${ARCH}.deb" \
&& dpkg -i /tmp/rclone.deb \
&& rm /tmp/rclone.deb
ARG TARGETARCH
# =========================
@@ -134,6 +143,12 @@ RUN apt-get update && apt-get install -y \
firebird3.0-utils \
&& rm -rf /var/lib/apt/lists/*
ARG RCLONE_VERSION=1.75.1
RUN ARCH=$(dpkg --print-architecture) \
&& curl -fsSL -o /tmp/rclone.deb "https://downloads.rclone.org/v${RCLONE_VERSION}/rclone-v${RCLONE_VERSION}-linux-${ARCH}.deb" \
&& dpkg -i /tmp/rclone.deb \
&& rm /tmp/rclone.deb
ENV DOTNET_ROOT=/usr/local/dotnet
RUN curl -sSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh \
&& chmod +x /tmp/dotnet-install.sh \
+15 -3
View File
@@ -1,5 +1,7 @@
use super::service::RestoreService;
use crate::services::backup::logger::JobLogger;
use crate::utils::retry::{RetryPolicy, retry};
use anyhow::Result;
use futures::StreamExt;
use reqwest::{Client, Url};
@@ -7,8 +9,6 @@ use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Instant;
use tokio::io::AsyncWriteExt;
use crate::services::backup::logger::JobLogger;
use crate::utils::retry::{RetryPolicy, retry};
fn human_size(bytes: u64) -> String {
if bytes >= 1024 * 1024 {
@@ -98,7 +98,9 @@ impl RestoreService {
format!(
"Downloading backup '{}' ({})",
filename,
total.map(human_size).unwrap_or_else(|| "unknown size".to_string())
total
.map(human_size)
.unwrap_or_else(|| "unknown size".to_string())
),
);
@@ -138,6 +140,16 @@ impl RestoreService {
);
}
if let Some(total) = total
&& downloaded < total
{
anyhow::bail!(
"Downloaded {} bytes but expected at least {} - backup appears truncated",
downloaded,
total
);
}
logger.log(
"info",
format!(
+2 -1
View File
@@ -9,6 +9,7 @@ use providers::azure_blob;
use providers::google_cloud_storage;
use providers::google_drive;
use providers::local;
use providers::rclone;
use providers::s3;
use std::sync::Arc;
use tracing::{error, info};
@@ -26,7 +27,6 @@ pub trait StorageProvider: Send + Sync {
) -> UploadResult;
}
/// Factory to create provider instance from storage config
pub fn get_provider(storage: &DatabaseStorage) -> Option<Box<dyn StorageProvider>> {
info!("Getting provider");
info!("{:#?}", storage.provider.as_str());
@@ -39,6 +39,7 @@ pub fn get_provider(storage: &DatabaseStorage) -> Option<Box<dyn StorageProvider
"google-cloud-storage" => Some(Box::new(
google_cloud_storage::GoogleCloudStorageProvider {},
)),
"rclone" => Some(Box::new(rclone::RcloneProvider {})),
_ => {
error!("Unknown storage provider: {}", storage.provider);
None
+1
View File
@@ -2,4 +2,5 @@ pub mod azure_blob;
pub mod google_cloud_storage;
pub mod google_drive;
pub mod local;
pub mod rclone;
pub mod s3;
@@ -0,0 +1,168 @@
use anyhow::{Context, Result, bail};
use bytes::Bytes;
use futures::{Stream, StreamExt};
use std::io::Write;
use std::path::Path;
use std::pin::Pin;
use std::process::Stdio;
use tempfile::NamedTempFile;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::process::Command;
use tracing::info;
const BLOCKED_BACKEND_TYPES: [&str; 13] = [
"local",
"alias",
"crypt",
"chunker",
"compress",
"union",
"combine",
"hasher",
"archive",
"cache",
"memory",
"http",
"googlephotos",
];
fn sections(config_text: &str) -> Vec<(String, Option<String>)> {
let mut out: Vec<(String, Option<String>)> = Vec::new();
for line in config_text.lines() {
let line = line.trim();
if line.starts_with('[') && line.ends_with(']') && line.len() > 2 {
out.push((line[1..line.len() - 1].trim().to_string(), None));
continue;
}
let Some((key, value)) = line.split_once('=') else {
continue;
};
if key.trim().eq_ignore_ascii_case("type")
&& let Some(current) = out.last_mut()
&& current.1.is_none()
{
current.1 = Some(value.trim().to_ascii_lowercase());
}
}
out
}
pub fn validate_config(config_text: &str, remote_name: &str) -> Result<()> {
let sections = sections(config_text);
if sections.is_empty() {
bail!("rclone config contains no remote sections");
}
for (name, backend) in &sections {
let Some(backend) = backend else { continue };
if BLOCKED_BACKEND_TYPES.contains(&backend.as_str()) {
bail!("rclone backend type '{backend}' is not allowed (remote '{name}')");
}
}
if !sections.iter().any(|(name, _)| name == remote_name) {
let available: Vec<&str> = sections.iter().map(|(name, _)| name.as_str()).collect();
bail!(
"remote '{remote_name}' is not defined in the rclone config (available: {})",
available.join(", ")
);
}
Ok(())
}
/// `<remote>:<remote_path>/<remote_file_path>`
pub fn remote_target(remote_name: &str, remote_path: &str, remote_file_path: &str) -> String {
let base = remote_path.trim().trim_matches('/');
if base.is_empty() {
format!("{remote_name}:{remote_file_path}")
} else {
format!("{remote_name}:{base}/{remote_file_path}")
}
}
pub type RcloneStream = Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>;
pub fn write_config(config_text: &str) -> Result<NamedTempFile> {
let mut file = NamedTempFile::new().context("failed to create rclone config temp file")?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(file.path(), std::fs::Permissions::from_mode(0o600))
.context("failed to restrict rclone config permissions")?;
}
file.write_all(config_text.as_bytes())
.context("failed to write rclone config")?;
file.flush().context("failed to flush rclone config")?;
Ok(file)
}
pub async fn rcat(config_path: &Path, target: &str, mut stream: RcloneStream) -> Result<()> {
info!("rclone rcat -> {}", target);
let mut child = Command::new("rclone")
.arg("--config")
.arg(config_path)
.arg("--contimeout")
.arg("30s")
.arg("--timeout")
.arg("5m")
.arg("--retries")
.arg("1")
.arg("--low-level-retries")
.arg("3")
.arg("rcat")
.arg(target)
.stdin(Stdio::piped())
.stdout(Stdio::null())
.stderr(Stdio::piped())
.spawn()
.context("failed to spawn rclone (is the binary installed in this image?)")?;
let mut stderr_pipe = child.stderr.take().context("rclone stderr unavailable")?;
let stderr_task = tokio::spawn(async move {
let mut buf = String::new();
let _ = stderr_pipe.read_to_string(&mut buf).await;
buf
});
let mut stdin = child.stdin.take().context("rclone stdin unavailable")?;
while let Some(chunk) = stream.next().await {
let chunk = match chunk {
Ok(c) => c,
Err(e) => {
let _ = child.start_kill();
let _ = child.wait().await;
return Err(e).context("backup stream failed");
}
};
if stdin.write_all(&chunk).await.is_err() {
break;
}
}
let _ = stdin.flush().await;
drop(stdin);
let status = child.wait().await.context("failed to wait for rclone")?;
let stderr = stderr_task.await.unwrap_or_default();
if !status.success() {
bail!("rclone rcat failed ({status}): {}", stderr.trim());
}
Ok(())
}
@@ -0,0 +1,112 @@
pub mod helpers;
pub mod models;
use crate::core::context::Context;
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::backup::models::{BackupResult, UploadResult};
use crate::services::storage::StorageProvider;
use crate::services::storage::providers::rclone::helpers::{
rcat, remote_target, validate_config, write_config,
};
use crate::services::storage::providers::rclone::models::RcloneProviderConfig;
use crate::utils::common::BackupMethod;
use crate::utils::file::{full_file_name, full_file_path};
use crate::utils::stream::build_stream;
use async_trait::async_trait;
use std::sync::Arc;
use tokio::fs;
use tracing::{error, info};
pub struct RcloneProvider {}
fn failed(storage_id: &str, error: impl ToString, total_size: Option<u64>) -> UploadResult {
UploadResult {
storage_id: storage_id.to_string(),
success: false,
error: Some(error.to_string()),
remote_file_path: None,
total_size,
}
}
#[async_trait]
impl StorageProvider for RcloneProvider {
async fn upload(
&self,
ctx: Arc<Context>,
result: BackupResult,
_method: BackupMethod,
storage: &DatabaseStorage,
encrypt: Option<bool>,
_backup_storage_id: &str,
) -> UploadResult {
let storage_id = storage.id.clone();
let Some(file_path) = result.backup_file else {
return failed(&storage_id, "Missing backup file path", None);
};
let total_size = match fs::metadata(&file_path).await {
Ok(meta) => meta.len(),
Err(e) => {
error!("Failed to get file size: {}", e);
return failed(&storage_id, e, None);
}
};
let config: RcloneProviderConfig = match storage.clone().config.try_into() {
Ok(c) => c,
Err(e) => {
error!("rclone config deserialization failed: {}", e);
return failed(&storage_id, e, Some(total_size));
}
};
if let Err(e) = validate_config(&config.config_text, &config.remote_name) {
error!("rclone config rejected: {}", e);
return failed(&storage_id, e, Some(total_size));
}
let encrypt = encrypt.unwrap_or(false);
let upload = match build_stream(&file_path, encrypt, &ctx.edge_key.master_key_b64).await {
Ok(u) => u,
Err(e) => {
error!("Stream build failed: {}", e);
return failed(&storage_id, e, Some(total_size));
}
};
let file_name = full_file_name(encrypt);
let remote_file_path = full_file_path(&file_name, storage.folder_name.as_deref());
let config_file = match write_config(&config.config_text) {
Ok(f) => f,
Err(e) => {
error!("rclone config write failed: {}", e);
return failed(&storage_id, e, Some(total_size));
}
};
let target = remote_target(&config.remote_name, &config.remote_path, &remote_file_path);
info!("Starting rclone upload to {}", target);
match rcat(config_file.path(), &target, upload.stream).await {
Ok(()) => {
info!("rclone upload successful: {}", remote_file_path);
UploadResult {
storage_id,
success: true,
error: None,
remote_file_path: Some(remote_file_path),
total_size: Some(total_size),
}
}
Err(e) => {
error!("rclone upload failed: {:?}", e);
failed(&storage_id, e, Some(total_size))
}
}
}
}
@@ -0,0 +1,8 @@
use serde::{Deserialize, Serialize};
#[derive(Debug, Deserialize, Serialize)]
pub struct RcloneProviderConfig {
pub config_text: String,
pub remote_name: String,
pub remote_path: String,
}
+1
View File
@@ -1,2 +1,3 @@
mod azure_blob;
mod google_cloud_storage;
mod rclone;
+433
View File
@@ -0,0 +1,433 @@
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::storage::providers::rclone::helpers::{remote_target, validate_config};
use crate::services::storage::providers::rclone::models::RcloneProviderConfig;
use crate::tests::init_tracing_for_test;
use crate::utils::file::full_file_path;
const OVH_CONFIG: &str = "[ovhcloud-rbx]\n\
type = s3\n\
provider = OVHcloud\n\
access_key_id = my_access\n\
secret_access_key = my_secret\n\
region = rbx\n\
endpoint = s3.rbx.io.cloud.ovh.net\n\
acl = private\n";
#[test]
fn config_deserializes_from_dashboard_camel_case() {
init_tracing_for_test();
let storage: DatabaseStorage = serde_json::from_value(serde_json::json!({
"id": "storage-1",
"provider": "rclone",
"folderName": "backups",
"config": {
"configText": OVH_CONFIG,
"remoteName": "ovhcloud-rbx",
"remotePath": "my-bucket",
}
}))
.unwrap();
let config: RcloneProviderConfig = storage.config.try_into().unwrap();
assert_eq!(config.remote_name, "ovhcloud-rbx");
assert_eq!(config.remote_path, "my-bucket");
assert!(config.config_text.contains("type = s3"));
}
#[test]
fn validate_config_accepts_the_target_remote() {
assert!(validate_config(OVH_CONFIG, "ovhcloud-rbx").is_ok());
}
#[test]
fn validate_config_rejects_an_unknown_remote_name() {
let err = validate_config(OVH_CONFIG, "typo").unwrap_err().to_string();
assert!(err.contains("typo"), "unexpected error: {err}");
assert!(err.contains("ovhcloud-rbx"), "error should list the available remotes: {err}");
}
#[test]
fn validate_config_rejects_local_backend() {
let cfg = "[disk]\ntype = local\n";
let err = validate_config(cfg, "disk").unwrap_err().to_string();
assert!(err.contains("local"), "unexpected error: {err}");
}
#[test]
fn validate_config_rejects_alias_backend() {
let cfg = "[shortcut]\ntype = alias\nremote = other:path\n";
let err = validate_config(cfg, "shortcut").unwrap_err().to_string();
assert!(err.contains("alias"), "unexpected error: {err}");
}
#[test]
fn validate_config_rejects_a_blocked_backend_in_a_chained_section() {
let cfg = "[secret]\ntype = crypt\nremote = disk:vault\n\n[disk]\ntype = local\n";
let err = validate_config(cfg, "secret").unwrap_err().to_string();
assert!(err.contains("crypt"), "unexpected error: {err}");
assert!(err.contains("secret"), "error should name the offending remote: {err}");
let cfg = "[outer]\ntype = s3\nprovider = Minio\n\n[disk]\ntype = local\n";
let err = validate_config(cfg, "outer").unwrap_err().to_string();
assert!(err.contains("local"), "unexpected error: {err}");
assert!(err.contains("disk"), "error should name the offending remote: {err}");
}
#[test]
fn validate_config_rejects_crypt_even_over_an_allowed_remote() {
let cfg = format!("[secret]\ntype = crypt\nremote = ovhcloud-rbx:bucket\n\n{OVH_CONFIG}");
let err = validate_config(&cfg, "secret").unwrap_err().to_string();
assert!(err.contains("crypt"), "unexpected error: {err}");
}
#[test]
fn validate_config_rejects_a_wrapping_backend_pointing_at_a_bare_local_path() {
for backend in ["crypt", "chunker", "compress", "union", "combine", "hasher"] {
let cfg = format!("[sneaky]\ntype = {backend}\nremote = /etc\n");
let err = validate_config(&cfg, "sneaky")
.unwrap_err()
.to_string();
assert!(err.contains(backend), "{backend} must be rejected: {err}");
}
}
#[test]
fn validate_config_rejects_backends_that_cannot_hold_a_backup() {
for backend in ["memory", "http", "googlephotos"] {
let cfg = format!("[nope]\ntype = {backend}\n");
let err = validate_config(&cfg, "nope").unwrap_err().to_string();
assert!(err.contains(backend), "{backend} must be rejected: {err}");
}
}
#[test]
fn remote_path_is_a_prefix_ahead_of_the_backup_folder() {
assert_eq!(
remote_target("ovhcloud-rbx", "my-bucket", "backups/2026-09-09/x.tar.gz"),
"ovhcloud-rbx:my-bucket/backups/2026-09-09/x.tar.gz"
);
// Deeper prefixes nest the same way.
assert_eq!(
remote_target("ovhcloud-rbx", "my-bucket/portabase", "backups/2026-09-09/x.tar.gz"),
"ovhcloud-rbx:my-bucket/portabase/backups/2026-09-09/x.tar.gz"
);
}
#[test]
fn remote_target_trims_surrounding_slashes_and_whitespace() {
assert_eq!(
remote_target("r", " /my-bucket/ ", "a/b.bin"),
"r:my-bucket/a/b.bin"
);
}
#[test]
fn remote_target_handles_an_empty_remote_path() {
assert_eq!(remote_target("r", "", "a/b.bin"), "r:a/b.bin");
assert_eq!(remote_target("r", " ", "a/b.bin"), "r:a/b.bin");
}
#[test]
fn an_empty_remote_path_falls_back_to_the_global_backup_folder() {
let remote_file_path = full_file_path(&"x.tar.gz".to_string(), None);
assert!(remote_file_path.starts_with("backups/"));
assert_eq!(
remote_target("ovhcloud-rbx", "", &remote_file_path),
format!("ovhcloud-rbx:{remote_file_path}")
);
assert_eq!(
remote_target("ovhcloud-rbx", "my-bucket", &remote_file_path),
format!("ovhcloud-rbx:my-bucket/{remote_file_path}")
);
}
use crate::services::storage::providers::rclone::helpers::{rcat, write_config};
use bytes::Bytes;
use futures::stream;
use std::process::Command;
use testcontainers::core::{IntoContainerPort, WaitFor};
use testcontainers::runners::AsyncRunner;
use testcontainers::{GenericImage, ImageExt};
const BUCKET: &str = "portabase";
async fn start_minio() -> (testcontainers::ContainerAsync<GenericImage>, String) {
let container = GenericImage::new("minio/minio", "latest")
.with_exposed_port(9000.tcp())
.with_wait_for(WaitFor::message_on_stderr("API:"))
.with_env_var("MINIO_ROOT_USER", "minioadmin")
.with_env_var("MINIO_ROOT_PASSWORD", "minioadmin")
.with_cmd(["server", "/data"])
.start()
.await
.unwrap();
let host = container.get_host().await.unwrap().to_string();
let port = container.get_host_port_ipv4(9000).await.unwrap();
(container, format!("http://{host}:{port}"))
}
fn minio_config(endpoint: &str) -> String {
format!(
"[minio]\n\
type = s3\n\
provider = Minio\n\
access_key_id = minioadmin\n\
secret_access_key = minioadmin\n\
endpoint = {endpoint}\n\
region = us-east-1\n\
force_path_style = true\n"
)
}
fn rclone_ok(config_path: &std::path::Path, args: &[&str]) -> Vec<u8> {
let out = Command::new("rclone")
.arg("--config")
.arg(config_path)
.args(args)
.output()
.expect("rclone binary not found — is it installed in this image?");
assert!(
out.status.success(),
"rclone {args:?} failed: {}",
String::from_utf8_lossy(&out.stderr)
);
out.stdout
}
#[test]
fn write_config_creates_an_owner_only_file_with_the_exact_text() {
use std::os::unix::fs::PermissionsExt;
let file = write_config(OVH_CONFIG).unwrap();
let mode = std::fs::metadata(file.path()).unwrap().permissions().mode();
assert_eq!(mode & 0o777, 0o600, "config file must not be group/world readable");
assert_eq!(std::fs::read_to_string(file.path()).unwrap(), OVH_CONFIG);
}
#[tokio::test]
async fn rcat_streams_a_multi_chunk_body_to_minio() {
init_tracing_for_test();
let (_container, endpoint) = start_minio().await;
let config = write_config(&minio_config(&endpoint)).unwrap();
rclone_ok(config.path(), &["mkdir", &format!("minio:{BUCKET}")]);
let data = vec![7u8; 10 * 1024];
let chunks: Vec<Result<Bytes, std::io::Error>> = data
.chunks(1024)
.map(|c| Ok(Bytes::copy_from_slice(c)))
.collect();
let target = remote_target("minio", BUCKET, "backups/2026-09-09/test.bin");
rcat(config.path(), &target, Box::pin(stream::iter(chunks)))
.await
.unwrap();
let got = rclone_ok(config.path(), &["cat", &target]);
assert_eq!(got, data);
}
#[tokio::test]
async fn rcat_reports_rclone_stderr_when_the_remote_is_unreachable() {
init_tracing_for_test();
let config = write_config(&minio_config("http://127.0.0.1:1")).unwrap();
let chunks: Vec<Result<Bytes, std::io::Error>> =
vec![Ok(Bytes::from_static(&[0u8; 4096]))];
let err = rcat(
config.path(),
"minio:portabase/x.bin",
Box::pin(stream::iter(chunks)),
)
.await
.expect_err("upload to an unreachable endpoint must fail");
let msg = err.to_string();
assert!(
msg.contains("rclone rcat failed"),
"the broken stdin pipe must not mask rclone's own error: {msg}"
);
let (_, stderr_part) = msg
.rsplit_once(": ")
.expect("bail message must carry rclone stderr after the exit status");
assert!(
!stderr_part.trim().is_empty(),
"rclone stderr must be included: {msg}"
);
}
#[tokio::test]
async fn rcat_aborts_the_upload_when_the_stream_fails() {
init_tracing_for_test();
let (_container, endpoint) = start_minio().await;
let config = write_config(&minio_config(&endpoint)).unwrap();
rclone_ok(config.path(), &["mkdir", &format!("minio:{BUCKET}")]);
let chunks: Vec<Result<Bytes, std::io::Error>> = vec![
Ok(Bytes::from_static(&[1u8; 1024])),
Err(std::io::Error::other("injected stream failure")),
];
let target = remote_target("minio", BUCKET, "backups/2026-09-09/aborted.bin");
let err = rcat(config.path(), &target, Box::pin(stream::iter(chunks)))
.await
.expect_err("a stream error must fail the upload");
assert!(
err.to_string().contains("backup stream failed"),
"unexpected error: {err}"
);
let stat_out = rclone_ok(config.path(), &["lsjson", "--stat", &target]);
let stat: serde_json::Value = serde_json::from_slice(&stat_out).unwrap();
assert_eq!(
stat["Name"], "",
"rclone must not have finalized the truncated object: {stat}"
);
assert_eq!(stat["IsDir"], true, "a miss reports IsDir: true: {stat}");
}
use crate::core::context::Context;
use crate::services::api::ApiClient;
use crate::services::backup::models::BackupResult;
use crate::services::config::DbType;
use crate::services::storage::providers::rclone::RcloneProvider;
use crate::services::storage::{StorageProvider, get_provider};
use crate::utils::common::BackupMethod;
use crate::utils::edge_key::EdgeKey;
use std::io::Write as _;
use std::sync::Arc;
use tempfile::NamedTempFile;
fn test_context() -> Arc<Context> {
Arc::new(Context {
edge_key: EdgeKey {
server_url: String::new(),
agent_id: "agent-1".to_string(),
master_key_b64: String::new(),
},
api: ApiClient::new(String::new()),
})
}
fn storage_for(config_text: &str, remote_path: &str) -> DatabaseStorage {
serde_json::from_value(serde_json::json!({
"id": "storage-1",
"provider": "rclone",
"folderName": "backups",
"config": {
"configText": config_text,
"remoteName": "minio",
"remotePath": remote_path,
}
}))
.unwrap()
}
#[test]
fn factory_resolves_the_rclone_provider_key() {
let storage = storage_for(OVH_CONFIG, "bucket");
assert!(
get_provider(&storage).is_some(),
"get_provider must recognise the \"rclone\" key"
);
}
#[tokio::test]
async fn provider_uploads_an_unencrypted_backup_to_minio() {
init_tracing_for_test();
let (_container, endpoint) = start_minio().await;
let config_text = minio_config(&endpoint);
let bootstrap = write_config(&config_text).unwrap();
rclone_ok(bootstrap.path(), &["mkdir", &format!("minio:{BUCKET}")]);
let payload = vec![42u8; 64 * 1024];
let mut backup_file = NamedTempFile::new().unwrap();
backup_file.write_all(&payload).unwrap();
backup_file.flush().unwrap();
let storage = storage_for(&config_text, BUCKET);
let result = RcloneProvider {}
.upload(
test_context(),
BackupResult {
generated_id: "db-1".to_string(),
db_type: DbType::Postgresql,
status: "success".to_string(),
backup_file: Some(backup_file.path().to_path_buf()),
code: None,
},
BackupMethod::Automatic,
&storage,
Some(false),
"backup-storage-1",
)
.await;
assert!(result.success, "upload failed: {:?}", result.error);
assert_eq!(result.total_size, Some(payload.len() as u64));
let remote_file_path = result.remote_file_path.expect("remote path must be reported");
assert!(
remote_file_path.starts_with("backups/"),
"folder_name must prefix the path: {remote_file_path}"
);
let target = remote_target("minio", BUCKET, &remote_file_path);
assert_eq!(rclone_ok(bootstrap.path(), &["cat", &target]), payload);
}
#[tokio::test]
async fn provider_refuses_a_blocked_backend_without_spawning_rclone() {
init_tracing_for_test();
let mut backup_file = NamedTempFile::new().unwrap();
backup_file.write_all(b"payload").unwrap();
backup_file.flush().unwrap();
let storage = storage_for("[minio]\ntype = local\n", "bucket");
let result = RcloneProvider {}
.upload(
test_context(),
BackupResult {
generated_id: "db-1".to_string(),
db_type: DbType::Postgresql,
status: "success".to_string(),
backup_file: Some(backup_file.path().to_path_buf()),
code: None,
},
BackupMethod::Automatic,
&storage,
Some(false),
"backup-storage-1",
)
.await;
assert!(!result.success);
assert!(
result.error.unwrap_or_default().contains("local"),
"the error must name the rejected backend type"
);
}
+6 -1
View File
@@ -79,7 +79,12 @@ pub async fn execute_task(
let ctx = Arc::new(Context::new());
let config_service = ConfigService::new(ctx.clone());
let backup_service = BackupService::new(ctx.clone());
let config = config_service.load(None).unwrap();
let local = config_service.load_optional(None);
let cache_path = std::path::PathBuf::from(&crate::settings::CONFIG.data_path)
.join("dashboard_databases.json");
let dashboard = crate::services::dashboard_config::load_cache(&cache_path);
let config = crate::services::dashboard_config::merge(&local.databases, &dashboard);
let metadata_obj = metadata
.into_iter()