mirror of
https://github.com/Portabase/agent.git
synced 2026-09-11 02:27:10 +00:00
Compare commits
13 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 9bb830327e | |||
| 31e55a7b4f | |||
| 576c055092 | |||
| ba83b6b2b7 | |||
| ea356ed54f | |||
| b222b13934 | |||
| 9409f0ff25 | |||
| 59312215e9 | |||
| 3ff360a971 | |||
| 5f92297108 | |||
| d1821f7045 | |||
| 0fe4d50cbd | |||
| 1f29466a28 |
+3
-5
@@ -1,11 +1,9 @@
|
||||
# Git
|
||||
.git
|
||||
.gitignore
|
||||
|
||||
# MD files
|
||||
CHANGELOG.md
|
||||
README.md
|
||||
RELEASE.md
|
||||
|
||||
#IDE configurations
|
||||
.idea
|
||||
target
|
||||
dump.rdb
|
||||
.superpowers
|
||||
|
||||
@@ -8,3 +8,4 @@
|
||||
.claude
|
||||
|
||||
/docs
|
||||
.superpowers
|
||||
|
||||
+1
-1
@@ -27,5 +27,5 @@ keywords:
|
||||
- self-hosted
|
||||
- portabase
|
||||
license: Apache-2.0
|
||||
version: 1.20.0
|
||||
version: 1.20.1
|
||||
date-released: '2026-02-24'
|
||||
|
||||
Generated
+1
-1
@@ -3503,7 +3503,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "portabase-agent"
|
||||
version = "1.20.0"
|
||||
version = "1.20.1"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"aes-gcm",
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "portabase-agent"
|
||||
version = "1.20.0"
|
||||
version = "1.20.1"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
@@ -19,7 +19,7 @@ log = "0.4.29"
|
||||
toml = "0.9.10"
|
||||
reqwest = { version = "0.13.1", features = ["json", "blocking", "multipart", "stream", "query"] }
|
||||
anyhow = "1.0.100"
|
||||
tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "fs"] }
|
||||
tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "fs", "process", "io-util"] }
|
||||
async-trait = "0.1.89"
|
||||
tempfile = "3.24.0"
|
||||
openssl = "0.10.75"
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@ services:
|
||||
LOG: debug
|
||||
TZ: "Europe/Paris"
|
||||
# TMPDIR: /scratch
|
||||
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiY2UxNjRiZDItZGZkMy00YzY4LThlZGItNmQ3OTczODAzZWEyIiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ=="
|
||||
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiMWNhYTY2ZjEtMWJjNi00MzQzLThiMmItNGEwZDFmM2UzMWI5IiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ=="
|
||||
#CHUNK_SIZE_MB: "1"
|
||||
#POOLING: 1
|
||||
#RETRY_ATTEMPTS: 3
|
||||
|
||||
@@ -44,6 +44,15 @@ RUN ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') \
|
||||
| tar -xjf - -C /usr/local/bin sqlcmd \
|
||||
&& chmod +x /usr/local/bin/sqlcmd
|
||||
|
||||
# =========================
|
||||
# rclone (all storage backends)
|
||||
# =========================
|
||||
ARG RCLONE_VERSION=1.75.1
|
||||
RUN ARCH=$(dpkg --print-architecture) \
|
||||
&& curl -fsSL -o /tmp/rclone.deb "https://downloads.rclone.org/v${RCLONE_VERSION}/rclone-v${RCLONE_VERSION}-linux-${ARCH}.deb" \
|
||||
&& dpkg -i /tmp/rclone.deb \
|
||||
&& rm /tmp/rclone.deb
|
||||
|
||||
ARG TARGETARCH
|
||||
|
||||
# =========================
|
||||
@@ -134,6 +143,12 @@ RUN apt-get update && apt-get install -y \
|
||||
firebird3.0-utils \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ARG RCLONE_VERSION=1.75.1
|
||||
RUN ARCH=$(dpkg --print-architecture) \
|
||||
&& curl -fsSL -o /tmp/rclone.deb "https://downloads.rclone.org/v${RCLONE_VERSION}/rclone-v${RCLONE_VERSION}-linux-${ARCH}.deb" \
|
||||
&& dpkg -i /tmp/rclone.deb \
|
||||
&& rm /tmp/rclone.deb
|
||||
|
||||
ENV DOTNET_ROOT=/usr/local/dotnet
|
||||
RUN curl -sSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh \
|
||||
&& chmod +x /tmp/dotnet-install.sh \
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
use super::service::RestoreService;
|
||||
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::utils::retry::{RetryPolicy, retry};
|
||||
use anyhow::Result;
|
||||
use futures::StreamExt;
|
||||
use reqwest::{Client, Url};
|
||||
@@ -7,8 +9,6 @@ use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::utils::retry::{RetryPolicy, retry};
|
||||
|
||||
fn human_size(bytes: u64) -> String {
|
||||
if bytes >= 1024 * 1024 {
|
||||
@@ -98,7 +98,9 @@ impl RestoreService {
|
||||
format!(
|
||||
"Downloading backup '{}' ({})",
|
||||
filename,
|
||||
total.map(human_size).unwrap_or_else(|| "unknown size".to_string())
|
||||
total
|
||||
.map(human_size)
|
||||
.unwrap_or_else(|| "unknown size".to_string())
|
||||
),
|
||||
);
|
||||
|
||||
@@ -138,6 +140,16 @@ impl RestoreService {
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(total) = total
|
||||
&& downloaded < total
|
||||
{
|
||||
anyhow::bail!(
|
||||
"Downloaded {} bytes but expected at least {} - backup appears truncated",
|
||||
downloaded,
|
||||
total
|
||||
);
|
||||
}
|
||||
|
||||
logger.log(
|
||||
"info",
|
||||
format!(
|
||||
|
||||
@@ -9,6 +9,7 @@ use providers::azure_blob;
|
||||
use providers::google_cloud_storage;
|
||||
use providers::google_drive;
|
||||
use providers::local;
|
||||
use providers::rclone;
|
||||
use providers::s3;
|
||||
use std::sync::Arc;
|
||||
use tracing::{error, info};
|
||||
@@ -26,7 +27,6 @@ pub trait StorageProvider: Send + Sync {
|
||||
) -> UploadResult;
|
||||
}
|
||||
|
||||
/// Factory to create provider instance from storage config
|
||||
pub fn get_provider(storage: &DatabaseStorage) -> Option<Box<dyn StorageProvider>> {
|
||||
info!("Getting provider");
|
||||
info!("{:#?}", storage.provider.as_str());
|
||||
@@ -39,6 +39,7 @@ pub fn get_provider(storage: &DatabaseStorage) -> Option<Box<dyn StorageProvider
|
||||
"google-cloud-storage" => Some(Box::new(
|
||||
google_cloud_storage::GoogleCloudStorageProvider {},
|
||||
)),
|
||||
"rclone" => Some(Box::new(rclone::RcloneProvider {})),
|
||||
_ => {
|
||||
error!("Unknown storage provider: {}", storage.provider);
|
||||
None
|
||||
|
||||
@@ -2,4 +2,5 @@ pub mod azure_blob;
|
||||
pub mod google_cloud_storage;
|
||||
pub mod google_drive;
|
||||
pub mod local;
|
||||
pub mod rclone;
|
||||
pub mod s3;
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
use anyhow::{Context, Result, bail};
|
||||
use bytes::Bytes;
|
||||
use futures::{Stream, StreamExt};
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
use std::pin::Pin;
|
||||
use std::process::Stdio;
|
||||
use tempfile::NamedTempFile;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::process::Command;
|
||||
use tracing::info;
|
||||
|
||||
const BLOCKED_BACKEND_TYPES: [&str; 13] = [
|
||||
"local",
|
||||
"alias",
|
||||
"crypt",
|
||||
"chunker",
|
||||
"compress",
|
||||
"union",
|
||||
"combine",
|
||||
"hasher",
|
||||
"archive",
|
||||
"cache",
|
||||
"memory",
|
||||
"http",
|
||||
"googlephotos",
|
||||
];
|
||||
|
||||
fn sections(config_text: &str) -> Vec<(String, Option<String>)> {
|
||||
let mut out: Vec<(String, Option<String>)> = Vec::new();
|
||||
|
||||
for line in config_text.lines() {
|
||||
let line = line.trim();
|
||||
|
||||
if line.starts_with('[') && line.ends_with(']') && line.len() > 2 {
|
||||
out.push((line[1..line.len() - 1].trim().to_string(), None));
|
||||
continue;
|
||||
}
|
||||
|
||||
let Some((key, value)) = line.split_once('=') else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if key.trim().eq_ignore_ascii_case("type")
|
||||
&& let Some(current) = out.last_mut()
|
||||
&& current.1.is_none()
|
||||
{
|
||||
current.1 = Some(value.trim().to_ascii_lowercase());
|
||||
}
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
|
||||
pub fn validate_config(config_text: &str, remote_name: &str) -> Result<()> {
|
||||
let sections = sections(config_text);
|
||||
|
||||
if sections.is_empty() {
|
||||
bail!("rclone config contains no remote sections");
|
||||
}
|
||||
|
||||
for (name, backend) in §ions {
|
||||
let Some(backend) = backend else { continue };
|
||||
if BLOCKED_BACKEND_TYPES.contains(&backend.as_str()) {
|
||||
bail!("rclone backend type '{backend}' is not allowed (remote '{name}')");
|
||||
}
|
||||
}
|
||||
|
||||
if !sections.iter().any(|(name, _)| name == remote_name) {
|
||||
let available: Vec<&str> = sections.iter().map(|(name, _)| name.as_str()).collect();
|
||||
bail!(
|
||||
"remote '{remote_name}' is not defined in the rclone config (available: {})",
|
||||
available.join(", ")
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `<remote>:<remote_path>/<remote_file_path>`
|
||||
pub fn remote_target(remote_name: &str, remote_path: &str, remote_file_path: &str) -> String {
|
||||
let base = remote_path.trim().trim_matches('/');
|
||||
|
||||
if base.is_empty() {
|
||||
format!("{remote_name}:{remote_file_path}")
|
||||
} else {
|
||||
format!("{remote_name}:{base}/{remote_file_path}")
|
||||
}
|
||||
}
|
||||
|
||||
pub type RcloneStream = Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>;
|
||||
|
||||
pub fn write_config(config_text: &str) -> Result<NamedTempFile> {
|
||||
let mut file = NamedTempFile::new().context("failed to create rclone config temp file")?;
|
||||
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
std::fs::set_permissions(file.path(), std::fs::Permissions::from_mode(0o600))
|
||||
.context("failed to restrict rclone config permissions")?;
|
||||
}
|
||||
|
||||
file.write_all(config_text.as_bytes())
|
||||
.context("failed to write rclone config")?;
|
||||
file.flush().context("failed to flush rclone config")?;
|
||||
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
pub async fn rcat(config_path: &Path, target: &str, mut stream: RcloneStream) -> Result<()> {
|
||||
info!("rclone rcat -> {}", target);
|
||||
|
||||
let mut child = Command::new("rclone")
|
||||
.arg("--config")
|
||||
.arg(config_path)
|
||||
.arg("--contimeout")
|
||||
.arg("30s")
|
||||
.arg("--timeout")
|
||||
.arg("5m")
|
||||
.arg("--retries")
|
||||
.arg("1")
|
||||
.arg("--low-level-retries")
|
||||
.arg("3")
|
||||
.arg("rcat")
|
||||
.arg(target)
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.context("failed to spawn rclone (is the binary installed in this image?)")?;
|
||||
|
||||
let mut stderr_pipe = child.stderr.take().context("rclone stderr unavailable")?;
|
||||
let stderr_task = tokio::spawn(async move {
|
||||
let mut buf = String::new();
|
||||
let _ = stderr_pipe.read_to_string(&mut buf).await;
|
||||
buf
|
||||
});
|
||||
|
||||
let mut stdin = child.stdin.take().context("rclone stdin unavailable")?;
|
||||
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = match chunk {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
let _ = child.start_kill();
|
||||
let _ = child.wait().await;
|
||||
return Err(e).context("backup stream failed");
|
||||
}
|
||||
};
|
||||
|
||||
if stdin.write_all(&chunk).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let _ = stdin.flush().await;
|
||||
drop(stdin);
|
||||
|
||||
let status = child.wait().await.context("failed to wait for rclone")?;
|
||||
let stderr = stderr_task.await.unwrap_or_default();
|
||||
|
||||
if !status.success() {
|
||||
bail!("rclone rcat failed ({status}): {}", stderr.trim());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
pub mod helpers;
|
||||
pub mod models;
|
||||
|
||||
use crate::core::context::Context;
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::services::backup::models::{BackupResult, UploadResult};
|
||||
use crate::services::storage::StorageProvider;
|
||||
use crate::services::storage::providers::rclone::helpers::{
|
||||
rcat, remote_target, validate_config, write_config,
|
||||
};
|
||||
use crate::services::storage::providers::rclone::models::RcloneProviderConfig;
|
||||
use crate::utils::common::BackupMethod;
|
||||
use crate::utils::file::{full_file_name, full_file_path};
|
||||
use crate::utils::stream::build_stream;
|
||||
use async_trait::async_trait;
|
||||
use std::sync::Arc;
|
||||
use tokio::fs;
|
||||
use tracing::{error, info};
|
||||
|
||||
pub struct RcloneProvider {}
|
||||
|
||||
fn failed(storage_id: &str, error: impl ToString, total_size: Option<u64>) -> UploadResult {
|
||||
UploadResult {
|
||||
storage_id: storage_id.to_string(),
|
||||
success: false,
|
||||
error: Some(error.to_string()),
|
||||
remote_file_path: None,
|
||||
total_size,
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl StorageProvider for RcloneProvider {
|
||||
async fn upload(
|
||||
&self,
|
||||
ctx: Arc<Context>,
|
||||
result: BackupResult,
|
||||
_method: BackupMethod,
|
||||
storage: &DatabaseStorage,
|
||||
encrypt: Option<bool>,
|
||||
_backup_storage_id: &str,
|
||||
) -> UploadResult {
|
||||
let storage_id = storage.id.clone();
|
||||
|
||||
let Some(file_path) = result.backup_file else {
|
||||
return failed(&storage_id, "Missing backup file path", None);
|
||||
};
|
||||
|
||||
let total_size = match fs::metadata(&file_path).await {
|
||||
Ok(meta) => meta.len(),
|
||||
Err(e) => {
|
||||
error!("Failed to get file size: {}", e);
|
||||
return failed(&storage_id, e, None);
|
||||
}
|
||||
};
|
||||
|
||||
let config: RcloneProviderConfig = match storage.clone().config.try_into() {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
error!("rclone config deserialization failed: {}", e);
|
||||
return failed(&storage_id, e, Some(total_size));
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(e) = validate_config(&config.config_text, &config.remote_name) {
|
||||
error!("rclone config rejected: {}", e);
|
||||
return failed(&storage_id, e, Some(total_size));
|
||||
}
|
||||
|
||||
let encrypt = encrypt.unwrap_or(false);
|
||||
|
||||
let upload = match build_stream(&file_path, encrypt, &ctx.edge_key.master_key_b64).await {
|
||||
Ok(u) => u,
|
||||
Err(e) => {
|
||||
error!("Stream build failed: {}", e);
|
||||
return failed(&storage_id, e, Some(total_size));
|
||||
}
|
||||
};
|
||||
|
||||
let file_name = full_file_name(encrypt);
|
||||
let remote_file_path = full_file_path(&file_name, storage.folder_name.as_deref());
|
||||
|
||||
let config_file = match write_config(&config.config_text) {
|
||||
Ok(f) => f,
|
||||
Err(e) => {
|
||||
error!("rclone config write failed: {}", e);
|
||||
return failed(&storage_id, e, Some(total_size));
|
||||
}
|
||||
};
|
||||
|
||||
let target = remote_target(&config.remote_name, &config.remote_path, &remote_file_path);
|
||||
|
||||
info!("Starting rclone upload to {}", target);
|
||||
|
||||
match rcat(config_file.path(), &target, upload.stream).await {
|
||||
Ok(()) => {
|
||||
info!("rclone upload successful: {}", remote_file_path);
|
||||
UploadResult {
|
||||
storage_id,
|
||||
success: true,
|
||||
error: None,
|
||||
remote_file_path: Some(remote_file_path),
|
||||
total_size: Some(total_size),
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("rclone upload failed: {:?}", e);
|
||||
failed(&storage_id, e, Some(total_size))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
pub struct RcloneProviderConfig {
|
||||
pub config_text: String,
|
||||
pub remote_name: String,
|
||||
pub remote_path: String,
|
||||
}
|
||||
@@ -1,2 +1,3 @@
|
||||
mod azure_blob;
|
||||
mod google_cloud_storage;
|
||||
mod rclone;
|
||||
|
||||
@@ -0,0 +1,433 @@
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::services::storage::providers::rclone::helpers::{remote_target, validate_config};
|
||||
use crate::services::storage::providers::rclone::models::RcloneProviderConfig;
|
||||
use crate::tests::init_tracing_for_test;
|
||||
use crate::utils::file::full_file_path;
|
||||
|
||||
const OVH_CONFIG: &str = "[ovhcloud-rbx]\n\
|
||||
type = s3\n\
|
||||
provider = OVHcloud\n\
|
||||
access_key_id = my_access\n\
|
||||
secret_access_key = my_secret\n\
|
||||
region = rbx\n\
|
||||
endpoint = s3.rbx.io.cloud.ovh.net\n\
|
||||
acl = private\n";
|
||||
|
||||
#[test]
|
||||
fn config_deserializes_from_dashboard_camel_case() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let storage: DatabaseStorage = serde_json::from_value(serde_json::json!({
|
||||
"id": "storage-1",
|
||||
"provider": "rclone",
|
||||
"folderName": "backups",
|
||||
"config": {
|
||||
"configText": OVH_CONFIG,
|
||||
"remoteName": "ovhcloud-rbx",
|
||||
"remotePath": "my-bucket",
|
||||
}
|
||||
}))
|
||||
.unwrap();
|
||||
|
||||
let config: RcloneProviderConfig = storage.config.try_into().unwrap();
|
||||
|
||||
assert_eq!(config.remote_name, "ovhcloud-rbx");
|
||||
assert_eq!(config.remote_path, "my-bucket");
|
||||
assert!(config.config_text.contains("type = s3"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_config_accepts_the_target_remote() {
|
||||
assert!(validate_config(OVH_CONFIG, "ovhcloud-rbx").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_config_rejects_an_unknown_remote_name() {
|
||||
let err = validate_config(OVH_CONFIG, "typo").unwrap_err().to_string();
|
||||
assert!(err.contains("typo"), "unexpected error: {err}");
|
||||
assert!(err.contains("ovhcloud-rbx"), "error should list the available remotes: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_config_rejects_local_backend() {
|
||||
let cfg = "[disk]\ntype = local\n";
|
||||
let err = validate_config(cfg, "disk").unwrap_err().to_string();
|
||||
assert!(err.contains("local"), "unexpected error: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_config_rejects_alias_backend() {
|
||||
let cfg = "[shortcut]\ntype = alias\nremote = other:path\n";
|
||||
let err = validate_config(cfg, "shortcut").unwrap_err().to_string();
|
||||
assert!(err.contains("alias"), "unexpected error: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_config_rejects_a_blocked_backend_in_a_chained_section() {
|
||||
let cfg = "[secret]\ntype = crypt\nremote = disk:vault\n\n[disk]\ntype = local\n";
|
||||
let err = validate_config(cfg, "secret").unwrap_err().to_string();
|
||||
assert!(err.contains("crypt"), "unexpected error: {err}");
|
||||
assert!(err.contains("secret"), "error should name the offending remote: {err}");
|
||||
|
||||
let cfg = "[outer]\ntype = s3\nprovider = Minio\n\n[disk]\ntype = local\n";
|
||||
let err = validate_config(cfg, "outer").unwrap_err().to_string();
|
||||
assert!(err.contains("local"), "unexpected error: {err}");
|
||||
assert!(err.contains("disk"), "error should name the offending remote: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_config_rejects_crypt_even_over_an_allowed_remote() {
|
||||
let cfg = format!("[secret]\ntype = crypt\nremote = ovhcloud-rbx:bucket\n\n{OVH_CONFIG}");
|
||||
let err = validate_config(&cfg, "secret").unwrap_err().to_string();
|
||||
assert!(err.contains("crypt"), "unexpected error: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_config_rejects_a_wrapping_backend_pointing_at_a_bare_local_path() {
|
||||
for backend in ["crypt", "chunker", "compress", "union", "combine", "hasher"] {
|
||||
let cfg = format!("[sneaky]\ntype = {backend}\nremote = /etc\n");
|
||||
let err = validate_config(&cfg, "sneaky")
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(err.contains(backend), "{backend} must be rejected: {err}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_config_rejects_backends_that_cannot_hold_a_backup() {
|
||||
for backend in ["memory", "http", "googlephotos"] {
|
||||
let cfg = format!("[nope]\ntype = {backend}\n");
|
||||
let err = validate_config(&cfg, "nope").unwrap_err().to_string();
|
||||
assert!(err.contains(backend), "{backend} must be rejected: {err}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_path_is_a_prefix_ahead_of_the_backup_folder() {
|
||||
assert_eq!(
|
||||
remote_target("ovhcloud-rbx", "my-bucket", "backups/2026-09-09/x.tar.gz"),
|
||||
"ovhcloud-rbx:my-bucket/backups/2026-09-09/x.tar.gz"
|
||||
);
|
||||
|
||||
// Deeper prefixes nest the same way.
|
||||
assert_eq!(
|
||||
remote_target("ovhcloud-rbx", "my-bucket/portabase", "backups/2026-09-09/x.tar.gz"),
|
||||
"ovhcloud-rbx:my-bucket/portabase/backups/2026-09-09/x.tar.gz"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_target_trims_surrounding_slashes_and_whitespace() {
|
||||
assert_eq!(
|
||||
remote_target("r", " /my-bucket/ ", "a/b.bin"),
|
||||
"r:my-bucket/a/b.bin"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_target_handles_an_empty_remote_path() {
|
||||
assert_eq!(remote_target("r", "", "a/b.bin"), "r:a/b.bin");
|
||||
assert_eq!(remote_target("r", " ", "a/b.bin"), "r:a/b.bin");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_remote_path_falls_back_to_the_global_backup_folder() {
|
||||
let remote_file_path = full_file_path(&"x.tar.gz".to_string(), None);
|
||||
assert!(remote_file_path.starts_with("backups/"));
|
||||
|
||||
assert_eq!(
|
||||
remote_target("ovhcloud-rbx", "", &remote_file_path),
|
||||
format!("ovhcloud-rbx:{remote_file_path}")
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
remote_target("ovhcloud-rbx", "my-bucket", &remote_file_path),
|
||||
format!("ovhcloud-rbx:my-bucket/{remote_file_path}")
|
||||
);
|
||||
}
|
||||
|
||||
use crate::services::storage::providers::rclone::helpers::{rcat, write_config};
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures::stream;
|
||||
use std::process::Command;
|
||||
use testcontainers::core::{IntoContainerPort, WaitFor};
|
||||
use testcontainers::runners::AsyncRunner;
|
||||
use testcontainers::{GenericImage, ImageExt};
|
||||
|
||||
const BUCKET: &str = "portabase";
|
||||
|
||||
async fn start_minio() -> (testcontainers::ContainerAsync<GenericImage>, String) {
|
||||
let container = GenericImage::new("minio/minio", "latest")
|
||||
.with_exposed_port(9000.tcp())
|
||||
.with_wait_for(WaitFor::message_on_stderr("API:"))
|
||||
.with_env_var("MINIO_ROOT_USER", "minioadmin")
|
||||
.with_env_var("MINIO_ROOT_PASSWORD", "minioadmin")
|
||||
.with_cmd(["server", "/data"])
|
||||
.start()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let host = container.get_host().await.unwrap().to_string();
|
||||
let port = container.get_host_port_ipv4(9000).await.unwrap();
|
||||
(container, format!("http://{host}:{port}"))
|
||||
}
|
||||
|
||||
fn minio_config(endpoint: &str) -> String {
|
||||
format!(
|
||||
"[minio]\n\
|
||||
type = s3\n\
|
||||
provider = Minio\n\
|
||||
access_key_id = minioadmin\n\
|
||||
secret_access_key = minioadmin\n\
|
||||
endpoint = {endpoint}\n\
|
||||
region = us-east-1\n\
|
||||
force_path_style = true\n"
|
||||
)
|
||||
}
|
||||
|
||||
fn rclone_ok(config_path: &std::path::Path, args: &[&str]) -> Vec<u8> {
|
||||
let out = Command::new("rclone")
|
||||
.arg("--config")
|
||||
.arg(config_path)
|
||||
.args(args)
|
||||
.output()
|
||||
.expect("rclone binary not found — is it installed in this image?");
|
||||
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"rclone {args:?} failed: {}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
|
||||
out.stdout
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_config_creates_an_owner_only_file_with_the_exact_text() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let file = write_config(OVH_CONFIG).unwrap();
|
||||
|
||||
let mode = std::fs::metadata(file.path()).unwrap().permissions().mode();
|
||||
assert_eq!(mode & 0o777, 0o600, "config file must not be group/world readable");
|
||||
|
||||
assert_eq!(std::fs::read_to_string(file.path()).unwrap(), OVH_CONFIG);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rcat_streams_a_multi_chunk_body_to_minio() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, endpoint) = start_minio().await;
|
||||
let config = write_config(&minio_config(&endpoint)).unwrap();
|
||||
|
||||
rclone_ok(config.path(), &["mkdir", &format!("minio:{BUCKET}")]);
|
||||
|
||||
let data = vec![7u8; 10 * 1024];
|
||||
let chunks: Vec<Result<Bytes, std::io::Error>> = data
|
||||
.chunks(1024)
|
||||
.map(|c| Ok(Bytes::copy_from_slice(c)))
|
||||
.collect();
|
||||
|
||||
let target = remote_target("minio", BUCKET, "backups/2026-09-09/test.bin");
|
||||
|
||||
rcat(config.path(), &target, Box::pin(stream::iter(chunks)))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let got = rclone_ok(config.path(), &["cat", &target]);
|
||||
assert_eq!(got, data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rcat_reports_rclone_stderr_when_the_remote_is_unreachable() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let config = write_config(&minio_config("http://127.0.0.1:1")).unwrap();
|
||||
|
||||
let chunks: Vec<Result<Bytes, std::io::Error>> =
|
||||
vec![Ok(Bytes::from_static(&[0u8; 4096]))];
|
||||
|
||||
let err = rcat(
|
||||
config.path(),
|
||||
"minio:portabase/x.bin",
|
||||
Box::pin(stream::iter(chunks)),
|
||||
)
|
||||
.await
|
||||
.expect_err("upload to an unreachable endpoint must fail");
|
||||
|
||||
let msg = err.to_string();
|
||||
assert!(
|
||||
msg.contains("rclone rcat failed"),
|
||||
"the broken stdin pipe must not mask rclone's own error: {msg}"
|
||||
);
|
||||
let (_, stderr_part) = msg
|
||||
.rsplit_once(": ")
|
||||
.expect("bail message must carry rclone stderr after the exit status");
|
||||
assert!(
|
||||
!stderr_part.trim().is_empty(),
|
||||
"rclone stderr must be included: {msg}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rcat_aborts_the_upload_when_the_stream_fails() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, endpoint) = start_minio().await;
|
||||
let config = write_config(&minio_config(&endpoint)).unwrap();
|
||||
|
||||
rclone_ok(config.path(), &["mkdir", &format!("minio:{BUCKET}")]);
|
||||
|
||||
let chunks: Vec<Result<Bytes, std::io::Error>> = vec![
|
||||
Ok(Bytes::from_static(&[1u8; 1024])),
|
||||
Err(std::io::Error::other("injected stream failure")),
|
||||
];
|
||||
|
||||
let target = remote_target("minio", BUCKET, "backups/2026-09-09/aborted.bin");
|
||||
|
||||
let err = rcat(config.path(), &target, Box::pin(stream::iter(chunks)))
|
||||
.await
|
||||
.expect_err("a stream error must fail the upload");
|
||||
assert!(
|
||||
err.to_string().contains("backup stream failed"),
|
||||
"unexpected error: {err}"
|
||||
);
|
||||
|
||||
let stat_out = rclone_ok(config.path(), &["lsjson", "--stat", &target]);
|
||||
let stat: serde_json::Value = serde_json::from_slice(&stat_out).unwrap();
|
||||
assert_eq!(
|
||||
stat["Name"], "",
|
||||
"rclone must not have finalized the truncated object: {stat}"
|
||||
);
|
||||
assert_eq!(stat["IsDir"], true, "a miss reports IsDir: true: {stat}");
|
||||
}
|
||||
|
||||
use crate::core::context::Context;
|
||||
use crate::services::api::ApiClient;
|
||||
use crate::services::backup::models::BackupResult;
|
||||
use crate::services::config::DbType;
|
||||
use crate::services::storage::providers::rclone::RcloneProvider;
|
||||
use crate::services::storage::{StorageProvider, get_provider};
|
||||
use crate::utils::common::BackupMethod;
|
||||
use crate::utils::edge_key::EdgeKey;
|
||||
|
||||
use std::io::Write as _;
|
||||
use std::sync::Arc;
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
fn test_context() -> Arc<Context> {
|
||||
Arc::new(Context {
|
||||
edge_key: EdgeKey {
|
||||
server_url: String::new(),
|
||||
agent_id: "agent-1".to_string(),
|
||||
master_key_b64: String::new(),
|
||||
},
|
||||
api: ApiClient::new(String::new()),
|
||||
})
|
||||
}
|
||||
|
||||
fn storage_for(config_text: &str, remote_path: &str) -> DatabaseStorage {
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"id": "storage-1",
|
||||
"provider": "rclone",
|
||||
"folderName": "backups",
|
||||
"config": {
|
||||
"configText": config_text,
|
||||
"remoteName": "minio",
|
||||
"remotePath": remote_path,
|
||||
}
|
||||
}))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn factory_resolves_the_rclone_provider_key() {
|
||||
let storage = storage_for(OVH_CONFIG, "bucket");
|
||||
assert!(
|
||||
get_provider(&storage).is_some(),
|
||||
"get_provider must recognise the \"rclone\" key"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn provider_uploads_an_unencrypted_backup_to_minio() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, endpoint) = start_minio().await;
|
||||
let config_text = minio_config(&endpoint);
|
||||
|
||||
let bootstrap = write_config(&config_text).unwrap();
|
||||
rclone_ok(bootstrap.path(), &["mkdir", &format!("minio:{BUCKET}")]);
|
||||
|
||||
let payload = vec![42u8; 64 * 1024];
|
||||
let mut backup_file = NamedTempFile::new().unwrap();
|
||||
backup_file.write_all(&payload).unwrap();
|
||||
backup_file.flush().unwrap();
|
||||
|
||||
let storage = storage_for(&config_text, BUCKET);
|
||||
|
||||
let result = RcloneProvider {}
|
||||
.upload(
|
||||
test_context(),
|
||||
BackupResult {
|
||||
generated_id: "db-1".to_string(),
|
||||
db_type: DbType::Postgresql,
|
||||
status: "success".to_string(),
|
||||
backup_file: Some(backup_file.path().to_path_buf()),
|
||||
code: None,
|
||||
},
|
||||
BackupMethod::Automatic,
|
||||
&storage,
|
||||
Some(false),
|
||||
"backup-storage-1",
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(result.success, "upload failed: {:?}", result.error);
|
||||
assert_eq!(result.total_size, Some(payload.len() as u64));
|
||||
|
||||
let remote_file_path = result.remote_file_path.expect("remote path must be reported");
|
||||
assert!(
|
||||
remote_file_path.starts_with("backups/"),
|
||||
"folder_name must prefix the path: {remote_file_path}"
|
||||
);
|
||||
|
||||
let target = remote_target("minio", BUCKET, &remote_file_path);
|
||||
assert_eq!(rclone_ok(bootstrap.path(), &["cat", &target]), payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn provider_refuses_a_blocked_backend_without_spawning_rclone() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let mut backup_file = NamedTempFile::new().unwrap();
|
||||
backup_file.write_all(b"payload").unwrap();
|
||||
backup_file.flush().unwrap();
|
||||
|
||||
let storage = storage_for("[minio]\ntype = local\n", "bucket");
|
||||
|
||||
let result = RcloneProvider {}
|
||||
.upload(
|
||||
test_context(),
|
||||
BackupResult {
|
||||
generated_id: "db-1".to_string(),
|
||||
db_type: DbType::Postgresql,
|
||||
status: "success".to_string(),
|
||||
backup_file: Some(backup_file.path().to_path_buf()),
|
||||
code: None,
|
||||
},
|
||||
BackupMethod::Automatic,
|
||||
&storage,
|
||||
Some(false),
|
||||
"backup-storage-1",
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(!result.success);
|
||||
assert!(
|
||||
result.error.unwrap_or_default().contains("local"),
|
||||
"the error must name the rejected backend type"
|
||||
);
|
||||
}
|
||||
@@ -79,7 +79,12 @@ pub async fn execute_task(
|
||||
let ctx = Arc::new(Context::new());
|
||||
let config_service = ConfigService::new(ctx.clone());
|
||||
let backup_service = BackupService::new(ctx.clone());
|
||||
let config = config_service.load(None).unwrap();
|
||||
|
||||
let local = config_service.load_optional(None);
|
||||
let cache_path = std::path::PathBuf::from(&crate::settings::CONFIG.data_path)
|
||||
.join("dashboard_databases.json");
|
||||
let dashboard = crate::services::dashboard_config::load_cache(&cache_path);
|
||||
let config = crate::services::dashboard_config::merge(&local.databases, &dashboard);
|
||||
|
||||
let metadata_obj = metadata
|
||||
.into_iter()
|
||||
|
||||
Reference in New Issue
Block a user