Compare commits

...

30 Commits

Author SHA1 Message Date
github-actions[bot] e0bc6556aa chore: release 1.21.0 2026-09-14 13:11:26 +00:00
Charles GTE 947cb9bed7 Merge pull request #104 from Portabase/dev
Dev
2026-09-14 15:09:04 +02:00
Charles GTE b5d0035a3b Merge pull request #106 from Portabase/poc/hmac
fix: openssl windows
2026-09-14 14:52:26 +02:00
Charles GTE d52c328a99 fix: refactoring 2026-09-14 14:51:58 +02:00
Charles GTE 9b79ac4850 Merge branch 'refs/heads/dev' into poc/hmac
# Conflicts:
#	Cargo.lock
2026-09-14 14:51:22 +02:00
Charles GTE 6e50718b65 fix: tests 2026-09-14 14:30:11 +02:00
Charles GTE ee5ed81d48 fix: tests 2026-09-14 14:12:29 +02:00
Charles GTE df078b6202 Merge pull request #105 from Portabase/feat/sftp-storage
feat: sftp-storage
2026-09-14 10:55:40 +02:00
Charles GTE 24e3e3098d chore: update packages 2026-09-13 17:25:13 +02:00
Charles GTE f6e1a0df98 fix: refactoring 2026-09-12 21:15:04 +02:00
Charles GTE 49366ee24e refactor(storage): generic rclone config builder and shared obscure util 2026-09-12 21:10:50 +02:00
Charles GTE 012a984973 fix(storage): reject line breaks in sftp host/username; drop unused import 2026-09-12 21:02:28 +02:00
Charles GTE 6d1d7c6891 fix: refactoring 2026-09-12 18:15:45 +02:00
Charles GTE c70cde8a9c feat(storage): sftp provider over rclone rcat 2026-09-12 17:57:04 +02:00
Charles GTE c353443bdd feat(storage): sftp config model and rclone-config builder 2026-09-12 17:57:04 +02:00
Charles GTE 9f9dedfe87 feat: rclone-privider (#103)
* build: install rclone 1.75.1 in agent images

* build: exclude target/ and local artifacts from the docker context

* feat(storage): add rclone config model and validation helpers

* feat(storage): stream backups into rclone rcat

* fix(storage): abort truncated rclone uploads and strengthen stderr test

* feat(storage): add RcloneProvider upload path

* fix(storage): bound rclone timeouts, detect restore truncation, track sdd workspace

* fix(storage): block virtual and non-viable rclone backends

* fix: refactoring
2026-09-12 17:09:57 +02:00
Charles GTE 9bb830327e fix: refactoring 2026-09-10 22:55:47 +02:00
Charles GTE 31e55a7b4f fix(storage): block virtual and non-viable rclone backends 2026-09-10 22:38:08 +02:00
Charles GTE 576c055092 fix(storage): bound rclone timeouts, detect restore truncation, track sdd workspace 2026-09-10 22:38:08 +02:00
Charles GTE ba83b6b2b7 feat(storage): add RcloneProvider upload path 2026-09-10 22:38:07 +02:00
Charles GTE ea356ed54f fix(storage): abort truncated rclone uploads and strengthen stderr test 2026-09-10 22:37:37 +02:00
Charles GTE b222b13934 feat(storage): stream backups into rclone rcat 2026-09-10 22:37:09 +02:00
Charles GTE 9409f0ff25 feat(storage): add rclone config model and validation helpers 2026-09-10 22:37:00 +02:00
Charles GTE 59312215e9 build: exclude target/ and local artifacts from the docker context 2026-09-10 22:36:46 +02:00
Charles GTE 3ff360a971 build: install rclone 1.75.1 in agent images 2026-09-10 22:36:38 +02:00
Théo LAGACHE cc1703592e poc 2026-09-09 11:57:56 +02:00
github-actions[bot] 5f92297108 chore: release 1.20.1 2026-09-03 15:41:20 +00:00
Charles GTE d1821f7045 Merge pull request #102 from Portabase/fix/cron-crash
fix: cron crash for databases setup from dashboard
2026-09-03 17:38:53 +02:00
charles-gauthereau 0fe4d50cbd fix: docker-compose.yml 2026-09-03 17:38:35 +02:00
charles-gauthereau 1f29466a28 fix: cron crash for databases setup from dashboard 2026-09-03 17:22:03 +02:00
22 changed files with 2333 additions and 1393 deletions
+3 -5
View File
@@ -1,11 +1,9 @@
# Git
.git
.gitignore
# MD files
CHANGELOG.md
README.md
RELEASE.md
#IDE configurations
.idea
target
dump.rdb
.superpowers
-15
View File
@@ -44,21 +44,6 @@ jobs:
- name: Cache cargo build
uses: Swatinem/rust-cache@v2
- name: Cache vcpkg installed packages
uses: actions/cache@v4
with:
path: C:\vcpkg\installed
key: vcpkg-openssl-x64-windows-v1
- name: Install OpenSSL (x64) via vcpkg
shell: pwsh
run: |
# windows-latest ships vcpkg preinstalled; the install is a no-op when the
# package is restored from cache.
& "$env:VCPKG_INSTALLATION_ROOT\vcpkg.exe" install openssl:x64-windows
'VCPKG_ROOT=C:\vcpkg' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
'OPENSSL_DIR=C:\vcpkg\installed\x64-windows' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
- name: Build (cargo release)
shell: pwsh
run: cargo build --release --bin app
+1
View File
@@ -8,3 +8,4 @@
.claude
/docs
.superpowers
+1 -1
View File
@@ -27,5 +27,5 @@ keywords:
- self-hosted
- portabase
license: Apache-2.0
version: 1.20.0
version: 1.21.0
date-released: '2026-02-24'
Generated
+1215 -1354
View File
File diff suppressed because it is too large Load Diff
+4 -3
View File
@@ -1,6 +1,6 @@
[package]
name = "portabase-agent"
version = "1.20.0"
version = "1.21.0"
edition = "2024"
[dependencies]
@@ -19,10 +19,11 @@ log = "0.4.29"
toml = "0.9.10"
reqwest = { version = "0.13.1", features = ["json", "blocking", "multipart", "stream", "query"] }
anyhow = "1.0.100"
tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "fs"] }
tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "fs", "process", "io-util"] }
async-trait = "0.1.89"
tempfile = "3.24.0"
openssl = "0.10.75"
hmac = "0.12"
sha2 = "0.10"
hex = "0.4.3"
flate2 = "1.1.5"
tar = "0.4.44"
+1 -1
View File
@@ -21,7 +21,7 @@ services:
LOG: debug
TZ: "Europe/Paris"
# TMPDIR: /scratch
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiY2UxNjRiZDItZGZkMy00YzY4LThlZGItNmQ3OTczODAzZWEyIiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ=="
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiYTQyNjQzNTQtZGE3Ni00OWFkLWJkYjctZDVjMjMwYzhjYmViIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ=="
#CHUNK_SIZE_MB: "1"
#POOLING: 1
#RETRY_ATTEMPTS: 3
+16 -1
View File
@@ -9,7 +9,7 @@ RUN mkdir -p /mysql-exports/bin /mysql-exports/lib \
# =========================
# Base image (shared)
# =========================
FROM rust:1.94.0 AS base
FROM rust:1.98 AS base
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
pkg-config \
@@ -44,6 +44,15 @@ RUN ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') \
| tar -xjf - -C /usr/local/bin sqlcmd \
&& chmod +x /usr/local/bin/sqlcmd
# =========================
# rclone (all storage backends)
# =========================
ARG RCLONE_VERSION=1.75.1
RUN ARCH=$(dpkg --print-architecture) \
&& curl -fsSL -o /tmp/rclone.deb "https://downloads.rclone.org/v${RCLONE_VERSION}/rclone-v${RCLONE_VERSION}-linux-${ARCH}.deb" \
&& dpkg -i /tmp/rclone.deb \
&& rm /tmp/rclone.deb
ARG TARGETARCH
# =========================
@@ -134,6 +143,12 @@ RUN apt-get update && apt-get install -y \
firebird3.0-utils \
&& rm -rf /var/lib/apt/lists/*
ARG RCLONE_VERSION=1.75.1
RUN ARCH=$(dpkg --print-architecture) \
&& curl -fsSL -o /tmp/rclone.deb "https://downloads.rclone.org/v${RCLONE_VERSION}/rclone-v${RCLONE_VERSION}-linux-${ARCH}.deb" \
&& dpkg -i /tmp/rclone.deb \
&& rm /tmp/rclone.deb
ENV DOTNET_ROOT=/usr/local/dotnet
RUN curl -sSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh \
&& chmod +x /tmp/dotnet-install.sh \
+15 -3
View File
@@ -1,5 +1,7 @@
use super::service::RestoreService;
use crate::services::backup::logger::JobLogger;
use crate::utils::retry::{RetryPolicy, retry};
use anyhow::Result;
use futures::StreamExt;
use reqwest::{Client, Url};
@@ -7,8 +9,6 @@ use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Instant;
use tokio::io::AsyncWriteExt;
use crate::services::backup::logger::JobLogger;
use crate::utils::retry::{RetryPolicy, retry};
fn human_size(bytes: u64) -> String {
if bytes >= 1024 * 1024 {
@@ -98,7 +98,9 @@ impl RestoreService {
format!(
"Downloading backup '{}' ({})",
filename,
total.map(human_size).unwrap_or_else(|| "unknown size".to_string())
total
.map(human_size)
.unwrap_or_else(|| "unknown size".to_string())
),
);
@@ -138,6 +140,16 @@ impl RestoreService {
);
}
if let Some(total) = total
&& downloaded < total
{
anyhow::bail!(
"Downloaded {} bytes but expected at least {} - backup appears truncated",
downloaded,
total
);
}
logger.log(
"info",
format!(
+4 -1
View File
@@ -9,7 +9,9 @@ use providers::azure_blob;
use providers::google_cloud_storage;
use providers::google_drive;
use providers::local;
use providers::rclone;
use providers::s3;
use providers::sftp;
use std::sync::Arc;
use tracing::{error, info};
@@ -26,7 +28,6 @@ pub trait StorageProvider: Send + Sync {
) -> UploadResult;
}
/// Factory to create provider instance from storage config
pub fn get_provider(storage: &DatabaseStorage) -> Option<Box<dyn StorageProvider>> {
info!("Getting provider");
info!("{:#?}", storage.provider.as_str());
@@ -39,6 +40,8 @@ pub fn get_provider(storage: &DatabaseStorage) -> Option<Box<dyn StorageProvider
"google-cloud-storage" => Some(Box::new(
google_cloud_storage::GoogleCloudStorageProvider {},
)),
"rclone" => Some(Box::new(rclone::RcloneProvider {})),
"sftp" => Some(Box::new(sftp::SftpProvider {})),
_ => {
error!("Unknown storage provider: {}", storage.provider);
None
@@ -2,9 +2,8 @@ use anyhow::{Context as _, Result, anyhow};
use base64::Engine;
use base64::engine::general_purpose::STANDARD;
use chrono::{Duration, Utc};
use openssl::hash::MessageDigest;
use openssl::pkey::PKey;
use openssl::sign::Signer;
use hmac::{Hmac, Mac};
use sha2::Sha256;
use url::Url;
use azure_core::http::RequestContent;
use azure_storage_blob::clients::{BlobClient, BlockBlobClient};
@@ -36,12 +35,12 @@ impl SasResource {
pub(crate) const SAS_VERSION: &str = "2022-11-02";
type HmacSha256 = Hmac<Sha256>;
pub(crate) fn hmac_sha256_b64(key: &[u8], data: &str) -> Result<String> {
let pkey = PKey::hmac(key).context("hmac key")?;
let mut signer = Signer::new(MessageDigest::sha256(), &pkey).context("signer")?;
signer.update(data.as_bytes()).context("signer update")?;
let sig = signer.sign_to_vec().context("sign")?;
Ok(STANDARD.encode(sig))
let mut mac = HmacSha256::new_from_slice(key).context("hmac key")?;
mac.update(data.as_bytes());
Ok(STANDARD.encode(mac.finalize().into_bytes()))
}
pub fn build_service_sas(
+2
View File
@@ -2,4 +2,6 @@ pub mod azure_blob;
pub mod google_cloud_storage;
pub mod google_drive;
pub mod local;
pub mod rclone;
pub mod s3;
pub mod sftp;
@@ -0,0 +1,205 @@
use anyhow::{Context, Result, bail};
use bytes::Bytes;
use futures::{Stream, StreamExt};
use std::io::Write;
use std::path::Path;
use std::pin::Pin;
use std::process::Stdio;
use tempfile::NamedTempFile;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::process::Command;
use tracing::info;
const BLOCKED_BACKEND_TYPES: [&str; 13] = [
"local",
"alias",
"crypt",
"chunker",
"compress",
"union",
"combine",
"hasher",
"archive",
"cache",
"memory",
"http",
"googlephotos",
];
fn sections(config_text: &str) -> Vec<(String, Option<String>)> {
let mut out: Vec<(String, Option<String>)> = Vec::new();
for line in config_text.lines() {
let line = line.trim();
if line.starts_with('[') && line.ends_with(']') && line.len() > 2 {
out.push((line[1..line.len() - 1].trim().to_string(), None));
continue;
}
let Some((key, value)) = line.split_once('=') else {
continue;
};
if key.trim().eq_ignore_ascii_case("type")
&& let Some(current) = out.last_mut()
&& current.1.is_none()
{
current.1 = Some(value.trim().to_ascii_lowercase());
}
}
out
}
pub fn validate_config(config_text: &str, remote_name: &str) -> Result<()> {
let sections = sections(config_text);
if sections.is_empty() {
bail!("rclone config contains no remote sections");
}
for (name, backend) in &sections {
let Some(backend) = backend else { continue };
if BLOCKED_BACKEND_TYPES.contains(&backend.as_str()) {
bail!("rclone backend type '{backend}' is not allowed (remote '{name}')");
}
}
if !sections.iter().any(|(name, _)| name == remote_name) {
let available: Vec<&str> = sections.iter().map(|(name, _)| name.as_str()).collect();
bail!(
"remote '{remote_name}' is not defined in the rclone config (available: {})",
available.join(", ")
);
}
Ok(())
}
pub fn obscure_password(password: &str) -> Result<String> {
let out = std::process::Command::new("rclone")
.arg("obscure")
.arg(password)
.output()
.context("failed to spawn rclone (is the binary installed in this image?)")?;
if !out.status.success() {
bail!(
"rclone obscure failed: {}",
String::from_utf8_lossy(&out.stderr).trim()
);
}
Ok(String::from_utf8_lossy(&out.stdout).trim().to_string())
}
pub fn build_rclone_config(remote_name: &str, fields: &[(&str, String)]) -> Result<String> {
if remote_name.contains(['\r', '\n']) {
bail!("rclone remote name must not contain line breaks");
}
let mut lines = vec![format!("[{remote_name}]")];
for (key, value) in fields {
let value = value.trim();
if value.is_empty() {
continue;
}
if value.contains(['\r', '\n']) {
bail!("rclone config value for '{key}' must not contain line breaks");
}
lines.push(format!("{key} = {value}"));
}
Ok(lines.join("\n") + "\n")
}
/// `<remote>:<remote_path>/<remote_file_path>`
pub fn remote_target(remote_name: &str, remote_path: &str, remote_file_path: &str) -> String {
let base = remote_path.trim().trim_matches('/');
if base.is_empty() {
format!("{remote_name}:{remote_file_path}")
} else {
format!("{remote_name}:{base}/{remote_file_path}")
}
}
pub type RcloneStream = Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>;
pub fn write_config(config_text: &str) -> Result<NamedTempFile> {
let mut file = NamedTempFile::new().context("failed to create rclone config temp file")?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(file.path(), std::fs::Permissions::from_mode(0o600))
.context("failed to restrict rclone config permissions")?;
}
file.write_all(config_text.as_bytes())
.context("failed to write rclone config")?;
file.flush().context("failed to flush rclone config")?;
Ok(file)
}
pub async fn rcat(config_path: &Path, target: &str, mut stream: RcloneStream) -> Result<()> {
info!("rclone rcat -> {}", target);
let mut child = Command::new("rclone")
.arg("--config")
.arg(config_path)
.arg("--contimeout")
.arg("30s")
.arg("--timeout")
.arg("5m")
.arg("--retries")
.arg("1")
.arg("--low-level-retries")
.arg("3")
.arg("rcat")
.arg(target)
.stdin(Stdio::piped())
.stdout(Stdio::null())
.stderr(Stdio::piped())
.spawn()
.context("failed to spawn rclone (is the binary installed in this image?)")?;
let mut stderr_pipe = child.stderr.take().context("rclone stderr unavailable")?;
let stderr_task = tokio::spawn(async move {
let mut buf = String::new();
let _ = stderr_pipe.read_to_string(&mut buf).await;
buf
});
let mut stdin = child.stdin.take().context("rclone stdin unavailable")?;
while let Some(chunk) = stream.next().await {
let chunk = match chunk {
Ok(c) => c,
Err(e) => {
let _ = child.start_kill();
let _ = child.wait().await;
return Err(e).context("backup stream failed");
}
};
if stdin.write_all(&chunk).await.is_err() {
break;
}
}
let _ = stdin.flush().await;
drop(stdin);
let status = child.wait().await.context("failed to wait for rclone")?;
let stderr = stderr_task.await.unwrap_or_default();
if !status.success() {
bail!("rclone rcat failed ({status}): {}", stderr.trim());
}
Ok(())
}
@@ -0,0 +1,112 @@
pub mod helpers;
pub mod models;
use crate::core::context::Context;
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::backup::models::{BackupResult, UploadResult};
use crate::services::storage::StorageProvider;
use crate::services::storage::providers::rclone::helpers::{
rcat, remote_target, validate_config, write_config,
};
use crate::services::storage::providers::rclone::models::RcloneProviderConfig;
use crate::utils::common::BackupMethod;
use crate::utils::file::{full_file_name, full_file_path};
use crate::utils::stream::build_stream;
use async_trait::async_trait;
use std::sync::Arc;
use tokio::fs;
use tracing::{error, info};
pub struct RcloneProvider {}
fn failed(storage_id: &str, error: impl ToString, total_size: Option<u64>) -> UploadResult {
UploadResult {
storage_id: storage_id.to_string(),
success: false,
error: Some(error.to_string()),
remote_file_path: None,
total_size,
}
}
#[async_trait]
impl StorageProvider for RcloneProvider {
async fn upload(
&self,
ctx: Arc<Context>,
result: BackupResult,
_method: BackupMethod,
storage: &DatabaseStorage,
encrypt: Option<bool>,
_backup_storage_id: &str,
) -> UploadResult {
let storage_id = storage.id.clone();
let Some(file_path) = result.backup_file else {
return failed(&storage_id, "Missing backup file path", None);
};
let total_size = match fs::metadata(&file_path).await {
Ok(meta) => meta.len(),
Err(e) => {
error!("Failed to get file size: {}", e);
return failed(&storage_id, e, None);
}
};
let config: RcloneProviderConfig = match storage.clone().config.try_into() {
Ok(c) => c,
Err(e) => {
error!("rclone config deserialization failed: {}", e);
return failed(&storage_id, e, Some(total_size));
}
};
if let Err(e) = validate_config(&config.config_text, &config.remote_name) {
error!("rclone config rejected: {}", e);
return failed(&storage_id, e, Some(total_size));
}
let encrypt = encrypt.unwrap_or(false);
let upload = match build_stream(&file_path, encrypt, &ctx.edge_key.master_key_b64).await {
Ok(u) => u,
Err(e) => {
error!("Stream build failed: {}", e);
return failed(&storage_id, e, Some(total_size));
}
};
let file_name = full_file_name(encrypt);
let remote_file_path = full_file_path(&file_name, storage.folder_name.as_deref());
let config_file = match write_config(&config.config_text) {
Ok(f) => f,
Err(e) => {
error!("rclone config write failed: {}", e);
return failed(&storage_id, e, Some(total_size));
}
};
let target = remote_target(&config.remote_name, &config.remote_path, &remote_file_path);
info!("Starting rclone upload to {}", target);
match rcat(config_file.path(), &target, upload.stream).await {
Ok(()) => {
info!("rclone upload successful: {}", remote_file_path);
UploadResult {
storage_id,
success: true,
error: None,
remote_file_path: Some(remote_file_path),
total_size: Some(total_size),
}
}
Err(e) => {
error!("rclone upload failed: {:?}", e);
failed(&storage_id, e, Some(total_size))
}
}
}
}
@@ -0,0 +1,8 @@
use serde::{Deserialize, Serialize};
#[derive(Debug, Deserialize, Serialize)]
pub struct RcloneProviderConfig {
pub config_text: String,
pub remote_name: String,
pub remote_path: String,
}
@@ -0,0 +1,71 @@
use crate::services::storage::providers::rclone::helpers::{build_rclone_config, obscure_password};
use crate::services::storage::providers::sftp::models::SftpProviderConfig;
use anyhow::{Context, Result, bail};
use std::io::Write;
use tempfile::NamedTempFile;
fn write_key(private_key: &str) -> Result<NamedTempFile> {
let mut file = NamedTempFile::new().context("failed to create sftp key temp file")?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(file.path(), std::fs::Permissions::from_mode(0o600))
.context("failed to restrict sftp key permissions")?;
}
file.write_all(private_key.as_bytes())
.context("failed to write sftp key")?;
file.flush().context("failed to flush sftp key")?;
Ok(file)
}
pub fn build_sftp_config(
config: &SftpProviderConfig,
) -> Result<(String, Option<NamedTempFile>)> {
if config.host.trim().is_empty() {
bail!("sftp host is required");
}
if config.username.trim().is_empty() {
bail!("sftp username is required");
}
let has_password = config.password.as_deref().is_some_and(|p| !p.trim().is_empty());
let has_key = config.private_key.as_deref().is_some_and(|k| !k.trim().is_empty());
if !has_password && !has_key {
bail!("sftp requires a password or a private key");
}
let mut key_file: Option<NamedTempFile> = None;
let mut key_file_path = String::new();
if has_key {
let file = write_key(config.private_key.as_deref().unwrap())?;
key_file_path = file.path().display().to_string();
key_file = Some(file);
}
let pass = if has_password {
obscure_password(config.password.as_deref().unwrap())?
} else {
String::new()
};
let port = config
.port
.as_deref()
.unwrap_or("")
.trim()
.to_string();
let fields: &[(&str, String)] = &[
("type", "sftp".to_string()),
("host", config.host.trim().to_string()),
("port", port),
("user", config.username.trim().to_string()),
("key_file", key_file_path),
("pass", pass),
];
let config_text = build_rclone_config("sftp", fields)?;
Ok((config_text, key_file))
}
+114
View File
@@ -0,0 +1,114 @@
pub mod helpers;
pub mod models;
use crate::core::context::Context;
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::backup::models::{BackupResult, UploadResult};
use crate::services::storage::StorageProvider;
use crate::services::storage::providers::rclone::helpers::{rcat, remote_target, write_config};
use crate::services::storage::providers::sftp::helpers::build_sftp_config;
use crate::services::storage::providers::sftp::models::SftpProviderConfig;
use crate::utils::common::BackupMethod;
use crate::utils::file::{full_file_name, full_file_path};
use crate::utils::stream::build_stream;
use async_trait::async_trait;
use std::sync::Arc;
use tokio::fs;
use tracing::{error, info};
pub struct SftpProvider {}
fn failed(storage_id: &str, error: impl ToString, total_size: Option<u64>) -> UploadResult {
UploadResult {
storage_id: storage_id.to_string(),
success: false,
error: Some(error.to_string()),
remote_file_path: None,
total_size,
}
}
#[async_trait]
impl StorageProvider for SftpProvider {
async fn upload(
&self,
ctx: Arc<Context>,
result: BackupResult,
_method: BackupMethod,
storage: &DatabaseStorage,
encrypt: Option<bool>,
_backup_storage_id: &str,
) -> UploadResult {
let storage_id = storage.id.clone();
let Some(file_path) = result.backup_file else {
return failed(&storage_id, "Missing backup file path", None);
};
let total_size = match fs::metadata(&file_path).await {
Ok(meta) => meta.len(),
Err(e) => {
error!("Failed to get file size: {}", e);
return failed(&storage_id, e, None);
}
};
let config: SftpProviderConfig = match storage.clone().config.try_into() {
Ok(c) => c,
Err(e) => {
error!("sftp config deserialization failed: {}", e);
return failed(&storage_id, e, Some(total_size));
}
};
let (config_text, _key_file) = match build_sftp_config(&config) {
Ok(v) => v,
Err(e) => {
error!("sftp config build failed: {}", e);
return failed(&storage_id, e, Some(total_size));
}
};
let encrypt = encrypt.unwrap_or(false);
let upload = match build_stream(&file_path, encrypt, &ctx.edge_key.master_key_b64).await {
Ok(u) => u,
Err(e) => {
error!("Stream build failed: {}", e);
return failed(&storage_id, e, Some(total_size));
}
};
let file_name = full_file_name(encrypt);
let remote_file_path = full_file_path(&file_name, storage.folder_name.as_deref());
let config_file = match write_config(&config_text) {
Ok(f) => f,
Err(e) => {
error!("sftp config write failed: {}", e);
return failed(&storage_id, e, Some(total_size));
}
};
let target = remote_target("sftp", &config.remote_path, &remote_file_path);
info!("Starting sftp (rclone) upload to {}", target);
match rcat(config_file.path(), &target, upload.stream).await {
Ok(()) => {
info!("sftp upload successful: {}", remote_file_path);
UploadResult {
storage_id,
success: true,
error: None,
remote_file_path: Some(remote_file_path),
total_size: Some(total_size),
}
}
Err(e) => {
error!("sftp upload failed: {:?}", e);
failed(&storage_id, e, Some(total_size))
}
}
}
}
@@ -0,0 +1,16 @@
use crate::utils::deserializer::string_or_number_to_string;
use serde::{Deserialize, Serialize};
#[derive(Debug, Deserialize, Serialize)]
pub struct SftpProviderConfig {
pub host: String,
#[serde(default, deserialize_with = "string_or_number_to_string")]
pub port: Option<String>,
pub username: String,
#[serde(default)]
pub password: Option<String>,
#[serde(default)]
pub private_key: Option<String>,
#[serde(default)]
pub remote_path: String,
}
+2
View File
@@ -1,2 +1,4 @@
mod azure_blob;
mod google_cloud_storage;
mod rclone;
mod sftp;
+455
View File
@@ -0,0 +1,455 @@
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::storage::providers::rclone::helpers::{remote_target, validate_config};
use crate::services::storage::providers::rclone::models::RcloneProviderConfig;
use crate::tests::init_tracing_for_test;
use crate::utils::file::full_file_path;
const OVH_CONFIG: &str = "[ovhcloud-rbx]\n\
type = s3\n\
provider = OVHcloud\n\
access_key_id = my_access\n\
secret_access_key = my_secret\n\
region = rbx\n\
endpoint = s3.rbx.io.cloud.ovh.net\n\
acl = private\n";
#[test]
fn config_deserializes_from_dashboard_camel_case() {
init_tracing_for_test();
let storage: DatabaseStorage = serde_json::from_value(serde_json::json!({
"id": "storage-1",
"provider": "rclone",
"folderName": "backups",
"config": {
"configText": OVH_CONFIG,
"remoteName": "ovhcloud-rbx",
"remotePath": "my-bucket",
}
}))
.unwrap();
let config: RcloneProviderConfig = storage.config.try_into().unwrap();
assert_eq!(config.remote_name, "ovhcloud-rbx");
assert_eq!(config.remote_path, "my-bucket");
assert!(config.config_text.contains("type = s3"));
}
#[test]
fn build_rclone_config_skips_empty_and_rejects_line_breaks() {
use crate::services::storage::providers::rclone::helpers::build_rclone_config;
let text = build_rclone_config(
"sftp",
&[
("type", "sftp".to_string()),
("host", "h".to_string()),
("port", "".to_string()), // empty -> skipped
("user", " u ".to_string()), // trimmed
],
)
.unwrap();
assert_eq!(text, "[sftp]\ntype = sftp\nhost = h\nuser = u\n");
let err = build_rclone_config("sftp", &[("host", "a\nkey = injected".to_string())])
.unwrap_err()
.to_string();
assert!(err.contains("line breaks"), "unexpected error: {err}");
}
#[test]
fn validate_config_accepts_the_target_remote() {
assert!(validate_config(OVH_CONFIG, "ovhcloud-rbx").is_ok());
}
#[test]
fn validate_config_rejects_an_unknown_remote_name() {
let err = validate_config(OVH_CONFIG, "typo").unwrap_err().to_string();
assert!(err.contains("typo"), "unexpected error: {err}");
assert!(err.contains("ovhcloud-rbx"), "error should list the available remotes: {err}");
}
#[test]
fn validate_config_rejects_local_backend() {
let cfg = "[disk]\ntype = local\n";
let err = validate_config(cfg, "disk").unwrap_err().to_string();
assert!(err.contains("local"), "unexpected error: {err}");
}
#[test]
fn validate_config_rejects_alias_backend() {
let cfg = "[shortcut]\ntype = alias\nremote = other:path\n";
let err = validate_config(cfg, "shortcut").unwrap_err().to_string();
assert!(err.contains("alias"), "unexpected error: {err}");
}
#[test]
fn validate_config_rejects_a_blocked_backend_in_a_chained_section() {
let cfg = "[secret]\ntype = crypt\nremote = disk:vault\n\n[disk]\ntype = local\n";
let err = validate_config(cfg, "secret").unwrap_err().to_string();
assert!(err.contains("crypt"), "unexpected error: {err}");
assert!(err.contains("secret"), "error should name the offending remote: {err}");
let cfg = "[outer]\ntype = s3\nprovider = Minio\n\n[disk]\ntype = local\n";
let err = validate_config(cfg, "outer").unwrap_err().to_string();
assert!(err.contains("local"), "unexpected error: {err}");
assert!(err.contains("disk"), "error should name the offending remote: {err}");
}
#[test]
fn validate_config_rejects_crypt_even_over_an_allowed_remote() {
let cfg = format!("[secret]\ntype = crypt\nremote = ovhcloud-rbx:bucket\n\n{OVH_CONFIG}");
let err = validate_config(&cfg, "secret").unwrap_err().to_string();
assert!(err.contains("crypt"), "unexpected error: {err}");
}
#[test]
fn validate_config_rejects_a_wrapping_backend_pointing_at_a_bare_local_path() {
for backend in ["crypt", "chunker", "compress", "union", "combine", "hasher"] {
let cfg = format!("[sneaky]\ntype = {backend}\nremote = /etc\n");
let err = validate_config(&cfg, "sneaky")
.unwrap_err()
.to_string();
assert!(err.contains(backend), "{backend} must be rejected: {err}");
}
}
#[test]
fn validate_config_rejects_backends_that_cannot_hold_a_backup() {
for backend in ["memory", "http", "googlephotos"] {
let cfg = format!("[nope]\ntype = {backend}\n");
let err = validate_config(&cfg, "nope").unwrap_err().to_string();
assert!(err.contains(backend), "{backend} must be rejected: {err}");
}
}
#[test]
fn remote_path_is_a_prefix_ahead_of_the_backup_folder() {
assert_eq!(
remote_target("ovhcloud-rbx", "my-bucket", "backups/2026-09-09/x.tar.gz"),
"ovhcloud-rbx:my-bucket/backups/2026-09-09/x.tar.gz"
);
// Deeper prefixes nest the same way.
assert_eq!(
remote_target("ovhcloud-rbx", "my-bucket/portabase", "backups/2026-09-09/x.tar.gz"),
"ovhcloud-rbx:my-bucket/portabase/backups/2026-09-09/x.tar.gz"
);
}
#[test]
fn remote_target_trims_surrounding_slashes_and_whitespace() {
assert_eq!(
remote_target("r", " /my-bucket/ ", "a/b.bin"),
"r:my-bucket/a/b.bin"
);
}
#[test]
fn remote_target_handles_an_empty_remote_path() {
assert_eq!(remote_target("r", "", "a/b.bin"), "r:a/b.bin");
assert_eq!(remote_target("r", " ", "a/b.bin"), "r:a/b.bin");
}
#[test]
fn an_empty_remote_path_falls_back_to_the_global_backup_folder() {
let remote_file_path = full_file_path(&"x.tar.gz".to_string(), None);
assert!(remote_file_path.starts_with("backups/"));
assert_eq!(
remote_target("ovhcloud-rbx", "", &remote_file_path),
format!("ovhcloud-rbx:{remote_file_path}")
);
assert_eq!(
remote_target("ovhcloud-rbx", "my-bucket", &remote_file_path),
format!("ovhcloud-rbx:my-bucket/{remote_file_path}")
);
}
use crate::services::storage::providers::rclone::helpers::{rcat, write_config};
use bytes::Bytes;
use futures::stream;
use std::process::Command;
use testcontainers::core::{IntoContainerPort, WaitFor};
use testcontainers::runners::AsyncRunner;
use testcontainers::{GenericImage, ImageExt};
const BUCKET: &str = "portabase";
async fn start_minio() -> (testcontainers::ContainerAsync<GenericImage>, String) {
let container = GenericImage::new("coollabsio/minio", "latest")
.with_exposed_port(9000.tcp())
.with_wait_for(WaitFor::message_on_stderr("API:"))
.with_env_var("MINIO_ROOT_USER", "minioadmin")
.with_env_var("MINIO_ROOT_PASSWORD", "minioadmin")
.with_cmd(["server", "/data"])
.start()
.await
.unwrap();
let host = container.get_host().await.unwrap().to_string();
let port = container.get_host_port_ipv4(9000).await.unwrap();
(container, format!("http://{host}:{port}"))
}
fn minio_config(endpoint: &str) -> String {
format!(
"[minio]\n\
type = s3\n\
provider = Minio\n\
access_key_id = minioadmin\n\
secret_access_key = minioadmin\n\
endpoint = {endpoint}\n\
region = us-east-1\n\
force_path_style = true\n"
)
}
fn rclone_ok(config_path: &std::path::Path, args: &[&str]) -> Vec<u8> {
let out = Command::new("rclone")
.arg("--config")
.arg(config_path)
.args(args)
.output()
.expect("rclone binary not found — is it installed in this image?");
assert!(
out.status.success(),
"rclone {args:?} failed: {}",
String::from_utf8_lossy(&out.stderr)
);
out.stdout
}
#[test]
fn write_config_creates_an_owner_only_file_with_the_exact_text() {
use std::os::unix::fs::PermissionsExt;
let file = write_config(OVH_CONFIG).unwrap();
let mode = std::fs::metadata(file.path()).unwrap().permissions().mode();
assert_eq!(mode & 0o777, 0o600, "config file must not be group/world readable");
assert_eq!(std::fs::read_to_string(file.path()).unwrap(), OVH_CONFIG);
}
#[tokio::test]
async fn rcat_streams_a_multi_chunk_body_to_minio() {
init_tracing_for_test();
let (_container, endpoint) = start_minio().await;
let config = write_config(&minio_config(&endpoint)).unwrap();
rclone_ok(config.path(), &["mkdir", &format!("minio:{BUCKET}")]);
let data = vec![7u8; 10 * 1024];
let chunks: Vec<Result<Bytes, std::io::Error>> = data
.chunks(1024)
.map(|c| Ok(Bytes::copy_from_slice(c)))
.collect();
let target = remote_target("minio", BUCKET, "backups/2026-09-09/test.bin");
rcat(config.path(), &target, Box::pin(stream::iter(chunks)))
.await
.unwrap();
let got = rclone_ok(config.path(), &["cat", &target]);
assert_eq!(got, data);
}
#[tokio::test]
async fn rcat_reports_rclone_stderr_when_the_remote_is_unreachable() {
init_tracing_for_test();
let config = write_config(&minio_config("http://127.0.0.1:1")).unwrap();
let chunks: Vec<Result<Bytes, std::io::Error>> =
vec![Ok(Bytes::from_static(&[0u8; 4096]))];
let err = rcat(
config.path(),
"minio:portabase/x.bin",
Box::pin(stream::iter(chunks)),
)
.await
.expect_err("upload to an unreachable endpoint must fail");
let msg = err.to_string();
assert!(
msg.contains("rclone rcat failed"),
"the broken stdin pipe must not mask rclone's own error: {msg}"
);
let (_, stderr_part) = msg
.rsplit_once(": ")
.expect("bail message must carry rclone stderr after the exit status");
assert!(
!stderr_part.trim().is_empty(),
"rclone stderr must be included: {msg}"
);
}
#[tokio::test]
async fn rcat_aborts_the_upload_when_the_stream_fails() {
init_tracing_for_test();
let (_container, endpoint) = start_minio().await;
let config = write_config(&minio_config(&endpoint)).unwrap();
rclone_ok(config.path(), &["mkdir", &format!("minio:{BUCKET}")]);
let chunks: Vec<Result<Bytes, std::io::Error>> = vec![
Ok(Bytes::from_static(&[1u8; 1024])),
Err(std::io::Error::other("injected stream failure")),
];
let target = remote_target("minio", BUCKET, "backups/2026-09-09/aborted.bin");
let err = rcat(config.path(), &target, Box::pin(stream::iter(chunks)))
.await
.expect_err("a stream error must fail the upload");
assert!(
err.to_string().contains("backup stream failed"),
"unexpected error: {err}"
);
let stat_out = rclone_ok(config.path(), &["lsjson", "--stat", &target]);
let stat: serde_json::Value = serde_json::from_slice(&stat_out).unwrap();
assert_eq!(
stat["Name"], "",
"rclone must not have finalized the truncated object: {stat}"
);
assert_eq!(stat["IsDir"], true, "a miss reports IsDir: true: {stat}");
}
use crate::core::context::Context;
use crate::services::api::ApiClient;
use crate::services::backup::models::BackupResult;
use crate::services::config::DbType;
use crate::services::storage::providers::rclone::RcloneProvider;
use crate::services::storage::{StorageProvider, get_provider};
use crate::utils::common::BackupMethod;
use crate::utils::edge_key::EdgeKey;
use std::io::Write as _;
use std::sync::Arc;
use tempfile::NamedTempFile;
fn test_context() -> Arc<Context> {
Arc::new(Context {
edge_key: EdgeKey {
server_url: String::new(),
agent_id: "agent-1".to_string(),
master_key_b64: String::new(),
},
api: ApiClient::new(String::new()),
})
}
fn storage_for(config_text: &str, remote_path: &str) -> DatabaseStorage {
serde_json::from_value(serde_json::json!({
"id": "storage-1",
"provider": "rclone",
"folderName": "backups",
"config": {
"configText": config_text,
"remoteName": "minio",
"remotePath": remote_path,
}
}))
.unwrap()
}
#[test]
fn factory_resolves_the_rclone_provider_key() {
let storage = storage_for(OVH_CONFIG, "bucket");
assert!(
get_provider(&storage).is_some(),
"get_provider must recognise the \"rclone\" key"
);
}
#[tokio::test]
async fn provider_uploads_an_unencrypted_backup_to_minio() {
init_tracing_for_test();
let (_container, endpoint) = start_minio().await;
let config_text = minio_config(&endpoint);
let bootstrap = write_config(&config_text).unwrap();
rclone_ok(bootstrap.path(), &["mkdir", &format!("minio:{BUCKET}")]);
let payload = vec![42u8; 64 * 1024];
let mut backup_file = NamedTempFile::new().unwrap();
backup_file.write_all(&payload).unwrap();
backup_file.flush().unwrap();
let storage = storage_for(&config_text, BUCKET);
let result = RcloneProvider {}
.upload(
test_context(),
BackupResult {
generated_id: "db-1".to_string(),
db_type: DbType::Postgresql,
status: "success".to_string(),
backup_file: Some(backup_file.path().to_path_buf()),
code: None,
},
BackupMethod::Automatic,
&storage,
Some(false),
"backup-storage-1",
)
.await;
assert!(result.success, "upload failed: {:?}", result.error);
assert_eq!(result.total_size, Some(payload.len() as u64));
let remote_file_path = result.remote_file_path.expect("remote path must be reported");
assert!(
remote_file_path.starts_with("backups/"),
"folder_name must prefix the path: {remote_file_path}"
);
let target = remote_target("minio", BUCKET, &remote_file_path);
assert_eq!(rclone_ok(bootstrap.path(), &["cat", &target]), payload);
}
#[tokio::test]
async fn provider_refuses_a_blocked_backend_without_spawning_rclone() {
init_tracing_for_test();
let mut backup_file = NamedTempFile::new().unwrap();
backup_file.write_all(b"payload").unwrap();
backup_file.flush().unwrap();
let storage = storage_for("[minio]\ntype = local\n", "bucket");
let result = RcloneProvider {}
.upload(
test_context(),
BackupResult {
generated_id: "db-1".to_string(),
db_type: DbType::Postgresql,
status: "success".to_string(),
backup_file: Some(backup_file.path().to_path_buf()),
code: None,
},
BackupMethod::Automatic,
&storage,
Some(false),
"backup-storage-1",
)
.await;
assert!(!result.success);
assert!(
result.error.unwrap_or_default().contains("local"),
"the error must name the rejected backend type"
);
}
+75
View File
@@ -0,0 +1,75 @@
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::storage::providers::sftp::helpers::build_sftp_config;
use crate::services::storage::providers::sftp::models::SftpProviderConfig;
use crate::services::storage::get_provider;
fn storage(config: serde_json::Value) -> DatabaseStorage {
serde_json::from_value(serde_json::json!({
"id": "storage-1",
"provider": "sftp",
"folderName": "backups",
"config": config,
}))
.unwrap()
}
#[test]
fn config_deserializes_from_dashboard_camel_case() {
let s = storage(serde_json::json!({
"host": "backup.example.com",
"port": 2222,
"username": "deploy",
"privateKey": "-----BEGIN KEY-----",
"remotePath": "/srv/backups",
}));
let config: SftpProviderConfig = s.config.try_into().unwrap();
assert_eq!(config.host, "backup.example.com");
assert_eq!(config.port.as_deref(), Some("2222"));
assert_eq!(config.username, "deploy");
assert_eq!(config.remote_path, "/srv/backups");
assert_eq!(config.private_key.as_deref(), Some("-----BEGIN KEY-----"));
}
#[test]
fn build_config_emits_key_file_for_key_auth() {
let config = SftpProviderConfig {
host: "h".into(),
port: Some("2222".into()),
username: "u".into(),
password: None,
private_key: Some("PEMDATA".into()),
remote_path: String::new(),
};
let (text, key) = build_sftp_config(&config).unwrap();
let key = key.expect("key auth must produce a key file");
assert!(text.contains("[sftp]"));
assert!(text.contains("type = sftp"));
assert!(text.contains("host = h"));
assert!(text.contains("port = 2222"));
assert!(text.contains("user = u"));
assert!(text.contains(&format!("key_file = {}", key.path().display())));
assert_eq!(std::fs::read_to_string(key.path()).unwrap(), "PEMDATA");
assert!(!text.contains("pass ="));
}
#[test]
fn build_config_omits_port_when_absent() {
let config = SftpProviderConfig {
host: "h".into(),
port: None,
username: "u".into(),
password: None,
private_key: Some("K".into()),
remote_path: String::new(),
};
let (text, _key) = build_sftp_config(&config).unwrap();
assert!(!text.contains("port ="));
}
#[test]
fn factory_resolves_the_sftp_provider_key() {
let s = storage(serde_json::json!({
"host": "h", "username": "u", "password": "pw",
}));
assert!(get_provider(&s).is_some());
}
+6 -1
View File
@@ -79,7 +79,12 @@ pub async fn execute_task(
let ctx = Arc::new(Context::new());
let config_service = ConfigService::new(ctx.clone());
let backup_service = BackupService::new(ctx.clone());
let config = config_service.load(None).unwrap();
let local = config_service.load_optional(None);
let cache_path = std::path::PathBuf::from(&crate::settings::CONFIG.data_path)
.join("dashboard_databases.json");
let dashboard = crate::services::dashboard_config::load_cache(&cache_path);
let config = crate::services::dashboard_config::merge(&local.databases, &dashboard);
let metadata_obj = metadata
.into_iter()