feat(files): files sources with Archive, Snapshots and Sync methods

Back up and restore plain directories, alongside the database sources.

Files source and Archive (P0)
- `files` type: absolute path (not /), options.exclude and
  options.one_file_system; walk + tar.gz with symlinks kept as links,
  special files skipped, unreadable entries fail the run.
- Mirror restore: archive validated first, then everything not excluded
  is wiped (mount points and special files kept) and the archive is
  extracted, retrying without metadata on filesystems refusing chown/chmod.
- Windows-safe platform helpers; helm extraVolumes/extraVolumeMounts.

Universal rclone mapper (P1)
- rclone_target(storage) for s3, blob, gcs, drive, rclone and sftp; sftp
  goes through it; rclone obscure reads the password from stdin.

Snapshots, restic (P2)
- restic in the image; repository per source and channel, password
  derived from the master key (HKDF); stable host, bs:<row> tags,
  translated excludes; exit 3 fails and forgets the snapshot.
- Mirror restore with `restic restore --delete`; restore payload carries
  the encrypted channel; snapshot ids must be full ids.
- Engine in the ping and in scheduled task metadata.

One locked method per source (spec A)
- databases.json declares options.method (archive, snapshot, sync),
  reported in the ping for local sources only.
- Sync: rclone sync per channel onto <folder>/sync/<id>/current, refuses
  an empty source, reports transferred/deleted counters.

Local channel (spec B)
- Snapshots and Sync reach the dashboard's local storage through its
  append-only restic REST server and its WebDAV server.

Security
- Strict rclone config validation: one section, no duplicate keys, no
  command, credential or host-file keys, no quoted values, valid backend
  type, blocked backends.
- Scheduled task metadata (decrypted storages) is no longer logged.
This commit is contained in:
Charles Gauthereau
2026-10-04 10:09:56 +02:00
parent bbca053a49
commit 5d87fb2d7d
79 changed files with 4762 additions and 127 deletions
Generated
+26
View File
@@ -994,6 +994,16 @@ dependencies = [
"uuid",
]
[[package]]
name = "bstr"
version = "1.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f"
dependencies = [
"memchr",
"serde_core",
]
[[package]]
name = "bumpalo"
version = "3.20.3"
@@ -1956,6 +1966,19 @@ dependencies = [
"polyval",
]
[[package]]
name = "globset"
version = "0.4.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07c34a9410465b45bd9787443bc7370f37735bad04b0f0cd57ff1a3186c98988"
dependencies = [
"aho-corasick",
"bstr",
"log",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "google-cloud-auth"
version = "1.16.0"
@@ -3664,11 +3687,13 @@ dependencies = [
"futures",
"futures-util",
"generic-array",
"globset",
"google-cloud-auth",
"google-cloud-storage",
"hex",
"hmac 0.12.1",
"hyper 1.11.1",
"libc",
"log",
"mockall",
"mongodb",
@@ -3702,6 +3727,7 @@ dependencies = [
"typenum",
"url",
"uuid",
"walkdir",
"wiremock",
]
+6 -1
View File
@@ -20,7 +20,7 @@ log = "0.4.29"
toml = "0.9.10"
reqwest = { version = "0.13.1", features = ["json", "blocking", "multipart", "stream", "query"] }
anyhow = "1.0.100"
tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "fs", "process", "io-util"] }
tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "fs", "process", "io-util", "time"] }
async-trait = "0.1.89"
tempfile = "3.24.0"
hmac = "0.12"
@@ -62,6 +62,11 @@ url = "2.5.8"
percent-encoding = "2.3.2"
bollard = "0.20.0"
rustls = "0.23"
globset = "0.4"
walkdir = "2.5"
[target.'cfg(unix)'.dependencies]
libc = "0.2"
[dev-dependencies]
tokio = { version = "1", features = ["full"] }
+23
View File
@@ -58,6 +58,17 @@ RUN ARCH=$(dpkg --print-architecture) \
&& dpkg -i /tmp/rclone.deb \
&& rm /tmp/rclone.deb
# =========================
# restic (Snapshots mode)
# =========================
ARG RESTIC_VERSION=0.19.1
RUN ARCH=$(dpkg --print-architecture) \
&& curl -fsSL -o /tmp/restic.bz2 "https://github.com/restic/restic/releases/download/v${RESTIC_VERSION}/restic_${RESTIC_VERSION}_linux_${ARCH}.bz2" \
&& bzip2 -d /tmp/restic.bz2 \
&& install -m 0755 /tmp/restic /usr/local/bin/restic \
&& rm /tmp/restic \
&& restic version
ARG TARGETARCH
# =========================
@@ -146,6 +157,7 @@ RUN apt-get update && apt-get install -y \
libncurses6 \
zlib1g \
curl \
bzip2 \
mariadb-client \
sqlite3 \
redis-tools \
@@ -159,6 +171,17 @@ RUN ARCH=$(dpkg --print-architecture) \
&& dpkg -i /tmp/rclone.deb \
&& rm /tmp/rclone.deb
# =========================
# restic (Snapshots mode)
# =========================
ARG RESTIC_VERSION=0.19.1
RUN ARCH=$(dpkg --print-architecture) \
&& curl -fsSL -o /tmp/restic.bz2 "https://github.com/restic/restic/releases/download/v${RESTIC_VERSION}/restic_${RESTIC_VERSION}_linux_${ARCH}.bz2" \
&& bzip2 -d /tmp/restic.bz2 \
&& install -m 0755 /tmp/restic /usr/local/bin/restic \
&& rm /tmp/restic \
&& restic version
ENV DOTNET_ROOT=/usr/local/dotnet
RUN curl -sSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh \
&& chmod +x /tmp/dotnet-install.sh \
+20
View File
@@ -57,3 +57,23 @@ kubectl logs portabase-agent-6f7d4f5c6b-abc12
``` bash
helm uninstall portabase-agent
```
## Mount a directory for a `files` source
A `files` source backs up a directory the agent can see. Mount it into the pod
with `extraVolumes` and `extraVolumeMounts`:
```yaml
extraVolumes:
- name: shared-files
hostPath:
path: /srv/files
type: Directory
extraVolumeMounts:
- name: shared-files
mountPath: /data/files
```
Mount it read-write: restores write into it. The `mountPath` (here `/data/files`)
is the `path` to enter for the source in Portabase.
+6
View File
@@ -44,6 +44,9 @@ spec:
# uses ConfigMap content
{{- end }}
{{- end }}
{{- with .Values.extraVolumeMounts }}
{{- toYaml . | nindent 12 }}
{{- end }}
volumes:
{{- if .Values.volume.configFile.enabled }}
{{- if .Values.volume.configFile.hostPath }}
@@ -59,4 +62,7 @@ spec:
- key: config.json
path: config.json
{{- end }}
{{- end }}
{{- with .Values.extraVolumes }}
{{- toYaml . | nindent 8 }}
{{- end }}
+11
View File
@@ -42,6 +42,17 @@ volume:
]
}
# Extra volumes for the agent pod, e.g. a host directory used by a `files`
# source. Mount it read-write: restores write into it.
extraVolumes: []
# - name: shared-files
# hostPath:
# path: /srv/files
# type: Directory
extraVolumeMounts: []
# - name: shared-files
# mountPath: /data/files # use this path as the source `path`
network:
hostAliases:
+4 -2
View File
@@ -4,7 +4,7 @@ use crate::core::context::Context;
use crate::services::backup::BackupService;
use crate::services::config::{ConfigService, DatabaseConfig};
use crate::services::cron::CronService;
use crate::services::dashboard_config::{collect_configs, load_cache, merge, persist_cache};
use crate::services::dashboard_config::{collect_configs, load_cache, local_only_ids, merge, persist_cache};
use crate::services::restore::RestoreService;
use crate::services::status::StatusService;
use crate::settings::CONFIG;
@@ -50,9 +50,10 @@ impl Agent {
pub async fn run(&mut self, method: BackupMethod) -> Result<(), Box<dyn std::error::Error>> {
let local = self.config_service.load_optional(None);
let local_ids = local_only_ids(&local.databases, &self.dashboard_cache);
let merged_in = merge(&local.databases, &self.dashboard_cache);
let ping_result = self.status_service.ping(&merged_in.databases).await?;
let ping_result = self.status_service.ping(&merged_in.databases, &local_ids).await?;
self.dashboard_cache = collect_configs(&ping_result);
if let Err(e) = persist_cache(&self.cache_path, &self.dashboard_cache) {
@@ -85,6 +86,7 @@ impl Agent {
method.clone(),
&db.storages,
db.encrypt,
db.data.backup.engine.as_deref().unwrap_or("archive"),
)
.await;
} else if db.data.restore.action {
+3
View File
@@ -1,4 +1,5 @@
use crate::domain::docker_volume::database::DockerVolumeDatabase;
use crate::domain::files::database::FilesDatabase;
use crate::domain::mongodb::database::MongoDatabase;
use crate::domain::mysql::database::MySQLDatabase;
use crate::domain::postgres::cluster::database::PostgresClusterDatabase;
@@ -43,6 +44,7 @@ impl DatabaseFactory {
DbType::Firebird => Arc::new(FirebirdDatabase::new(cfg)),
DbType::Mssql => Arc::new(MssqlDatabase::new(cfg)),
DbType::DockerVolume => Arc::new(DockerVolumeDatabase::new(cfg)),
DbType::Files => Arc::new(FilesDatabase::new(cfg)),
}
}
@@ -62,6 +64,7 @@ impl DatabaseFactory {
DbType::Firebird => Arc::new(FirebirdDatabase::new(cfg)),
DbType::Mssql => Arc::new(MssqlDatabase::new(cfg)),
DbType::DockerVolume => Arc::new(DockerVolumeDatabase::new(cfg)),
DbType::Files => Arc::new(FilesDatabase::new(cfg)),
}
}
}
+133
View File
@@ -0,0 +1,133 @@
use super::matcher::ExcludeMatcher;
use super::platform::dev;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result, bail};
use flate2::Compression;
use flate2::write::GzEncoder;
use std::fs::File;
use std::io::Read;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use walkdir::WalkDir;
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
logger: Arc<JobLogger>,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || archive(&cfg, &backup_dir, &logger)).await?
}
fn archive(cfg: &DatabaseConfig, backup_dir: &Path, logger: &JobLogger) -> Result<PathBuf> {
// Canonical paths so the output location can be recognised while walking.
let root = std::fs::canonicalize(&cfg.path)
.with_context(|| format!("cannot read source directory {}", cfg.path))?;
if !root.is_dir() {
bail!("source path {} is not a directory", root.display());
}
let backup_real = std::fs::canonicalize(backup_dir)
.with_context(|| format!("cannot read backup directory {}", backup_dir.display()))?;
let matcher = ExcludeMatcher::from_config(cfg)?;
let one_file_system = cfg
.options
.get("one_file_system")
.and_then(|v| v.as_bool())
.unwrap_or(false);
let root_dev = dev(&std::fs::metadata(&root)
.with_context(|| format!("cannot read source directory {}", root.display()))?);
let file_name = format!("{}.tar.gz", cfg.generated_id);
let out = backup_dir.join(&file_name);
let out_real = backup_real.join(&file_name);
let gz = GzEncoder::new(File::create(&out)?, Compression::default());
let mut tar = tar::Builder::new(gz);
tar.follow_symlinks(false);
logger.log("info", format!("Archiving {}", root.display()));
let mut files = 0u64;
// Entries that could not be read: a mirror restore of this archive would delete them.
let mut unreadable = 0usize;
let mut walker = WalkDir::new(&root)
.follow_links(false)
.min_depth(1)
.into_iter();
while let Some(entry) = walker.next() {
let entry = match entry {
Ok(e) => e,
Err(e) => {
logger.log("warn", format!("Skipping unreadable entry: {e}"));
unreadable += 1;
continue;
}
};
let rel = entry.path().strip_prefix(&root)?.to_path_buf();
let ft = entry.file_type();
// Never archive our own output (backup dir under the source, or the archive itself).
let mut skip =
matcher.is_excluded(&rel) || entry.path() == backup_real || entry.path() == out_real;
if !skip && one_file_system && ft.is_dir() {
match entry.metadata() {
Ok(m) => skip = dev(&m) != root_dev,
Err(e) => {
logger.log("warn", format!("Skipping {}: {e}", rel.display()));
unreadable += 1;
skip = true;
}
}
}
if skip {
if ft.is_dir() {
walker.skip_current_dir();
}
continue;
}
if ft.is_file() {
// Only a failure to open is skippable: it happens before anything is
// written. Every later error leaves the tar stream misaligned and must abort.
let mut f = match File::open(entry.path()) {
Ok(f) => f,
Err(e) => {
logger.log("warn", format!("Skipping {}: {e}", rel.display()));
unreadable += 1;
continue;
}
};
let meta = f.metadata()?;
let size = meta.len();
let mut header = tar::Header::new_gnu();
header.set_metadata(&meta);
// The header size is fixed up front: cap the copy if the file grew and
// zero-pad if it shrank, so the entry always matches its header.
let reader = (&mut f).take(size).chain(std::io::repeat(0)).take(size);
tar.append_data(&mut header, &rel, reader)
.with_context(|| format!("failed to archive {}", rel.display()))?;
files += 1;
} else if ft.is_dir() || ft.is_symlink() {
// These fail on lstat/readlink before the header is written.
if let Err(e) = tar.append_path_with_name(entry.path(), &rel) {
logger.log("warn", format!("Skipping {}: {e}", rel.display()));
unreadable += 1;
}
} else {
// Sockets, fifos and devices hold no data to restore; restore keeps them too.
logger.log("warn", format!("Skipping special file {}", rel.display()));
}
}
if unreadable > 0 {
let noun = if unreadable == 1 { "entry" } else { "entries" };
bail!(
"{unreadable} {noun} could not be read; backup aborted so a mirror restore cannot delete unarchived data (fix permissions or add an exclude)"
);
}
tar.into_inner()?.finish()?;
logger.log(
"info",
format!("Archived {files} file(s) into {}", out.display()),
);
Ok(out)
}
+45
View File
@@ -0,0 +1,45 @@
use anyhow::Result;
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
pub struct FilesDatabase {
cfg: DatabaseConfig,
}
impl FilesDatabase {
pub fn new(cfg: DatabaseConfig) -> Self {
Self { cfg }
}
}
#[async_trait]
impl Database for FilesDatabase {
fn file_extension(&self) -> &'static str {
".tar.gz"
}
async fn ping(&self) -> Result<bool> {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+73
View File
@@ -0,0 +1,73 @@
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use globset::{GlobBuilder, GlobSet, GlobSetBuilder};
use std::path::{Path, PathBuf};
/// Exclude patterns, matched against paths relative to the source root.
/// No leading `/` matches at any depth; a leading `/` anchors at the root.
pub struct ExcludeMatcher {
set: GlobSet,
literal: Vec<PathBuf>,
}
impl ExcludeMatcher {
pub fn new(patterns: &[String]) -> Result<Self> {
let mut builder = GlobSetBuilder::new();
for raw in patterns {
let pattern = raw.trim();
if pattern.is_empty() {
continue;
}
let glob = match pattern.strip_prefix('/') {
Some(anchored) => anchored.trim_end_matches('/').to_string(),
None => format!("**/{}", pattern.trim_end_matches('/')),
};
builder.add(
GlobBuilder::new(&glob)
.literal_separator(true)
.build()
.with_context(|| format!("invalid exclude pattern '{pattern}'"))?,
);
}
Ok(Self {
set: builder.build()?,
literal: Vec::new(),
})
}
pub fn from_config(cfg: &DatabaseConfig) -> Result<Self> {
Self::new(&exclude_patterns(cfg))
}
/// Also exclude exactly this root-relative path (and its subtree), compared
/// literally so glob characters in the name mean nothing.
pub fn exclude_path(&mut self, relative: PathBuf) {
self.literal.push(relative);
}
pub fn is_excluded(&self, relative: &Path) -> bool {
self.set.is_match(relative) || self.literal.iter().any(|p| p == relative)
}
}
/// The `options.exclude` patterns of a files source, as configured.
pub fn exclude_patterns(cfg: &DatabaseConfig) -> Vec<String> {
cfg.options
.get("exclude")
.and_then(|v| v.as_array())
.map(|items| {
items
.iter()
.filter_map(|v| v.as_str().map(str::to_string))
.collect()
})
.unwrap_or_default()
}
/// `options.one_file_system` of a files source.
pub fn one_file_system(cfg: &DatabaseConfig) -> bool {
cfg.options
.get("one_file_system")
.and_then(|v| v.as_bool())
.unwrap_or(false)
}
+6
View File
@@ -0,0 +1,6 @@
pub mod backup;
pub mod database;
pub mod matcher;
pub mod ping;
pub mod platform;
pub mod restore;
+16
View File
@@ -0,0 +1,16 @@
use crate::services::config::DatabaseConfig;
use anyhow::Result;
use std::path::Path;
use tokio::time::{Duration, timeout};
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
// A hung mount (NFS) blocks is_dir/read_dir: keep it off the runtime and bounded.
let check = tokio::task::spawn_blocking(move || {
let root = Path::new(&cfg.path);
root.is_dir() && std::fs::read_dir(root).is_ok()
});
Ok(matches!(
timeout(Duration::from_secs(10), check).await,
Ok(Ok(true))
))
}
+26
View File
@@ -0,0 +1,26 @@
//! Unix-only metadata behind `cfg`, with inert fallbacks so the Windows release
//! build compiles.
/// Device id, used to spot mount points. Off unix every entry reports 0, so no
/// directory is ever treated as a mount point.
#[cfg(unix)]
pub fn dev(meta: &std::fs::Metadata) -> u64 {
use std::os::unix::fs::MetadataExt;
meta.dev()
}
#[cfg(not(unix))]
pub fn dev(_meta: &std::fs::Metadata) -> u64 {
0
}
/// Whether the agent can restore file ownership.
#[cfg(unix)]
pub fn is_root() -> bool {
unsafe { libc::geteuid() == 0 }
}
#[cfg(not(unix))]
pub fn is_root() -> bool {
false
}
+247
View File
@@ -0,0 +1,247 @@
use super::matcher::ExcludeMatcher;
use super::platform::{dev, is_root};
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result, bail};
use flate2::read::GzDecoder;
use std::fs::{self, File};
use std::io::{self, Read, Seek};
use std::path::{Path, PathBuf};
use std::sync::Arc;
pub async fn run(cfg: DatabaseConfig, archive: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
tokio::task::spawn_blocking(move || restore(&cfg, &archive, &logger)).await?
}
fn restore(cfg: &DatabaseConfig, archive: &Path, logger: &JobLogger) -> Result<()> {
// Everything below works on the canonical path, like backup.rs.
let root = ensure_restorable(Path::new(&cfg.path))?;
let mut matcher = ExcludeMatcher::from_config(cfg)?;
let one_file_system = cfg
.options
.get("one_file_system")
.and_then(|v| v.as_bool())
.unwrap_or(false);
// Opened once: the wipe may delete the archive's path, never an open handle.
let file = File::open(archive)
.with_context(|| format!("cannot open archive {}", archive.display()))?;
// The wipe is irreversible: prove the whole archive reads back before touching data.
logger.log("info", format!("Validating archive {}", archive.display()));
validate_archive(&file).context("archive is corrupt or truncated, nothing was changed")?;
// Keep the archive's own directory (or the archive itself when it sits directly
// in the target), like backup.rs prunes its output.
let archive_real = fs::canonicalize(archive)?;
let keep = match archive_real.parent() {
Some(dir) if dir != root => dir,
_ => archive_real.as_path(),
};
if let Ok(rel) = keep.strip_prefix(&root) {
matcher.exclude_path(rel.to_path_buf());
}
logger.log(
"info",
format!("Removing non-excluded content of {}", root.display()),
);
// Best effort: whatever could not be removed is kept, and extraction always runs.
let failures = wipe_except_excluded(&root, &matcher, one_file_system, logger)?;
logger.log(
"info",
format!("Extracting archive into {}", root.display()),
);
unpack(&file, &root, is_root(), logger)?;
if failures > 0 {
let noun = if failures == 1 { "entry" } else { "entries" };
bail!(
"restore extracted the archive but {failures} existing {noun} could not be removed; see warnings"
);
}
logger.log("info", "Files restore finished".to_string());
Ok(())
}
/// Extract the whole archive into `root`. Filesystems that refuse chown/chmod/utime
/// (NFS root_squash, CIFS, NFSv4 idmap mismatch) fail the first pass, after the wipe,
/// so retry once without metadata: the content matters more than owner and mode.
pub fn unpack(
file: &File,
root: &Path,
preserve_ownerships: bool,
logger: &JobLogger,
) -> Result<()> {
if let Err(e) = unpack_once(file, root, true, preserve_ownerships) {
logger.log(
"warn",
format!("extraction with ownership/permissions failed: {e:#}; retrying without metadata preservation"),
);
// ponytail: tar-rs still chmods to `mode & 0o777` with preservation off, so a
// filesystem that rejects chmod on files the agent just created fails this pass
// too; per-entry extraction with deferred directories if that shows up.
unpack_once(file, root, false, false)?;
}
Ok(())
}
fn unpack_once(
mut file: &File,
root: &Path,
preserve: bool,
preserve_ownerships: bool,
) -> Result<()> {
file.rewind()?;
let mut ar = tar::Archive::new(GzDecoder::new(file));
ar.set_preserve_permissions(preserve);
ar.set_preserve_mtime(preserve);
ar.set_preserve_ownerships(preserve_ownerships);
ar.set_overwrite(true);
ar.unpack(root)?;
Ok(())
}
/// Read every entry to the end, then drain the gzip stream so its CRC is checked.
fn validate_archive(file: &File) -> Result<()> {
let mut gz = GzDecoder::new(file);
let mut ar = tar::Archive::new(&mut gz);
for entry in ar.entries()? {
io::copy(&mut entry?, &mut io::sink())?;
}
drop(ar);
// tar stops at its end marker, before the gzip trailer.
gz.read_to_end(&mut Vec::new())?;
Ok(())
}
/// Refuse targets a mirror restore must never touch, and return the canonical path to
/// restore into. The canonical path is checked too, because `/data/..` style escapes
/// resolve to `/`.
pub fn ensure_restorable(root: &Path) -> Result<PathBuf> {
if root.as_os_str().is_empty() || !root.is_absolute() || root == Path::new("/") {
bail!(
"refusing to restore into {:?}: path must be absolute and not /",
root
);
}
if !root.is_dir() {
bail!(
"restore target {} is not an existing directory",
root.display()
);
}
let canonical = fs::canonicalize(root)
.with_context(|| format!("cannot resolve restore target {}", root.display()))?;
if canonical == Path::new("/") {
bail!("refusing to restore into {:?}: it resolves to /", root);
}
Ok(canonical)
}
/// Delete everything under `root` that does not match an exclude pattern.
/// Excluded entries (and their subtrees) are kept; `root` itself is kept.
///
/// Mount points (a directory on a different device than its parent): with
/// `one_file_system` they are kept untouched like an excluded path, matching what
/// backup never archived. Without it their contents are wiped but the mount point
/// itself is never removed (`remove_dir` on it fails with EBUSY) and counts as kept.
/// Special files (sockets, fifos, devices) are kept: backup never archives them.
///
/// Best effort: an entry that cannot be read or removed is logged, kept (so its
/// parents are kept too) and counted; the returned count is how many failed.
/// Only an unreadable `root` is an error, and then nothing was deleted.
pub fn wipe_except_excluded(
root: &Path,
matcher: &ExcludeMatcher,
one_file_system: bool,
logger: &JobLogger,
) -> Result<usize> {
let meta = fs::symlink_metadata(root).with_context(|| format!("reading {}", root.display()))?;
let mut wipe = Wipe {
matcher,
one_file_system,
logger,
failures: 0,
};
wipe.dir(root, Path::new(""), dev(&meta));
Ok(wipe.failures)
}
struct Wipe<'a> {
matcher: &'a ExcludeMatcher,
one_file_system: bool,
logger: &'a JobLogger,
failures: usize,
}
impl Wipe<'_> {
fn fail(&mut self, what: &str, path: &Path, e: io::Error) {
self.logger
.log("warn", format!("Could not {what} {}: {e}", path.display()));
self.failures += 1;
}
/// Returns true when something was kept inside `dir`. `rel` is `dir` relative to
/// the root and `dir_dev` its device.
fn dir(&mut self, dir: &Path, rel: &Path, dir_dev: u64) -> bool {
let entries = match fs::read_dir(dir) {
Ok(entries) => entries,
Err(e) => {
self.fail("read", dir, e);
return true;
}
};
let mut kept = false;
for entry in entries {
let entry = match entry {
Ok(entry) => entry,
Err(e) => {
self.fail("read", dir, e);
kept = true;
continue;
}
};
let path = entry.path();
let rel = rel.join(entry.file_name());
if self.matcher.is_excluded(&rel) {
kept = true;
continue;
}
// symlink_metadata: a symlinked directory is removed as a link, never followed.
let meta = match fs::symlink_metadata(&path) {
Ok(meta) => meta,
Err(e) => {
self.fail("read", &path, e);
kept = true;
continue;
}
};
let ft = meta.file_type();
let removed = if ft.is_dir() {
let mount_point = dev(&meta) != dir_dev;
if mount_point && self.one_file_system {
kept = true;
continue;
}
if self.dir(&path, &rel, dev(&meta)) || mount_point {
kept = true;
continue;
}
fs::remove_dir(&path)
} else if ft.is_file() || ft.is_symlink() {
fs::remove_file(&path)
} else {
// Socket, fifo or device: backup never archives them, so keep them.
kept = true;
continue;
};
if let Err(e) = removed {
self.fail("remove", &path, e);
kept = true;
}
}
kept
}
}
+1
View File
@@ -1,4 +1,5 @@
pub mod docker_volume;
pub mod files;
pub mod factory;
pub mod mongodb;
pub mod mysql;
@@ -12,6 +12,9 @@ pub struct InitUploadRequest {
pub storage_channel_id: String,
#[serde(rename = "backupId")]
pub backup_id: String,
/// "restic" for snapshots; omitted for archives (older dashboards ignore it).
#[serde(skip_serializing_if = "Option::is_none")]
pub engine: Option<String>,
}
impl ApiClient {
@@ -21,11 +24,13 @@ impl ApiClient {
generated_id: impl Into<String>,
storage_channel_id: impl Into<String>,
backup_id: impl Into<String>,
engine: Option<&str>,
) -> Result<Option<BackupUploadResponse>, ApiError> {
let body = InitUploadRequest {
generated_id: generated_id.into(),
storage_channel_id: storage_channel_id.into(),
backup_id: backup_id.into(),
engine: engine.map(str::to_string),
};
let agent_id = agent_id.into();
@@ -1,5 +1,6 @@
use crate::services::api::models::agent::backup::BackupUploadResponse;
use crate::services::api::{ApiClient, ApiError};
use crate::services::sync::stats::SyncStats;
use anyhow::Result;
use reqwest::Method;
use serde::Serialize;
@@ -15,6 +16,10 @@ pub struct StatusUploadRequest {
pub size: u64,
#[serde(rename = "backupId")]
pub backup_id: String,
#[serde(rename = "filesTransferred", skip_serializing_if = "Option::is_none")]
pub files_transferred: Option<u64>,
#[serde(rename = "filesDeleted", skip_serializing_if = "Option::is_none")]
pub files_deleted: Option<u64>,
}
impl ApiClient {
@@ -35,12 +40,41 @@ impl ApiClient {
path: remote_path.into(),
size: total_size.into(),
backup_id: backup_id.into(),
files_transferred: None,
files_deleted: None,
};
self.send_upload_status(agent_id.into(), &body).await
}
let agent_id = agent_id.into();
/// Success status of one sync replica, with rclone's counters (`size` = bytes transferred).
pub async fn backup_upload_sync_status(
&self,
agent_id: impl Into<String>,
generated_id: impl Into<String>,
backup_storage_id: impl Into<String>,
path: String,
stats: &SyncStats,
backup_id: impl Into<String>,
) -> Result<Option<BackupUploadResponse>, ApiError> {
let body = StatusUploadRequest {
generated_id: generated_id.into(),
backup_storage_id: backup_storage_id.into(),
status: "success".into(),
path,
size: stats.bytes,
backup_id: backup_id.into(),
files_transferred: Some(stats.transfers),
files_deleted: Some(stats.deletes),
};
self.send_upload_status(agent_id.into(), &body).await
}
async fn send_upload_status(
&self,
agent_id: String,
body: &StatusUploadRequest,
) -> Result<Option<BackupUploadResponse>, ApiError> {
let path = format!("/agent/{}/backup/upload/status", agent_id);
self.request_with_body(Method::PATCH, path.as_str(), &body)
.await
self.request_with_body(Method::PATCH, path.as_str(), body).await
}
}
+3 -1
View File
@@ -12,7 +12,9 @@ pub struct DatabasePayload<'a> {
#[serde(rename = "generatedId")]
pub generated_id: &'a str,
#[serde(rename = "pingStatus")]
pub ping_status: bool
pub ping_status: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub method: Option<&'a str>,
}
#[derive(Serialize)]
+16 -1
View File
@@ -61,14 +61,29 @@ pub struct DatabaseData {
pub struct BackupInfo {
pub action: bool,
pub cron: Option<String>,
/// "archive" | "restic"; absent from dashboards older than P2 (→ archive).
#[serde(default)]
pub engine: Option<String>,
}
#[derive(Debug, Deserialize)]
pub struct RestoreInfo {
pub action: bool,
#[serde(default)]
pub file: Option<String>,
#[serde(rename = "metaFile")]
#[serde(default, rename = "metaFile")]
pub meta_file: Option<String>,
#[serde(default, deserialize_with = "string_or_number_to_string")]
pub size: Option<String>,
/// "restic" for a snapshot restore; absent for archives.
#[serde(default)]
pub engine: Option<String>,
#[serde(default, rename = "snapshotId")]
pub snapshot_id: Option<String>,
/// AES-GCM JSON array holding the snapshot's storage channel.
#[serde(default, rename = "storageCiphertext")]
pub storage_ciphertext: Option<String>,
/// Filled in memory from `storage_ciphertext`; never on the wire.
#[serde(skip)]
pub storage: Option<DatabaseStorage>,
}
+3 -1
View File
@@ -12,6 +12,7 @@ impl BackupService {
method: BackupMethod,
storages: &Vec<DatabaseStorage>,
encrypt: bool,
engine: &str,
) {
let Some(cfg) = config
.databases
@@ -29,10 +30,11 @@ impl BackupService {
let db_cfg = cfg.clone();
let storages = storages.clone();
let generated_id = generated_id.clone();
let engine = engine.to_string();
tokio::spawn(async move {
if let Err(e) = service
.execute_backup(generated_id, db_cfg, method, storages, encrypt)
.execute_backup(generated_id, db_cfg, method, storages, encrypt, engine)
.await
{
error!("Backup execution failed: {}", e);
+30 -1
View File
@@ -1,7 +1,10 @@
use super::logger::JobLogger;
use super::models::BackupResult;
use super::service::BackupService;
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::config::DatabaseConfig;
use crate::services::config::{DatabaseConfig, DbType};
use crate::services::restic;
use crate::services::sync;
use crate::utils::common::BackupMethod;
use crate::utils::locks::FileLock;
@@ -18,6 +21,7 @@ impl BackupService {
method: BackupMethod,
storages: Vec<DatabaseStorage>,
encrypt: bool,
engine: String,
) -> Result<()> {
let logger = Arc::new(JobLogger::new());
@@ -30,6 +34,31 @@ impl BackupService {
let backup = self.create_backup_record(&generated_id, &method).await?;
let backup_id = backup.backup.id;
if engine == "restic" || engine == "sync" {
if matches!(db_cfg.db_type, DbType::Files) {
let uploads = if engine == "restic" {
restic::backup::run(&self.ctx, &db_cfg, &storages, &backup_id, &logger).await
} else {
sync::backup::run(&self.ctx, &db_cfg, &storages, &backup_id, &logger).await
};
logger.log("info", "Database backup job finished".to_string());
let result = BackupResult {
generated_id: generated_id.clone(),
db_type: db_cfg.db_type.clone(),
status: "success".into(),
backup_file: None,
code: None,
};
let duration_ms = start.elapsed().as_millis() as f64;
let logs = Arc::try_unwrap(logger).unwrap_or_else(|_| JobLogger::new()).into_entries();
// fileSize = average logical size across storages (send_result): restic reports
// the size each snapshot processed; sync reports none, so it stays null.
self.send_result(result, uploads, &backup_id, logs, duration_ms).await?;
return Ok(());
}
logger.log("warn", format!("The {engine} method only applies to files sources; making an archive"));
}
let temp_dir = TempDir::new()?;
let tmp_path = temp_dir.path();
+3 -5
View File
@@ -21,11 +21,9 @@ impl BackupService {
"failed"
};
let file_size = upload_results
.iter()
.filter_map(|r| r.total_size)
.reduce(|a, b| a + b)
.map(|sum| sum / upload_results.len() as u64);
// Average over the uploads that reported a size: failed ones have none.
let sizes: Vec<u64> = upload_results.iter().filter_map(|r| r.total_size).collect();
let file_size = (!sizes.is_empty()).then(|| sizes.iter().sum::<u64>() / sizes.len() as u64);
self.ctx
.api
+1
View File
@@ -49,6 +49,7 @@ impl BackupService {
generated_id.clone(),
storage_id.clone(),
backup_id,
None,
)
.await
{
+38 -3
View File
@@ -28,6 +28,7 @@ pub enum DbType {
Mssql,
#[serde(rename = "docker-volume")]
DockerVolume,
Files,
}
impl DbType {
@@ -44,6 +45,7 @@ impl DbType {
DbType::Firebird => "firebird",
DbType::Mssql => "mssql",
DbType::DockerVolume => "docker-volume",
DbType::Files => "files",
}
}
}
@@ -112,6 +114,17 @@ fn optional<T: Clone + Default>(opt: &Option<T>) -> T {
opt.clone().unwrap_or_default()
}
pub const FILES_METHODS: [&str; 3] = ["archive", "snapshot", "sync"];
/// The method a files source declares in `options.method` (`archive` when absent).
pub fn files_method(cfg: &DatabaseConfig) -> &'static str {
match cfg.options.get("method").and_then(|v| v.as_str()) {
Some("snapshot") => "snapshot",
Some("sync") => "sync",
_ => "archive",
}
}
pub fn build_config(db: InputDatabaseConfig) -> Result<DatabaseConfig, String> {
if Uuid::parse_str(&db.generated_id).is_err() {
return Err(format!("Invalid UUID for database '{}'", db.name));
@@ -143,7 +156,7 @@ pub fn build_config(db: InputDatabaseConfig) -> Result<DatabaseConfig, String> {
| DbType::Firebird
| DbType::Valkey
| DbType::Mssql => required(&db.host, &db.name, "host")?,
DbType::Sqlite | DbType::DockerVolume => optional(&db.host),
DbType::Sqlite | DbType::DockerVolume | DbType::Files => optional(&db.host),
};
let port = match db.db_type {
@@ -155,11 +168,13 @@ pub fn build_config(db: InputDatabaseConfig) -> Result<DatabaseConfig, String> {
| DbType::Firebird
| DbType::Valkey
| DbType::Mssql => required(&db.port, &db.name, "port")?,
DbType::MongoDB | DbType::Sqlite | DbType::DockerVolume => db.port.unwrap_or(0),
DbType::MongoDB | DbType::Sqlite | DbType::DockerVolume | DbType::Files => {
db.port.unwrap_or(0)
}
};
let database_name = match db.db_type {
DbType::Sqlite | DbType::Redis | DbType::Valkey | DbType::DockerVolume => {
DbType::Sqlite | DbType::Redis | DbType::Valkey | DbType::DockerVolume | DbType::Files => {
optional(&db.database)
}
DbType::PostgresqlCluster => db
@@ -170,8 +185,28 @@ pub fn build_config(db: InputDatabaseConfig) -> Result<DatabaseConfig, String> {
};
let path_val = match db.db_type {
DbType::Sqlite => required(&db.path, &db.name, "path")?,
DbType::Files => {
let p = required(&db.path, &db.name, "path")?;
if !p.starts_with('/') || p.trim_end_matches('/').is_empty() {
return Err(format!(
"Path for files source '{}' must be absolute and not '/'",
db.name
));
}
p
}
_ => optional(&db.path),
};
if matches!(db.db_type, DbType::Files) {
if let Some(method) = db.options.as_ref().and_then(|o| o.get("method")) {
if !method.as_str().is_some_and(|m| FILES_METHODS.contains(&m)) {
return Err(format!(
"Unknown method {method} for files source '{}' (expected archive, snapshot or sync)",
db.name
));
}
}
}
let max_packet_size = match db.db_type {
DbType::Mysql | DbType::Mariadb => db.max_packet_size.unwrap_or_else(|| "512M".to_string()),
_ => String::new(),
+2 -1
View File
@@ -29,7 +29,8 @@ impl CronService {
let encrypt: bool = database.encrypt;
let metadata = json!({
"storages": storages,
"encrypt": encrypt
"encrypt": encrypt,
"engine": database.data.backup.engine.as_deref().unwrap_or("archive"),
});
check_and_update_cron(
+10
View File
@@ -2,6 +2,7 @@
use crate::services::api::models::agent::status::PingResult;
use crate::services::config::{DatabaseConfig, DatabasesConfig};
use std::collections::HashSet;
use std::path::Path;
pub fn merge(local: &[DatabaseConfig], dashboard: &[DatabaseConfig]) -> DatabasesConfig {
@@ -19,6 +20,15 @@ pub fn merge(local: &[DatabaseConfig], dashboard: &[DatabaseConfig]) -> Database
DatabasesConfig { databases }
}
/// `generated_id`s of local (databases.json) sources that no dashboard config replaces.
pub fn local_only_ids(local: &[DatabaseConfig], dashboard: &[DatabaseConfig]) -> HashSet<String> {
local
.iter()
.filter(|c| !dashboard.iter().any(|d| d.generated_id == c.generated_id))
.map(|c| c.generated_id.clone())
.collect()
}
pub fn collect_configs(ping: &PingResult) -> Vec<DatabaseConfig> {
ping.databases
.iter()
+2
View File
@@ -3,6 +3,8 @@ pub mod backup;
pub mod config;
pub mod cron;
pub mod dashboard_config;
pub mod restic;
pub mod restore;
pub mod status;
pub mod storage;
pub mod sync;
+201
View File
@@ -0,0 +1,201 @@
use super::command::{ResticRepo, short};
use super::excludes::backup_excludes;
use super::json::BackupSummary;
use crate::core::context::Context as CoreContext;
use crate::domain::files::matcher::{exclude_patterns, one_file_system};
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::backup::logger::JobLogger;
use crate::services::backup::models::UploadResult;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
use anyhow::{Context, Result, bail};
/// One snapshot of `cfg.path`, tagged with its `backup_storage` id.
/// Exit 3 (unreadable files) fails the run and forgets the snapshot: a mirror
/// restore of it would delete the files it is missing (P0 rule R21). On an
/// append-only repository the agent cannot forget it, so the dashboard does.
pub async fn snapshot(
repo: &ResticRepo,
cfg: &DatabaseConfig,
backup_storage_id: &str,
logger: &JobLogger,
) -> Result<BackupSummary> {
let root = std::fs::canonicalize(&cfg.path)
.with_context(|| format!("cannot read source directory {}", cfg.path))?;
if !root.is_dir() {
bail!("source path {} is not a directory", root.display());
}
repo.ensure_initialized(logger).await?;
let mut args: Vec<String> = vec![
"backup".into(),
root.to_string_lossy().into_owned(),
"--json".into(),
// Stable host: container hostnames change on recreate and would break
// parent-snapshot detection (full rescan every time).
"--host".into(),
format!("portabase-{}", cfg.generated_id),
"--tag".into(),
"portabase".into(),
"--tag".into(),
format!("bs:{backup_storage_id}"),
];
for pattern in backup_excludes(&root, &exclude_patterns(cfg))? {
args.push("--exclude".into());
args.push(pattern);
}
if one_file_system(cfg) {
args.push("--one-file-system".into());
}
logger.log("info", format!("Snapshotting {} into {}", root.display(), repo.repository()));
let run = repo.run(&args, logger).await?;
for error in &run.errors {
logger.log("warn", format!("Unreadable: {error}"));
}
let summary: Option<BackupSummary> = run
.summary
.clone()
.map(serde_json::from_value)
.transpose()
.context("unexpected restic backup summary")?;
match (run.code, summary) {
(0, Some(s)) => {
logger.log(
"info",
format!(
"Snapshot {}: {} new, {} changed, {} unmodified file(s), {} byte(s) added",
short(&s.snapshot_id), s.files_new, s.files_changed, s.files_unmodified, s.data_added_packed
),
);
Ok(s)
}
(0, None) => bail!("restic backup returned no summary"),
(3, summary) => {
let mut fate = "snapshot discarded";
if let Some(s) = summary {
if repo.is_append_only() {
logger.log(
"warn",
format!("Incomplete snapshot {} left for the dashboard to forget", short(&s.snapshot_id)),
);
fate = "snapshot not restorable (the dashboard forgets it)";
} else {
let forget = repo.run(["forget", s.snapshot_id.as_str()], logger).await?;
if forget.code != 0 {
logger.log(
"error",
format!("Could not forget incomplete snapshot {}: {}", s.snapshot_id, forget.error("forget")),
);
}
}
}
bail!(
"{} file(s) could not be read; {fate} so a mirror restore cannot delete them (fix permissions or add an exclude)",
run.errors.len()
)
}
_ => Err(run.error("backup")),
}
}
/// Snapshots every storage in turn (one scan each, v1). Never errors: each
/// storage reports its own `backup_storage` row, like the archive uploader.
pub async fn run(
ctx: &CoreContext,
cfg: &DatabaseConfig,
storages: &[DatabaseStorage],
backup_id: &str,
logger: &JobLogger,
) -> Vec<UploadResult> {
if let Err(e) = FileLock::acquire(&cfg.generated_id, DbOpLock::Backup.as_str()).await {
logger.log("error", format!("Snapshot aborted: {e}"));
return Vec::new();
}
let mut results = Vec::with_capacity(storages.len());
for storage in storages {
results.push(one_storage(ctx, cfg, storage, backup_id, logger).await);
}
if let Err(e) = FileLock::release(&cfg.generated_id).await {
logger.log("warn", format!("Failed to release the backup lock: {e}"));
}
results
}
pub(crate) async fn one_storage(
ctx: &CoreContext,
cfg: &DatabaseConfig,
storage: &DatabaseStorage,
backup_id: &str,
logger: &JobLogger,
) -> UploadResult {
let agent_id = ctx.edge_key.agent_id.clone();
let failed = |error: String| UploadResult {
storage_id: storage.id.clone(),
success: false,
error: Some(error),
remote_file_path: None,
total_size: None,
};
let backup_storage_id = match ctx
.api
.backup_upload_init(agent_id.clone(), cfg.generated_id.clone(), storage.id.clone(), backup_id, Some("restic"))
.await
{
Ok(Some(response)) => response.backup_storage.id,
Ok(None) => {
logger.log("error", "Upload init returned empty response");
return failed("backup_upload_init returned empty response".into());
}
Err(e) => {
logger.log("error", format!("Upload init failed: {e}"));
return failed("backup_upload_init failed".into());
}
};
let outcome = match ResticRepo::open(storage, &cfg.generated_id, &ctx.edge_key) {
Ok(repo) => snapshot(&repo, cfg, &backup_storage_id, logger).await,
Err(e) => Err(e),
};
match outcome {
Ok(summary) => match ctx
.api
.backup_upload_status(
agent_id,
cfg.generated_id.clone(),
backup_storage_id,
"success",
summary.snapshot_id.clone(),
summary.data_added_packed,
backup_id,
)
.await
{
Ok(_) => UploadResult {
storage_id: storage.id.clone(),
success: true,
error: None,
remote_file_path: Some(summary.snapshot_id),
total_size: Some(summary.total_bytes_processed),
},
Err(e) => {
logger.log("error", format!("Upload status update failed for {}: {e}", storage.id));
failed(e.to_string())
}
},
Err(e) => {
logger.log("error", format!("Snapshot to storage {} failed: {e:#}", storage.id));
if let Err(err) = ctx
.api
.backup_upload_status(agent_id, cfg.generated_id.clone(), backup_storage_id, "failed", String::new(), 0u64, backup_id)
.await
{
logger.log("error", format!("Failed-status update failed for {}: {err}", storage.id));
}
failed(format!("{e:#}"))
}
}
}
+254
View File
@@ -0,0 +1,254 @@
use super::json::{self, Line};
use super::password::{derive_password, local_storage_password, LOCAL_STORAGE_USER};
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::backup::logger::JobLogger;
use crate::services::storage::providers::rclone::helpers::{remote_target, write_config};
use crate::services::storage::providers::rclone::target::{RcloneTarget, rclone_target};
use crate::settings::CONFIG;
use crate::utils::edge_key::EdgeKey;
use anyhow::{Context, Result, anyhow};
use serde_json::Value;
use std::ffi::{OsStr, OsString};
use std::path::PathBuf;
use std::process::Stdio;
use tempfile::NamedTempFile;
use tokio::io::{AsyncBufReadExt, BufReader};
use tokio::process::Command;
/// One source's repository on one storage channel:
/// `rclone:<remote>:<base>/<folder>/restic/<generated_id>`, or
/// `rest:<server_url>/storage/restic/<generated_id>/` on the dashboard's local storage.
pub struct ResticRepo {
repository: String,
password: String,
/// `None` when the cache directory cannot be created: restic then runs `--no-cache`.
cache_dir: Option<PathBuf>,
/// rclone config of `rclone:` repositories; `None` on the dashboard's REST server.
config: Option<NamedTempFile>,
/// HTTP basic auth of `rest:` repositories (the dashboard's local storage).
rest_auth: Option<(String, String)>,
/// Keeps temp files referenced by the config (sftp key) alive.
_target: Option<RcloneTarget>,
}
/// What a finished restic command left behind.
pub struct ResticRun {
pub code: i32,
/// stdout lines that are neither `status` nor `summary` (e.g. `snapshots --json`).
pub stdout: String,
/// The last `summary` line, if any.
pub summary: Option<Value>,
/// `error` lines as "item: message" (unreadable files and the like).
pub errors: Vec<String>,
/// Last plain-text / `exit_error` stderr lines, for failure messages.
pub stderr_tail: String,
}
impl ResticRun {
pub fn error(&self, op: &str) -> anyhow::Error {
match self.code {
10 => anyhow!("restic {op}: repository does not exist"),
11 => anyhow!("restic {op}: repository is locked by another operation (waited 5 minutes)"),
12 => anyhow!("restic {op}: wrong repository password (did the master key change?)"),
code => anyhow!("restic {op} failed (exit {code}): {}", self.stderr_tail.trim()),
}
}
}
/// A full snapshot id. Ids from the dashboard reach restic as positional arguments:
/// anything else could be parsed as a flag (e.g. `--password-command=<shell>`).
pub fn is_snapshot_id(id: &str) -> bool {
id.len() == 64 && id.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f'))
}
pub fn short(id: &str) -> &str {
id.get(..8).unwrap_or(id)
}
/// restic fails hard (not warns) on an unusable cache dir, so probe it up front.
fn usable_cache_dir(cache_dir: PathBuf) -> Option<PathBuf> {
match std::fs::create_dir_all(&cache_dir) {
Ok(()) => Some(cache_dir),
Err(e) => {
tracing::warn!("restic cache dir {} unusable ({e}); running without cache", cache_dir.display());
None
}
}
}
impl ResticRepo {
pub fn open(storage: &DatabaseStorage, generated_id: &str, edge_key: &EdgeKey) -> Result<Self> {
let password = derive_password(&edge_key.master_key_b64, generated_id)?;
let cache_dir = PathBuf::from(&CONFIG.data_path).join("cache/restic");
if storage.provider == "local" {
return Ok(Self::rest(
&format!("{}/storage/restic", edge_key.server_url.trim_end_matches('/')),
generated_id,
&password,
&local_storage_password(&edge_key.master_key_b64)?,
cache_dir,
));
}
let target = rclone_target(storage)?;
let folder = storage
.folder_name
.as_deref()
.map(|f| f.trim().trim_matches('/'))
.filter(|f| !f.is_empty())
.unwrap_or("backups");
let mut repo = Self::new(
&target.config_text,
&target.remote_name,
&target.base_path,
folder,
generated_id,
&password,
cache_dir,
)?;
repo._target = Some(target);
Ok(repo)
}
pub fn new(
config_text: &str,
remote_name: &str,
base_path: &str,
folder: &str,
generated_id: &str,
password: &str,
cache_dir: PathBuf,
) -> Result<Self> {
let path = remote_target(remote_name, base_path, &format!("{folder}/restic/{generated_id}"));
Ok(Self {
repository: format!("rclone:{path}"),
password: password.to_string(),
cache_dir: usable_cache_dir(cache_dir),
config: Some(write_config(config_text)?),
rest_auth: None,
_target: None,
})
}
/// The dashboard's local storage (`rclone serve restic --append-only`):
/// `rest:<base_url>/<generated_id>/`, basic auth only in the child env.
pub fn rest(base_url: &str, generated_id: &str, password: &str, server_password: &str, cache_dir: PathBuf) -> Self {
Self {
repository: format!("rest:{}/{generated_id}/", base_url.trim_end_matches('/')),
password: password.to_string(),
cache_dir: usable_cache_dir(cache_dir),
config: None,
rest_auth: Some((LOCAL_STORAGE_USER.to_string(), server_password.to_string())),
_target: None,
}
}
/// The dashboard's server refuses deletions: the agent cannot forget there.
pub fn is_append_only(&self) -> bool {
self.rest_auth.is_some()
}
pub fn repository(&self) -> &str {
&self.repository
}
/// `restic <args> --retry-lock 5m` (+ `--no-cache` without a cache dir); secrets only in the child env. `status`
/// lines are logged every 10 %.
pub async fn run<I, S>(&self, args: I, logger: &JobLogger) -> Result<ResticRun>
where
I: IntoIterator<Item = S>,
S: AsRef<OsStr>,
{
let args: Vec<OsString> = args.into_iter().map(|a| a.as_ref().to_owned()).collect();
let shown: Vec<String> = args.iter().map(|a| a.to_string_lossy().into_owned()).collect();
logger.log("debug", format!("restic {}", shown.join(" ")));
let mut cmd = Command::new("restic");
cmd.args(&args).args(["--retry-lock", "5m"]);
match &self.cache_dir {
Some(dir) => cmd.env("RESTIC_CACHE_DIR", dir),
None => cmd.arg("--no-cache"),
};
if let Some(config) = &self.config {
cmd.env("RCLONE_CONFIG", config.path());
}
if let Some((user, pass)) = &self.rest_auth {
cmd.env("RESTIC_REST_USERNAME", user).env("RESTIC_REST_PASSWORD", pass);
}
let mut child = cmd
.env("RESTIC_REPOSITORY", &self.repository)
.env("RESTIC_PASSWORD", &self.password)
// No TTY: without this restic prints no status lines at all.
.env("RESTIC_PROGRESS_FPS", "0.2")
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.kill_on_drop(true)
.spawn()
.map_err(|e| match e.kind() {
std::io::ErrorKind::NotFound => anyhow!("restic binary not found"),
_ => anyhow!(e).context("failed to start restic"),
})?;
let stderr = child.stderr.take().context("restic stderr unavailable")?;
let stderr_task = tokio::spawn(async move {
let mut errors = Vec::new();
let mut tail = Vec::new();
let mut lines = BufReader::new(stderr).lines();
while let Ok(Some(line)) = lines.next_line().await {
match json::parse(&line) {
Line::Error { message, item } => errors.push(match item {
Some(item) => format!("{item}: {message}"),
None => message,
}),
Line::ExitError { message, .. } => tail.push(message),
Line::Text(text) if !text.trim().is_empty() => tail.push(text),
_ => {}
}
}
(errors, tail)
});
let mut run = ResticRun { code: -1, stdout: String::new(), summary: None, errors: Vec::new(), stderr_tail: String::new() };
let stdout = child.stdout.take().context("restic stdout unavailable")?;
let mut lines = BufReader::new(stdout).lines();
let mut next_pct = 10u32;
while let Some(line) = lines.next_line().await.context("failed to read restic output")? {
match json::parse(&line) {
Line::Status { percent_done } => {
let pct = (percent_done * 100.0) as u32;
if pct >= next_pct {
logger.log("info", format!("restic: {pct}% done"));
next_pct = pct / 10 * 10 + 10;
}
}
Line::Summary(value) => run.summary = Some(value),
_ => {
run.stdout.push_str(&line);
run.stdout.push('\n');
}
}
}
let status = child.wait().await.context("failed to wait for restic")?;
let (errors, tail) = stderr_task.await.unwrap_or_default();
run.code = status.code().unwrap_or(-1);
run.errors = errors;
run.stderr_tail = tail[tail.len().saturating_sub(10)..].join("\n");
Ok(run)
}
/// `restic cat config`; exit 10 (no repository) → `restic init`. Callers hold
/// the source's FileLock, so two inits never race.
pub async fn ensure_initialized(&self, logger: &JobLogger) -> Result<()> {
let cat = self.run(["cat", "config"], logger).await?;
match cat.code {
0 => Ok(()),
10 => {
logger.log("info", format!("Initializing restic repository {}", self.repository));
let init = self.run(["init"], logger).await?;
if init.code == 0 { Ok(()) } else { Err(init.error("init")) }
}
_ => Err(cat.error("cat config")),
}
}
}
+79
View File
@@ -0,0 +1,79 @@
use anyhow::{Result, bail};
use std::path::Path;
/// Escapes restic/filepath.Match metacharacters so a literal path matches only itself.
pub fn glob_escape(literal: &str) -> String {
let mut out = String::with_capacity(literal.len());
for c in literal.chars() {
if matches!(c, '\\' | '*' | '?' | '[' | ']') {
out.push('\\');
}
out.push(c);
}
out
}
/// globset (the P0 matcher) reads `[!x]` as bracket negation; restic (Go
/// `filepath.Match`) and rclone only know `[^x]` and take `!` literally.
pub(crate) fn bracket_negation(pattern: &str) -> String {
let mut out = String::with_capacity(pattern.len());
let mut chars = pattern.chars();
let mut in_class = false;
while let Some(c) = chars.next() {
out.push(c);
match c {
'\\' => out.extend(chars.next()),
'[' if !in_class => {
in_class = true;
if chars.as_str().starts_with('!') {
chars.next();
out.push('^');
}
}
']' => in_class = false,
_ => {}
}
}
out
}
/// P0 patterns, trimmed like the P0 matcher, as `(anchored, pattern)`; a leading `/` marks anchored ones.
fn normalized(patterns: &[String]) -> Result<Vec<(bool, String)>> {
let mut out = Vec::new();
for raw in patterns {
let pattern = raw.trim();
if pattern.contains(['{', '}']) {
bail!("exclude pattern '{pattern}': brace patterns are not supported in Snapshots mode");
}
let (anchored, rest) = match pattern.strip_prefix('/') {
Some(rest) => (true, rest),
None => (false, pattern),
};
let rest = rest.trim_end_matches('/');
if !rest.is_empty() {
out.push((anchored, bracket_negation(rest)));
}
}
Ok(out)
}
/// `restic backup` matches patterns against absolute paths, so every pattern is
/// rooted at the source; unanchored ones would otherwise also match directories
/// above it and exclude the whole source (spike S5).
pub fn backup_excludes(root: &Path, patterns: &[String]) -> Result<Vec<String>> {
let root = glob_escape(root.to_string_lossy().trim_end_matches('/'));
Ok(normalized(patterns)?
.into_iter()
.map(|(anchored, p)| {
if anchored { format!("{root}/{p}") } else { format!("{root}/**/{p}") }
})
.collect())
}
/// `restic restore <id>:<path>` matches patterns relative to that subfolder (spike S1).
pub fn restore_excludes(patterns: &[String]) -> Result<Vec<String>> {
Ok(normalized(patterns)?
.into_iter()
.map(|(anchored, p)| if anchored { format!("/{p}") } else { format!("/**/{p}") })
.collect())
}
+52
View File
@@ -0,0 +1,52 @@
use serde::Deserialize;
use serde_json::Value;
/// One line of `restic --json` output. stdout carries `status` / `summary`,
/// stderr carries `error` / `exit_error` and plain-text fatal messages.
#[derive(Debug, PartialEq)]
pub enum Line {
Status { percent_done: f64 },
Summary(Value),
Error { message: String, item: Option<String> },
ExitError { code: i32, message: String },
Text(String),
}
pub fn parse(line: &str) -> Line {
let Ok(value) = serde_json::from_str::<Value>(line) else {
return Line::Text(line.to_string());
};
match value.get("message_type").and_then(Value::as_str) {
Some("status") => Line::Status {
percent_done: value["percent_done"].as_f64().unwrap_or(0.0),
},
Some("summary") => Line::Summary(value),
Some("error") => Line::Error {
message: value["error"]["message"].as_str().unwrap_or("unknown error").to_string(),
item: value["item"].as_str().map(str::to_string),
},
Some("exit_error") => Line::ExitError {
code: value["code"].as_i64().unwrap_or(1) as i32,
message: value["message"].as_str().unwrap_or_default().to_string(),
},
_ => Line::Text(line.to_string()),
}
}
#[derive(Debug, Deserialize)]
pub struct BackupSummary {
pub snapshot_id: String,
pub data_added_packed: u64,
pub total_bytes_processed: u64,
pub files_new: u64,
pub files_changed: u64,
pub files_unmodified: u64,
}
/// One entry of `restic snapshots --json`.
#[derive(Debug, Deserialize)]
pub struct Snapshot {
#[allow(dead_code)] // part of the `restic snapshots` shape; restore() only needs `paths`
pub id: String,
pub paths: Vec<String>,
}
+7
View File
@@ -0,0 +1,7 @@
//! Snapshots mode: restic repositories reached through the P1 rclone mapper.
pub mod backup;
pub mod command;
pub mod excludes;
pub mod json;
pub mod password;
pub mod restore;
+42
View File
@@ -0,0 +1,42 @@
use anyhow::{Context, Result};
use base64::{Engine as _, engine::general_purpose};
use hmac::{Hmac, Mac};
use sha2::Sha256;
type HmacSha256 = Hmac<Sha256>;
/// Repository password: hex(HKDF-SHA256(master key, salt = "",
/// info = "portabase/restic/v1/<generated_id>", 32 bytes)). The dashboard derives
/// the same value (`src/lib/restic/repo.ts`), so it is never stored nor sent.
pub fn derive_password(master_key_b64: &str, generated_id: &str) -> Result<String> {
let master_key = general_purpose::STANDARD
.decode(master_key_b64.trim())
.context("master key is not valid base64")?;
let info = format!("portabase/restic/v1/{generated_id}");
Ok(hex::encode(hkdf_sha256_32(&master_key, info.as_bytes())))
}
/// User of the dashboard's local-storage servers (`/storage/restic`, `/storage/sync`).
pub const LOCAL_STORAGE_USER: &str = "portabase";
/// Their password: hex(HKDF-SHA256(master key, salt = "", info = "portabase/local-storage/v1",
/// 32 bytes)). The dashboard derives the same value (`src/lib/local-storage/credential.ts`).
pub fn local_storage_password(master_key_b64: &str) -> Result<String> {
let master_key = general_purpose::STANDARD
.decode(master_key_b64.trim())
.context("master key is not valid base64")?;
Ok(hex::encode(hkdf_sha256_32(&master_key, b"portabase/local-storage/v1")))
}
/// RFC 5869 with an empty salt and a single expand block (L = HashLen = 32).
pub fn hkdf_sha256_32(ikm: &[u8], info: &[u8]) -> [u8; 32] {
// An empty HMAC key is zero-padded: the RFC's default salt of HashLen zeros.
let mut extract = HmacSha256::new_from_slice(&[]).expect("HMAC takes any key length");
extract.update(ikm);
let prk = extract.finalize().into_bytes();
let mut expand = HmacSha256::new_from_slice(&prk).expect("HMAC takes any key length");
expand.update(info);
expand.update(&[1]);
expand.finalize().into_bytes().into()
}
+124
View File
@@ -0,0 +1,124 @@
use super::command::{ResticRepo, is_snapshot_id, short};
use super::excludes::{glob_escape, restore_excludes};
use super::json::Snapshot;
use crate::domain::files::matcher::{exclude_patterns, one_file_system};
use crate::domain::files::platform::dev;
use crate::domain::files::restore::ensure_restorable;
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::backup::logger::JobLogger;
use crate::services::config::{DatabaseConfig, DbType};
use crate::utils::edge_key::EdgeKey;
use crate::utils::locks::{DbOpLock, FileLock};
use anyhow::{Context, Result, bail};
use std::path::Path;
use walkdir::WalkDir;
/// Mirror restore of `snapshot_id` into `cfg.path`: files created since are
/// deleted, paths matching the current exclude patterns are kept (spike S1).
pub async fn restore(
repo: &ResticRepo,
cfg: &DatabaseConfig,
snapshot_id: &str,
logger: &JobLogger,
) -> Result<()> {
if !is_snapshot_id(snapshot_id) {
bail!("invalid snapshot id in the restore payload");
}
let target = ensure_restorable(Path::new(&cfg.path))?;
let listed = repo.run(["snapshots", snapshot_id, "--json"], logger).await?;
if listed.code != 0 {
return Err(listed.error("snapshots"));
}
let snapshots: Vec<Snapshot> = serde_json::from_str(listed.stdout.trim())
.context("unexpected `restic snapshots` output")?;
// The original path inside the snapshot; the mount point may have moved since.
let Some(original) = snapshots.first().and_then(|s| s.paths.first()) else {
bail!("snapshot {} not found in {}", short(snapshot_id), repo.repository());
};
let mut excludes = restore_excludes(&exclude_patterns(cfg))?;
if one_file_system(cfg) {
excludes.extend(mount_points(&target).into_iter().map(|rel| format!("/{}", glob_escape(&rel))));
}
let mut args: Vec<String> = vec![
"restore".into(),
format!("{snapshot_id}:{original}"),
"--target".into(),
target.to_string_lossy().into_owned(),
"--delete".into(),
"--json".into(),
];
for pattern in excludes {
args.push("--exclude".into());
args.push(pattern);
}
logger.log(
"info",
format!("Restoring snapshot {} ({original}) into {}", short(snapshot_id), target.display()),
);
let run = repo.run(&args, logger).await?;
for error in &run.errors {
logger.log("warn", format!("Restore error: {error}"));
}
if run.code != 0 {
return Err(run.error("restore"));
}
if let Some(s) = &run.summary {
logger.log(
"info",
format!(
"Restored {} file(s), deleted {} file(s)",
s["files_restored"].as_u64().unwrap_or(0),
s["files_deleted"].as_u64().unwrap_or(0)
),
);
}
Ok(())
}
/// Directories under `root` on another device. With `one_file_system` backup
/// never entered them, so the mirror restore must not delete their content.
fn mount_points(root: &Path) -> Vec<String> {
let Ok(meta) = std::fs::metadata(root) else { return Vec::new() };
let root_dev = dev(&meta);
let mut out = Vec::new();
let mut walker = WalkDir::new(root).min_depth(1).follow_links(false).into_iter();
while let Some(entry) = walker.next() {
let Ok(entry) = entry else { continue };
if !entry.file_type().is_dir() {
continue;
}
if matches!(entry.metadata(), Ok(m) if dev(&m) != root_dev) {
if let Ok(rel) = entry.path().strip_prefix(root) {
out.push(rel.to_string_lossy().into_owned());
}
walker.skip_current_dir();
}
}
out
}
/// Snapshot restore job: the source's FileLock is held for the whole restore.
pub async fn run(
edge_key: &EdgeKey,
cfg: &DatabaseConfig,
snapshot_id: &str,
storage: &DatabaseStorage,
logger: &JobLogger,
) -> Result<()> {
if !matches!(cfg.db_type, DbType::Files) {
bail!("snapshot restore needs a files source, got {}", cfg.db_type.as_str());
}
FileLock::acquire(&cfg.generated_id, DbOpLock::Restore.as_str())
.await
.context("another backup or restore of this source is running")?;
let result = async {
let repo = ResticRepo::open(storage, &cfg.generated_id, edge_key)?;
restore(&repo, cfg, snapshot_id, logger).await
}
.await;
FileLock::release(&cfg.generated_id).await?;
result
}
+5
View File
@@ -60,6 +60,11 @@ impl RestoreService {
archive = decrypted;
}
if matches!(db_type, DbType::Files) {
logger.log("info", "Files archive ready for extraction".to_string());
return Ok(archive);
}
if matches!(db_type, DbType::DockerVolume) {
let raw_tar = tmp_path.join("volume.tar");
crate::utils::compress::gunzip_to_file(archive.as_path(), &raw_tar).await?;
+13
View File
@@ -15,6 +15,19 @@ impl RestoreService {
return;
};
if db.data.restore.engine.as_deref() == Some("restic") {
let snapshot_id = db.data.restore.snapshot_id.clone();
let storage = db.data.restore.storage.clone();
let service = Self { ctx: self.ctx.clone() };
let db_cfg = cfg.clone();
tokio::spawn(async move {
if let Err(e) = service.execute_restic_restore(db_cfg, snapshot_id, storage).await {
error!("Snapshot restore failed: {}", e);
}
});
return;
}
let Some(file_to_restore) = db.data.restore.file.clone() else {
error!("restore file not found");
return;
+39
View File
@@ -1,6 +1,9 @@
use super::models::RestoreResult;
use super::service::RestoreService;
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::services::restic;
use anyhow::Result;
use std::sync::Arc;
use std::time::Instant;
@@ -43,4 +46,40 @@ impl RestoreService {
Ok(())
}
pub async fn execute_restic_restore(
&self,
cfg: DatabaseConfig,
snapshot_id: Option<String>,
storage: Option<DatabaseStorage>,
) -> Result<()> {
let logger = Arc::new(JobLogger::new());
let start = Instant::now();
logger.log("info", "Snapshot restore job started".to_string());
let outcome = match (snapshot_id, storage) {
(Some(id), Some(storage)) => {
restic::restore::run(&self.ctx.edge_key, &cfg, &id, &storage, &logger).await
}
_ => Err(anyhow::anyhow!(
"incomplete snapshot restore payload (snapshotId or storage missing)"
)),
};
let status = match outcome {
Ok(()) => {
logger.log("info", "Snapshot restore job finished".to_string());
"success"
}
Err(e) => {
logger.log("error", format!("Snapshot restore failed: {e:#}"));
"failed"
}
};
let duration_ms = start.elapsed().as_millis() as f64;
let logs = Arc::try_unwrap(logger).unwrap_or_else(|_| JobLogger::new()).into_entries();
let result = RestoreResult { generated_id: cfg.generated_id.clone(), status: status.into() };
self.send_result(result, logs, duration_ms).await?;
Ok(())
}
}
+34 -2
View File
@@ -6,11 +6,12 @@ use crate::services::api::endpoints::status::DatabasePayload;
use crate::services::api::models::agent::status::DatabaseStatus;
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::api::models::agent::status::PingResult;
use crate::services::config::{build_config, DatabaseConfig, InputDatabaseConfig};
use crate::services::config::{build_config, files_method, DatabaseConfig, DbType, InputDatabaseConfig};
use crate::settings::CONFIG;
use crate::utils::file::decrypt_json_gcm;
use futures_util::future::try_join_all;
use reqwest::Client;
use std::collections::HashSet;
use std::error::Error;
use std::sync::Arc;
use tracing::info;
@@ -35,6 +36,29 @@ pub fn resolve_dashboard_config(
Ok(())
}
/// A storage channel sent encrypted as a one-element JSON array.
fn decrypt_storage(ciphertext: &str, master_key_b64: &str, what: &str) -> Result<Option<DatabaseStorage>, String> {
let plaintext = decrypt_json_gcm(ciphertext, master_key_b64)
.map_err(|e| format!("Failed to decrypt {what} storage: {e}"))?;
let storages: Vec<DatabaseStorage> = serde_json::from_slice(&plaintext)
.map_err(|e| format!("Failed to parse {what} storage: {e}"))?;
Ok(storages.into_iter().next())
}
/// Decrypts the storage channel of a snapshot restore (`restore.storageCiphertext`).
pub fn resolve_restore_storage(status: &mut DatabaseStatus, master_key_b64: &str) -> Result<(), String> {
if let Some(ciphertext) = status.data.restore.storage_ciphertext.clone() {
status.data.restore.storage = decrypt_storage(&ciphertext, master_key_b64, "restore")?;
}
Ok(())
}
/// `method` reported in the ping for a files source declared in databases.json;
/// `None` for dashboard-managed sources (the dashboard owns their method) and other dbms.
pub fn payload_method(db: &DatabaseConfig, local_ids: &HashSet<String>) -> Option<&'static str> {
(matches!(db.db_type, DbType::Files) && local_ids.contains(&db.generated_id)).then(|| files_method(db))
}
pub struct StatusService {
ctx: Arc<Context>,
client: Client,
@@ -48,7 +72,11 @@ impl StatusService {
}
}
pub async fn ping(&self, databases: &[DatabaseConfig]) -> Result<PingResult, Box<dyn Error>> {
pub async fn ping(
&self,
databases: &[DatabaseConfig],
local_ids: &HashSet<String>,
) -> Result<PingResult, Box<dyn Error>> {
let edge_key = &self.ctx.edge_key;
let databases_payload: Vec<DatabasePayload> =
@@ -63,6 +91,7 @@ impl StatusService {
dbms: &db.db_type.as_str(),
generated_id: &db.generated_id,
ping_status: reachable,
method: payload_method(db, local_ids),
})
}))
.await?;
@@ -92,6 +121,9 @@ impl StatusService {
if let Err(e) = resolve_dashboard_config(db, &edge_key.master_key_b64) {
tracing::warn!("Skipping dashboard config for {}: {e}", db.generated_id);
}
if let Err(e) = resolve_restore_storage(db, &edge_key.master_key_b64) {
tracing::warn!("Snapshot restore storage unreadable for {}: {e}", db.generated_id);
}
}
Ok(result)
}
@@ -8,25 +8,26 @@ use google_cloud_storage::client::Storage;
use google_cloud_storage::streaming_source::{SizeHint, StreamingSource};
use std::pin::Pin;
pub fn build_credentials(cfg: &GoogleCloudStorageProviderConfig) -> Result<Credentials> {
/// Service-account key JSON (keys in alphabetical order so the rclone mapper
/// emits byte-identical output to the dashboard's).
pub fn service_account_key(cfg: &GoogleCloudStorageProviderConfig) -> serde_json::Value {
// Service-account JSON stores the PEM with `\n` escape sequences. When the key is
// carried through config as a JSON string those can arrive as literal two-char `\n`
// sequences rather than real newlines, so the PEM parser finds no `-----BEGIN-----`
// line ("no items found"). Normalize them back to real newlines. A PEM that already
// has real newlines contains no literal `\n` pairs, so this is a no-op for it.
// sequences rather than real newlines. Normalize them back to real newlines.
let private_key = cfg.private_key.replace("\\n", "\n");
let key = serde_json::json!({
"type": "service_account",
"project_id": cfg.project_id,
serde_json::json!({
"client_email": cfg.client_email,
"private_key": private_key,
"private_key_id": "",
"project_id": cfg.project_id,
"token_uri": "https://oauth2.googleapis.com/token",
"type": "service_account",
"universe_domain": "googleapis.com",
});
})
}
google_cloud_auth::credentials::service_account::Builder::new(key)
pub fn build_credentials(cfg: &GoogleCloudStorageProviderConfig) -> Result<Credentials> {
google_cloud_auth::credentials::service_account::Builder::new(service_account_key(cfg))
.build()
.context("failed to build GCS service account credentials")
}
@@ -1,5 +1,5 @@
pub mod helpers;
mod models;
pub mod models;
use crate::core::context::Context;
use crate::services::api::models::agent::status::DatabaseStorage;
@@ -1,5 +1,5 @@
mod helpers;
mod models;
pub mod models;
use crate::core::context::Context;
use crate::services::api::models::agent::status::DatabaseStorage;
+111 -43
View File
@@ -10,7 +10,9 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::process::Command;
use tracing::info;
const BLOCKED_BACKEND_TYPES: [&str; 13] = [
/// Entries must not contain spaces: the type is compared with its spaces removed (rclone resolves a
/// backend by name, registered prefix, or name without spaces: `google photos` = `googlephotos`).
const BLOCKED_BACKEND_TYPES: [&str; 14] = [
"local",
"alias",
"crypt",
@@ -24,67 +26,133 @@ const BLOCKED_BACKEND_TYPES: [&str; 13] = [
"memory",
"http",
"googlephotos",
"gphotos",
];
fn sections(config_text: &str) -> Vec<(String, Option<String>)> {
let mut out: Vec<(String, Option<String>)> = Vec::new();
for line in config_text.lines() {
let line = line.trim();
if line.starts_with('[') && line.ends_with(']') && line.len() > 2 {
out.push((line[1..line.len() - 1].trim().to_string(), None));
continue;
}
let Some((key, value)) = line.split_once('=') else {
continue;
};
if key.trim().eq_ignore_ascii_case("type")
&& let Some(current) = out.last_mut()
&& current.1.is_none()
{
current.1 = Some(value.trim().to_ascii_lowercase());
}
}
out
fn valid_remote_name(name: &str) -> bool {
let mut chars = name.chars();
chars.next().is_some_and(|c| c.is_ascii_alphanumeric() || c == '_')
&& chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | '+' | '@' | '-'))
}
fn valid_key(key: &str) -> bool {
!key.is_empty() && key.chars().all(|c| c.is_ascii_alphanumeric() || c == '_')
}
fn forbidden_key(key: &str) -> bool {
let key = key.to_ascii_lowercase();
matches!(
key.as_str(),
"ssh" | "env_auth" | "use_msi" | "use_az" | "use_kerberos" | "kerberos_ccache" | "key_use_agent" | "set_env" | "unix_socket"
) || key.ends_with("_command")
|| key.ends_with("_file")
|| key.ends_with("_path")
}
/// Strict check for user-pasted rclone configs (never for configs Portabase builds itself).
/// Exactly one `[remote]` of plain `key = value` lines: rclone and a naive parser must read it
/// identically, and nothing may run a local command or read host credentials.
pub fn validate_config(config_text: &str, remote_name: &str) -> Result<()> {
let sections = sections(config_text);
let mut names = Vec::new();
let mut pairs = Vec::new();
let mut before_header = None;
if sections.is_empty() {
bail!("rclone config contains no remote sections");
}
for (name, backend) in &sections {
let Some(backend) = backend else { continue };
if BLOCKED_BACKEND_TYPES.contains(&backend.as_str()) {
bail!("rclone backend type '{backend}' is not allowed (remote '{name}')");
for (i, line) in config_text.split('\n').enumerate() {
let (n, line) = (i + 1, line.trim());
if line.is_empty() || line.starts_with(['#', ';']) {
continue;
}
if line.starts_with('[') && line.ends_with(']') {
names.push(line[1..line.len() - 1].trim());
} else if let Some((key, value)) = line.split_once('=').filter(|(k, _)| !k.is_empty()) {
if names.is_empty() {
before_header.get_or_insert(n);
}
pairs.push((key.trim(), value.trim()));
} else {
bail!("rclone config line {n} is not a [section] header or a 'key = value' pair");
}
}
if !sections.iter().any(|(name, _)| name == remote_name) {
let available: Vec<&str> = sections.iter().map(|(name, _)| name.as_str()).collect();
bail!(
"remote '{remote_name}' is not defined in the rclone config (available: {})",
available.join(", ")
);
if let Some(n) = before_header {
bail!("rclone config line {n} appears before any [section]");
}
if names.len() != 1 {
bail!("rclone config must contain exactly one [section] (found {})", names.len());
}
let name = names[0];
if !valid_remote_name(name) {
bail!("invalid rclone remote name '{name}'");
}
for (key, _) in &pairs {
if !valid_key(key) {
bail!("invalid rclone config key '{key}' (remote '{name}')");
}
}
// rclone (goconfig) unquotes `value` and `"""value"""` and drops what follows the closing quote.
for (key, value) in &pairs {
if value.starts_with('`') || value.starts_with("\"\"\"") {
bail!("rclone config value for key '{key}' must not be quoted (remote '{name}')");
}
}
let mut seen = std::collections::HashSet::new();
for (key, _) in &pairs {
if !seen.insert(key.to_ascii_lowercase()) {
bail!("duplicate rclone config key '{key}' (remote '{name}')");
}
}
for (key, _) in &pairs {
if forbidden_key(key) {
bail!("rclone config key '{key}' is not allowed (remote '{name}')");
}
}
let Some((_, backend)) = pairs.iter().find(|(k, _)| k.eq_ignore_ascii_case("type")) else {
bail!("rclone remote '{name}' has no type");
};
let backend = backend.to_ascii_lowercase();
if backend.is_empty() || !backend.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == ' ') {
bail!("invalid rclone backend type '{backend}' (remote '{name}')");
}
let compact = backend.replace(' ', "");
if BLOCKED_BACKEND_TYPES.contains(&compact.as_str()) {
bail!("rclone backend type '{backend}' is not allowed (remote '{name}')");
}
// The other OCI providers read the host's OCI config or instance identity. Exact match on
// purpose: rclone 1.75.1 compares the key and the value case-sensitively, and falls back to
// host credentials for `PROVIDER = no_auth` (key ignored) and `provider = NO_AUTH` (unknown value).
if compact == "oracleobjectstorage" {
if !pairs.contains(&("provider", "no_auth")) {
bail!("rclone oracleobjectstorage remotes must use provider = no_auth (remote '{name}')");
}
}
if remote_name != name {
bail!("remote '{remote_name}' is not defined in the rclone config (available: {name})");
}
Ok(())
}
/// `rclone obscure -` reads the password from stdin, so it never shows up in argv.
pub fn obscure_password(password: &str) -> Result<String> {
let out = std::process::Command::new("rclone")
let mut child = std::process::Command::new("rclone")
.arg("obscure")
.arg(password)
.output()
.arg("-")
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.context("failed to spawn rclone (is the binary installed in this image?)")?;
// Dropping the handle closes stdin, which is what ends rclone's read.
child
.stdin
.take()
.context("rclone stdin unavailable")?
.write_all(password.as_bytes())
.context("failed to write the password to rclone obscure")?;
let out = child.wait_with_output().context("failed to wait for rclone obscure")?;
if !out.status.success() {
bail!(
"rclone obscure failed: {}",
@@ -1,5 +1,6 @@
pub mod helpers;
pub mod models;
pub mod target;
use crate::core::context::Context;
use crate::services::api::models::agent::status::DatabaseStorage;
@@ -0,0 +1,137 @@
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::storage::providers::azure_blob::models::AzureBlobProviderConfig;
use crate::services::storage::providers::google_cloud_storage::helpers::service_account_key;
use crate::services::storage::providers::google_cloud_storage::models::GoogleCloudStorageProviderConfig;
use crate::services::storage::providers::google_drive::models::GoogleDriveProviderConfig;
use crate::services::storage::providers::rclone::helpers::{build_rclone_config, validate_config};
use crate::services::storage::providers::rclone::models::RcloneProviderConfig;
use crate::services::storage::providers::s3::models::S3ProviderConfig;
use crate::services::storage::providers::sftp::helpers::build_sftp_config;
use crate::services::storage::providers::sftp::models::SftpProviderConfig;
use anyhow::{Context, Result, bail};
use tempfile::TempPath;
/// A storage channel expressed as an rclone remote. Temp files the config refers
/// to (e.g. an sftp key) live as long as this value.
pub struct RcloneTarget {
pub config_text: String,
pub remote_name: String,
pub base_path: String,
_guards: Vec<TempPath>,
}
impl RcloneTarget {
fn new(config_text: String, remote_name: &str, base_path: String) -> Self {
Self { config_text, remote_name: remote_name.to_string(), base_path, _guards: Vec::new() }
}
}
pub fn rclone_target(storage: &DatabaseStorage) -> Result<RcloneTarget> {
let config = storage.config.clone();
match storage.provider.as_str() {
"s3" => s3(config.try_into().context("invalid s3 storage config")?),
"blob" => blob(config.try_into().context("invalid azure blob storage config")?),
"google-cloud-storage" => gcs(config.try_into().context("invalid google cloud storage config")?),
"google-drive" => drive(config.try_into().context("invalid google drive storage config")?),
"rclone" => user_rclone(config.try_into().context("invalid rclone storage config")?),
"sftp" => sftp(config.try_into().context("invalid sftp storage config")?),
"local" => bail!("a local storage channel cannot be used as an rclone target"),
other => bail!("unknown storage provider '{other}'"),
}
}
fn s3(c: S3ProviderConfig) -> Result<RcloneTarget> {
let scheme = if c.ssl { "https" } else { "http" };
let endpoint = match c.port.as_deref().map(str::trim).filter(|p| !p.is_empty()) {
Some(port) => format!("{scheme}://{}:{port}", c.end_point_url),
None => format!("{scheme}://{}", c.end_point_url),
};
let region = c.region.as_deref().map(str::trim).filter(|r| !r.is_empty()).unwrap_or("us-east-1");
let fields = [
("type", "s3".to_string()),
("provider", "Other".to_string()),
("access_key_id", c.access_key),
("secret_access_key", c.secret_key),
("endpoint", endpoint),
("region", region.to_string()),
("force_path_style", "true".to_string()),
("no_check_bucket", "true".to_string()),
];
Ok(RcloneTarget::new(build_rclone_config("s3", &fields)?, "s3", c.bucket_name))
}
fn blob(c: AzureBlobProviderConfig) -> Result<RcloneTarget> {
let resolved = c.resolve().context("invalid azure blob storage config")?;
if resolved.account_key.trim().is_empty() {
bail!("azure blob needs an account key to be used as an rclone target (SAS-only connection strings are not supported)");
}
let fields = [
("type", "azureblob".to_string()),
("account", resolved.account_name),
("key", resolved.account_key),
("endpoint", resolved.blob_endpoint),
// Native uploads never create the container; neither should rclone.
("no_check_container", "true".to_string()),
];
Ok(RcloneTarget::new(build_rclone_config("blob", &fields)?, "blob", c.container_name))
}
fn gcs(c: GoogleCloudStorageProviderConfig) -> Result<RcloneTarget> {
let mut fields = vec![
("type", "google cloud storage".to_string()),
("project_number", c.project_id.clone()),
];
match c.api_endpoint.as_deref().map(str::trim).filter(|e| !e.is_empty()) {
// A custom endpoint means the fake-gcs emulator, which does not verify credentials.
// rclone's `endpoint` is the JSON API base path, not the bare host.
Some(endpoint) => {
fields.push(("anonymous", "true".to_string()));
fields.push(("endpoint", format!("{}/storage/v1/", endpoint.trim_end_matches('/'))));
}
None => fields.push((
"service_account_credentials",
serde_json::to_string(&service_account_key(&c))?,
)),
}
// Uniform bucket-level access (the GCS default) rejects rclone's default
// `predefinedAcl=private`; and native uploads never create the bucket.
fields.push(("bucket_policy_only", "true".to_string()));
fields.push(("no_check_bucket", "true".to_string()));
Ok(RcloneTarget::new(
build_rclone_config("google-cloud-storage", &fields)?,
"google-cloud-storage",
c.bucket_name,
))
}
fn drive(c: GoogleDriveProviderConfig) -> Result<RcloneTarget> {
let token = serde_json::json!({
"access_token": "",
"expiry": "0001-01-01T00:00:00Z",
"refresh_token": c.refresh_token,
"token_type": "Bearer",
});
let fields = [
("type", "drive".to_string()),
("client_id", c.client_id),
("client_secret", c.client_secret),
("scope", "drive.file".to_string()),
("token", serde_json::to_string(&token)?),
("root_folder_id", c.folder_id),
// Deletes (retention, restic prune) must free space, not fill the Drive trash.
("use_trash", "false".to_string()),
];
Ok(RcloneTarget::new(build_rclone_config("google-drive", &fields)?, "google-drive", String::new()))
}
fn user_rclone(c: RcloneProviderConfig) -> Result<RcloneTarget> {
validate_config(&c.config_text, &c.remote_name)?;
Ok(RcloneTarget::new(c.config_text, &c.remote_name, c.remote_path))
}
fn sftp(c: SftpProviderConfig) -> Result<RcloneTarget> {
let (config_text, key_file) = build_sftp_config(&c)?;
let mut target = RcloneTarget::new(config_text, "sftp", c.remote_path);
target._guards.extend(key_file.map(|f| f.into_temp_path()));
Ok(target)
}
+1 -1
View File
@@ -1,4 +1,4 @@
mod models;
pub mod models;
use crate::core::context::Context;
use crate::services::api::models::agent::status::DatabaseStorage;
+11 -18
View File
@@ -6,8 +6,7 @@ use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::backup::models::{BackupResult, UploadResult};
use crate::services::storage::StorageProvider;
use crate::services::storage::providers::rclone::helpers::{rcat, remote_target, write_config};
use crate::services::storage::providers::sftp::helpers::build_sftp_config;
use crate::services::storage::providers::sftp::models::SftpProviderConfig;
use crate::services::storage::providers::rclone::target::rclone_target;
use crate::utils::common::BackupMethod;
use crate::utils::file::{full_file_name, full_file_path};
use crate::utils::stream::build_stream;
@@ -53,19 +52,13 @@ impl StorageProvider for SftpProvider {
}
};
let config: SftpProviderConfig = match storage.clone().config.try_into() {
Ok(c) => c,
// `target` owns the sftp key file: keep it alive until `rcat` returns.
let target = match rclone_target(storage) {
Ok(t) => t,
Err(e) => {
error!("sftp config deserialization failed: {}", e);
return failed(&storage_id, e, Some(total_size));
}
};
let (config_text, _key_file) = match build_sftp_config(&config) {
Ok(v) => v,
Err(e) => {
error!("sftp config build failed: {}", e);
return failed(&storage_id, e, Some(total_size));
// `{:#}` keeps the whole anyhow chain (the serde cause), which is what the user sees.
error!("sftp config build failed: {e:#}");
return failed(&storage_id, format!("{e:#}"), Some(total_size));
}
};
@@ -82,7 +75,7 @@ impl StorageProvider for SftpProvider {
let file_name = full_file_name(encrypt);
let remote_file_path = full_file_path(&file_name, storage.folder_name.as_deref());
let config_file = match write_config(&config_text) {
let config_file = match write_config(&target.config_text) {
Ok(f) => f,
Err(e) => {
error!("sftp config write failed: {}", e);
@@ -90,11 +83,11 @@ impl StorageProvider for SftpProvider {
}
};
let target = remote_target("sftp", &config.remote_path, &remote_file_path);
let remote = remote_target(&target.remote_name, &target.base_path, &remote_file_path);
info!("Starting sftp (rclone) upload to {}", target);
info!("Starting sftp (rclone) upload to {}", remote);
match rcat(config_file.path(), &target, upload.stream).await {
match rcat(config_file.path(), &remote, upload.stream).await {
Ok(()) => {
info!("sftp upload successful: {}", remote_file_path);
UploadResult {
+147
View File
@@ -0,0 +1,147 @@
//! Sync as a backup method: one `rclone sync` per storage channel, each reported
//! as its own `backup_storage` row (`engine = sync`), like restic snapshots.
use super::rclone::{replica_target, sync_dir};
use super::stats::SyncStats;
use crate::core::context::Context as CoreContext;
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::backup::logger::JobLogger;
use crate::services::backup::models::UploadResult;
use crate::services::config::DatabaseConfig;
use crate::services::restic::password::{LOCAL_STORAGE_USER, local_storage_password};
use crate::services::storage::providers::rclone::helpers::{build_rclone_config, obscure_password};
use crate::services::storage::providers::rclone::target::rclone_target;
use crate::utils::edge_key::EdgeKey;
use crate::utils::locks::{DbOpLock, FileLock};
use anyhow::Result;
/// Syncs every storage in turn, under the source's backup lock. Never errors:
/// each storage reports its own `backup_storage` row.
pub async fn run(
ctx: &CoreContext,
cfg: &DatabaseConfig,
storages: &[DatabaseStorage],
backup_id: &str,
logger: &JobLogger,
) -> Vec<UploadResult> {
if let Err(e) = FileLock::acquire(&cfg.generated_id, DbOpLock::Backup.as_str()).await {
logger.log("error", format!("Sync aborted: {e}"));
return Vec::new();
}
let mut results = Vec::with_capacity(storages.len());
for storage in storages {
results.push(one_storage(ctx, cfg, storage, backup_id, logger).await);
}
if let Err(e) = FileLock::release(&cfg.generated_id).await {
logger.log("warn", format!("Failed to release the backup lock: {e}"));
}
results
}
fn folder(storage: &DatabaseStorage) -> &str {
storage
.folder_name
.as_deref()
.map(|f| f.trim().trim_matches('/'))
.filter(|f| !f.is_empty())
.unwrap_or("backups")
}
/// `<folder>/sync/<generated_id>/current`, relative to the channel's base path.
pub fn replica_path(storage: &DatabaseStorage, generated_id: &str) -> String {
format!("{}/sync/{generated_id}/current", folder(storage))
}
const LOCAL_REMOTE: &str = "pblocal";
/// rclone config of the dashboard's local storage (`rclone serve webdav` at `/storage/sync`).
pub fn local_sync_config(edge_key: &EdgeKey) -> Result<String> {
build_rclone_config(
LOCAL_REMOTE,
&[
("type", "webdav".into()),
("url", format!("{}/storage/sync", edge_key.server_url.trim_end_matches('/'))),
// rclone's own WebDAV extensions keep modification times: unchanged files are skipped.
("vendor", "rclone".into()),
("user", LOCAL_STORAGE_USER.into()),
("pass", obscure_password(&local_storage_password(&edge_key.master_key_b64)?)?),
],
)
}
async fn replicate(ctx: &CoreContext, cfg: &DatabaseConfig, storage: &DatabaseStorage, logger: &JobLogger) -> Result<SyncStats> {
if storage.provider == "local" {
let dest = format!("{LOCAL_REMOTE}:{}/current", cfg.generated_id);
return sync_dir(&local_sync_config(&ctx.edge_key)?, &dest, cfg, logger).await;
}
let target = rclone_target(storage)?;
let dest = replica_target(&target.remote_name, &target.base_path, folder(storage), &cfg.generated_id);
sync_dir(&target.config_text, &dest, cfg, logger).await
}
pub(crate) async fn one_storage(
ctx: &CoreContext,
cfg: &DatabaseConfig,
storage: &DatabaseStorage,
backup_id: &str,
logger: &JobLogger,
) -> UploadResult {
let agent_id = ctx.edge_key.agent_id.clone();
let failed = |error: String| UploadResult {
storage_id: storage.id.clone(),
success: false,
error: Some(error),
remote_file_path: None,
total_size: None,
};
let backup_storage_id = match ctx
.api
.backup_upload_init(agent_id.clone(), cfg.generated_id.clone(), storage.id.clone(), backup_id, Some("sync"))
.await
{
Ok(Some(response)) => response.backup_storage.id,
Ok(None) => {
logger.log("error", "Upload init returned empty response");
return failed("backup_upload_init returned empty response".into());
}
Err(e) => {
logger.log("error", format!("Upload init failed: {e}"));
return failed("backup_upload_init failed".into());
}
};
match replicate(ctx, cfg, storage, logger).await {
Ok(stats) => {
let path = replica_path(storage, &cfg.generated_id);
match ctx
.api
.backup_upload_sync_status(agent_id, cfg.generated_id.clone(), backup_storage_id, path.clone(), &stats, backup_id)
.await
{
Ok(_) => UploadResult {
storage_id: storage.id.clone(),
success: true,
error: None,
remote_file_path: Some(path),
// No total size: rclone does not report the replica's size (backups.file_size stays null).
total_size: None,
},
Err(e) => {
logger.log("error", format!("Upload status update failed for {}: {e}", storage.id));
failed(e.to_string())
}
}
}
Err(e) => {
logger.log("error", format!("Sync to storage {} failed: {e:#}", storage.id));
if let Err(err) = ctx
.api
.backup_upload_status(agent_id, cfg.generated_id.clone(), backup_storage_id, "failed", String::new(), 0u64, backup_id)
.await
{
logger.log("error", format!("Failed-status update failed for {}: {err}", storage.id));
}
failed(format!("{e:#}"))
}
}
}
+65
View File
@@ -0,0 +1,65 @@
use crate::services::restic::excludes::bracket_negation;
/// P0 exclude patterns → `rclone sync --exclude` values. rclone filters are
/// relative to the sync root (spike 2026-10-03: 19/19 patterns give the P0 file set).
pub fn sync_excludes(patterns: &[String]) -> Vec<String> {
let mut out = Vec::new();
for raw in patterns {
let pattern = raw.trim();
let (anchored, rest) = match pattern.strip_prefix('/') {
Some(rest) => (true, rest),
None => (false, pattern),
};
let mut rest = rest.trim_end_matches('/');
if !anchored {
// Unanchored rclone patterns already match at any depth; a leading
// `**/` would stop them from matching at the root.
while let Some(stripped) = rest.strip_prefix("**/") {
rest = stripped;
}
}
if rest.is_empty() {
continue;
}
let rest = bracket_negation(rest);
// Collapse repeated /**/ into a single /**/
let rest = collapse_double_stars(&rest);
let full = if anchored { format!("/{rest}") } else { rest };
for variant in zero_dir_variants(&full) {
out.push(variant.clone());
out.push(format!("{variant}/**"));
}
}
out
}
/// Collapse repeated `/**/` sequences into a single `/**/`.
/// E.g., `a/**/**/b` becomes `a/**/b`.
fn collapse_double_stars(pattern: &str) -> String {
let mut result = pattern.to_string();
while result.contains("/**/**/") {
result = result.replace("/**/**/", "/**/");
}
result
}
/// rclone's `**` between slashes needs at least one directory; globset's also
/// matches none. Emit every combination of each `/**/` kept or collapsed to `/`.
fn zero_dir_variants(pattern: &str) -> Vec<String> {
let parts: Vec<&str> = pattern.split("/**/").collect();
let gaps = parts.len() - 1;
// ponytail: 2^gaps variants; past 4 gaps only "all kept" and "all collapsed".
if gaps > 4 {
return vec![pattern.to_string(), pattern.replace("/**/", "/")];
}
(0..1u32 << gaps)
.map(|mask| {
let mut variant = parts[0].to_string();
for (i, part) in parts[1..].iter().enumerate() {
variant.push_str(if mask & (1 << i) == 0 { "/**/" } else { "/" });
variant.push_str(part);
}
variant
})
.collect()
}
+6
View File
@@ -0,0 +1,6 @@
//! Sync mode: a destination kept identical to a files source (`rclone sync`).
//! Replication, not backup: deletions propagate.
pub mod backup;
pub mod excludes;
pub mod stats;
pub mod rclone;
+86
View File
@@ -0,0 +1,86 @@
use super::excludes::sync_excludes;
use super::stats::{SyncStats, parse_log};
use crate::domain::files::matcher::{exclude_patterns, one_file_system};
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::services::storage::providers::rclone::helpers::{remote_target, write_config};
use anyhow::{Context, Result, anyhow, bail};
use std::process::Stdio;
use tokio::process::Command;
/// `<remote>:<base>/<folder>/sync/<generated_id>/current`
pub fn replica_target(remote_name: &str, base_path: &str, folder: &str, generated_id: &str) -> String {
remote_target(remote_name, base_path, &format!("{folder}/sync/{generated_id}/current"))
}
/// Mirrors `cfg.path` onto `dest` (`<remote>:<path>`). rclone skips its deletions when an error
/// happens before the delete phase ("not deleting files as there were IO errors");
/// an error during the delete phase can leave the replica partially updated.
/// An empty source is refused so an unmounted volume cannot wipe the replica.
pub async fn sync_dir(config_text: &str, dest: &str, cfg: &DatabaseConfig, logger: &JobLogger) -> Result<SyncStats> {
let root = std::fs::canonicalize(&cfg.path)
.with_context(|| format!("cannot read source directory {}", cfg.path))?;
if !root.is_dir() {
bail!("source path {} is not a directory", root.display());
}
if std::fs::read_dir(&root)
.with_context(|| format!("cannot read source directory {}", root.display()))?
.next()
.is_none()
{
bail!(
"source directory {} is empty; refusing to sync so an unmounted volume cannot wipe the replica",
root.display()
);
}
let config = write_config(config_text)?;
let mut cmd = Command::new("rclone");
cmd.arg("--config")
.arg(config.path())
.arg("sync")
.arg(&root)
.arg(dest)
.args([
"--links", "--use-json-log", "--stats", "1h", "--stats-log-level", "NOTICE", "--retries", "1",
// An unreachable endpoint fails in ~1–2 min instead of blocking the next runs.
"--contimeout", "30s", "--low-level-retries", "3",
]);
if one_file_system(cfg) {
cmd.arg("--one-file-system");
}
for pattern in sync_excludes(&exclude_patterns(cfg)) {
cmd.arg("--exclude").arg(pattern);
}
logger.log("info", format!("Syncing {} to {dest}", root.display()));
let out = cmd
.stdin(Stdio::null())
.kill_on_drop(true)
.output()
.await
.map_err(|e| match e.kind() {
std::io::ErrorKind::NotFound => anyhow!("rclone binary not found"),
_ => anyhow!(e).context("failed to start rclone"),
})?;
let log = parse_log(&String::from_utf8_lossy(&out.stderr));
for error in &log.errors {
logger.log("warn", format!("rclone: {error}"));
}
if !out.status.success() {
let reason = log
.stats
.as_ref()
.and_then(|s| s.last_error.clone())
.or_else(|| log.errors.last().cloned())
.unwrap_or_else(|| "no error message".to_string());
bail!("rclone sync failed ({}): {reason}; the replica may be partially updated", out.status);
}
let stats = log.stats.unwrap_or_default();
logger.log(
"info",
format!("Synced: {} file(s) transferred ({} bytes), {} deleted", stats.transfers, stats.bytes, stats.deletes),
);
Ok(stats)
}
+56
View File
@@ -0,0 +1,56 @@
use serde::Deserialize;
use serde_json::Value;
/// Final `stats` object of `rclone --use-json-log --stats 1h --stats-log-level NOTICE`.
#[derive(Debug, Default, Deserialize, PartialEq)]
pub struct SyncStats {
#[serde(default)]
pub bytes: u64,
#[serde(default)]
pub transfers: u64,
#[serde(default)]
pub deletes: u64,
#[serde(default)]
pub errors: u64,
#[serde(default, rename = "lastError")]
pub last_error: Option<String>,
}
/// What `rclone sync` printed on stderr: its last stats object and its error
/// messages (JSON `level: error|critical|alert|emergency` lines and any plain-text line).
#[derive(Debug, Default)]
pub struct SyncLog {
pub stats: Option<SyncStats>,
pub errors: Vec<String>,
}
pub fn parse_log(stderr: &str) -> SyncLog {
let mut log = SyncLog::default();
for line in stderr.lines().map(str::trim).filter(|l| !l.is_empty()) {
let Ok(value) = serde_json::from_str::<Value>(line) else {
if !log.errors.contains(&line.to_string()) {
log.errors.push(line.to_string());
}
continue;
};
if let Some(stats) = value.get("stats").and_then(|s| serde_json::from_value(s.clone()).ok()) {
log.stats = Some(stats);
}
if let Some(level) = value.get("level").and_then(Value::as_str) {
if matches!(level, "error" | "critical" | "alert" | "emergency") {
if let Some(msg) = value.get("msg").and_then(Value::as_str) {
let msg_str = msg.to_string();
if !log.errors.contains(&msg_str) {
log.errors.push(msg_str);
}
}
}
}
}
// Keep only the last 20 messages
if log.errors.len() > 20 {
log.errors = log.errors.into_iter().rev().take(20).collect::<Vec<_>>();
log.errors.reverse();
}
log
}
+616
View File
@@ -0,0 +1,616 @@
use crate::domain::files::matcher::ExcludeMatcher;
use std::path::Path;
fn matcher(patterns: &[&str]) -> ExcludeMatcher {
let owned: Vec<String> = patterns.iter().map(|p| p.to_string()).collect();
ExcludeMatcher::new(&owned).unwrap()
}
#[test]
fn matcher_unanchored_matches_any_depth() {
let m = matcher(&["node_modules", "*.log"]);
assert!(m.is_excluded(Path::new("node_modules")));
assert!(m.is_excluded(Path::new("app/node_modules")));
assert!(m.is_excluded(Path::new("x.log")));
assert!(m.is_excluded(Path::new("a/b/x.log")));
assert!(!m.is_excluded(Path::new("src/main.rs")));
}
#[test]
fn matcher_anchored_matches_root_only() {
let m = matcher(&["/cache", "/build/"]);
assert!(m.is_excluded(Path::new("cache")));
assert!(!m.is_excluded(Path::new("app/cache")));
assert!(m.is_excluded(Path::new("build")));
}
#[test]
fn matcher_star_does_not_cross_separator_but_double_star_does() {
let m = matcher(&["/logs/*.txt", "/data/**/tmp"]);
assert!(m.is_excluded(Path::new("logs/a.txt")));
assert!(!m.is_excluded(Path::new("logs/sub/a.txt")));
assert!(m.is_excluded(Path::new("data/tmp")));
assert!(m.is_excluded(Path::new("data/a/b/tmp")));
}
#[test]
fn matcher_ignores_blank_patterns_and_rejects_invalid_ones() {
let m = matcher(&["", " "]);
assert!(!m.is_excluded(Path::new("anything")));
assert!(ExcludeMatcher::new(&["[".to_string()]).is_err());
}
use crate::domain::files::backup;
use crate::services::backup::logger::JobLogger;
use crate::services::config::{DatabaseConfig, DbType};
use flate2::read::GzDecoder;
use std::collections::HashMap;
use std::fs;
use std::io::Read;
use std::sync::Arc;
pub(crate) fn files_config(root: &Path, exclude: &[&str]) -> DatabaseConfig {
let mut options = HashMap::new();
options.insert("exclude".to_string(), serde_json::json!(exclude));
DatabaseConfig {
name: "files-test".to_string(),
database: String::new(),
db_type: DbType::Files,
username: String::new(),
password: String::new(),
port: 0,
host: String::new(),
generated_id: uuid::Uuid::new_v4().to_string(),
path: root.to_string_lossy().into_owned(),
max_packet_size: String::new(),
volume_name: String::new(),
container_name: None,
options,
}
}
fn sample_tree(root: &Path) {
fs::create_dir_all(root.join("docs")).unwrap();
fs::write(root.join("docs/a.txt"), "v1").unwrap();
fs::create_dir_all(root.join("app/node_modules")).unwrap();
fs::write(root.join("app/node_modules/dep.js"), "dep").unwrap();
fs::write(root.join("app/index.js"), "app").unwrap();
std::os::unix::fs::symlink("docs/a.txt", root.join("link")).unwrap();
}
#[tokio::test]
async fn backup_writes_tar_gz_without_excluded_paths() {
let src = tempfile::TempDir::new().unwrap();
sample_tree(src.path());
let cfg = files_config(src.path(), &["node_modules"]);
let out = tempfile::TempDir::new().unwrap();
let archive = backup::run(
cfg.clone(),
out.path().to_path_buf(),
Arc::new(JobLogger::new()),
)
.await
.unwrap();
assert_eq!(
archive.file_name().unwrap().to_string_lossy(),
format!("{}.tar.gz", cfg.generated_id)
);
let mut ar = tar::Archive::new(GzDecoder::new(fs::File::open(&archive).unwrap()));
let mut files = Vec::new();
let mut link_is_symlink = false;
let mut a_txt = String::new();
for entry in ar.entries().unwrap() {
let mut entry = entry.unwrap();
let name = entry
.path()
.unwrap()
.to_string_lossy()
.trim_end_matches('/')
.to_string();
if name == "link" {
link_is_symlink = entry.header().entry_type().is_symlink();
}
if name == "docs/a.txt" {
entry.read_to_string(&mut a_txt).unwrap();
}
files.push(name);
}
assert!(files.contains(&"docs/a.txt".to_string()));
assert!(files.contains(&"app/index.js".to_string()));
assert!(
!files.iter().any(|n| n.contains("node_modules")),
"excluded path archived: {files:?}"
);
assert!(link_is_symlink, "symlink must be stored as a link");
assert_eq!(a_txt, "v1", "file content must be archived");
}
fn entry_names(archive: &Path) -> Vec<String> {
let mut ar = tar::Archive::new(GzDecoder::new(fs::File::open(archive).unwrap()));
ar.entries()
.unwrap()
.map(|e| {
e.unwrap()
.path()
.unwrap()
.to_string_lossy()
.trim_end_matches('/')
.to_string()
})
.collect()
}
#[tokio::test]
async fn backup_does_not_archive_its_own_output() {
let src = tempfile::TempDir::new().unwrap();
sample_tree(src.path());
let out_dir = src.path().join("backups");
fs::create_dir_all(&out_dir).unwrap();
let cfg = files_config(src.path(), &[]);
let archive = backup::run(cfg, out_dir.clone(), Arc::new(JobLogger::new()))
.await
.unwrap();
assert!(archive.exists());
let names = entry_names(&archive);
assert!(names.contains(&"docs/a.txt".to_string()));
assert!(
!names.iter().any(|n| n.starts_with("backups")),
"own output archived: {names:?}"
);
}
#[tokio::test]
async fn backup_skips_sockets_with_a_warning() {
let src = tempfile::TempDir::new().unwrap();
sample_tree(src.path());
let _listener = std::os::unix::net::UnixListener::bind(src.path().join("sock")).unwrap();
let cfg = files_config(src.path(), &[]);
let out = tempfile::TempDir::new().unwrap();
let archive = backup::run(cfg, out.path().to_path_buf(), Arc::new(JobLogger::new()))
.await
.unwrap();
let names = entry_names(&archive);
assert!(names.contains(&"docs/a.txt".to_string()));
assert!(
!names.iter().any(|n| n == "sock"),
"socket archived: {names:?}"
);
}
#[tokio::test]
async fn backup_fails_when_source_is_not_a_directory() {
let src = tempfile::TempDir::new().unwrap();
let file = src.path().join("plain.txt");
fs::write(&file, "x").unwrap();
let cfg = files_config(&file, &[]);
let out = tempfile::TempDir::new().unwrap();
let result = backup::run(cfg, out.path().to_path_buf(), Arc::new(JobLogger::new())).await;
assert!(
result.is_err(),
"a file path must not produce an empty archive"
);
}
use crate::domain::files::restore;
#[tokio::test]
async fn restore_is_a_mirror_that_keeps_excluded_paths() {
let src = tempfile::TempDir::new().unwrap();
let root = src.path();
sample_tree(root);
let cfg = files_config(root, &["node_modules"]);
let out = tempfile::TempDir::new().unwrap();
let logger = Arc::new(JobLogger::new());
let archive = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone())
.await
.unwrap();
fs::write(root.join("docs/a.txt"), "v2").unwrap();
fs::write(root.join("new.txt"), "new").unwrap();
fs::create_dir_all(root.join("newdir/sub")).unwrap();
fs::write(root.join("app/node_modules/dep.js"), "changed").unwrap();
restore::run(cfg, archive, logger).await.unwrap();
assert_eq!(fs::read_to_string(root.join("docs/a.txt")).unwrap(), "v1");
assert_eq!(
fs::read_to_string(root.join("app/index.js")).unwrap(),
"app"
);
assert!(
!root.join("new.txt").exists(),
"file created after backup must be deleted"
);
assert!(
!root.join("newdir").exists(),
"dir created after backup must be deleted"
);
assert_eq!(
fs::read_to_string(root.join("app/node_modules/dep.js")).unwrap(),
"changed",
"excluded path must be left untouched"
);
assert!(
fs::symlink_metadata(root.join("link"))
.unwrap()
.file_type()
.is_symlink()
);
}
#[test]
fn ensure_restorable_rejects_dangerous_targets() {
assert!(restore::ensure_restorable(Path::new("/")).is_err());
assert!(restore::ensure_restorable(Path::new("")).is_err());
assert!(restore::ensure_restorable(Path::new("relative/dir")).is_err());
assert!(restore::ensure_restorable(Path::new("/definitely/not/here")).is_err());
let dir = tempfile::TempDir::new().unwrap();
assert!(restore::ensure_restorable(dir.path()).is_ok());
let escapes_to_root = dir.path().join("../../../../../../../../../..");
assert!(restore::ensure_restorable(&escapes_to_root).is_err());
}
#[tokio::test]
async fn restore_refuses_corrupt_archive_without_touching_data() {
let src = tempfile::TempDir::new().unwrap();
let root = src.path();
sample_tree(root);
let cfg = files_config(root, &[]);
let out = tempfile::TempDir::new().unwrap();
let logger = Arc::new(JobLogger::new());
let archive = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone())
.await
.unwrap();
let len = fs::metadata(&archive).unwrap().len();
fs::OpenOptions::new()
.write(true)
.open(&archive)
.unwrap()
.set_len(len / 2)
.unwrap();
fs::write(root.join("keep.txt"), "keep").unwrap();
let result = restore::run(cfg, archive, logger).await;
assert!(result.is_err(), "a truncated archive must be refused");
assert!(
root.join("keep.txt").exists(),
"data must survive a refused restore"
);
assert!(
root.join("docs/a.txt").exists(),
"data must survive a refused restore"
);
}
#[tokio::test]
async fn ping_reports_directory_reachability() {
let dir = tempfile::TempDir::new().unwrap();
let ok = files_config(dir.path(), &[]);
assert!(crate::domain::files::ping::run(ok).await.unwrap());
let missing = files_config(Path::new("/definitely/not/here"), &[]);
assert!(!crate::domain::files::ping::run(missing).await.unwrap());
}
#[tokio::test]
async fn restore_survives_archive_stored_inside_the_source() {
let src = tempfile::TempDir::new().unwrap();
let root = src.path();
sample_tree(root);
let cfg = files_config(root, &[]);
let tmp_dir = root.join(".tmp-restore");
fs::create_dir_all(&tmp_dir).unwrap();
let logger = Arc::new(JobLogger::new());
let archive = backup::run(cfg.clone(), tmp_dir.clone(), logger.clone())
.await
.unwrap();
fs::write(root.join("docs/a.txt"), "v2").unwrap();
fs::write(root.join("new.txt"), "new").unwrap();
restore::run(cfg, archive.clone(), logger).await.unwrap();
assert_eq!(fs::read_to_string(root.join("docs/a.txt")).unwrap(), "v1");
assert_eq!(
fs::read_to_string(root.join("app/node_modules/dep.js")).unwrap(),
"dep"
);
assert!(!root.join("new.txt").exists());
assert!(
archive.exists(),
"the directory holding the archive must not be wiped"
);
}
#[tokio::test]
async fn restore_survives_archive_stored_directly_in_the_source() {
let src = tempfile::TempDir::new().unwrap();
let root = src.path();
sample_tree(root);
let cfg = files_config(root, &[]);
let out = tempfile::TempDir::new().unwrap();
let logger = Arc::new(JobLogger::new());
let outside = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone())
.await
.unwrap();
let archive = root.join("snapshot.tar.gz");
fs::copy(&outside, &archive).unwrap();
fs::write(root.join("new.txt"), "new").unwrap();
restore::run(cfg, archive.clone(), logger).await.unwrap();
assert_eq!(fs::read_to_string(root.join("docs/a.txt")).unwrap(), "v1");
assert!(!root.join("new.txt").exists());
assert!(archive.exists(), "the archive itself must not be wiped");
}
#[tokio::test]
async fn restore_never_follows_symlinked_directories() {
let outside = tempfile::TempDir::new().unwrap();
fs::write(outside.path().join("secret.txt"), "outside").unwrap();
let src = tempfile::TempDir::new().unwrap();
let root = src.path();
sample_tree(root);
std::os::unix::fs::symlink(outside.path(), root.join("old_link")).unwrap();
let cfg = files_config(root, &[]);
let out = tempfile::TempDir::new().unwrap();
let logger = Arc::new(JobLogger::new());
let archive = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone())
.await
.unwrap();
std::os::unix::fs::symlink(outside.path(), root.join("late_link")).unwrap();
restore::run(cfg, archive, logger).await.unwrap();
assert_eq!(
fs::read_to_string(outside.path().join("secret.txt")).unwrap(),
"outside"
);
assert!(
fs::symlink_metadata(root.join("late_link")).is_err(),
"link created after backup must be removed"
);
assert!(
fs::symlink_metadata(root.join("old_link"))
.unwrap()
.file_type()
.is_symlink()
);
}
#[tokio::test]
async fn restore_refuses_archive_with_corrupt_gzip_trailer() {
let src = tempfile::TempDir::new().unwrap();
let root = src.path();
sample_tree(root);
let cfg = files_config(root, &[]);
let out = tempfile::TempDir::new().unwrap();
let logger = Arc::new(JobLogger::new());
let archive = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone())
.await
.unwrap();
// The last 8 bytes are CRC32 + ISIZE; tar itself never reads them.
let mut bytes = fs::read(&archive).unwrap();
let idx = bytes.len() - 6;
bytes[idx] ^= 0xff;
fs::write(&archive, bytes).unwrap();
fs::write(root.join("keep.txt"), "keep").unwrap();
assert!(restore::run(cfg, archive, logger).await.is_err());
assert_eq!(fs::read_to_string(root.join("keep.txt")).unwrap(), "keep");
assert_eq!(fs::read_to_string(root.join("docs/a.txt")).unwrap(), "v1");
}
#[tokio::test]
async fn restore_anchored_exclude_applies_through_a_symlinked_path() {
let base = tempfile::TempDir::new().unwrap();
let real = base.path().join("real");
fs::create_dir_all(real.join("app/node_modules")).unwrap();
fs::create_dir_all(real.join("node_modules")).unwrap();
fs::write(real.join("app/node_modules/x"), "x1").unwrap();
fs::write(real.join("node_modules/y"), "y1").unwrap();
let link = base.path().join("link");
std::os::unix::fs::symlink(&real, &link).unwrap();
let cfg = files_config(&link, &["/app/node_modules"]);
let out = tempfile::TempDir::new().unwrap();
let logger = Arc::new(JobLogger::new());
let archive = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone())
.await
.unwrap();
fs::write(real.join("app/node_modules/x"), "x2").unwrap();
fs::write(real.join("node_modules/y"), "y2").unwrap();
restore::run(cfg, archive, logger).await.unwrap();
assert_eq!(
fs::read_to_string(real.join("app/node_modules/x")).unwrap(),
"x2",
"anchored exclude must be kept"
);
assert_eq!(
fs::read_to_string(real.join("node_modules/y")).unwrap(),
"y1",
"top-level node_modules is not excluded"
);
}
#[test]
fn wipe_keeps_only_excluded_paths_on_a_single_filesystem() {
for one_file_system in [false, true] {
let src = tempfile::TempDir::new().unwrap();
let root = src.path();
sample_tree(root);
let m = matcher(&["node_modules"]);
let failures =
restore::wipe_except_excluded(root, &m, one_file_system, &JobLogger::new()).unwrap();
assert_eq!(failures, 0);
assert!(root.join("app/node_modules/dep.js").exists());
assert!(!root.join("app/index.js").exists());
assert!(!root.join("docs").exists());
assert!(fs::symlink_metadata(root.join("link")).is_err());
}
}
use crate::domain::files::platform::is_root;
use std::os::unix::fs::PermissionsExt;
#[test]
fn unpack_retries_without_metadata_when_ownership_cannot_be_set() {
if is_root() {
return; // root may chown to uid 0, so the first pass would not fail
}
let dir = tempfile::TempDir::new().unwrap();
let archive = dir.path().join("root-owned.tar.gz");
let gz = flate2::write::GzEncoder::new(
fs::File::create(&archive).unwrap(),
flate2::Compression::default(),
);
let mut builder = tar::Builder::new(gz);
for (name, body) in [("a.txt", "alpha"), ("sub/b.txt", "beta")] {
let mut header = tar::Header::new_gnu();
header.set_path(name).unwrap();
header.set_uid(0);
header.set_gid(0);
header.set_mode(0o644);
header.set_size(body.len() as u64);
header.set_cksum();
builder.append(&header, body.as_bytes()).unwrap();
}
builder.into_inner().unwrap().finish().unwrap();
let target = tempfile::TempDir::new().unwrap();
let logger = JobLogger::new();
restore::unpack(
&fs::File::open(&archive).unwrap(),
target.path(),
true,
&logger,
)
.unwrap();
assert_eq!(
fs::read_to_string(target.path().join("a.txt")).unwrap(),
"alpha"
);
assert_eq!(
fs::read_to_string(target.path().join("sub/b.txt")).unwrap(),
"beta"
);
assert!(
logger
.into_entries()
.iter()
.any(|e| e.level == "warn" && e.message.contains("retrying without metadata")),
"the chown failure must go through the fallback"
);
}
#[tokio::test]
async fn restore_extracts_even_when_some_entries_cannot_be_removed() {
if is_root() {
return; // root ignores directory permissions
}
let src = tempfile::TempDir::new().unwrap();
let root = src.path();
sample_tree(root);
let cfg = files_config(root, &[]);
let out = tempfile::TempDir::new().unwrap();
let logger = Arc::new(JobLogger::new());
let archive = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone())
.await
.unwrap();
fs::write(root.join("docs/a.txt"), "v2").unwrap();
fs::write(root.join("new.txt"), "new").unwrap();
let locked = root.join("locked");
fs::create_dir(&locked).unwrap();
fs::write(locked.join("stale.txt"), "stale").unwrap();
fs::set_permissions(&locked, fs::Permissions::from_mode(0o555)).unwrap();
let result = restore::run(cfg, archive, logger).await;
fs::set_permissions(&locked, fs::Permissions::from_mode(0o755)).unwrap();
let err = result.expect_err("an entry that could not be removed must fail the restore");
assert!(
err.to_string().contains("could not be removed"),
"unexpected error: {err:#}"
);
assert_eq!(
fs::read_to_string(root.join("docs/a.txt")).unwrap(),
"v1",
"the archive must still be extracted"
);
assert_eq!(
fs::read_to_string(root.join("app/index.js")).unwrap(),
"app"
);
assert!(
!root.join("new.txt").exists(),
"removable entries must still be wiped"
);
assert!(locked.join("stale.txt").exists());
}
#[tokio::test]
async fn backup_fails_when_an_entry_cannot_be_read() {
if is_root() {
return; // root reads mode 0o000 files
}
let src = tempfile::TempDir::new().unwrap();
sample_tree(src.path());
let secret = src.path().join("secret.txt");
fs::write(&secret, "secret").unwrap();
fs::set_permissions(&secret, fs::Permissions::from_mode(0o000)).unwrap();
let cfg = files_config(src.path(), &[]);
let out = tempfile::TempDir::new().unwrap();
let result = backup::run(cfg, out.path().to_path_buf(), Arc::new(JobLogger::new())).await;
fs::set_permissions(&secret, fs::Permissions::from_mode(0o644)).unwrap();
let err = result.expect_err("an unreadable entry must fail the backup");
assert!(
err.to_string().contains("1 entry could not be read"),
"unexpected error: {err:#}"
);
}
#[tokio::test]
async fn restore_keeps_special_files() {
let src = tempfile::TempDir::new().unwrap();
let root = src.path();
sample_tree(root);
let cfg = files_config(root, &[]);
let out = tempfile::TempDir::new().unwrap();
let logger = Arc::new(JobLogger::new());
let archive = backup::run(cfg.clone(), out.path().to_path_buf(), logger.clone())
.await
.unwrap();
let _listener = std::os::unix::net::UnixListener::bind(root.join("sock")).unwrap();
restore::run(cfg, archive, logger).await.unwrap();
assert!(
fs::symlink_metadata(root.join("sock")).is_ok(),
"socket must survive a restore"
);
assert_eq!(fs::read_to_string(root.join("docs/a.txt")).unwrap(), "v1");
}
+1
View File
@@ -8,3 +8,4 @@ mod valkey;
mod firebird;
mod mssql;
mod docker_volume;
pub(crate) mod files;
+102
View File
@@ -228,6 +228,72 @@ fn resolve_dashboard_config_noop_when_not_encrypted() {
assert!(status.resolved_config.is_none());
}
#[test]
fn ping_without_engine_fields_defaults_to_none() {
let status: DatabaseStatus = serde_json::from_value(json!({
"dbms": "files",
"generatedId": "16678159-ff7e-4c97-8c83-0adeff214681",
"encrypt": false,
"data": { "backup": { "action": false, "cron": null },
"restore": { "action": false, "file": null, "metaFile": null, "size": null } }
}))
.unwrap();
assert!(status.data.backup.engine.is_none());
assert!(status.data.restore.engine.is_none());
assert!(status.data.restore.snapshot_id.is_none());
assert!(status.data.restore.storage.is_none());
}
#[test]
fn resolve_restore_storage_decrypts_the_snapshot_channel() {
use crate::services::status::resolve_restore_storage;
use base64::{engine::general_purpose, Engine};
let master_key_b64 = general_purpose::STANDARD.encode([7u8; 32]);
let channel = r#"[{"id":"ch-1","provider":"s3","folderName":"backups","config":{"endPointUrl":"s3.example.com","bucketName":"b"}}]"#;
let snapshot_id = "ab".repeat(32);
let mut status: DatabaseStatus = serde_json::from_value(json!({
"dbms": "files",
"generatedId": "16678159-ff7e-4c97-8c83-0adeff214681",
"encrypt": false,
"data": {
"backup": { "action": false, "cron": "0 * * * *", "engine": "restic" },
"restore": { "action": true, "file": null, "metaFile": null, "size": null,
"engine": "restic", "snapshotId": snapshot_id,
"storageCiphertext": encrypt_json_gcm(channel.as_bytes(), &master_key_b64) }
}
}))
.unwrap();
assert_eq!(status.data.backup.engine.as_deref(), Some("restic"));
assert_eq!(status.data.restore.snapshot_id.as_deref(), Some(snapshot_id.as_str()));
resolve_restore_storage(&mut status, &master_key_b64).unwrap();
let storage = status.data.restore.storage.expect("decrypted");
assert_eq!(storage.id, "ch-1");
assert_eq!(storage.provider, "s3");
assert_eq!(storage.folder_name.as_deref(), Some("backups"));
}
#[test]
fn resolve_restore_storage_rejects_a_bad_ciphertext() {
use crate::services::status::resolve_restore_storage;
use base64::{engine::general_purpose, Engine};
let mut status: DatabaseStatus = serde_json::from_value(json!({
"dbms": "files",
"generatedId": "16678159-ff7e-4c97-8c83-0adeff214681",
"encrypt": false,
"data": { "backup": { "action": false, "cron": null },
"restore": { "action": true, "engine": "restic", "storageCiphertext": "bm9wZQ==" } }
}))
.unwrap();
assert!(resolve_restore_storage(&mut status, &general_purpose::STANDARD.encode([7u8; 32])).is_err());
assert!(status.data.restore.storage.is_none());
}
fn encrypt_json_gcm(plaintext: &[u8], master_key_b64: &str) -> String {
use aes_gcm::aead::{Aead, KeyInit};
use aes_gcm::{Aes256Gcm, Key, Nonce};
@@ -243,3 +309,39 @@ fn encrypt_json_gcm(plaintext: &[u8], master_key_b64: &str) -> String {
data.extend_from_slice(&ct);
general_purpose::STANDARD.encode(data)
}
#[test]
fn ping_payload_sends_method_only_when_set() {
use crate::services::api::endpoints::status::DatabasePayload;
let payload = |method: Option<&'static str>| DatabasePayload {
name: "docs",
dbms: "files",
generated_id: "g",
ping_status: true,
method,
};
let with = serde_json::to_value(payload(Some("sync"))).unwrap();
assert_eq!(with["method"], "sync");
let without = serde_json::to_value(payload(None)).unwrap();
assert!(without.get("method").is_none(), "{without}");
}
#[test]
fn upload_status_sends_sync_counters_only_when_set() {
use crate::services::api::endpoints::agent::backup::upload::status::StatusUploadRequest;
let request = |files_transferred: Option<u64>, files_deleted: Option<u64>| StatusUploadRequest {
generated_id: "g".into(),
backup_storage_id: "bs".into(),
status: "success".into(),
path: "p".into(),
size: 3,
backup_id: "b".into(),
files_transferred,
files_deleted,
};
let with = serde_json::to_value(request(Some(2), Some(1))).unwrap();
assert_eq!(with["filesTransferred"], 2);
assert_eq!(with["filesDeleted"], 1);
let without = serde_json::to_value(request(None, None)).unwrap();
assert!(without.get("filesTransferred").is_none() && without.get("filesDeleted").is_none(), "{without}");
}
+1 -1
View File
@@ -20,7 +20,7 @@ use tempfile::NamedTempFile;
use wiremock::matchers::{body_partial_json, method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
fn ctx_pointing_at(base_url: String) -> Context {
pub(crate) fn ctx_pointing_at(base_url: String) -> Context {
Context {
edge_key: EdgeKey {
server_url: String::new(),
+101 -8
View File
@@ -1,7 +1,7 @@
use crate::core::context::Context;
use crate::services::api::ApiClient;
use crate::services::config::ConfigService;
use crate::services::config::{build_config, DatabasesConfig, InputDatabaseConfig};
use crate::services::config::{DatabasesConfig, InputDatabaseConfig, build_config, files_method};
use crate::utils::edge_key::EdgeKey;
use std::io::Write;
use std::sync::Arc;
@@ -176,24 +176,39 @@ fn keep_ownership_extraction_logic() {
// true → keep ownership
let mut opts: HashMap<String, Value> = HashMap::new();
opts.insert("keep_ownership".to_string(), Value::Bool(true));
let keep = opts.get("keep_ownership").and_then(|v| v.as_bool()).unwrap_or(false);
let keep = opts
.get("keep_ownership")
.and_then(|v| v.as_bool())
.unwrap_or(false);
assert!(keep, "should keep ownership when flag is true");
// false → strip
let mut opts2: HashMap<String, Value> = HashMap::new();
opts2.insert("keep_ownership".to_string(), Value::Bool(false));
let keep2 = opts2.get("keep_ownership").and_then(|v| v.as_bool()).unwrap_or(false);
let keep2 = opts2
.get("keep_ownership")
.and_then(|v| v.as_bool())
.unwrap_or(false);
assert!(!keep2, "should strip when flag is false");
// missing → strip
let opts3: HashMap<String, Value> = HashMap::new();
let keep3 = opts3.get("keep_ownership").and_then(|v| v.as_bool()).unwrap_or(false);
let keep3 = opts3
.get("keep_ownership")
.and_then(|v| v.as_bool())
.unwrap_or(false);
assert!(!keep3, "should strip when key absent");
// wrong type → strip
let mut opts4: HashMap<String, Value> = HashMap::new();
opts4.insert("keep_ownership".to_string(), Value::String("yes".to_string()));
let keep4 = opts4.get("keep_ownership").and_then(|v| v.as_bool()).unwrap_or(false);
opts4.insert(
"keep_ownership".to_string(),
Value::String("yes".to_string()),
);
let keep4 = opts4
.get("keep_ownership")
.and_then(|v| v.as_bool())
.unwrap_or(false);
assert!(!keep4, "should strip when value is not bool");
}
@@ -258,7 +273,9 @@ fn docker_volume_requires_volume_name() {
);
let service = ConfigService::new(test_context());
let err = service.load(Some(file.path().to_str().unwrap())).unwrap_err();
let err = service
.load(Some(file.path().to_str().unwrap()))
.unwrap_err();
assert!(err.contains("volume_name"), "error was: {err}");
}
@@ -323,7 +340,9 @@ fn databases_config_roundtrips_through_serde() {
)
.unwrap();
let cfg = build_config(input).unwrap();
let wrapped = DatabasesConfig { databases: vec![cfg] };
let wrapped = DatabasesConfig {
databases: vec![cfg],
};
let json = serde_json::to_string(&wrapped).unwrap();
let back: DatabasesConfig = serde_json::from_str(&json).unwrap();
@@ -331,3 +350,77 @@ fn databases_config_roundtrips_through_serde() {
assert_eq!(back.databases[0].db_type.as_str(), "postgresql");
assert_eq!(back.databases[0].password, "secret");
}
#[test]
fn files_type_parses_with_options() {
let input: InputDatabaseConfig = serde_json::from_str(
r#"{
"name": "shared",
"type": "files",
"path": "/data/files",
"generated_id": "16678159-ff7e-4c97-8c83-0adeff214681",
"options": { "exclude": ["node_modules", "/cache"], "one_file_system": true }
}"#,
)
.unwrap();
let cfg = build_config(input).unwrap();
assert_eq!(cfg.db_type.as_str(), "files");
assert_eq!(cfg.path, "/data/files");
assert_eq!(
cfg.options["exclude"],
serde_json::json!(["node_modules", "/cache"])
);
assert_eq!(cfg.options["one_file_system"], serde_json::json!(true));
}
#[test]
fn files_type_requires_path() {
let input: InputDatabaseConfig = serde_json::from_str(
r#"{ "name": "shared", "type": "files", "generated_id": "16678159-ff7e-4c97-8c83-0adeff214681" }"#,
)
.unwrap();
let err = build_config(input).unwrap_err();
assert!(err.contains("path"), "unexpected error: {err}");
}
#[test]
fn files_type_rejects_relative_and_root_paths() {
for bad in ["data/files", "/", "//"] {
let json = format!(
r#"{{ "name": "shared", "type": "files", "path": "{bad}", "generated_id": "16678159-ff7e-4c97-8c83-0adeff214681" }}"#
);
let input: InputDatabaseConfig = serde_json::from_str(&json).unwrap();
let err = build_config(input).unwrap_err();
assert!(err.contains("absolute"), "path {bad:?} gave: {err}");
}
}
fn files_input(options: &str) -> InputDatabaseConfig {
serde_json::from_str(&format!(
r#"{{ "name": "docs", "type": "files", "path": "/data/files",
"generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"{options} }}"#
))
.unwrap()
}
#[test]
fn files_method_defaults_to_archive() {
let cfg = build_config(files_input("")).unwrap();
assert_eq!(files_method(&cfg), "archive");
}
#[test]
fn files_method_reads_snapshot_and_sync() {
let snap = build_config(files_input(r#", "options": { "method": "snapshot" }"#)).unwrap();
assert_eq!(files_method(&snap), "snapshot");
let sync = build_config(files_input(r#", "options": { "method": "sync" }"#)).unwrap();
assert_eq!(files_method(&sync), "sync");
}
#[test]
fn unknown_files_method_refuses_the_source() {
let err = build_config(files_input(r#", "options": { "method": "mirror" }"#)).unwrap_err();
assert!(err.contains("Unknown method"), "{err}");
assert!(err.contains("docs"), "{err}");
}
@@ -82,3 +82,38 @@ fn persist_cache_leaves_no_tmp_file() {
assert!(!tmp.exists(), "temp file should have been renamed away");
assert!(path.exists());
}
use crate::services::dashboard_config::local_only_ids;
use crate::services::status::payload_method;
fn files_cfg(gen_id: &str, method: Option<&str>) -> DatabaseConfig {
let options = method
.map(|m| format!(r#", "options": {{ "method": "{m}" }}"#))
.unwrap_or_default();
let json = format!(
r#"{{ "name": "docs", "type": "files", "path": "/data/files", "generated_id": "{gen_id}"{options} }}"#
);
build_config(serde_json::from_str::<InputDatabaseConfig>(&json).unwrap()).unwrap()
}
#[test]
fn local_only_ids_skips_sources_replaced_by_the_dashboard() {
let local = vec![cfg("local-a", ID_A, "h"), cfg("local-b", ID_B, "h")];
let dashboard = vec![cfg("dash-b", ID_B, "h")];
let ids = local_only_ids(&local, &dashboard);
assert!(ids.contains(ID_A));
assert!(!ids.contains(ID_B));
}
#[test]
fn payload_method_is_sent_for_local_files_sources_only() {
let local = files_cfg(ID_A, Some("sync"));
let ids = local_only_ids(&[local.clone()], &[]);
assert_eq!(payload_method(&local, &ids), Some("sync"));
assert_eq!(payload_method(&files_cfg(ID_A, None), &ids), Some("archive"));
// Replaced by a dashboard config: the dashboard owns the method.
let replaced = local_only_ids(&[local.clone()], &[local.clone()]);
assert_eq!(payload_method(&local, &replaced), None);
// Not a files source.
assert_eq!(payload_method(&cfg("pg", ID_A, "h"), &ids), None);
}
+190
View File
@@ -0,0 +1,190 @@
//! Spec B: the dashboard's local storage, served here by `rclone serve restic
//! --append-only` and `rclone serve webdav` on free ports. The restic tests need
//! restic on PATH (test container); the sync test only needs rclone.
use crate::domain::files::platform::is_root;
use crate::services::backup::logger::JobLogger;
use crate::services::restic::backup::snapshot;
use crate::services::restic::command::ResticRepo;
use crate::services::restic::restore::restore;
use crate::tests::domain::files::files_config;
use std::fs;
use std::path::Path;
use std::process::{Child, Command, Stdio};
use std::time::Duration;
use tempfile::TempDir;
pub(crate) const MASTER_KEY_B64: &str = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=";
pub(crate) struct RcloneServer {
child: Child,
pub base_url: String,
}
impl Drop for RcloneServer {
fn drop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
}
}
/// `rclone serve <kind> --baseurl /storage/<prefix> <root>` on a free port, user
/// `portabase`, credentials through the env like the dashboard does.
pub(crate) fn serve(kind: &str, prefix: &str, root: &Path, extra: &[&str], password: &str) -> RcloneServer {
let port = std::net::TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port();
let child = Command::new("rclone")
.args(["serve", kind, "--addr", &format!("127.0.0.1:{port}"), "--baseurl", &format!("/storage/{prefix}")])
.args(extra)
.arg(root)
.env("RCLONE_USER", "portabase")
.env("RCLONE_PASS", password)
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
.expect("rclone binary not found");
for _ in 0..100 {
if std::net::TcpStream::connect(("127.0.0.1", port)).is_ok() {
break;
}
std::thread::sleep(Duration::from_millis(50));
}
RcloneServer { child, base_url: format!("http://127.0.0.1:{port}") }
}
fn tree(root: &Path) {
fs::create_dir_all(root.join("docs")).unwrap();
fs::write(root.join("a.txt"), "v1").unwrap();
fs::write(root.join("docs/b.txt"), "b").unwrap();
}
fn local_repo(server: &RcloneServer, generated_id: &str, cache: &Path) -> ResticRepo {
ResticRepo::rest(&format!("{}/storage/restic", server.base_url), generated_id, "test-password", "s3cret", cache.to_path_buf())
}
#[tokio::test]
async fn local_snapshot_and_restore_go_through_the_append_only_server() {
let store = TempDir::new().unwrap();
let work = TempDir::new().unwrap();
let server = serve("restic", "restic", store.path(), &["--append-only"], "s3cret");
let src = work.path().join("src");
tree(&src);
let cfg = files_config(&src, &[]);
let repo = local_repo(&server, &cfg.generated_id, &work.path().join("cache"));
let logger = JobLogger::new();
let snap = snapshot(&repo, &cfg, "bs-1", &logger).await.unwrap();
let repo_dir = store.path().join(&cfg.generated_id);
assert!(repo_dir.join("config").exists(), "repository written on the served disk");
assert_eq!(fs::read_dir(repo_dir.join("locks")).unwrap().count(), 0, "lock removed despite --append-only");
fs::write(src.join("a.txt"), "changed").unwrap();
fs::write(src.join("extra.txt"), "x").unwrap();
restore(&repo, &cfg, &snap.snapshot_id, &logger).await.unwrap();
assert_eq!(fs::read_to_string(src.join("a.txt")).unwrap(), "v1");
assert!(!src.join("extra.txt").exists());
}
#[tokio::test]
async fn the_agent_cannot_forget_on_the_local_server() {
let store = TempDir::new().unwrap();
let work = TempDir::new().unwrap();
let server = serve("restic", "restic", store.path(), &["--append-only"], "s3cret");
let src = work.path().join("src");
tree(&src);
let cfg = files_config(&src, &[]);
let repo = local_repo(&server, &cfg.generated_id, &work.path().join("cache"));
let logger = JobLogger::new();
let snap = snapshot(&repo, &cfg, "bs-1", &logger).await.unwrap();
let forget = repo.run(["forget", snap.snapshot_id.as_str()], &logger).await.unwrap();
assert_ne!(forget.code, 0, "append-only must refuse forget");
let listed = repo.run(["snapshots", "--json"], &logger).await.unwrap();
assert!(listed.stdout.contains(&snap.snapshot_id), "{}", listed.stdout);
}
#[tokio::test]
async fn unreadable_file_on_a_local_repo_keeps_the_snapshot_for_the_dashboard() {
if is_root() {
return; // root reads a 000 file anyway
}
use std::os::unix::fs::PermissionsExt;
let store = TempDir::new().unwrap();
let work = TempDir::new().unwrap();
let server = serve("restic", "restic", store.path(), &["--append-only"], "s3cret");
let src = work.path().join("src");
tree(&src);
let locked = src.join("docs/b.txt");
fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).unwrap();
let cfg = files_config(&src, &[]);
let repo = local_repo(&server, &cfg.generated_id, &work.path().join("cache"));
let logger = JobLogger::new();
let err = snapshot(&repo, &cfg, "bs-1", &logger).await.unwrap_err();
fs::set_permissions(&locked, fs::Permissions::from_mode(0o644)).unwrap();
assert!(err.to_string().contains("the dashboard forgets it"), "{err:#}");
let listed = repo.run(["snapshots", "--json"], &logger).await.unwrap();
assert!(listed.stdout.contains("bs:bs-1"), "{}", listed.stdout);
}
#[tokio::test]
async fn local_sync_mirrors_through_the_webdav_server() {
use crate::core::context::Context;
use crate::services::api::ApiClient;
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::restic::password::local_storage_password;
use crate::services::sync::backup::one_storage;
use crate::utils::edge_key::EdgeKey;
use serde_json::json;
use wiremock::matchers::{body_partial_json, method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let store = TempDir::new().unwrap();
let work = TempDir::new().unwrap();
let password = local_storage_password(MASTER_KEY_B64).unwrap();
let server = serve("webdav", "sync", store.path(), &[], &password);
let api = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/agent/agent-1/backup/upload/init"))
.and(body_partial_json(json!({ "engine": "sync" })))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({ "message": "ok", "backupStorage": { "id": "bs" } })))
.expect(2)
.mount(&api)
.await;
Mock::given(method("PATCH"))
.and(path("/agent/agent-1/backup/upload/status"))
.and(body_partial_json(json!({ "status": "success" })))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({ "message": "ok", "backupStorage": { "id": "bs" } })))
.expect(2)
.mount(&api)
.await;
let ctx = Context {
edge_key: EdgeKey {
server_url: server.base_url.clone(),
agent_id: "agent-1".into(),
master_key_b64: MASTER_KEY_B64.into(),
},
api: ApiClient::new(api.uri()),
};
let storage: DatabaseStorage =
serde_json::from_value(json!({ "id": "ch-local", "provider": "local", "folderName": "backups", "config": {} })).unwrap();
let src = work.path().join("src");
tree(&src);
let cfg = files_config(&src, &[]);
let replica = store.path().join(&cfg.generated_id).join("current");
let first = one_storage(&ctx, &cfg, &storage, "backup-1", &JobLogger::new()).await;
assert!(first.success, "{:?}", first.error);
assert_eq!(first.remote_file_path.as_deref(), Some(format!("backups/sync/{}/current", cfg.generated_id).as_str()));
assert_eq!(fs::read_to_string(replica.join("docs/b.txt")).unwrap(), "b");
fs::remove_file(src.join("docs/b.txt")).unwrap();
let second = one_storage(&ctx, &cfg, &storage, "backup-2", &JobLogger::new()).await;
assert!(second.success, "{:?}", second.error);
assert!(!replica.join("docs/b.txt").exists(), "deletion propagated");
assert!(replica.join("a.txt").exists());
}
+4 -1
View File
@@ -1,6 +1,9 @@
mod api_models_tests;
mod backup_runner_tests;
mod backup_uploader_tests;
pub(crate) mod backup_uploader_tests;
mod config_tests;
mod dashboard_config_tests;
pub(crate) mod local_storage_tests;
mod restic_tests;
mod restore_downloader_tests;
mod sync_tests;
+348
View File
@@ -0,0 +1,348 @@
//! Snapshots mode (restic). Pure helpers first; the integration tests further
//! down need `restic` and `rclone` on PATH.
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::backup::logger::JobLogger;
use crate::services::restic::command::is_snapshot_id;
use crate::services::restic::excludes::{backup_excludes, glob_escape, restore_excludes};
use crate::services::restic::json::{self, BackupSummary, Line};
use crate::services::restic::password::{derive_password, hkdf_sha256_32};
use crate::tests::domain::files::files_config;
use serde_json::json;
use std::path::Path;
fn strings(items: &[&str]) -> Vec<String> {
items.iter().map(|s| s.to_string()).collect()
}
#[test]
fn snapshot_ids_are_full_lowercase_hex() {
assert!(is_snapshot_id(&"a1".repeat(32)));
for bad in ["--password-command=touch /tmp/pwned", &"A".repeat(64), &"a".repeat(63), &"a".repeat(65), ""] {
assert!(!is_snapshot_id(bad), "{bad}");
}
}
#[test]
fn hkdf_matches_rfc5869_case_3_first_block() {
let okm = hkdf_sha256_32(&[0x0b; 22], b"");
assert_eq!(
hex::encode(okm),
"8da4e775a563c18f715f802a063c5a31b8a11f5c5ee1879ec3454e5f3c738d2d"
);
}
#[test]
fn password_matches_the_dashboard_vector() {
// Same vector as src/lib/restic/repo.ts in the dashboard (spike S4).
let password = derive_password(
"AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=",
"test-generated-id",
)
.unwrap();
assert_eq!(
password,
"577e00efe6c953e8583d90e14d78109d805b616164279b8182282db76033b090"
);
}
#[test]
fn password_rejects_a_non_base64_master_key() {
assert!(derive_password("not base64!", "x").is_err());
}
#[test]
fn backup_excludes_are_rooted_at_the_source() {
let got = backup_excludes(
Path::new("/data/files"),
&strings(&["*.log", "node_modules/", "/build/", "/docs/**/*.md", " ", "/"]),
)
.unwrap();
assert_eq!(
got,
strings(&[
"/data/files/**/*.log",
"/data/files/**/node_modules",
"/data/files/build",
"/data/files/docs/**/*.md",
])
);
}
#[test]
fn backup_excludes_escape_glob_characters_in_the_root() {
let got = backup_excludes(Path::new("/data/src[1]"), &strings(&["tmp"])).unwrap();
assert_eq!(got, strings(&["/data/src\\[1\\]/**/tmp"]));
}
#[test]
fn restore_excludes_are_relative_to_the_snapshot_subfolder() {
let got = restore_excludes(&strings(&["*.log", "/build/", "a/b"])).unwrap();
assert_eq!(got, strings(&["/**/*.log", "/build", "/**/a/b"]));
}
#[test]
fn bracket_negation_is_rewritten_for_restic() {
// globset reads `[!a]`; restic (Go filepath.Match) only knows `[^a]`.
assert_eq!(
backup_excludes(Path::new("/data"), &strings(&["[!a]*.log", "/x/[!0-9]", r"\[!a]", "[a[!]"])).unwrap(),
strings(&["/data/**/[^a]*.log", "/data/x/[^0-9]", r"/data/**/\[!a]", "/data/**/[a[!]"])
);
assert_eq!(restore_excludes(&strings(&["/[!a]*", "[!a]"])).unwrap(), strings(&["/[^a]*", "/**/[^a]"]));
}
#[test]
fn brace_patterns_are_rejected_on_both_sides() {
let patterns = strings(&["{x,y}.dat"]);
let err = backup_excludes(Path::new("/data"), &patterns).unwrap_err();
assert!(err.to_string().contains("brace patterns are not supported in Snapshots mode"));
assert!(restore_excludes(&patterns).is_err());
}
#[test]
fn glob_escape_escapes_every_metacharacter() {
assert_eq!(glob_escape(r"a*b?c[d]e\f"), r"a\*b\?c\[d\]e\\f");
}
#[test]
fn json_lines_are_classified() {
// Recorded from restic 0.19.1 (paths shortened).
let status = r#"{"message_type":"status","percent_done":0.9999997777778271,"total_files":4,"files_done":2,"total_bytes":9000002,"bytes_done":9000000,"error_count":1}"#;
let error = r#"{"message_type":"error","error":{"message":"open /src/u: permission denied"},"during":"archival","item":"/src/u"}"#;
let exit = r#"{"message_type":"exit_error","code":3,"message":"Warning: at least one source file could not be read"}"#;
assert_eq!(json::parse(status), Line::Status { percent_done: 0.9999997777778272 });
assert_eq!(
json::parse(error),
Line::Error { message: "open /src/u: permission denied".into(), item: Some("/src/u".into()) }
);
assert_eq!(
json::parse(exit),
Line::ExitError { code: 3, message: "Warning: at least one source file could not be read".into() }
);
assert_eq!(json::parse("Fatal: wrong password"), Line::Text("Fatal: wrong password".into()));
assert_eq!(json::parse("[]"), Line::Text("[]".into()));
}
#[test]
fn backup_summary_deserializes_from_a_recorded_line() {
let line = r#"{"message_type":"summary","files_new":3,"files_changed":0,"files_unmodified":0,"dirs_new":9,"dirs_changed":0,"dirs_unmodified":0,"data_blobs":8,"tree_blobs":10,"data_added":9005859,"data_added_packed":9004808,"total_files_processed":3,"total_bytes_processed":9000000,"total_duration":0.78928,"backup_start":"2026-10-03T16:59:06.569467+02:00","backup_end":"2026-10-03T16:59:07.359124+02:00","snapshot_id":"05ba7db8d08e4e097b869dbc511a8b8936298b2fcb01b221115789491204a470"}"#;
let Line::Summary(value) = json::parse(line) else { panic!("not a summary") };
let summary: BackupSummary = serde_json::from_value(value).unwrap();
assert_eq!(summary.snapshot_id.len(), 64);
assert_eq!(summary.data_added_packed, 9004808);
assert_eq!(summary.total_bytes_processed, 9000000);
assert_eq!(summary.files_new, 3);
}
#[test]
fn a_local_channel_opens_the_dashboard_rest_repository() {
use crate::utils::edge_key::EdgeKey;
let storage: DatabaseStorage =
serde_json::from_value(json!({ "id": "storage-1", "provider": "local", "config": {} })).unwrap();
let edge_key = EdgeKey {
server_url: "http://dash:8887/".into(),
agent_id: "agent-1".into(),
master_key_b64: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=".into(),
};
let repo = ResticRepo::open(&storage, "g1", &edge_key).unwrap();
assert_eq!(repo.repository(), "rest:http://dash:8887/storage/restic/g1/");
assert!(repo.is_append_only());
}
#[test]
fn local_storage_password_matches_the_dashboard_vector() {
use crate::services::restic::password::local_storage_password;
// Same vector as src/lib/local-storage/credential.ts in the dashboard.
assert_eq!(
local_storage_password("AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=").unwrap(),
"8abfb788701b8c502658154746af6af768ea08f579381ad73b0f1e7dd43caea2"
);
}
// ---------- integration: restic + rclone on PATH ----------
use crate::domain::files::platform::is_root;
use crate::services::restic::backup::snapshot;
use crate::services::restic::command::ResticRepo;
use crate::services::restic::restore::restore;
use std::fs;
use tempfile::TempDir;
/// A repository on an rclone `alias` remote pointing at a temp dir (test-only:
/// production channels always go through `rclone_target`).
fn local_repo(dir: &Path, generated_id: &str) -> ResticRepo {
fs::create_dir_all(dir).unwrap();
let config = format!("[pb]\ntype = alias\nremote = {}\n", dir.display());
ResticRepo::new(&config, "pb", "", "backups", generated_id, "test-password", dir.join("cache")).unwrap()
}
fn tree(root: &Path) {
fs::create_dir_all(root.join("docs")).unwrap();
fs::write(root.join("docs/a.txt"), "v1").unwrap();
fs::write(root.join("docs/b.txt"), "b").unwrap();
fs::create_dir_all(root.join("build")).unwrap();
fs::write(root.join("build/out.bin"), "out").unwrap();
fs::write(root.join("app.log"), "log").unwrap();
}
/// `relative/path=content` for every regular file, sorted.
fn files(root: &Path) -> Vec<String> {
let mut out: Vec<String> = walkdir::WalkDir::new(root)
.min_depth(1)
.into_iter()
.filter_map(Result::ok)
.filter(|e| e.file_type().is_file())
.map(|e| {
let rel = e.path().strip_prefix(root).unwrap().display().to_string();
format!("{rel}={}", fs::read_to_string(e.path()).unwrap())
})
.collect();
out.sort();
out
}
#[tokio::test]
async fn first_snapshot_initializes_the_repository_and_the_second_reuses_its_parent() {
let base = TempDir::new().unwrap();
let src = TempDir::new().unwrap();
tree(src.path());
let cfg = files_config(src.path(), &[]);
let repo = local_repo(&base.path().join("repo"), &cfg.generated_id);
let logger = JobLogger::new();
let first = snapshot(&repo, &cfg, "bs-1", &logger).await.unwrap();
assert_eq!(first.snapshot_id.len(), 64);
assert_eq!(first.files_new, 4);
assert!(base.path().join("repo/backups/restic").join(&cfg.generated_id).join("config").exists());
fs::write(src.path().join("docs/a.txt"), "v2").unwrap();
let second = snapshot(&repo, &cfg, "bs-2", &logger).await.unwrap();
assert_eq!(second.files_changed, 1);
assert_eq!(second.files_unmodified, 3);
let tagged = repo.run(["snapshots", "--json", "--tag", "bs:bs-2"], &logger).await.unwrap();
// The tag selects only the second snapshot, and its `parent` is the first
// (a plain `contains(first)` check would match that `parent` field).
let listed: serde_json::Value = serde_json::from_str(tagged.stdout.trim()).unwrap();
let listed = listed.as_array().unwrap();
assert_eq!(listed.len(), 1, "{}", tagged.stdout);
assert_eq!(listed[0]["id"], second.snapshot_id.as_str());
assert_eq!(listed[0]["parent"], first.snapshot_id.as_str());
}
#[tokio::test]
async fn restore_is_a_mirror_that_keeps_excluded_paths() {
let base = TempDir::new().unwrap();
let src = TempDir::new().unwrap();
tree(src.path());
let cfg = files_config(src.path(), &["*.log", "/build"]);
let repo = local_repo(&base.path().join("repo"), &cfg.generated_id);
let logger = JobLogger::new();
let snap = snapshot(&repo, &cfg, "bs-1", &logger).await.unwrap();
fs::write(src.path().join("docs/a.txt"), "v2").unwrap();
fs::remove_file(src.path().join("docs/b.txt")).unwrap();
fs::write(src.path().join("new.txt"), "new").unwrap();
fs::write(src.path().join("docs/new.log"), "kept").unwrap();
fs::write(src.path().join("build/new.bin"), "kept").unwrap();
restore(&repo, &cfg, &snap.snapshot_id, &logger).await.unwrap();
assert_eq!(
files(src.path()),
vec![
"app.log=log",
"build/new.bin=kept",
"build/out.bin=out",
"docs/a.txt=v1",
"docs/b.txt=b",
"docs/new.log=kept",
]
);
}
#[tokio::test]
async fn unreadable_file_fails_the_snapshot_and_forgets_it() {
if is_root() {
return; // root reads a 000 file anyway
}
use std::os::unix::fs::PermissionsExt;
let base = TempDir::new().unwrap();
let src = TempDir::new().unwrap();
tree(src.path());
let locked = src.path().join("docs/b.txt");
fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).unwrap();
let cfg = files_config(src.path(), &[]);
let repo = local_repo(&base.path().join("repo"), &cfg.generated_id);
let logger = JobLogger::new();
let err = snapshot(&repo, &cfg, "bs-1", &logger).await.unwrap_err();
fs::set_permissions(&locked, fs::Permissions::from_mode(0o644)).unwrap();
assert!(err.to_string().contains("could not be read"), "{err:#}");
let listed = repo.run(["snapshots", "--json"], &logger).await.unwrap();
assert_eq!(listed.stdout.trim(), "[]");
}
#[tokio::test]
async fn unanchored_exclude_never_matches_directories_above_the_source() {
// Regression for spike S5: the source lives under a directory named like the pattern.
let base = TempDir::new().unwrap();
let src = base.path().join("tmp/src");
fs::create_dir_all(src.join("tmp")).unwrap();
fs::write(src.join("kept.txt"), "k").unwrap();
fs::write(src.join("tmp/dropped.txt"), "d").unwrap();
let cfg = files_config(&src, &["tmp"]);
let repo = local_repo(&base.path().join("repo"), &cfg.generated_id);
let snap = snapshot(&repo, &cfg, "bs-1", &JobLogger::new()).await.unwrap();
assert_eq!(snap.files_new, 1);
}
#[tokio::test]
async fn restoring_an_unknown_snapshot_fails_without_touching_the_target() {
let base = TempDir::new().unwrap();
let src = TempDir::new().unwrap();
tree(src.path());
let cfg = files_config(src.path(), &[]);
let repo = local_repo(&base.path().join("repo"), &cfg.generated_id);
let logger = JobLogger::new();
snapshot(&repo, &cfg, "bs-1", &logger).await.unwrap();
fs::write(src.path().join("new.txt"), "new").unwrap();
let before = files(src.path());
let err = restore(&repo, &cfg, &"deadbeef".repeat(8), &logger).await.unwrap_err();
assert!(err.to_string().contains("not found"), "{err:#}");
assert_eq!(files(src.path()), before);
}
#[tokio::test]
async fn wrong_password_is_reported_explicitly() {
let base = TempDir::new().unwrap();
let src = TempDir::new().unwrap();
tree(src.path());
let cfg = files_config(src.path(), &[]);
let dir = base.path().join("repo");
snapshot(&local_repo(&dir, &cfg.generated_id), &cfg, "bs-1", &JobLogger::new()).await.unwrap();
let config = format!("[pb]\ntype = alias\nremote = {}\n", dir.display());
let other = ResticRepo::new(&config, "pb", "", "backups", &cfg.generated_id, "other", dir.join("cache")).unwrap();
let err = snapshot(&other, &cfg, "bs-2", &JobLogger::new()).await.unwrap_err();
assert!(err.to_string().contains("wrong repository password"), "{err:#}");
}
#[tokio::test]
async fn an_unusable_cache_dir_runs_restic_without_cache() {
let base = TempDir::new().unwrap();
let src = TempDir::new().unwrap();
tree(src.path());
let cfg = files_config(src.path(), &[]);
let dir = base.path().join("repo");
fs::create_dir_all(&dir).unwrap();
let blocker = tempfile::NamedTempFile::new().unwrap(); // a regular file: nothing can be created below it
let config = format!("[pb]\ntype = alias\nremote = {}\n", dir.display());
let repo = ResticRepo::new(&config, "pb", "", "backups", &cfg.generated_id, "test-password", blocker.path().join("cache")).unwrap();
snapshot(&repo, &cfg, "bs-1", &JobLogger::new()).await.unwrap();
}
+250
View File
@@ -0,0 +1,250 @@
//! Sync mode (rclone replica). Pure helpers first; integration tests need `rclone` on PATH.
use crate::services::sync::excludes::sync_excludes;
use crate::services::sync::stats::{SyncStats, parse_log};
fn strings(items: &[&str]) -> Vec<String> {
items.iter().map(|s| s.to_string()).collect()
}
#[test]
fn sync_excludes_translate_p0_patterns() {
let got = sync_excludes(&strings(&[
"*.log", "node_modules/", "/build/", "**/tmp", "/docs/**/*.md", "[!a].txt", "{x,y}.dat", " ", "/",
]));
assert_eq!(
got,
strings(&[
"*.log", "*.log/**",
"node_modules", "node_modules/**",
"/build", "/build/**",
"tmp", "tmp/**",
"/docs/**/*.md", "/docs/**/*.md/**", "/docs/*.md", "/docs/*.md/**",
"[^a].txt", "[^a].txt/**",
"{x,y}.dat", "{x,y}.dat/**",
])
);
}
#[test]
fn every_double_star_gap_gets_a_zero_directory_variant() {
assert_eq!(
sync_excludes(&strings(&["a/**/b/**/c"])),
strings(&[
"a/**/b/**/c", "a/**/b/**/c/**",
"a/b/**/c", "a/b/**/c/**",
"a/**/b/c", "a/**/b/c/**",
"a/b/c", "a/b/c/**",
])
);
}
#[test]
fn anchored_leading_double_star_also_matches_the_root() {
assert_eq!(sync_excludes(&strings(&["/**/x"])), strings(&["/**/x", "/**/x/**", "/x", "/x/**"]));
}
#[test]
fn escaped_bracket_is_left_alone() {
assert_eq!(sync_excludes(&strings(&[r"\[!a]"])), strings(&[r"\[!a]", r"\[!a]/**"]));
}
#[test]
fn parse_log_reads_the_final_stats_and_error_lines() {
let stderr = concat!(
r#"{"level":"error","msg":"Failed to copy: couldn't copy from /src/secret: errno -1","source":"operations/copy.go:1","time":"2026-10-03T19:44:59+02:00"}"#, "\n",
r#"{"level":"error","msg":"not deleting files as there were IO errors","source":"sync/sync.go:1","time":"2026-10-03T19:44:59+02:00"}"#, "\n",
r#"{"level":"notice","msg":"\nTransferred: 0 B\n","source":"accounting/stats.go:1","stats":{"bytes":0,"checks":12,"deletes":0,"errors":1,"fatalError":false,"lastError":"couldn't copy from /src/secret: errno -1","transfers":0},"time":"2026-10-03T19:44:59+02:00"}"#, "\n",
);
let log = parse_log(stderr);
assert_eq!(
log.stats,
Some(SyncStats { bytes: 0, transfers: 0, deletes: 0, errors: 1, last_error: Some("couldn't copy from /src/secret: errno -1".into()) })
);
assert_eq!(log.errors.len(), 2);
assert_eq!(log.errors[1], "not deleting files as there were IO errors");
}
#[test]
fn parse_log_reads_a_successful_run() {
let stderr = r#"{"level":"notice","msg":"\nTransferred: 2 B\n","source":"accounting/stats.go:1","stats":{"bytes":2,"checks":5,"deletes":1,"errors":0,"fatalError":false,"transfers":1},"time":"2026-10-03T19:45:10+02:00"}"#;
let log = parse_log(stderr);
assert_eq!(log.stats, Some(SyncStats { bytes: 2, transfers: 1, deletes: 1, errors: 0, last_error: None }));
assert!(log.errors.is_empty());
}
#[test]
fn parse_log_keeps_plain_text_lines_as_errors() {
let log = parse_log("Error: unknown flag: --bogus\n");
assert!(log.stats.is_none());
assert_eq!(log.errors, strings(&["Error: unknown flag: --bogus"]));
}
#[test]
fn parse_log_accepts_critical_error_level() {
let stderr = r#"{"level":"critical","msg":"Failed to create file system for \":sftp,host=127.0.0.1,port=1,user=x:/tmp/dst\": NewFs: couldn't connect SSH: dial tcp 127.0.0.1:1: connect: connection refused","source":"cmd/cmd.go:148"}"#;
let log = parse_log(stderr);
assert!(log.stats.is_none());
assert_eq!(log.errors.len(), 1);
assert!(log.errors[0].starts_with("Failed to create file system"));
}
#[test]
fn parse_log_deduplicates_repeated_error_lines() {
let stderr = concat!(
r#"{"level":"error","msg":"connection timeout","source":"operations/copy.go:1","time":"2026-10-03T19:44:59+02:00"}"#, "\n",
r#"{"level":"error","msg":"connection timeout","source":"operations/copy.go:1","time":"2026-10-03T19:44:59+02:00"}"#, "\n",
r#"{"level":"error","msg":"connection timeout","source":"operations/copy.go:1","time":"2026-10-03T19:44:59+02:00"}"#, "\n",
);
let log = parse_log(stderr);
assert_eq!(log.errors.len(), 1);
assert_eq!(log.errors[0], "connection timeout");
}
#[test]
fn sync_excludes_collapses_repeated_double_stars() {
assert_eq!(
sync_excludes(&strings(&["a/**/**/b"])),
strings(&["a/**/b", "a/**/b/**", "a/b", "a/b/**"])
);
}
// ---------- integration: rclone on PATH, `alias` remote under a temp dir ----------
use crate::domain::files::platform::is_root;
use crate::services::backup::logger::JobLogger;
use crate::services::sync::rclone::{replica_target, sync_dir};
use crate::tests::domain::files::files_config;
use std::fs;
use std::path::{Path, PathBuf};
use tempfile::TempDir;
fn alias(dir: &Path) -> String {
fs::create_dir_all(dir).unwrap();
format!("[pb]\ntype = alias\nremote = {}\n", dir.display())
}
fn replica(store: &Path, generated_id: &str) -> PathBuf {
store.join("backups/sync").join(generated_id).join("current")
}
/// `relative/path=content` for every regular file, sorted.
fn files(root: &Path) -> Vec<String> {
let mut out: Vec<String> = walkdir::WalkDir::new(root)
.min_depth(1)
.into_iter()
.filter_map(Result::ok)
.filter(|e| e.file_type().is_file())
.map(|e| format!("{}={}", e.path().strip_prefix(root).unwrap().display(), fs::read_to_string(e.path()).unwrap()))
.collect();
out.sort();
out
}
fn tree(root: &Path) {
fs::create_dir_all(root.join("docs")).unwrap();
fs::write(root.join("docs/a.txt"), "v1").unwrap();
fs::write(root.join("docs/b.txt"), "b").unwrap();
fs::create_dir_all(root.join("build")).unwrap();
fs::write(root.join("build/out.bin"), "out").unwrap();
fs::write(root.join("app.log"), "log").unwrap();
}
#[test]
fn replica_target_follows_the_path_convention() {
assert_eq!(replica_target("s3", "bucket", "backups", "g1"), "s3:bucket/backups/sync/g1/current");
assert_eq!(replica_target("pb", "", "backups", "g1"), "pb:backups/sync/g1/current");
}
#[tokio::test]
async fn sync_mirrors_the_source_and_propagates_changes() {
let store = TempDir::new().unwrap();
let src = TempDir::new().unwrap();
tree(src.path());
let cfg = files_config(src.path(), &["*.log", "/build"]);
let config = alias(store.path());
let logger = JobLogger::new();
let first = sync_dir(&config, &replica_target("pb", "", "backups", &cfg.generated_id), &cfg, &logger).await.unwrap();
assert_eq!(first.transfers, 2);
assert_eq!(files(&replica(store.path(), &cfg.generated_id)), vec!["docs/a.txt=v1", "docs/b.txt=b"]);
fs::write(src.path().join("docs/a.txt"), "v2").unwrap();
fs::remove_file(src.path().join("docs/b.txt")).unwrap();
fs::write(src.path().join("new.txt"), "new").unwrap();
let second = sync_dir(&config, &replica_target("pb", "", "backups", &cfg.generated_id), &cfg, &logger).await.unwrap();
assert_eq!(second.transfers, 2);
assert_eq!(second.deletes, 1);
assert_eq!(files(&replica(store.path(), &cfg.generated_id)), vec!["docs/a.txt=v2", "new.txt=new"]);
}
#[tokio::test]
async fn failed_sync_keeps_files_the_failed_run_would_delete() {
if is_root() {
return; // root reads a 000 file anyway
}
use std::os::unix::fs::PermissionsExt;
let store = TempDir::new().unwrap();
let src = TempDir::new().unwrap();
tree(src.path());
let cfg = files_config(src.path(), &[]);
let config = alias(store.path());
sync_dir(&config, &replica_target("pb", "", "backups", &cfg.generated_id), &cfg, &JobLogger::new()).await.unwrap();
fs::remove_file(src.path().join("docs/b.txt")).unwrap();
let locked = src.path().join("docs/a.txt");
// rclone only opens a file whose size/mtime changed, so the unreadable file must differ from the replica's copy.
fs::write(&locked, "changed and now unreadable").unwrap();
fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).unwrap();
let err = sync_dir(&config, &replica_target("pb", "", "backups", &cfg.generated_id), &cfg, &JobLogger::new()).await.unwrap_err();
fs::set_permissions(&locked, fs::Permissions::from_mode(0o644)).unwrap();
assert!(err.to_string().contains("rclone sync failed"), "{err:#}");
assert!(replica(store.path(), &cfg.generated_id).join("docs/b.txt").exists());
}
#[tokio::test]
async fn empty_source_is_refused_and_the_replica_is_untouched() {
let store = TempDir::new().unwrap();
let src = TempDir::new().unwrap();
let cfg = files_config(src.path(), &[]);
let err = sync_dir(&alias(store.path()), &replica_target("pb", "", "backups", &cfg.generated_id), &cfg, &JobLogger::new()).await.unwrap_err();
assert!(err.to_string().contains("is empty"), "{err:#}");
assert!(!store.path().join("backups").exists());
}
// ---------- sync as a backup method ----------
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::sync::backup::replica_path;
use serde_json::json;
#[test]
fn replica_path_uses_the_channel_folder() {
let custom: DatabaseStorage =
serde_json::from_value(json!({ "id": "s", "provider": "s3", "folderName": "/pb/", "config": {} })).unwrap();
assert_eq!(replica_path(&custom, "g1"), "pb/sync/g1/current");
let default: DatabaseStorage =
serde_json::from_value(json!({ "id": "s", "provider": "s3", "config": {} })).unwrap();
assert_eq!(replica_path(&default, "g1"), "backups/sync/g1/current");
}
#[test]
fn local_sync_config_points_at_the_dashboard_webdav_server() {
use crate::services::sync::backup::local_sync_config;
use crate::utils::edge_key::EdgeKey;
let edge_key = EdgeKey {
server_url: "http://dash:8887/".into(),
agent_id: "agent-1".into(),
master_key_b64: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=".into(),
};
let text = local_sync_config(&edge_key).unwrap();
assert!(
text.starts_with(
"[pblocal]\ntype = webdav\nurl = http://dash:8887/storage/sync\nvendor = rclone\nuser = portabase\npass = "
),
"{text}"
);
assert!(!text.contains("8abfb788"), "the password must be obscured: {text}");
}
@@ -0,0 +1,6 @@
[blob]
type = azureblob
account = myaccount
key = a2V5LWJhc2U2NA==
endpoint = https://myaccount.blob.core.windows.net
no_check_container = true
@@ -0,0 +1,6 @@
[blob]
type = azureblob
account = devstoreaccount1
key = a2V5
endpoint = http://127.0.0.1:10000/devstoreaccount1
no_check_container = true
@@ -0,0 +1,6 @@
[blob]
type = azureblob
account = devstoreaccount1
key = a2V5
endpoint = http://127.0.0.1:10000/devstoreaccount1
no_check_container = true
@@ -0,0 +1,7 @@
[google-cloud-storage]
type = google cloud storage
project_number = test
anonymous = true
endpoint = http://127.0.0.1:4443/storage/v1/
bucket_policy_only = true
no_check_bucket = true
@@ -0,0 +1,6 @@
[google-cloud-storage]
type = google cloud storage
project_number = my-project
service_account_credentials = {"client_email":"svc@my-project.iam.gserviceaccount.com","private_key":"-----BEGIN PRIVATE KEY-----\nMIIEexample\n-----END PRIVATE KEY-----\n","private_key_id":"","project_id":"my-project","token_uri":"https://oauth2.googleapis.com/token","type":"service_account","universe_domain":"googleapis.com"}
bucket_policy_only = true
no_check_bucket = true
@@ -0,0 +1,8 @@
[google-drive]
type = drive
client_id = cid.apps.googleusercontent.com
client_secret = csecret
scope = drive.file
token = {"access_token":"","expiry":"0001-01-01T00:00:00Z","refresh_token":"1//refresh","token_type":"Bearer"}
root_folder_id = folder123
use_trash = false
@@ -0,0 +1,52 @@
{
"s3": {
"provider": "s3",
"config": { "endPointUrl": "s3.example.com", "accessKey": "AKIAEXAMPLE", "secretKey": "secretEXAMPLE", "bucketName": "my-bucket", "ssl": true, "region": "eu-west-1" },
"expected": { "remoteName": "s3", "basePath": "my-bucket" }
},
"s3-port-no-ssl": {
"provider": "s3",
"config": { "endPointUrl": "minio.local", "port": 9000, "accessKey": "minioadmin", "secretKey": "minioadmin", "bucketName": "backups", "ssl": false, "region": " " },
"expected": { "remoteName": "s3", "basePath": "backups" }
},
"blob-account-key": {
"provider": "blob",
"config": { "authMode": "accountKey", "accountName": "myaccount", "accountKey": "a2V5LWJhc2U2NA==", "containerName": "backups" },
"expected": { "remoteName": "blob", "basePath": "backups" }
},
"blob-endpoint-url": {
"provider": "blob",
"config": { "authMode": "accountKey", "accountName": "devstoreaccount1", "accountKey": "a2V5", "containerName": "backups", "endpointUrl": "http://127.0.0.1:10000" },
"expected": { "remoteName": "blob", "basePath": "backups" }
},
"blob-connection-string": {
"provider": "blob",
"config": { "authMode": "connectionString", "connectionString": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=a2V5;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;", "containerName": "backups" },
"expected": { "remoteName": "blob", "basePath": "backups" }
},
"google-cloud-storage": {
"provider": "google-cloud-storage",
"config": { "projectId": "my-project", "bucketName": "my-gcs-bucket", "clientEmail": "svc@my-project.iam.gserviceaccount.com", "privateKey": "-----BEGIN PRIVATE KEY-----\\nMIIEexample\\n-----END PRIVATE KEY-----\\n", "apiEndpoint": "" },
"expected": { "remoteName": "google-cloud-storage", "basePath": "my-gcs-bucket" }
},
"google-cloud-storage-emulator": {
"provider": "google-cloud-storage",
"config": { "projectId": "test", "bucketName": "bucket", "clientEmail": "x@test", "privateKey": "", "apiEndpoint": "http://127.0.0.1:4443" },
"expected": { "remoteName": "google-cloud-storage", "basePath": "bucket" }
},
"google-drive": {
"provider": "google-drive",
"config": { "clientId": "cid.apps.googleusercontent.com", "clientSecret": "csecret", "refreshToken": "1//refresh", "folderId": "folder123" },
"expected": { "remoteName": "google-drive", "basePath": "" }
},
"rclone": {
"provider": "rclone",
"config": { "configText": "[ovh]\ntype = s3\nprovider = OVHcloud\naccess_key_id = a\nsecret_access_key = b\nendpoint = s3.gra.io.cloud.ovh.net\n", "remoteName": "ovh", "remotePath": "my-bucket" },
"expected": { "remoteName": "ovh", "basePath": "my-bucket" }
},
"sftp-key": {
"provider": "sftp",
"config": { "host": "sftp.example.com", "port": 2222, "username": "backup", "privateKey": "-----BEGIN OPENSSH PRIVATE KEY-----\nabc\n-----END OPENSSH PRIVATE KEY-----\n", "remotePath": "/srv/backups" },
"expected": { "remoteName": "sftp", "basePath": "/srv/backups" }
}
}
@@ -0,0 +1,6 @@
[ovh]
type = s3
provider = OVHcloud
access_key_id = a
secret_access_key = b
endpoint = s3.gra.io.cloud.ovh.net
@@ -0,0 +1,9 @@
[s3]
type = s3
provider = Other
access_key_id = minioadmin
secret_access_key = minioadmin
endpoint = http://minio.local:9000
region = us-east-1
force_path_style = true
no_check_bucket = true
@@ -0,0 +1,9 @@
[s3]
type = s3
provider = Other
access_key_id = AKIAEXAMPLE
secret_access_key = secretEXAMPLE
endpoint = https://s3.example.com
region = eu-west-1
force_path_style = true
no_check_bucket = true
@@ -0,0 +1,6 @@
[sftp]
type = sftp
host = sftp.example.com
port = 2222
user = backup
key_file = <KEY_FILE>
@@ -0,0 +1,345 @@
[
{
"name": "ovh-s3",
"configText": "[ovhcloud-rbx]\ntype = s3\nprovider = OVHcloud\naccess_key_id = my_access\nsecret_access_key = my_secret\nregion = rbx\nendpoint = s3.rbx.io.cloud.ovh.net\nacl = private\n",
"remoteName": "ovhcloud-rbx",
"ok": true,
"errorContains": null
},
{
"name": "comments-crlf",
"configText": "# comment\r\n[r]\r\ntype = s3\r\n; note\r\nprovider = AWS\r\n",
"remoteName": "r",
"ok": true,
"errorContains": null
},
{
"name": "empty-value",
"configText": "[r]\ntype = webdav\nurl = https://dav.example.com\nuser =\n",
"remoteName": "r",
"ok": true,
"errorContains": null
},
{
"name": "duplicate-type",
"configText": "[x]\ntype = s3\ntype = local\n",
"remoteName": "x",
"ok": false,
"errorContains": "duplicate rclone config key 'type'"
},
{
"name": "duplicate-type-case",
"configText": "[x]\ntype = s3\nTYPE = local\n",
"remoteName": "x",
"ok": false,
"errorContains": "duplicate rclone config key 'TYPE'"
},
{
"name": "colon-separator",
"configText": "[x]\ntype: local\n",
"remoteName": "x",
"ok": false,
"errorContains": "line 2 is not a [section] header"
},
{
"name": "on-the-fly",
"configText": "[:local]\ntype = s3\n",
"remoteName": ":local",
"ok": false,
"errorContains": "invalid rclone remote name ':local'"
},
{
"name": "sftp-ssh",
"configText": "[x]\ntype = sftp\nhost = h\nssh = touch /tmp/pwned\n",
"remoteName": "x",
"ok": false,
"errorContains": "key 'ssh' is not allowed"
},
{
"name": "webdav-command",
"configText": "[x]\ntype = webdav\nurl = http://h\nbearer_token_command = id\n",
"remoteName": "x",
"ok": false,
"errorContains": "key 'bearer_token_command' is not allowed"
},
{
"name": "env-auth",
"configText": "[x]\ntype = s3\nenv_auth = true\n",
"remoteName": "x",
"ok": false,
"errorContains": "key 'env_auth' is not allowed"
},
{
"name": "use-msi",
"configText": "[x]\ntype = azureblob\naccount = a\nuse_msi = true\n",
"remoteName": "x",
"ok": false,
"errorContains": "key 'use_msi' is not allowed"
},
{
"name": "service-account-file",
"configText": "[x]\ntype = google cloud storage\nservice_account_file = /etc/passwd\n",
"remoteName": "x",
"ok": false,
"errorContains": "key 'service_account_file' is not allowed"
},
{
"name": "blocked-local",
"configText": "[x]\ntype = local\n",
"remoteName": "x",
"ok": false,
"errorContains": "backend type 'local' is not allowed"
},
{
"name": "two-sections",
"configText": "[a]\ntype = s3\n[b]\ntype = s3\n",
"remoteName": "a",
"ok": false,
"errorContains": "exactly one [section] (found 2)"
},
{
"name": "key-before-section",
"configText": "type = s3\n[x]\ntype = s3\n",
"remoteName": "x",
"ok": false,
"errorContains": "line 1 appears before any [section]"
},
{
"name": "no-type",
"configText": "[x]\nprovider = AWS\n",
"remoteName": "x",
"ok": false,
"errorContains": "has no type"
},
{
"name": "remote-mismatch",
"configText": "[a]\ntype = s3\n",
"remoteName": "b",
"ok": false,
"errorContains": "remote 'b' is not defined"
},
{
"name": "bad-key",
"configText": "[x]\ntype = s3\nbad key = 1\n",
"remoteName": "x",
"ok": false,
"errorContains": "invalid rclone config key 'bad key'"
},
{
"name": "empty-config",
"configText": "",
"remoteName": "r",
"ok": false,
"errorContains": "exactly one [section] (found 0)"
},
{
"name": "empty-section-name",
"configText": "[]\ntype = s3\n",
"remoteName": "",
"ok": false,
"errorContains": "invalid rclone remote name ''"
},
{
"name": "name-with-space",
"configText": "[a b]\ntype = s3\n",
"remoteName": "a b",
"ok": false,
"errorContains": "invalid rclone remote name 'a b'"
},
{
"name": "blocked-type-uppercase",
"configText": "[x]\ntype = LOCAL\n",
"remoteName": "x",
"ok": false,
"errorContains": "backend type 'local' is not allowed"
},
{
"name": "key-file-uppercase",
"configText": "[x]\ntype = sftp\nKEY_FILE = /etc/passwd\n",
"remoteName": "x",
"ok": false,
"errorContains": "key 'KEY_FILE' is not allowed"
},
{
"name": "equals-at-line-start",
"configText": "[x]\n= v\n",
"remoteName": "x",
"ok": false,
"errorContains": "line 2 is not a [section] header"
},
{
"name": "whitespace-tolerant",
"configText": " [r] \n type=s3 \n\n",
"remoteName": "r",
"ok": true,
"errorContains": null
},
{
"name": "quoted-type-backtick",
"configText": "[x]\ntype = `local`\n",
"remoteName": "x",
"ok": false,
"errorContains": "value for key 'type' must not be quoted"
},
{
"name": "quoted-type-triple",
"configText": "[x]\ntype = \"\"\"local\"\"\"\n",
"remoteName": "x",
"ok": false,
"errorContains": "value for key 'type' must not be quoted"
},
{
"name": "quoted-type-trailing-junk",
"configText": "[x]\ntype = `alias` junk\nremote = /\n",
"remoteName": "x",
"ok": false,
"errorContains": "value for key 'type' must not be quoted"
},
{
"name": "quoted-other-value",
"configText": "[x]\ntype = s3\nendpoint = `x`\n",
"remoteName": "x",
"ok": false,
"errorContains": "value for key 'endpoint' must not be quoted (remote 'x')"
},
{
"name": "blocked-google-photos",
"configText": "[x]\ntype = google photos\n",
"remoteName": "x",
"ok": false,
"errorContains": "backend type 'google photos' is not allowed"
},
{
"name": "blocked-gphotos",
"configText": "[x]\ntype = gphotos\n",
"remoteName": "x",
"ok": false,
"errorContains": "backend type 'gphotos' is not allowed"
},
{
"name": "invalid-type-syntax",
"configText": "[x]\ntype = s3!\n",
"remoteName": "x",
"ok": false,
"errorContains": "invalid rclone backend type 's3!' (remote 'x')"
},
{
"name": "forbidden-use-az",
"configText": "[x]\ntype = azureblob\nuse_az = v\n",
"remoteName": "x",
"ok": false,
"errorContains": "key 'use_az' is not allowed (remote 'x')"
},
{
"name": "forbidden-use-kerberos",
"configText": "[x]\ntype = hdfs\nuse_kerberos = v\n",
"remoteName": "x",
"ok": false,
"errorContains": "key 'use_kerberos' is not allowed (remote 'x')"
},
{
"name": "forbidden-kerberos-ccache",
"configText": "[x]\ntype = hdfs\nkerberos_ccache = v\n",
"remoteName": "x",
"ok": false,
"errorContains": "key 'kerberos_ccache' is not allowed (remote 'x')"
},
{
"name": "forbidden-key-use-agent",
"configText": "[x]\ntype = sftp\nkey_use_agent = v\n",
"remoteName": "x",
"ok": false,
"errorContains": "key 'key_use_agent' is not allowed (remote 'x')"
},
{
"name": "forbidden-set-env",
"configText": "[x]\ntype = sftp\nset_env = v\n",
"remoteName": "x",
"ok": false,
"errorContains": "key 'set_env' is not allowed (remote 'x')"
},
{
"name": "forbidden-unix-socket",
"configText": "[x]\ntype = sftp\nunix_socket = v\n",
"remoteName": "x",
"ok": false,
"errorContains": "key 'unix_socket' is not allowed (remote 'x')"
},
{
"name": "forbidden-client-certificate-path",
"configText": "[x]\ntype = azureblob\nclient_certificate_path = v\n",
"remoteName": "x",
"ok": false,
"errorContains": "key 'client_certificate_path' is not allowed (remote 'x')"
},
{
"name": "oracle-env-auth",
"configText": "[x]\ntype = oracleobjectstorage\nprovider = env_auth\nnamespace = n\n",
"remoteName": "x",
"ok": false,
"errorContains": "oracleobjectstorage remotes must use provider = no_auth (remote 'x')"
},
{
"name": "oracle-missing-provider",
"configText": "[x]\ntype = oracleobjectstorage\nnamespace = n\n",
"remoteName": "x",
"ok": false,
"errorContains": "oracleobjectstorage remotes must use provider = no_auth (remote 'x')"
},
{
"name": "oracle-spaced-type",
"configText": "[x]\ntype = oracle object storage\nprovider = instance_principal_auth\n",
"remoteName": "x",
"ok": false,
"errorContains": "oracleobjectstorage remotes must use provider = no_auth (remote 'x')"
},
{
"name": "oracle-provider-uppercase-value",
"configText": "[x]\ntype = oracleobjectstorage\nprovider = NO_AUTH\nnamespace = n\n",
"remoteName": "x",
"ok": false,
"errorContains": "oracleobjectstorage remotes must use provider = no_auth (remote 'x')"
},
{
"name": "oracle-provider-uppercase-key",
"configText": "[x]\ntype = oracleobjectstorage\nPROVIDER = no_auth\nnamespace = n\n",
"remoteName": "x",
"ok": false,
"errorContains": "oracleobjectstorage remotes must use provider = no_auth (remote 'x')"
},
{
"name": "oracle-no-auth",
"configText": "[x]\ntype = oracleobjectstorage\nprovider = no_auth\nnamespace = n\nregion = eu-frankfurt-1\n",
"remoteName": "x",
"ok": true,
"errorContains": null
},
{
"name": "nel-in-type",
"configText": "[x]\ntype = local\u0085\n",
"remoteName": "x",
"ok": false,
"errorContains": "backend type 'local' is not allowed (remote 'x')"
},
{
"name": "order-duplicate-before-forbidden",
"configText": "[x]\ntype = s3\nssh = a\nssh = b\n",
"remoteName": "x",
"ok": false,
"errorContains": "duplicate rclone config key 'ssh' (remote 'x')"
},
{
"name": "order-bad-line-before-pair-before-header",
"configText": "type = s3\n[x]\nfoo\n",
"remoteName": "x",
"ok": false,
"errorContains": "rclone config line 3 is not a [section] header"
},
{
"name": "order-invalid-key-before-duplicate",
"configText": "[x]\ntype = s3\nbad key = 1\nbad key = 2\n",
"remoteName": "x",
"ok": false,
"errorContains": "invalid rclone config key 'bad key' (remote 'x')"
}
]
+1 -1
View File
@@ -12,7 +12,7 @@ use testcontainers::{GenericImage, ImageExt};
const BUCKET: &str = "portabase";
async fn start_fake_gcs() -> (testcontainers::ContainerAsync<GenericImage>, String) {
pub(super) async fn start_fake_gcs() -> (testcontainers::ContainerAsync<GenericImage>, String) {
// Natural random host port (no port-80 pin). The provider forces a single-shot
// upload for custom endpoints, which issues one request to this endpoint and never
// follows a server-built `Location` — so it works on any port, unlike the resumable
+1
View File
@@ -1,4 +1,5 @@
mod azure_blob;
mod google_cloud_storage;
mod rclone;
mod rclone_target;
mod sftp;
+50 -10
View File
@@ -63,6 +63,41 @@ fn validate_config_accepts_the_target_remote() {
assert!(validate_config(OVH_CONFIG, "ovhcloud-rbx").is_ok());
}
#[derive(serde::Deserialize)]
#[serde(rename_all = "camelCase")]
struct ValidationCase {
name: String,
config_text: String,
remote_name: String,
ok: bool,
error_contains: Option<String>,
}
/// The same file is verified against the dashboard validator: keep both in sync.
#[test]
fn validate_config_matches_the_shared_test_vectors() {
let cases: Vec<ValidationCase> =
serde_json::from_str(include_str!("fixtures/rclone-validation/cases.json")).unwrap();
assert!(!cases.is_empty());
let mut failures = Vec::new();
for case in &cases {
match (validate_config(&case.config_text, &case.remote_name), case.ok) {
(Ok(()), true) => {}
(Ok(()), false) => failures.push(format!("[{}] expected an error, got Ok", case.name)),
(Err(e), true) => failures.push(format!("[{}] expected Ok, got: {e:#}", case.name)),
(Err(e), false) => {
let msg = format!("{e:#}");
let want = case.error_contains.as_deref().expect("ok:false needs errorContains");
if !msg.contains(want) {
failures.push(format!("[{}] error {msg:?} does not contain {want:?}", case.name));
}
}
}
}
assert!(failures.is_empty(), "{} case(s) failed:\n{}", failures.len(), failures.join("\n"));
}
#[test]
fn validate_config_rejects_an_unknown_remote_name() {
let err = validate_config(OVH_CONFIG, "typo").unwrap_err().to_string();
@@ -85,23 +120,28 @@ fn validate_config_rejects_alias_backend() {
}
#[test]
fn validate_config_rejects_a_blocked_backend_in_a_chained_section() {
fn validate_config_rejects_chained_sections() {
// A blocked backend hidden behind an allowed (or wrapping) remote can no longer be expressed:
// anything beyond a single [section] is refused outright.
let cfg = "[secret]\ntype = crypt\nremote = disk:vault\n\n[disk]\ntype = local\n";
let err = validate_config(cfg, "secret").unwrap_err().to_string();
assert!(err.contains("crypt"), "unexpected error: {err}");
assert!(err.contains("secret"), "error should name the offending remote: {err}");
assert!(err.contains("exactly one [section] (found 2)"), "unexpected error: {err}");
let cfg = "[outer]\ntype = s3\nprovider = Minio\n\n[disk]\ntype = local\n";
let err = validate_config(cfg, "outer").unwrap_err().to_string();
assert!(err.contains("local"), "unexpected error: {err}");
assert!(err.contains("disk"), "error should name the offending remote: {err}");
assert!(err.contains("exactly one [section] (found 2)"), "unexpected error: {err}");
let cfg = format!("[secret]\ntype = crypt\nremote = ovhcloud-rbx:bucket\n\n{OVH_CONFIG}");
let err = validate_config(&cfg, "secret").unwrap_err().to_string();
assert!(err.contains("exactly one [section] (found 2)"), "unexpected error: {err}");
}
#[test]
fn validate_config_rejects_crypt_even_over_an_allowed_remote() {
let cfg = format!("[secret]\ntype = crypt\nremote = ovhcloud-rbx:bucket\n\n{OVH_CONFIG}");
let err = validate_config(&cfg, "secret").unwrap_err().to_string();
fn validate_config_rejects_crypt_over_an_allowed_remote_name() {
let cfg = "[secret]\ntype = crypt\nremote = ovhcloud-rbx:bucket\n";
let err = validate_config(cfg, "secret").unwrap_err().to_string();
assert!(err.contains("crypt"), "unexpected error: {err}");
assert!(err.contains("secret"), "error should name the offending remote: {err}");
}
#[test]
@@ -179,7 +219,7 @@ use testcontainers::{GenericImage, ImageExt};
const BUCKET: &str = "portabase";
async fn start_minio() -> (testcontainers::ContainerAsync<GenericImage>, String) {
pub(super) async fn start_minio() -> (testcontainers::ContainerAsync<GenericImage>, String) {
let container = GenericImage::new("coollabsio/minio", "latest")
.with_exposed_port(9000.tcp())
.with_wait_for(WaitFor::message_on_stderr("API:"))
@@ -208,7 +248,7 @@ fn minio_config(endpoint: &str) -> String {
)
}
fn rclone_ok(config_path: &std::path::Path, args: &[&str]) -> Vec<u8> {
pub(super) fn rclone_ok(config_path: &std::path::Path, args: &[&str]) -> Vec<u8> {
let out = Command::new("rclone")
.arg("--config")
.arg(config_path)
+301
View File
@@ -0,0 +1,301 @@
use crate::services::api::models::agent::status::DatabaseStorage;
use super::google_cloud_storage::start_fake_gcs;
use super::rclone::{rclone_ok, start_minio};
use crate::services::storage::providers::rclone::helpers::{obscure_password, write_config};
use crate::services::storage::providers::rclone::target::rclone_target;
use serde_json::{Value, json};
const INPUTS: &str = include_str!("fixtures/rclone-target/inputs.json");
pub(super) fn expected_ini(name: &str) -> &'static str {
match name {
"s3" => include_str!("fixtures/rclone-target/s3.ini"),
"s3-port-no-ssl" => include_str!("fixtures/rclone-target/s3-port-no-ssl.ini"),
"blob-account-key" => include_str!("fixtures/rclone-target/blob-account-key.ini"),
"blob-endpoint-url" => include_str!("fixtures/rclone-target/blob-endpoint-url.ini"),
"blob-connection-string" => include_str!("fixtures/rclone-target/blob-connection-string.ini"),
"google-cloud-storage" => include_str!("fixtures/rclone-target/google-cloud-storage.ini"),
"google-cloud-storage-emulator" => {
include_str!("fixtures/rclone-target/google-cloud-storage-emulator.ini")
}
"google-drive" => include_str!("fixtures/rclone-target/google-drive.ini"),
"rclone" => include_str!("fixtures/rclone-target/rclone.ini"),
"sftp-key" => include_str!("fixtures/rclone-target/sftp-key.ini"),
other => panic!("no expected fixture for {other}"),
}
}
pub(super) fn storage(provider: &str, config: Value) -> DatabaseStorage {
serde_json::from_value(json!({
"id": "storage-1",
"provider": provider,
"folderName": "backups",
"config": config,
}))
.unwrap()
}
/// Temp-file paths differ per run; normalize them before comparing. Keeps the exact
/// trailing newline so a missing final `\n` fails the golden test.
pub(super) fn normalize(config_text: &str) -> String {
config_text
.split('\n')
.map(|l| {
if l.starts_with("key_file = ") {
"key_file = <KEY_FILE>".to_string()
} else {
l.to_string()
}
})
.collect::<Vec<_>>()
.join("\n")
}
#[test]
fn rclone_target_matches_golden_fixtures() {
let inputs: Value = serde_json::from_str(INPUTS).unwrap();
for (name, case) in inputs.as_object().unwrap() {
let target = rclone_target(&storage(
case["provider"].as_str().unwrap(),
case["config"].clone(),
))
.unwrap_or_else(|e| panic!("{name}: {e:#}"));
assert_eq!(normalize(&target.config_text), expected_ini(name), "{name}: config_text");
assert_eq!(target.remote_name, case["expected"]["remoteName"].as_str().unwrap(), "{name}: remote_name");
assert_eq!(target.base_path, case["expected"]["basePath"].as_str().unwrap(), "{name}: base_path");
}
}
#[test]
fn rclone_target_rejects_local_and_unknown_providers() {
let err = rclone_target(&storage("local", json!({}))).err().unwrap();
assert!(format!("{err:#}").contains("a local storage channel cannot be used as an rclone target"));
let err = rclone_target(&storage("ftp-of-doom", json!({}))).err().unwrap();
assert!(format!("{err:#}").contains("ftp-of-doom"));
}
#[test]
fn rclone_target_rejects_azure_without_account_key() {
let err = rclone_target(&storage(
"blob",
json!({
"authMode": "connectionString",
"connectionString": "BlobEndpoint=https://acct.blob.core.windows.net;SharedAccessSignature=sv=2022&sig=x",
"containerName": "c"
}),
))
.err()
.unwrap();
assert!(format!("{err:#}").contains("account key"), "{err:#}");
}
#[test]
fn rclone_target_applies_the_rclone_backend_blocklist() {
let err = rclone_target(&storage(
"rclone",
json!({ "configText": "[disk]\ntype = local\n", "remoteName": "disk", "remotePath": "" }),
))
.err()
.unwrap();
assert!(format!("{err:#}").contains("not allowed"), "{err:#}");
}
#[test]
fn sftp_key_file_lives_exactly_as_long_as_the_target() {
let target = rclone_target(&storage(
"sftp",
json!({ "host": "h", "username": "u", "privateKey": "KEY", "remotePath": "" }),
))
.unwrap();
let key_path = target
.config_text
.lines()
.find_map(|l| l.strip_prefix("key_file = "))
.unwrap()
.to_string();
assert_eq!(std::fs::read_to_string(&key_path).unwrap(), "KEY");
drop(target);
assert!(!std::path::Path::new(&key_path).exists(), "key file must be removed with the target");
}
#[test]
fn sftp_config_error_keeps_the_root_cause() {
let err = rclone_target(&storage("sftp", json!({ "username": "u", "password": "p" })))
.err()
.unwrap();
let text = format!("{err:#}");
assert!(text.contains("invalid sftp storage config"), "{text}");
assert!(text.contains("host"), "{text}");
}
#[test]
fn sftp_password_only_has_pass_and_no_key_file() {
let target = rclone_target(&storage(
"sftp",
json!({ "host": "h", "username": "u", "password": "secret", "remotePath": "" }),
))
.unwrap();
assert!(!target.config_text.contains("key_file"), "{}", target.config_text);
assert!(target.config_text.lines().any(|l| l.starts_with("pass = ")), "{}", target.config_text);
}
#[test]
fn obscured_password_round_trips_through_rclone_reveal() {
let obscured = obscure_password("s3cr3t").unwrap();
assert_ne!(obscured, "s3cr3t");
let out = std::process::Command::new("rclone").args(["reveal", &obscured]).output().unwrap();
assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr));
assert_eq!(String::from_utf8_lossy(&out.stdout).trim(), "s3cr3t");
}
#[tokio::test]
async fn s3_target_reaches_minio_through_rclone() {
let (_container, endpoint) = start_minio().await; // "http://host:port"
let host_port = endpoint.trim_start_matches("http://");
let (host, port) = host_port.rsplit_once(':').unwrap();
let target = rclone_target(&storage(
"s3",
json!({ "endPointUrl": host, "port": port.parse::<u16>().unwrap(), "ssl": false,
"accessKey": "minioadmin", "secretKey": "minioadmin", "bucketName": "portabase" }),
))
.unwrap();
let config = write_config(&target.config_text).unwrap();
// The mapper sets `no_check_bucket = true`, so create the bucket with a per-call override.
let create = format!("{},no_check_bucket=false:{}", target.remote_name, target.base_path);
rclone_ok(config.path(), &["mkdir", &create]);
let listing = String::from_utf8(rclone_ok(config.path(), &["lsd", &format!("{}:", target.remote_name)])).unwrap();
assert!(listing.contains("portabase"), "lsd output: {listing}");
}
#[tokio::test]
async fn gcs_emulator_target_reaches_fake_gcs_through_rclone() {
let (_container, endpoint) = start_fake_gcs().await; // "http://host:port"
let target = rclone_target(&storage(
"google-cloud-storage",
json!({ "projectId": "test", "bucketName": "bucket1", "clientEmail": "x@test",
"privateKey": "", "apiEndpoint": endpoint }),
))
.unwrap();
let config = write_config(&target.config_text).unwrap();
// Same per-call override as the s3 test: the mapper never creates buckets on its own.
let create = format!("{},no_check_bucket=false:{}", target.remote_name, target.base_path);
rclone_ok(config.path(), &["mkdir", &create]);
let listing = String::from_utf8(rclone_ok(config.path(), &["lsd", &format!("{}:", target.remote_name)])).unwrap();
assert!(listing.contains("bucket1"), "lsd output: {listing}");
}
#[tokio::test]
async fn s3_target_hosts_a_restic_repository() {
use crate::services::backup::logger::JobLogger;
use crate::services::restic::backup::snapshot;
use crate::services::restic::command::ResticRepo;
use base64::{Engine as _, engine::general_purpose};
let (_container, endpoint) = start_minio().await; // "http://host:port"
let host_port = endpoint.trim_start_matches("http://");
let (host, port) = host_port.rsplit_once(':').unwrap();
let channel = storage(
"s3",
json!({ "endPointUrl": host, "port": port.parse::<u16>().unwrap(), "ssl": false,
"accessKey": "minioadmin", "secretKey": "minioadmin", "bucketName": "portabase" }),
);
let target = rclone_target(&channel).unwrap();
let config = write_config(&target.config_text).unwrap();
let create = format!("{},no_check_bucket=false:{}", target.remote_name, target.base_path);
rclone_ok(config.path(), &["mkdir", &create]);
let src = tempfile::TempDir::new().unwrap();
std::fs::write(src.path().join("a.txt"), "a").unwrap();
let cfg = crate::tests::domain::files::files_config(src.path(), &[]);
let master_key_b64 = general_purpose::STANDARD.encode([7u8; 32]);
let edge_key = crate::utils::edge_key::EdgeKey {
server_url: String::new(),
agent_id: "agent-1".into(),
master_key_b64: master_key_b64.clone(),
};
let repo = ResticRepo::open(&channel, &cfg.generated_id, &edge_key).unwrap();
assert_eq!(repo.repository(), format!("rclone:s3:portabase/backups/restic/{}", cfg.generated_id));
let snap = snapshot(&repo, &cfg, "bs-1", &JobLogger::new()).await.unwrap();
let listed = String::from_utf8(rclone_ok(
config.path(),
&["lsf", &format!("s3:portabase/backups/restic/{}/snapshots", cfg.generated_id)],
))
.unwrap();
assert!(listed.contains(&snap.snapshot_id), "{listed}");
}
#[tokio::test]
async fn s3_targets_host_one_sync_replica_per_channel() {
use crate::services::backup::logger::JobLogger;
use crate::services::sync::backup::one_storage;
use crate::tests::services::backup_uploader_tests::ctx_pointing_at;
use wiremock::matchers::{body_partial_json, method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
let (_container, endpoint) = start_minio().await; // "http://host:port"
let host_port = endpoint.trim_start_matches("http://");
let (host, port) = host_port.rsplit_once(':').unwrap();
let channel = |id: &str, bucket: &str| -> DatabaseStorage {
serde_json::from_value(json!({
"id": id, "provider": "s3", "folderName": "backups",
"config": { "endPointUrl": host, "port": port.parse::<u16>().unwrap(), "ssl": false,
"accessKey": "minioadmin", "secretKey": "minioadmin", "bucketName": bucket }
}))
.unwrap()
};
let channels = [channel("ch-a", "portabase"), channel("ch-b", "portabase2")];
for c in &channels {
let target = rclone_target(c).unwrap();
let config = write_config(&target.config_text).unwrap();
let create = format!("{},no_check_bucket=false:{}", target.remote_name, target.base_path);
rclone_ok(config.path(), &["mkdir", &create]);
}
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/agent/agent-1/backup/upload/init"))
.and(body_partial_json(json!({ "engine": "sync" })))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({ "message": "ok", "backupStorage": { "id": "bs" } })))
.expect(2)
.mount(&server)
.await;
Mock::given(method("PATCH"))
.and(path("/agent/agent-1/backup/upload/status"))
.and(body_partial_json(json!({ "status": "success", "filesTransferred": 2, "filesDeleted": 0 })))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({ "message": "ok", "backupStorage": { "id": "bs" } })))
.expect(2)
.mount(&server)
.await;
let src = tempfile::TempDir::new().unwrap();
std::fs::create_dir_all(src.path().join("docs")).unwrap();
std::fs::write(src.path().join("a.txt"), "a").unwrap();
std::fs::write(src.path().join("docs/b.txt"), "b").unwrap();
let cfg = crate::tests::domain::files::files_config(src.path(), &[]);
let ctx = ctx_pointing_at(server.uri());
for c in &channels {
let result = one_storage(&ctx, &cfg, c, "backup-1", &JobLogger::new()).await;
assert!(result.success, "{:?}", result.error);
assert_eq!(result.remote_file_path.as_deref(), Some(format!("backups/sync/{}/current", cfg.generated_id).as_str()));
}
for (c, bucket) in channels.iter().zip(["portabase", "portabase2"]) {
let target = rclone_target(c).unwrap();
let config = write_config(&target.config_text).unwrap();
let listed = String::from_utf8(rclone_ok(
config.path(),
&["lsf", "-R", &format!("s3:{bucket}/backups/sync/{}/current", cfg.generated_id)],
))
.unwrap();
assert!(listed.contains("a.txt") && listed.contains("docs/b.txt"), "{bucket}: {listed}");
}
}
+14 -10
View File
@@ -32,10 +32,8 @@ pub async fn scheduler_loop(mut conn: MultiplexedConnection) {
let mut conn_clone = conn.clone();
tokio::spawn(async move {
info!(
"Executing task={} args={:?} metadata={:?}",
task_clone.task, task_clone.args, task_clone.metadata
);
// Never log metadata: it carries decrypted storage channels (secrets).
info!("Executing task={} args={:?}", task_clone.task, task_clone.args);
if let Err(e) = execute_task(
task_clone.task.as_str(),
task_clone.args,
@@ -65,6 +63,14 @@ pub async fn scheduler_loop(mut conn: MultiplexedConnection) {
}
}
/// Local config merged with the dashboard-pushed sources, as the agent loop sees them.
fn merged_config(ctx: &Arc<Context>) -> crate::services::config::DatabasesConfig {
let local = ConfigService::new(ctx.clone()).load_optional(None);
let cache_path = std::path::PathBuf::from(&crate::settings::CONFIG.data_path).join("dashboard_databases.json");
let dashboard = crate::services::dashboard_config::load_cache(&cache_path);
crate::services::dashboard_config::merge(&local.databases, &dashboard)
}
pub async fn execute_task(
task: &str,
args: Vec<String>,
@@ -77,14 +83,9 @@ pub async fn execute_task(
info!("{} | {}", generated_id, dbms);
let ctx = Arc::new(Context::new());
let config_service = ConfigService::new(ctx.clone());
let backup_service = BackupService::new(ctx.clone());
let local = config_service.load_optional(None);
let cache_path = std::path::PathBuf::from(&crate::settings::CONFIG.data_path)
.join("dashboard_databases.json");
let dashboard = crate::services::dashboard_config::load_cache(&cache_path);
let config = crate::services::dashboard_config::merge(&local.databases, &dashboard);
let config = merged_config(&ctx);
let metadata_obj = metadata
.into_iter()
@@ -101,6 +102,8 @@ pub async fn execute_task(
let storages: Vec<DatabaseStorage> = serde_json::from_value(storages_value.clone())?;
let encrypt: bool = serde_json::from_value(encrypt_value.clone())?;
// Tasks stored before P2 have no engine: archive.
let engine = metadata_obj.get("engine").and_then(Value::as_str).unwrap_or("archive");
backup_service
.dispatch(
@@ -109,6 +112,7 @@ pub async fn execute_task(
BackupMethod::Automatic,
&storages,
encrypt,
engine,
)
.await;