Compare commits

...

23 Commits

Author SHA1 Message Date
github-actions[bot] f95f0aa73a chore: release 1.19.2 2026-08-28 07:59:57 +00:00
Charles GTE 2177dfd44b Merge pull request #100 from Portabase/fix/kill-on-drop
fix: kill_on_drop on firebird, mariadb, mysql, redis, valkey
2026-08-28 09:57:51 +02:00
charles-gauthereau 0a53eec184 fix: kill_on_drop on firebird, mariadb, mysql, redis, valkey 2026-08-28 09:43:08 +02:00
github-actions[bot] f7f5f7e141 chore: release 1.19.1 2026-08-21 17:09:15 +00:00
Charles GTE 2170f96a72 Merge pull request #99 from Portabase/fix/config-file
fix: config directory creation
2026-08-21 19:06:48 +02:00
Charles GTE 298d46ba81 fix: config directory creation 2026-08-21 13:50:40 +02:00
github-actions[bot] 6537e9df53 chore: release 1.19.0 2026-08-20 11:56:59 +00:00
Charles GTE b8d869d5a6 Merge pull request #97 from Portabase/feat/dashboard-databases
feat: dashboard-databases
2026-08-20 13:54:46 +02:00
Charles GTE 90941ea67d fix 2026-08-20 13:41:14 +02:00
charles-gauthereau 046d593e2b fix 2026-08-18 18:20:21 +02:00
charles-gauthereau cf9a59a138 fix: config.rs 2026-08-18 13:07:22 +02:00
charles-gauthereau 2464f6dfb0 Merge branch 'main' into feat/dashboard-databases
# Conflicts:
#	docker-compose.yml
#	src/services/config.rs
2026-08-18 13:05:56 +02:00
github-actions[bot] 069067ca55 chore: release 1.18.6 2026-08-17 09:10:08 +00:00
Charles GTE 04b654d219 Merge pull request #96 from Portabase/fix/mongo-cloud
fix: mongodb cloud cluster issue
2026-08-17 11:07:35 +02:00
charles-gauthereau d83511cb64 fix: mongodb cloud cluster issue 2026-08-17 10:52:21 +02:00
charles-gauthereau ca294e968c fix 2026-08-13 22:02:17 +02:00
charles-gauthereau 1be88ffdb9 Merge branch 'main' into feat/dashboard-databases
# Conflicts:
#	docker-compose.yml
2026-08-13 21:56:38 +02:00
charles-gauthereau 9d393a96a4 fix 2026-08-13 21:55:59 +02:00
charles-gauthereau 044bf80633 feat(agent): ingest dashboard databases via merged cycle + cache 2026-08-13 18:47:18 +02:00
charles-gauthereau 0a6eb6db22 feat(dashboard_config): atomic cache load/persist 2026-08-13 18:40:56 +02:00
charles-gauthereau b26ff81889 feat(dashboard_config): merge (dashboard-wins) + collect_configs 2026-08-13 18:39:54 +02:00
charles-gauthereau 4e2f29f4ca feat(status): decrypt dashboard config_ciphertext into resolved_config 2026-08-13 18:38:15 +02:00
charles-gauthereau d1c8df4cac refactor(config): extract build_config, add load_optional + Serialize 2026-08-13 18:35:52 +02:00
25 changed files with 640 additions and 172 deletions
+1 -1
View File
@@ -27,5 +27,5 @@ keywords:
- self-hosted
- portabase
license: Apache-2.0
version: 1.18.5
version: 1.19.2
date-released: '2026-02-24'
Generated
+2 -1
View File
@@ -3503,7 +3503,7 @@ dependencies = [
[[package]]
name = "portabase-agent"
version = "1.18.5"
version = "1.19.2"
dependencies = [
"aes",
"aes-gcm",
@@ -3535,6 +3535,7 @@ dependencies = [
"oauth2",
"once_cell",
"openssl",
"percent-encoding",
"postgres",
"rand 0.9.2",
"redis",
+2 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "portabase-agent"
version = "1.18.5"
version = "1.19.2"
edition = "2024"
[dependencies]
@@ -57,6 +57,7 @@ testcontainers = "0.27.1"
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "valkey", "mysql", "mariadb", "mongo"] }
postgres = "0.19.12"
url = "2.5.8"
percent-encoding = "2.3.2"
bollard = "0.20.0"
[dev-dependencies]
+2 -2
View File
@@ -9,7 +9,7 @@ services:
- .:/app
- cargo-registry:/usr/local/cargo/registry
- cargo-git:/usr/local/cargo/git
- ./databases.json:/config/config.json
# - ./databases.json:/config/config.json
#- ./databases.toml:/config/config.toml
- /var/run/docker.sock:/var/run/docker.sock
# - cargo-target:/app/target
@@ -21,7 +21,7 @@ services:
LOG: debug
TZ: "Europe/Paris"
# TMPDIR: /scratch
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiODY1Mjk2NDgtYmQ0Zi00MWMxLWFmNDItNGM1MzE3ZDEzY2JhIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ=="
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiZjlkZjhiNWYtM2I0MC00NWM3LWI3N2UtYzY4NzQ1YmU2NjMwIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ=="
#CHUNK_SIZE_MB: "1"
#POOLING: 1
#DATABASES_CONFIG_FILE: "config.toml"
+2
View File
@@ -142,6 +142,8 @@ RUN curl -sSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh \
WORKDIR /app
RUN mkdir -p /config
COPY --from=builder /app/target/release/app /usr/local/bin/app
COPY --from=builder /app/version.env /app/version.env
COPY entrypoint.sh /entrypoint.sh
+30 -8
View File
@@ -2,13 +2,16 @@
use crate::core::context::Context;
use crate::services::backup::BackupService;
use crate::services::config::ConfigService;
use crate::services::config::{ConfigService, DatabaseConfig};
use crate::services::cron::CronService;
use crate::services::dashboard_config::{collect_configs, load_cache, merge, persist_cache};
use crate::services::restore::RestoreService;
use crate::services::status::StatusService;
use crate::settings::CONFIG;
use crate::utils::common::BackupMethod;
use std::path::PathBuf;
use std::sync::Arc;
use tracing::info;
use tracing::{error, info, warn};
pub struct Agent {
ctx: Arc<Context>,
@@ -17,6 +20,8 @@ pub struct Agent {
cron_service: CronService,
backup_service: BackupService,
restore_service: RestoreService,
dashboard_cache: Vec<DatabaseConfig>,
cache_path: PathBuf,
}
impl Agent {
@@ -28,6 +33,9 @@ impl Agent {
let backup_service = BackupService::new(ctx.clone());
let restore_service = RestoreService::new(ctx.clone());
let cache_path = PathBuf::from(&CONFIG.data_path).join("dashboard_databases.json");
let dashboard_cache = load_cache(&cache_path);
Agent {
ctx,
config_service,
@@ -35,19 +43,33 @@ impl Agent {
cron_service,
backup_service,
restore_service,
dashboard_cache,
cache_path,
}
}
pub async fn run(&mut self, method: BackupMethod) -> Result<(), Box<dyn std::error::Error>> {
let config = self.config_service.load(None)?;
let ping_result = self.status_service.ping(&config.databases).await?;
let local = self.config_service.load_optional(None);
let merged_in = merge(&local.databases, &self.dashboard_cache);
let ping_result = self.status_service.ping(&merged_in.databases).await?;
self.dashboard_cache = collect_configs(&ping_result);
if let Err(e) = persist_cache(&self.cache_path, &self.dashboard_cache) {
error!("Failed to persist dashboard cache: {e}");
}
let merged = merge(&local.databases, &self.dashboard_cache);
for db in ping_result.databases.iter() {
let database = config
let Some(database) = merged
.databases
.iter()
.find(|cfg_db| cfg_db.generated_id == db.generated_id)
.unwrap();
else {
warn!("No config for returned database {}; skipping", db.generated_id);
continue;
};
info!(
"Generated Id: {} | backup action: {} | restore action: {} | Database Name: {}",
db.generated_id, db.data.backup.action, db.data.restore.action, database.name,
@@ -59,14 +81,14 @@ impl Agent {
.backup_service
.dispatch(
&db.generated_id,
&config,
&merged,
method.clone(),
&db.storages,
db.encrypt,
)
.await;
} else if db.data.restore.action {
let _ = self.restore_service.dispatch(db, &config).await;
let _ = self.restore_service.dispatch(db, &merged).await;
}
}
+1
View File
@@ -20,6 +20,7 @@ pub async fn run(cfg: DatabaseConfig) -> anyhow::Result<bool> {
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.kill_on_drop(true)
.spawn()?;
let query = b"SELECT 1 FROM RDB$DATABASE;\nQUIT;\n";
+2 -1
View File
@@ -12,7 +12,8 @@ pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::R
.arg("--user")
.arg(cfg.username)
.arg("ping")
.envs(env);
.envs(env)
.kill_on_drop(true);
let result = timeout(Duration::from_secs(10), cmd.output()).await;
+106 -11
View File
@@ -1,6 +1,13 @@
use crate::services::config::DatabaseConfig;
use anyhow::Result;
use mongodb::Client;
use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
const USERINFO_ENCODE: &AsciiSet = &NON_ALPHANUMERIC
.remove(b'-')
.remove(b'_')
.remove(b'.')
.remove(b'~');
pub async fn connect(cfg: DatabaseConfig) -> Result<Client> {
let uri = get_mongo_uri(cfg)?;
@@ -16,19 +23,40 @@ pub fn select_mongo_path() -> std::path::PathBuf {
}
pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result<String> {
if cfg.username.is_empty() || cfg.password.is_empty() {
Ok(format!(
"mongodb://{}:{}/{}",
cfg.host, cfg.port, cfg.database
))
} else {
Ok(format!(
"mongodb://{}:{}@{}:{}/{}?authSource=admin",
cfg.username, cfg.password, cfg.host, cfg.port, cfg.database
))
}
Ok(build_mongo_uri(&cfg, true))
}
pub fn build_mongo_uri(cfg: &DatabaseConfig, include_db: bool) -> String {
let is_srv = cfg.port == 0;
let scheme = if is_srv { "mongodb+srv" } else { "mongodb" };
let has_auth = !cfg.username.is_empty() && !cfg.password.is_empty();
let credentials = if has_auth {
format!(
"{}:{}@",
utf8_percent_encode(&cfg.username, USERINFO_ENCODE),
utf8_percent_encode(&cfg.password, USERINFO_ENCODE)
)
} else {
String::new()
};
let authority = if is_srv {
cfg.host.clone()
} else {
format!("{}:{}", cfg.host, cfg.port)
};
let path = if include_db {
format!("/{}", cfg.database)
} else {
"/".to_string()
};
let query = if has_auth { "?authSource=admin" } else { "" };
format!("{}://{}{}{}{}", scheme, credentials, authority, path, query)
}
pub fn extract_db_name(dry_output: &str) -> Option<String> {
let mut dbs = std::collections::HashSet::new();
@@ -43,3 +71,70 @@ pub fn extract_db_name(dry_output: &str) -> Option<String> {
}
dbs.into_iter().next()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::services::config::{DatabaseConfig, DbType};
use std::collections::HashMap;
fn cfg(host: &str, port: u16, user: &str, pass: &str) -> DatabaseConfig {
DatabaseConfig {
name: "t".into(),
database: "mydb".into(),
db_type: DbType::MongoDB,
username: user.into(),
password: pass.into(),
port,
host: host.into(),
generated_id: "id".into(),
path: String::new(),
max_packet_size: String::new(),
volume_name: String::new(),
container_name: None,
options: HashMap::new(),
}
}
#[test]
fn standard_with_auth() {
let c = cfg("localhost", 27017, "user", "pass");
assert_eq!(
build_mongo_uri(&c, true),
"mongodb://user:pass@localhost:27017/mydb?authSource=admin"
);
}
#[test]
fn standard_no_auth() {
let c = cfg("localhost", 27017, "", "");
assert_eq!(build_mongo_uri(&c, true), "mongodb://localhost:27017/mydb");
}
#[test]
fn srv_with_auth() {
let c = cfg("cluster.example.mongodb.net", 0, "user", "pass");
assert_eq!(
build_mongo_uri(&c, true),
"mongodb+srv://user:pass@cluster.example.mongodb.net/mydb?authSource=admin"
);
}
#[test]
fn srv_no_db_for_dryrun() {
let c = cfg("cluster.example.mongodb.net", 0, "user", "pass");
assert_eq!(
build_mongo_uri(&c, false),
"mongodb+srv://user:pass@cluster.example.mongodb.net/?authSource=admin"
);
}
#[test]
fn encodes_special_chars_in_credentials() {
let c = cfg("cluster.example.mongodb.net", 0, "user", "p@ss:w/rd?");
assert_eq!(
build_mongo_uri(&c, true),
"mongodb+srv://user:p%40ss%3Aw%2Frd%3F@cluster.example.mongodb.net/mydb?authSource=admin"
);
}
}
+5 -1
View File
@@ -19,7 +19,11 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
Ok(_) => Ok(true),
Err(e) => {
error!("--- MongoDB Connection Error Details ---");
error!("Target Host: {}:{}", cfg.host, cfg.port);
if cfg.port == 0 {
error!("Target Host: {} (srv)", cfg.host);
} else {
error!("Target Host: {}:{}", cfg.host, cfg.port);
}
error!("Error Kind: {:?}", e.kind);
error!("Full Error: {}", e);
error!("Check you database network connectivity");
+4 -8
View File
@@ -1,4 +1,6 @@
use crate::domain::mongodb::connection::{extract_db_name, get_mongo_uri, select_mongo_path};
use crate::domain::mongodb::connection::{
build_mongo_uri, extract_db_name, get_mongo_uri, select_mongo_path,
};
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
@@ -16,13 +18,7 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogg
let dry_start = Instant::now();
let dry_run = Command::new(&mongorestore)
.arg(format!(
"--uri={}",
format!(
"mongodb://{}:{}@{}:{}/?authSource=admin",
cfg.username, cfg.password, cfg.host, cfg.port
)
))
.arg(format!("--uri={}", build_mongo_uri(&cfg, false)))
.arg(format!("--archive={}", restore_file.display()))
.arg("--gzip")
.arg("--dryRun")
+2 -1
View File
@@ -12,7 +12,8 @@ pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::R
.arg("--user")
.arg(cfg.username)
.arg("ping")
.envs(env);
.envs(env)
.kill_on_drop(true);
let result = timeout(Duration::from_secs(10), cmd.output()).await;
+2
View File
@@ -21,6 +21,8 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
cmd.arg("PING");
cmd.kill_on_drop(true);
debug!("Command Ping Redis: {:?}", cmd);
let result = timeout(Duration::from_secs(10), cmd.output()).await;
+1
View File
@@ -20,6 +20,7 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
}
cmd.arg("PING");
cmd.kill_on_drop(true);
debug!("Command Ping Valkey: {:?}", cmd);
-1
View File
@@ -22,7 +22,6 @@ async fn main() {
eprintln!("Failed to clean locks on startup: {:?}", e);
}
// Best-effort cleanup of ephemeral helper containers orphaned by a crash.
match crate::domain::docker_volume::docker::client() {
Ok(docker) => match crate::domain::docker_volume::docker::sweep_ephemeral(&docker).await {
Ok(n) if n > 0 => tracing::info!("Removed {n} orphaned ephemeral helper container(s)"),
+8
View File
@@ -1,5 +1,6 @@
#![allow(dead_code)]
use crate::services::config::DatabaseConfig;
use crate::utils::deserializer::{deserialize_snake_case, string_or_number_to_string};
use serde::{Deserialize, Serialize};
use toml::Value;
@@ -39,6 +40,13 @@ pub struct DatabaseStatus {
pub storages_encrypted: Option<bool>,
#[serde(default)]
pub storages_ciphertext: Option<String>,
#[serde(default)]
pub config_encrypted: Option<bool>,
#[serde(default)]
pub config_ciphertext: Option<String>,
/// Filled in memory after decrypting `config_ciphertext`; never on the wire.
#[serde(skip)]
pub resolved_config: Option<DatabaseConfig>,
pub encrypt: bool,
pub data: DatabaseData,
}
+136 -128
View File
@@ -1,7 +1,7 @@
#![allow(dead_code)]
use crate::core::context::Context;
use serde::Deserialize;
use serde::{Deserialize, Serialize};
use serde_json;
use std::collections::HashMap;
use std::fs::File;
@@ -12,7 +12,7 @@ use toml;
use tracing::info;
use uuid::Uuid;
#[derive(Debug, Deserialize, Clone)]
#[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(rename_all = "lowercase")]
pub enum DbType {
Mysql,
@@ -49,7 +49,7 @@ impl DbType {
}
#[allow(dead_code)]
#[derive(Debug, Deserialize, Clone)]
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct DatabaseConfig {
pub name: String,
pub database: String,
@@ -68,7 +68,7 @@ pub struct DatabaseConfig {
}
#[allow(dead_code)]
#[derive(Debug, Deserialize, Clone)]
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct DatabasesConfig {
pub databases: Vec<DatabaseConfig>,
}
@@ -98,6 +98,106 @@ pub struct InputDatabasesConfig {
pub databases: Vec<InputDatabaseConfig>,
}
fn required<T: Clone>(opt: &Option<T>, db_name: &str, field_name: &str) -> Result<T, String> {
match opt {
Some(v) => Ok(v.clone()),
None => Err(format!(
"Missing required field '{}' for database '{}'",
field_name, db_name
)),
}
}
fn optional<T: Clone + Default>(opt: &Option<T>) -> T {
opt.clone().unwrap_or_default()
}
pub fn build_config(db: InputDatabaseConfig) -> Result<DatabaseConfig, String> {
if Uuid::parse_str(&db.generated_id).is_err() {
return Err(format!("Invalid UUID for database '{}'", db.name));
}
let username = match db.db_type {
DbType::Postgresql
| DbType::PostgresqlCluster
| DbType::Mysql
| DbType::Mariadb
| DbType::Mssql => required(&db.username, &db.name, "username")?,
_ => optional(&db.username),
};
let password = match db.db_type {
DbType::Postgresql
| DbType::PostgresqlCluster
| DbType::Mysql
| DbType::Mariadb
| DbType::Mssql => required(&db.password, &db.name, "password")?,
_ => optional(&db.password),
};
let host = match db.db_type {
DbType::Postgresql
| DbType::PostgresqlCluster
| DbType::Mysql
| DbType::Mariadb
| DbType::MongoDB
| DbType::Redis
| DbType::Firebird
| DbType::Valkey
| DbType::Mssql => required(&db.host, &db.name, "host")?,
DbType::Sqlite | DbType::DockerVolume => optional(&db.host),
};
let port = match db.db_type {
DbType::Postgresql
| DbType::PostgresqlCluster
| DbType::Mysql
| DbType::Mariadb
| DbType::Redis
| DbType::Firebird
| DbType::Valkey
| DbType::Mssql => required(&db.port, &db.name, "port")?,
DbType::MongoDB | DbType::Sqlite | DbType::DockerVolume => db.port.unwrap_or(0),
};
let database_name = match db.db_type {
DbType::Sqlite | DbType::Redis | DbType::Valkey | DbType::DockerVolume => {
optional(&db.database)
}
DbType::PostgresqlCluster => db
.database
.clone()
.unwrap_or_else(|| "postgres".to_string()),
_ => required(&db.database, &db.name, "database")?,
};
let path_val = match db.db_type {
DbType::Sqlite => required(&db.path, &db.name, "path")?,
_ => optional(&db.path),
};
let max_packet_size = match db.db_type {
DbType::Mysql | DbType::Mariadb => db.max_packet_size.unwrap_or_else(|| "512M".to_string()),
_ => String::new(),
};
let volume_name = match db.db_type {
DbType::DockerVolume => required(&db.volume_name, &db.name, "volume_name")?,
_ => optional(&db.volume_name),
};
Ok(DatabaseConfig {
name: db.name,
database: database_name,
db_type: db.db_type,
username,
password,
host,
port,
generated_id: db.generated_id,
path: path_val,
max_packet_size,
volume_name,
container_name: db.container_name.clone(),
options: db.options.unwrap_or_default(),
})
}
pub struct ConfigService {
ctx: Arc<Context>,
}
@@ -107,16 +207,19 @@ impl ConfigService {
ConfigService { ctx }
}
pub fn load(&self, file_path: Option<&str>) -> Result<DatabasesConfig, String> {
let path: String = if let Some(fp) = file_path {
fp.to_string()
} else {
format!(
fn resolve_path(file_path: Option<&str>) -> String {
match file_path {
Some(fp) => fp.to_string(),
None => format!(
"{}/{}",
crate::settings::CONFIG.data_path,
crate::settings::CONFIG.databases_config_file
)
};
),
}
}
pub fn load(&self, file_path: Option<&str>) -> Result<DatabasesConfig, String> {
let path = Self::resolve_path(file_path);
info!("Loading databases config from: {}", path);
@@ -150,128 +253,33 @@ impl ConfigService {
_ => return Err("Unsupported config file format. Use .json or .toml".to_string()),
};
fn required<T: Clone>(
opt: &Option<T>,
db_name: &str,
field_name: &str,
) -> Result<T, String> {
match opt {
Some(v) => Ok(v.clone()),
None => {
let msg = format!(
"Missing required field '{}' for database '{}'",
field_name, db_name
);
Err(msg)
}
}
}
fn optional<T: Clone>(opt: &Option<T>) -> T
where
T: Default,
{
opt.clone().unwrap_or_default()
}
let mut databases = Vec::with_capacity(input_config.databases.len());
for db in input_config.databases {
if Uuid::parse_str(&db.generated_id).is_err() {
return Err(format!("Invalid UUID for database '{}'", db.name));
}
let username = match db.db_type {
DbType::Postgresql
| DbType::PostgresqlCluster
| DbType::Mysql
| DbType::Mariadb
| DbType::Mssql => required(&db.username, &db.name, "username")?,
_ => optional(&db.username),
};
let password = match db.db_type {
DbType::Postgresql
| DbType::PostgresqlCluster
| DbType::Mysql
| DbType::Mariadb
| DbType::Mssql => required(&db.password, &db.name, "password")?,
_ => optional(&db.password),
};
let host = match db.db_type {
DbType::Postgresql
| DbType::PostgresqlCluster
| DbType::Mysql
| DbType::Mariadb
| DbType::MongoDB
| DbType::Redis
| DbType::Firebird
| DbType::Valkey
| DbType::Mssql => required(&db.host, &db.name, "host")?,
DbType::Sqlite | DbType::DockerVolume => optional(&db.host),
};
let port = match db.db_type {
DbType::Postgresql
| DbType::PostgresqlCluster
| DbType::Mysql
| DbType::Mariadb
| DbType::MongoDB
| DbType::Redis
| DbType::Firebird
| DbType::Valkey
| DbType::Mssql => required(&db.port, &db.name, "port")?,
DbType::Sqlite | DbType::DockerVolume => db.port.unwrap_or(0),
};
let database_name = match db.db_type {
DbType::Sqlite | DbType::Redis | DbType::Valkey | DbType::DockerVolume => {
optional(&db.database)
}
DbType::PostgresqlCluster => db
.database
.clone()
.unwrap_or_else(|| "postgres".to_string()),
_ => required(&db.database, &db.name, "database")?,
};
let path_val = match db.db_type {
DbType::Sqlite => required(&db.path, &db.name, "path")?,
_ => optional(&db.path),
};
let max_packet_size = match db.db_type {
DbType::Mysql | DbType::Mariadb => {
db.max_packet_size.unwrap_or_else(|| "512M".to_string())
}
_ => String::new(),
};
let volume_name = match db.db_type {
DbType::DockerVolume => required(&db.volume_name, &db.name, "volume_name")?,
_ => optional(&db.volume_name),
};
let container_name = db.container_name.clone();
databases.push(DatabaseConfig {
name: db.name,
database: database_name,
db_type: db.db_type,
username,
password,
host,
port,
generated_id: db.generated_id,
path: path_val,
max_packet_size,
volume_name,
container_name,
options: db.options.unwrap_or_default(),
});
databases.push(build_config(db)?);
}
info!("Databases: {} instances loaded", databases.len());
Ok(DatabasesConfig { databases })
}
pub fn load_optional(&self, file_path: Option<&str>) -> DatabasesConfig {
let path = Self::resolve_path(file_path);
if !Path::new(&path).exists() {
info!(
"No local databases config at {}; using dashboard-defined databases only",
path
);
return DatabasesConfig {
databases: Vec::new(),
};
}
self.load(file_path).unwrap_or_else(|e| {
tracing::warn!(
"Local databases config unavailable ({e}); continuing with dashboard-defined databases only"
);
DatabasesConfig { databases: Vec::new() }
})
}
}
+56
View File
@@ -0,0 +1,56 @@
#![allow(dead_code)]
use crate::services::api::models::agent::status::PingResult;
use crate::services::config::{DatabaseConfig, DatabasesConfig};
use std::path::Path;
pub fn merge(local: &[DatabaseConfig], dashboard: &[DatabaseConfig]) -> DatabasesConfig {
let mut databases: Vec<DatabaseConfig> = local.to_vec();
for d in dashboard {
if let Some(slot) = databases
.iter_mut()
.find(|c| c.generated_id == d.generated_id)
{
*slot = d.clone();
} else {
databases.push(d.clone());
}
}
DatabasesConfig { databases }
}
pub fn collect_configs(ping: &PingResult) -> Vec<DatabaseConfig> {
ping.databases
.iter()
.filter_map(|db| db.resolved_config.clone())
.collect()
}
pub fn load_cache(path: &Path) -> Vec<DatabaseConfig> {
let contents = match std::fs::read_to_string(path) {
Ok(c) => c,
Err(_) => return Vec::new(),
};
match serde_json::from_str::<DatabasesConfig>(&contents) {
Ok(cfg) => cfg.databases,
Err(e) => {
tracing::warn!("Dashboard cache at {:?} is corrupt ({e}); ignoring", path);
Vec::new()
}
}
}
pub fn persist_cache(path: &Path, databases: &[DatabaseConfig]) -> std::io::Result<()> {
let wrapper = DatabasesConfig {
databases: databases.to_vec(),
};
let json = serde_json::to_string_pretty(&wrapper)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
let tmp = path.with_extension("json.tmp");
std::fs::write(&tmp, json)?;
std::fs::rename(&tmp, path)?;
Ok(())
}
+1
View File
@@ -2,6 +2,7 @@ pub mod api;
pub mod backup;
pub mod config;
pub mod cron;
pub mod dashboard_config;
pub mod restore;
pub mod status;
pub mod storage;
+26 -1
View File
@@ -3,9 +3,10 @@
use crate::core::context::Context;
use crate::domain::factory::DatabaseFactory;
use crate::services::api::endpoints::status::DatabasePayload;
use crate::services::api::models::agent::status::DatabaseStatus;
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::api::models::agent::status::PingResult;
use crate::services::config::DatabaseConfig;
use crate::services::config::{build_config, DatabaseConfig, InputDatabaseConfig};
use crate::settings::CONFIG;
use crate::utils::file::decrypt_json_gcm;
use futures_util::future::try_join_all;
@@ -14,6 +15,26 @@ use std::error::Error;
use std::sync::Arc;
use tracing::info;
pub fn resolve_dashboard_config(
status: &mut DatabaseStatus,
master_key_b64: &str,
) -> Result<(), String> {
if status.config_encrypted != Some(true) {
return Ok(());
}
let ciphertext = status
.config_ciphertext
.as_deref()
.ok_or("config_encrypted set but config_ciphertext missing")?;
let plaintext = decrypt_json_gcm(ciphertext, master_key_b64)
.map_err(|e| format!("Failed to decrypt config: {e}"))?;
let input: InputDatabaseConfig = serde_json::from_slice(&plaintext)
.map_err(|e| format!("Failed to parse decrypted config: {e}"))?;
status.resolved_config = Some(build_config(input)?);
Ok(())
}
pub struct StatusService {
ctx: Arc<Context>,
client: Client,
@@ -67,6 +88,10 @@ impl StatusService {
db.storages = serde_json::from_slice::<Vec<DatabaseStorage>>(&plaintext)
.map_err(|e| format!("Failed to parse decrypted storages: {e}"))?;
}
if let Err(e) = resolve_dashboard_config(db, &edge_key.master_key_b64) {
tracing::warn!("Skipping dashboard config for {}: {e}", db.generated_id);
}
}
Ok(result)
}
-3
View File
@@ -137,9 +137,6 @@ impl StorageProvider for S3Provider {
.credentials_provider(credentials)
.region(region)
.force_path_style(true)
// S3-compatible endpoints (MinIO, Garage, RustFS, Synology, ...) reject the
// default CRC32 integrity checksums the SDK attaches to multipart uploads.
// Only send checksums when the operation actually requires them.
.request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
.endpoint_url(endpoint)
.behavior_version(BehaviorVersion::latest())
+95
View File
@@ -148,3 +148,98 @@ fn database_status_encrypted_envelope() {
assert_eq!(status.storages_encrypted, Some(true));
assert_eq!(status.storages_ciphertext.as_deref(), Some("AQIDBA=="));
}
#[test]
fn database_status_defaults_config_fields_absent() {
let json = r#"{
"dbms": "postgresql",
"generatedId": "16678159-ff7e-4c97-8c83-0adeff214681",
"encrypt": false,
"data": { "backup": { "action": false, "cron": null },
"restore": { "action": false, "file": null, "metaFile": null, "size": null } }
}"#;
let status: crate::services::api::models::agent::status::DatabaseStatus =
serde_json::from_str(json).unwrap();
assert_eq!(status.config_encrypted, None);
assert!(status.config_ciphertext.is_none());
assert!(status.resolved_config.is_none());
}
#[test]
fn resolve_dashboard_config_decrypts_full_entry() {
use crate::services::status::resolve_dashboard_config;
use base64::{engine::general_purpose, Engine};
// 32-byte master key, base64 STANDARD (matches decrypt_json_gcm).
let master_key_b64 = general_purpose::STANDARD.encode([7u8; 32]);
// Full agent-entry shape the dashboard encrypts.
let entry = r#"{
"name": "Dashboard PG",
"type": "postgresql",
"database": "app",
"username": "postgres",
"password": "s3cret",
"port": 5432,
"host": "10.0.0.10",
"generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"
}"#;
let ciphertext = encrypt_json_gcm(entry.as_bytes(), &master_key_b64);
let mut status: crate::services::api::models::agent::status::DatabaseStatus =
serde_json::from_str(
r#"{
"dbms": "postgresql",
"generatedId": "16678159-ff7e-4c97-8c83-0adeff214681",
"encrypt": false,
"config_encrypted": true,
"config_ciphertext": "PLACEHOLDER",
"data": { "backup": { "action": false, "cron": null },
"restore": { "action": false, "file": null, "metaFile": null, "size": null } }
}"#,
)
.unwrap();
status.config_ciphertext = Some(ciphertext);
resolve_dashboard_config(&mut status, &master_key_b64).unwrap();
let cfg = status.resolved_config.expect("resolved");
assert_eq!(cfg.name, "Dashboard PG");
assert_eq!(cfg.password, "s3cret");
assert_eq!(cfg.host, "10.0.0.10");
assert_eq!(cfg.db_type.as_str(), "postgresql");
}
#[test]
fn resolve_dashboard_config_noop_when_not_encrypted() {
use crate::services::status::resolve_dashboard_config;
let mut status: crate::services::api::models::agent::status::DatabaseStatus =
serde_json::from_str(
r#"{
"dbms": "postgresql",
"generatedId": "16678159-ff7e-4c97-8c83-0adeff214681",
"encrypt": false,
"data": { "backup": { "action": false, "cron": null },
"restore": { "action": false, "file": null, "metaFile": null, "size": null } }
}"#,
)
.unwrap();
resolve_dashboard_config(&mut status, "unused").unwrap();
assert!(status.resolved_config.is_none());
}
fn encrypt_json_gcm(plaintext: &[u8], master_key_b64: &str) -> String {
use aes_gcm::aead::{Aead, KeyInit};
use aes_gcm::{Aes256Gcm, Key, Nonce};
use base64::{engine::general_purpose, Engine};
let key_bytes = general_purpose::STANDARD.decode(master_key_b64).unwrap();
let key = Key::<Aes256Gcm>::try_from(key_bytes.as_slice()).unwrap();
let cipher = Aes256Gcm::new(&key);
let nonce_bytes = [0u8; 12];
let nonce = Nonce::try_from(&nonce_bytes[..]).unwrap();
let ct = cipher.encrypt(&nonce, plaintext).unwrap();
let mut data = nonce_bytes.to_vec();
data.extend_from_slice(&ct);
general_purpose::STANDARD.encode(data)
}
+71 -4
View File
@@ -1,15 +1,12 @@
use crate::core::context::Context;
use crate::services::api::ApiClient;
use crate::services::config::ConfigService;
use crate::services::config::{build_config, DatabasesConfig, InputDatabaseConfig};
use crate::utils::edge_key::EdgeKey;
use std::io::Write;
use std::sync::Arc;
use tempfile::NamedTempFile;
// `ConfigService::load` never touches `self.ctx` on the `Some(file_path)` path,
// so the values here don't matter — but `Context::new()` panics without an
// `EDGE_KEY` env var, so build the struct directly (mirrors
// backup_uploader_tests.rs's `ctx_pointing_at`).
fn test_context() -> Arc<Context> {
Arc::new(Context {
edge_key: EdgeKey {
@@ -264,3 +261,73 @@ fn docker_volume_requires_volume_name() {
let err = service.load(Some(file.path().to_str().unwrap())).unwrap_err();
assert!(err.contains("volume_name"), "error was: {err}");
}
#[test]
fn build_config_applies_type_defaults() {
let input: InputDatabaseConfig = serde_json::from_str(
r#"{
"name": "cluster1",
"type": "postgresql-cluster",
"username": "postgres",
"password": "p",
"port": 5432,
"host": "localhost",
"generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"
}"#,
)
.unwrap();
let cfg = build_config(input).unwrap();
assert_eq!(cfg.db_type.as_str(), "postgresql-cluster");
assert_eq!(cfg.database, "postgres"); // cluster default
}
#[test]
fn build_config_rejects_missing_required_field() {
let input: InputDatabaseConfig = serde_json::from_str(
r#"{
"name": "pg",
"type": "postgresql",
"username": "postgres",
"port": 5432,
"host": "localhost",
"generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"
}"#,
)
.unwrap();
let err = build_config(input).unwrap_err();
assert!(err.contains("password"), "unexpected error: {err}");
}
#[test]
fn load_optional_returns_empty_when_file_missing() {
let service = ConfigService::new(test_context());
let cfg = service.load_optional(Some("/nonexistent/path/does-not-exist.json"));
assert!(cfg.databases.is_empty());
}
#[test]
fn databases_config_roundtrips_through_serde() {
let input: InputDatabaseConfig = serde_json::from_str(
r#"{
"name": "pg",
"type": "postgresql",
"database": "app",
"username": "postgres",
"password": "secret",
"port": 5432,
"host": "localhost",
"generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"
}"#,
)
.unwrap();
let cfg = build_config(input).unwrap();
let wrapped = DatabasesConfig { databases: vec![cfg] };
let json = serde_json::to_string(&wrapped).unwrap();
let back: DatabasesConfig = serde_json::from_str(&json).unwrap();
assert_eq!(back.databases[0].name, "pg");
assert_eq!(back.databases[0].db_type.as_str(), "postgresql");
assert_eq!(back.databases[0].password, "secret");
}
@@ -0,0 +1,84 @@
use crate::services::config::{build_config, DatabaseConfig, InputDatabaseConfig};
use crate::services::dashboard_config::merge;
use crate::services::dashboard_config::{load_cache, persist_cache};
fn cfg(name: &str, gen_id: &str, host: &str) -> DatabaseConfig {
let json = format!(
r#"{{ "name": "{name}", "type": "postgresql", "database": "app",
"username": "u", "password": "p", "port": 5432,
"host": "{host}", "generated_id": "{gen_id}" }}"#
);
let input: InputDatabaseConfig = serde_json::from_str(&json).unwrap();
build_config(input).unwrap()
}
const ID_A: &str = "16678159-ff7e-4c97-8c83-0adeff214681";
const ID_B: &str = "16678124-ff7e-4c97-8c83-0adeff214681";
#[test]
fn merge_keeps_local_only_databases() {
let local = vec![cfg("local-a", ID_A, "local-host")];
let merged = merge(&local, &[]);
assert_eq!(merged.databases.len(), 1);
assert_eq!(merged.databases[0].host, "local-host");
}
#[test]
fn merge_appends_dashboard_only_databases() {
let local = vec![cfg("local-a", ID_A, "local-host")];
let dashboard = vec![cfg("dash-b", ID_B, "dash-host")];
let merged = merge(&local, &dashboard);
assert_eq!(merged.databases.len(), 2);
assert!(merged.databases.iter().any(|d| d.generated_id == ID_B));
}
#[test]
fn merge_dashboard_wins_on_id_collision() {
let local = vec![cfg("local-a", ID_A, "local-host")];
let dashboard = vec![cfg("dash-a", ID_A, "dash-host")];
let merged = merge(&local, &dashboard);
assert_eq!(merged.databases.len(), 1);
assert_eq!(merged.databases[0].host, "dash-host"); // dashboard wins
assert_eq!(merged.databases[0].name, "dash-a");
}
#[test]
fn cache_roundtrips() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("dashboard_databases.json");
let dbs = vec![cfg("dash-a", ID_A, "dash-host")];
persist_cache(&path, &dbs).unwrap();
let loaded = load_cache(&path);
assert_eq!(loaded.len(), 1);
assert_eq!(loaded[0].generated_id, ID_A);
assert_eq!(loaded[0].host, "dash-host");
}
#[test]
fn load_cache_missing_file_is_empty() {
let loaded = load_cache(std::path::Path::new("/nonexistent/dashboard_databases.json"));
assert!(loaded.is_empty());
}
#[test]
fn load_cache_corrupt_file_is_empty() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("dashboard_databases.json");
std::fs::write(&path, b"{ this is not valid json").unwrap();
let loaded = load_cache(&path);
assert!(loaded.is_empty());
}
#[test]
fn persist_cache_leaves_no_tmp_file() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("dashboard_databases.json");
persist_cache(&path, &[cfg("dash-a", ID_A, "h")]).unwrap();
let tmp = path.with_extension("json.tmp");
assert!(!tmp.exists(), "temp file should have been renamed away");
assert!(path.exists());
}
+1
View File
@@ -1,3 +1,4 @@
mod api_models_tests;
mod backup_uploader_tests;
mod config_tests;
mod dashboard_config_tests;