diff --git a/databases.json b/databases.json index 4f42d3a..2e15f61 100644 --- a/databases.json +++ b/databases.json @@ -34,19 +34,12 @@ "name": "Test database 5 - MongoDB", "database": "testdb", "type": "mongodb", - "username": "", - "password": "", "port": 27017, "host": "db-mongodb", "generated_id": "16678147-ff7e-4c97-8c83-0adeff214681" }, { "name": "Test database 6 - SQLite DB", - "database": "", - "username": "", - "password": "", - "port": 0, - "host": "", "type": "sqlite", "path": "/sqlite-data/workspace/data/app.db", "generated_id": "16678178-ff7e-4c97-8c83-0adeff214681" diff --git a/docker-compose.yml b/docker-compose.yml index 6a8a992..e0c7a38 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,73 +25,72 @@ services: networks: - portabase - # db-postgres: - # container_name: db-postgres - # image: postgres:17-alpine - # ports: - # - "5436:5432" - # volumes: - # - postgres-data:/var/lib/postgresql/data - # environment: - # - POSTGRES_DB=devdb - # - POSTGRES_USER=devuser - # - POSTGRES_PASSWORD=changeme - # networks: - # - portabase - # - # db-mariadb: - # container_name: db-mariadb - # image: mariadb:latest - # ports: - # - "3311:3306" - # environment: - # - MYSQL_DATABASE=mariadb - # - MYSQL_USER=mariadb - # - MYSQL_PASSWORD=changeme - # - MYSQL_RANDOM_ROOT_PASSWORD=yes - # volumes: - # - mariadb-data:/var/lib/mysql - # networks: - # - portabase - # - # - # db-mongodb-auth: - # container_name: db-mongodb-auth - # image: mongo:latest - # ports: - # - "27082:27017" - # environment: - # MONGO_INITDB_ROOT_USERNAME: root - # MONGO_INITDB_ROOT_PASSWORD: rootpassword - # MONGO_INITDB_DATABASE: testdbauth - # command: mongod --auth - # networks: - # - portabase - # volumes: - # - mongodb-data-auth:/data/db - # healthcheck: - # test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] - # interval: 5s - # timeout: 5s - # retries: 10 - # - # db-mongodb: - # container_name: db-mongodb - # image: mongo:latest - # ports: - # - "27083:27017" - # volumes: - # - mongodb-data:/data/db - # healthcheck: - # test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ] - # interval: 5s - # timeout: 5s - # retries: 10 - # environment: - # MONGO_INITDB_DATABASE: testdb - # networks: - # - portabase + db-postgres: + container_name: db-postgres + image: postgres:17-alpine + ports: + - "5436:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + environment: + - POSTGRES_DB=devdb + - POSTGRES_USER=devuser + - POSTGRES_PASSWORD=changeme + networks: + - portabase + db-mariadb: + container_name: db-mariadb + image: mariadb:latest + ports: + - "3311:3306" + environment: + - MYSQL_DATABASE=mariadb + - MYSQL_USER=mariadb + - MYSQL_PASSWORD=changeme + - MYSQL_RANDOM_ROOT_PASSWORD=yes + volumes: + - mariadb-data:/var/lib/mysql + networks: + - portabase + + + db-mongodb-auth: + container_name: db-mongodb-auth + image: mongo:latest + ports: + - "27082:27017" + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: rootpassword + MONGO_INITDB_DATABASE: testdbauth + command: mongod --auth + networks: + - portabase + volumes: + - mongodb-data-auth:/data/db + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + timeout: 5s + retries: 10 + + db-mongodb: + container_name: db-mongodb + image: mongo:latest + ports: + - "27083:27017" + volumes: + - mongodb-data:/data/db + healthcheck: + test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ] + interval: 5s + timeout: 5s + retries: 10 + environment: + MONGO_INITDB_DATABASE: testdb + networks: + - portabase sqlite: container_name: db-sqlite @@ -103,15 +102,16 @@ services: stdin_open: true tty: true + volumes: cargo-registry: cargo-git: # cargo-target: - # postgres-data: - # mariadb-data: - # mongodb-data: - # mongodb-data-auth: + postgres-data: + mariadb-data: + mongodb-data: + mongodb-data-auth: sqlite-data: networks: diff --git a/src/domain/mongodb/backup.rs b/src/domain/mongodb/backup.rs index 1737ab2..d4724b3 100644 --- a/src/domain/mongodb/backup.rs +++ b/src/domain/mongodb/backup.rs @@ -15,7 +15,7 @@ pub async fn run( let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension)); let mongodump = select_mongo_path().join("mongodump"); - let uri = get_mongo_uri(cfg.clone()); + let uri = get_mongo_uri(cfg.clone())?; let output = Command::new(mongodump) .arg(format!("--uri={}", uri)) diff --git a/src/domain/mongodb/connection.rs b/src/domain/mongodb/connection.rs index ef760f2..dcc48d4 100644 --- a/src/domain/mongodb/connection.rs +++ b/src/domain/mongodb/connection.rs @@ -3,7 +3,7 @@ use anyhow::Result; use mongodb::Client; pub async fn connect(cfg: DatabaseConfig) -> Result { - let uri = get_mongo_uri(cfg); + let uri = get_mongo_uri(cfg)?; let mut options = mongodb::options::ClientOptions::parse(&uri).await?; options.server_selection_timeout = Some(std::time::Duration::from_secs(3)); options.connect_timeout = Some(std::time::Duration::from_secs(3)); @@ -15,14 +15,16 @@ pub fn select_mongo_path() -> std::path::PathBuf { "/usr/local/mongodb/bin".to_string().into() } -pub fn get_mongo_uri(cfg: DatabaseConfig) -> String { - if cfg.username.is_empty() { - format!("mongodb://{}:{}/{}", cfg.host, cfg.port, cfg.database) +pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result { + if cfg.username.is_empty() && cfg.password.is_empty() { + Ok(format!("mongodb://{}:{}/{}", cfg.host, cfg.port, cfg.database)) } else { - format!( + Ok(format!( "mongodb://{}:{}@{}:{}/{}?authSource=admin", cfg.username, cfg.password, cfg.host, cfg.port, cfg.database - ) + )) } } + + diff --git a/src/domain/mongodb/ping.rs b/src/domain/mongodb/ping.rs index 0d288ad..6a3478b 100644 --- a/src/domain/mongodb/ping.rs +++ b/src/domain/mongodb/ping.rs @@ -1,14 +1,20 @@ #![allow(dead_code)] +use crate::domain::mongodb::connection::connect; use crate::services::config::DatabaseConfig; use anyhow::Result; use mongodb::bson::doc; -use tracing::{error}; -use crate::domain::mongodb::connection::connect; +use tracing::error; pub async fn run(cfg: DatabaseConfig) -> Result { let client = connect(cfg.clone()).await?; - let db_name = if cfg.username.is_empty() { &cfg.database } else { "admin" }; + + let db_name = if cfg.username.is_empty() && cfg.password.is_empty() { + &cfg.database + } else { + "admin" + }; + match client.database(db_name).run_command(doc! {"ping": 1}).await { Ok(_) => Ok(true), Err(e) => { diff --git a/src/domain/mongodb/restore.rs b/src/domain/mongodb/restore.rs index cb37a46..617834d 100644 --- a/src/domain/mongodb/restore.rs +++ b/src/domain/mongodb/restore.rs @@ -10,7 +10,7 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> { debug!("Starting MongoDB restore for database {}", cfg.name); let mongorestore = select_mongo_path().join("mongorestore"); - let uri = get_mongo_uri(cfg.clone()); + let uri = get_mongo_uri(cfg.clone())?; let output = Command::new(mongorestore) .arg(format!("--uri={}", uri)) diff --git a/src/domain/mysql/backup.rs b/src/domain/mysql/backup.rs index 2ec7531..1413bd4 100644 --- a/src/domain/mysql/backup.rs +++ b/src/domain/mysql/backup.rs @@ -1,4 +1,4 @@ -use crate::domain::mysql::connection::server_version; +use crate::domain::mysql::connection::{server_version}; use crate::services::config::DatabaseConfig; use anyhow::{Context, Result}; use std::collections::HashMap; diff --git a/src/domain/mysql/connection.rs b/src/domain/mysql/connection.rs index b89a0f8..db26bbb 100644 --- a/src/domain/mysql/connection.rs +++ b/src/domain/mysql/connection.rs @@ -3,6 +3,7 @@ use std::process::Command; use anyhow::Result; pub async fn server_version(cfg: &DatabaseConfig) -> Result { + let output = Command::new("mysql") .arg("--host").arg(&cfg.host) .arg("--port").arg(cfg.port.to_string()) @@ -25,3 +26,4 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result { Ok(version) } + diff --git a/src/domain/mysql/database.rs b/src/domain/mysql/database.rs index 4c129ed..13fa06f 100644 --- a/src/domain/mysql/database.rs +++ b/src/domain/mysql/database.rs @@ -21,7 +21,7 @@ impl MySQLDatabase { fn build_env(&self) -> HashMap { let mut envs = std::env::vars().collect::>(); - envs.insert("MYSQL_PWD".to_string(), self.cfg.password.clone()); + envs.insert("MYSQL_PWD".to_string(), self.cfg.password.to_string()); envs } } diff --git a/src/domain/mysql/ping.rs b/src/domain/mysql/ping.rs index fde0407..dd2491f 100644 --- a/src/domain/mysql/ping.rs +++ b/src/domain/mysql/ping.rs @@ -4,6 +4,7 @@ use tokio::process::Command; use tokio::time::{Duration, timeout}; pub async fn run(cfg: DatabaseConfig, env: HashMap) -> anyhow::Result { + let mut cmd = Command::new("mysqladmin"); cmd.arg("--host") .arg(cfg.host) diff --git a/src/domain/mysql/restore.rs b/src/domain/mysql/restore.rs index 2039b1a..f662a50 100644 --- a/src/domain/mysql/restore.rs +++ b/src/domain/mysql/restore.rs @@ -1,11 +1,10 @@ +use crate::services::config::DatabaseConfig; +use anyhow::{Context, Result}; use std::fs::File; use std::io::{Read, Write}; -use anyhow::{Context, Result}; -use tracing::{debug, error, info}; use std::path::PathBuf; use std::process::Command; - -use crate::services::config::DatabaseConfig; +use tracing::{debug, error, info}; pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> { let handle = tokio::task::spawn_blocking(move || -> Result<()> { @@ -55,7 +54,8 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> { .with_context(|| format!("Failed to start mysql restore for {}", cfg.name))?; let mut stdin = child.stdin.take().context("Failed to open child stdin")?; - stdin.write_all(sql_content.as_bytes()) + stdin + .write_all(sql_content.as_bytes()) .context("Failed to write SQL content to mysql stdin")?; stdin.flush()?; drop(stdin); @@ -74,8 +74,7 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> { Ok(()) }); - handle - .await??; + handle.await??; Ok(()) } diff --git a/src/domain/postgres/backup.rs b/src/domain/postgres/backup.rs index 107f30d..c7dbfcb 100644 --- a/src/domain/postgres/backup.rs +++ b/src/domain/postgres/backup.rs @@ -1,7 +1,7 @@ use anyhow::Result; -use tracing::{debug, error, info}; use std::path::PathBuf; use std::process::Command; +use tracing::{debug, error, info}; use super::connection::{select_pg_path, server_version}; use super::format::PostgresDumpFormat; diff --git a/src/domain/postgres/connection.rs b/src/domain/postgres/connection.rs index e14fcda..92648f3 100644 --- a/src/domain/postgres/connection.rs +++ b/src/domain/postgres/connection.rs @@ -1,11 +1,12 @@ -use std::path::Path; use crate::domain::postgres::format::PostgresDumpFormat; use crate::services::config::DatabaseConfig; use anyhow::Result; +use std::path::Path; use tokio_postgres::{Client, NoTls}; -use tracing::info; +use tracing::{error, info}; pub async fn connect(cfg: &DatabaseConfig) -> Result { + info!("Connecting to postgres database {}:{}", cfg.host, cfg.port); let dsn = format!( "host={} port={} user={} password={} dbname={}", cfg.host, cfg.port, cfg.username, cfg.password, cfg.database @@ -14,7 +15,7 @@ pub async fn connect(cfg: &DatabaseConfig) -> Result { let (client, connection) = tokio_postgres::connect(&dsn, NoTls).await?; tokio::spawn(async move { if let Err(e) = connection.await { - tracing::error!("Postgres connection error: {}", e); + error!("Postgres connection error: {}", e); } }); Ok(client) @@ -34,7 +35,7 @@ pub fn select_pg_path(version: &str) -> std::path::PathBuf { pub async fn terminate_connections(cfg: &DatabaseConfig) -> Result<()> { let mut admin = cfg.clone(); - admin.database = "postgres".into(); + admin.database = "postgres".to_string().into(); let client = connect(&admin).await?; diff --git a/src/domain/postgres/database.rs b/src/domain/postgres/database.rs index f2ac8cb..618c1c3 100644 --- a/src/domain/postgres/database.rs +++ b/src/domain/postgres/database.rs @@ -39,7 +39,6 @@ impl Database for PostgresDatabase { FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; let res = backup::run(self.cfg.clone(), self.format, dir.to_path_buf()).await; FileLock::release(&self.cfg.generated_id).await?; - res } diff --git a/src/domain/postgres/restore.rs b/src/domain/postgres/restore.rs index cb3d130..0458d82 100644 --- a/src/domain/postgres/restore.rs +++ b/src/domain/postgres/restore.rs @@ -1,7 +1,7 @@ use anyhow::Result; -use tracing::{debug, error, info}; use std::path::PathBuf; use std::process::Command; +use tracing::{debug, error, info}; use super::connection::{select_pg_path, server_version, terminate_connections}; use super::format::PostgresDumpFormat; @@ -14,6 +14,7 @@ pub async fn run( ) -> Result<()> { tokio::task::spawn_blocking(move || -> Result<()> { debug!("Starting restore for database {}", cfg.name); + let version = match futures::executor::block_on(server_version(&cfg)) { Ok(v) => { debug!("Postgres version detected: {}", v); @@ -91,8 +92,6 @@ pub async fn run( let dec = flate2::read::GzDecoder::new(tar_gz); let mut archive = tar::Archive::new(dec); - - let tmp_dir = match tempfile::TempDir::new() { Ok(d) => d, Err(e) => { @@ -104,8 +103,6 @@ pub async fn run( } }; - - if let Err(e) = archive.unpack(tmp_dir.path()) { error!("Failed to unpack FD archive for {}: {:?}", cfg.name, e); return Err(e.into()); @@ -113,7 +110,6 @@ pub async fn run( debug!("Listing contents of temp dir: {}", tmp_dir.path().display()); - for entry in std::fs::read_dir(tmp_dir.path())? { if let Ok(entry) = entry { let path = entry.path(); diff --git a/src/domain/sqlite/backup.rs b/src/domain/sqlite/backup.rs index 86e894d..578cfd6 100644 --- a/src/domain/sqlite/backup.rs +++ b/src/domain/sqlite/backup.rs @@ -12,9 +12,11 @@ pub async fn run( tokio::task::spawn_blocking(move || -> Result { debug!("Starting SQLite backup for database {}", cfg.name); - let db_path_str = cfg.path - .as_ref() - .ok_or_else(|| anyhow::anyhow!("Database path not configured"))?; + let db_path_str = if cfg.path.is_empty() { + anyhow::bail!("Database path not configured"); + } else { + cfg.path.as_str().to_string() + }; let db_path = PathBuf::from(db_path_str); diff --git a/src/domain/sqlite/restore.rs b/src/domain/sqlite/restore.rs index 147a8fb..a386a5d 100644 --- a/src/domain/sqlite/restore.rs +++ b/src/domain/sqlite/restore.rs @@ -8,19 +8,16 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> { tokio::task::spawn_blocking(move || -> Result<()> { debug!("Starting SQLite restore for database {}", cfg.name); - - let db_path_str = cfg.path - .as_ref() - .ok_or_else(|| anyhow::anyhow!("Database path not configured"))?; + let db_path_str = if cfg.path.is_empty() { + anyhow::bail!("Database path not configured"); + } else { + cfg.path.as_str().to_string() + }; let db_path = PathBuf::from(db_path_str); - if !restore_file.exists() { - anyhow::bail!( - "Restore file not found: {}", - restore_file.display() - ); + anyhow::bail!("Restore file not found: {}", restore_file.display()); } if db_path.exists() { @@ -43,5 +40,5 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> { info!("SQLite restore completed for {}", cfg.name); Ok(()) }) - .await? -} \ No newline at end of file + .await? +} diff --git a/src/services/config.rs b/src/services/config.rs index dab2834..af98f07 100644 --- a/src/services/config.rs +++ b/src/services/config.rs @@ -9,6 +9,7 @@ use std::path::Path; use std::sync::Arc; use toml; use tracing::info; +use uuid::Uuid; #[derive(Debug, Deserialize, Clone)] #[serde(rename_all = "lowercase")] @@ -17,7 +18,7 @@ pub enum DbType { Mariadb, Postgresql, MongoDB, - Sqlite + Sqlite, // Add other DB types if needed } @@ -45,7 +46,7 @@ pub struct DatabaseConfig { pub port: u16, pub host: String, pub generated_id: String, - pub path: Option + pub path: String, } #[allow(dead_code)] @@ -54,6 +55,29 @@ pub struct DatabasesConfig { pub databases: Vec, } + +#[allow(dead_code)] +#[derive(Debug, Deserialize, Clone)] +pub struct InputDatabaseConfig { + pub name: String, + pub database: Option, + #[serde(rename = "type")] + pub db_type: DbType, + pub username: Option, + pub password: Option, + pub port: Option, + pub host: Option, + pub generated_id: String, + pub path: Option, +} + +#[allow(dead_code)] +#[derive(Debug, Deserialize, Clone)] +pub struct InputDatabasesConfig { + pub databases: Vec, +} + + pub struct ConfigService { ctx: Arc, } @@ -96,7 +120,7 @@ impl ConfigService { file.read_to_string(&mut contents) .map_err(|e| format!("Failed to read config file: {}", e))?; - let config: DatabasesConfig = match extension { + let input_config: InputDatabasesConfig = match extension { "json" => { serde_json::from_str(&contents).map_err(|e| format!("JSON parsing error: {}", e))? } @@ -106,8 +130,71 @@ impl ConfigService { _ => return Err("Unsupported config file format. Use .json or .toml".to_string()), }; - info!("Databases : {:?} instances loaded", config.databases.len()); + fn required(opt: &Option, db_name: &str, field_name: &str) -> Result { + match opt { + Some(v) => Ok(v.clone()), + None => { + let msg = format!("Missing required field '{}' for database '{}'", field_name, db_name); + Err(msg) + } + } + } - Ok(config) + fn optional(opt: &Option) -> T where T: Default { + opt.clone().unwrap_or_default() + } + + let mut databases = Vec::with_capacity(input_config.databases.len()); + + for db in input_config.databases { + if Uuid::parse_str(&db.generated_id).is_err() { + return Err(format!("Invalid UUID for database '{}'", db.name)); + } + + let username = match db.db_type { + DbType::Postgresql | DbType::Mysql | DbType::Mariadb => required(&db.username, &db.name, "username")?, + _ => optional(&db.username), + }; + + let password = match db.db_type { + DbType::Postgresql | DbType::Mysql | DbType::Mariadb => required(&db.password, &db.name, "password")?, + _ => optional(&db.password), + }; + + let host = match db.db_type { + DbType::Postgresql | DbType::Mysql | DbType::Mariadb | DbType::MongoDB => required(&db.host, &db.name, "host")?, + DbType::Sqlite => optional(&db.host), + }; + + let port = match db.db_type { + DbType::Postgresql | DbType::Mysql | DbType::Mariadb | DbType::MongoDB => required(&db.port, &db.name, "port")?, + DbType::Sqlite => db.port.unwrap_or(0), + }; + + let database_name = match db.db_type { + DbType::Sqlite => optional(&db.database), + _ => required(&db.database, &db.name, "database")? + }; + + let path_val = match db.db_type { + DbType::Sqlite => required(&db.path, &db.name, "path")?, + _ => optional(&db.path), + }; + + databases.push(DatabaseConfig { + name: db.name, + database: database_name, + db_type: db.db_type, + username, + password, + host, + port, + generated_id: db.generated_id, + path: path_val, + }); + } + + info!("Databases: {} instances loaded", databases.len()); + Ok(DatabasesConfig { databases }) } } diff --git a/src/services/storage/providers/s3/mod.rs b/src/services/storage/providers/s3/mod.rs index fb23c63..bc1430d 100644 --- a/src/services/storage/providers/s3/mod.rs +++ b/src/services/storage/providers/s3/mod.rs @@ -94,12 +94,15 @@ impl StorageProvider for S3Provider { ); let region = Region::new(config.region.clone().unwrap_or("us-east-1".to_string())); - - let scheme = if config.ssl { "https" } else { "http" }; - - let endpoint = match config.port { - Some(port) => format!("{scheme}://{}:{port}", config.end_point_url), - None => format!("{scheme}://{}", config.end_point_url), + + let endpoint = if let Some(port) = &config.port { + if port.trim().is_empty() { + format!("{}://{}", if config.ssl { "https" } else { "http" }, config.end_point_url) + } else { + format!("{}://{}:{}", if config.ssl { "https" } else { "http" }, config.end_point_url, port) + } + } else { + format!("{}://{}", if config.ssl { "https" } else { "http" }, config.end_point_url) }; info!("S3 endpoint to {}", &endpoint);