From 178628cb1cb69234190586aa657ff0f82d74c705 Mon Sep 17 00:00:00 2001 From: charlesgauthereau Date: Sun, 22 Feb 2026 21:46:16 +0100 Subject: [PATCH 1/2] feat: add sqlite support, first working version --- Makefile | 17 ++++ databases.json | 11 +++ docker-compose.yml | 154 ++++++++++++++++++---------------- docker/Dockerfile | 2 + scripts/README.md | 17 +++- scripts/sqlite/seed-big.sql | 74 ++++++++++++++++ scripts/sqlite/seed.sql | 59 +++++++++++++ src/domain/factory.rs | 3 + src/domain/mod.rs | 1 + src/domain/sqlite/backup.rs | 43 ++++++++++ src/domain/sqlite/database.rs | 43 ++++++++++ src/domain/sqlite/mod.rs | 4 + src/domain/sqlite/ping.rs | 5 ++ src/domain/sqlite/restore.rs | 47 +++++++++++ src/services/config.rs | 7 +- 15 files changed, 411 insertions(+), 76 deletions(-) create mode 100644 scripts/sqlite/seed-big.sql create mode 100644 scripts/sqlite/seed.sql create mode 100644 src/domain/sqlite/backup.rs create mode 100644 src/domain/sqlite/database.rs create mode 100644 src/domain/sqlite/mod.rs create mode 100644 src/domain/sqlite/ping.rs create mode 100644 src/domain/sqlite/restore.rs diff --git a/Makefile b/Makefile index 56a9060..a363fa2 100644 --- a/Makefile +++ b/Makefile @@ -30,4 +30,21 @@ seed-postgres-1gb: docker exec -i -e PGPASSWORD=$$PG_PASSWORD $$PG_CONTAINER \ psql -U $$PG_USER -d $$PG_DB < ./scripts/postgres/seed-1gb.sql + +SQLITE_SEED_FILE := $(if $(filter big,$(SEED)),./scripts/sqlite/seed-big.sql,./scripts/sqlite/seed.sql) + +seed-sqlite: + @echo "Seeding Sqlite..." + @echo "Run as root to fix permissions inside the volume" + docker exec -u 0 -it db-sqlite sh -c "chmod -R 777 /workspace/data" + @echo "Create the database file (if it doesn’t exist)" + docker exec -u 0 -it db-sqlite sh -c "touch /workspace/data/app.db" + @echo "Seed the database" + docker exec -i db-sqlite sh -c "sqlite3 /workspace/data/app.db" < $(SQLITE_SEED_FILE) + @echo "Verify" + docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT name FROM sqlite_master WHERE type='table';" + @echo "Done" + + + seed-all: seed-mongo seed-mysql seed-postgres seed-postgres-1gb diff --git a/databases.json b/databases.json index 7b83c99..4f42d3a 100644 --- a/databases.json +++ b/databases.json @@ -39,6 +39,17 @@ "port": 27017, "host": "db-mongodb", "generated_id": "16678147-ff7e-4c97-8c83-0adeff214681" + }, + { + "name": "Test database 6 - SQLite DB", + "database": "", + "username": "", + "password": "", + "port": 0, + "host": "", + "type": "sqlite", + "path": "/sqlite-data/workspace/data/app.db", + "generated_id": "16678178-ff7e-4c97-8c83-0adeff214681" } ] } diff --git a/docker-compose.yml b/docker-compose.yml index 066a66a..6a8a992 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,8 @@ services: # - ./databases.toml:/config/config.toml - cargo-registry:/usr/local/cargo/registry - cargo-git:/usr/local/cargo/git -# - cargo-target:/app/target + # - cargo-target:/app/target + - sqlite-data:/sqlite-data/workspace/data environment: APP_ENV: development LOG: debug @@ -24,83 +25,94 @@ services: networks: - portabase - db-postgres: - container_name: db-postgres - image: postgres:17-alpine - ports: - - "5436:5432" - volumes: - - postgres-data:/var/lib/postgresql/data - environment: - - POSTGRES_DB=devdb - - POSTGRES_USER=devuser - - POSTGRES_PASSWORD=changeme - networks: - - portabase + # db-postgres: + # container_name: db-postgres + # image: postgres:17-alpine + # ports: + # - "5436:5432" + # volumes: + # - postgres-data:/var/lib/postgresql/data + # environment: + # - POSTGRES_DB=devdb + # - POSTGRES_USER=devuser + # - POSTGRES_PASSWORD=changeme + # networks: + # - portabase + # + # db-mariadb: + # container_name: db-mariadb + # image: mariadb:latest + # ports: + # - "3311:3306" + # environment: + # - MYSQL_DATABASE=mariadb + # - MYSQL_USER=mariadb + # - MYSQL_PASSWORD=changeme + # - MYSQL_RANDOM_ROOT_PASSWORD=yes + # volumes: + # - mariadb-data:/var/lib/mysql + # networks: + # - portabase + # + # + # db-mongodb-auth: + # container_name: db-mongodb-auth + # image: mongo:latest + # ports: + # - "27082:27017" + # environment: + # MONGO_INITDB_ROOT_USERNAME: root + # MONGO_INITDB_ROOT_PASSWORD: rootpassword + # MONGO_INITDB_DATABASE: testdbauth + # command: mongod --auth + # networks: + # - portabase + # volumes: + # - mongodb-data-auth:/data/db + # healthcheck: + # test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + # interval: 5s + # timeout: 5s + # retries: 10 + # + # db-mongodb: + # container_name: db-mongodb + # image: mongo:latest + # ports: + # - "27083:27017" + # volumes: + # - mongodb-data:/data/db + # healthcheck: + # test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ] + # interval: 5s + # timeout: 5s + # retries: 10 + # environment: + # MONGO_INITDB_DATABASE: testdb + # networks: + # - portabase - db-mariadb: - container_name: db-mariadb - image: mariadb:latest - ports: - - "3311:3306" - environment: - - MYSQL_DATABASE=mariadb - - MYSQL_USER=mariadb - - MYSQL_PASSWORD=changeme - - MYSQL_RANDOM_ROOT_PASSWORD=yes - volumes: - - mariadb-data:/var/lib/mysql - networks: - - portabase -# -# db-mongodb-auth: -# container_name: db-mongodb-auth -# image: mongo:latest -# ports: -# - "27082:27017" -# environment: -# MONGO_INITDB_ROOT_USERNAME: root -# MONGO_INITDB_ROOT_PASSWORD: rootpassword -# MONGO_INITDB_DATABASE: testdbauth -# command: mongod --auth -# networks: -# - portabase -# volumes: -# - mongodb-data-auth:/data/db -# healthcheck: -# test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] -# interval: 5s -# timeout: 5s -# retries: 10 -# -# db-mongodb: -# container_name: db-mongodb -# image: mongo:latest -# ports: -# - "27083:27017" -# volumes: -# - mongodb-data:/data/db -# healthcheck: -# test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ] -# interval: 5s -# timeout: 5s -# retries: 10 -# environment: -# MONGO_INITDB_DATABASE: testdb -# networks: -# - portabase + sqlite: + container_name: db-sqlite + image: keinos/sqlite3 + volumes: + - sqlite-data:/workspace/data + working_dir: /workspace + command: tail -f /dev/null + stdin_open: true + tty: true volumes: cargo-registry: cargo-git: -# cargo-target: + # cargo-target: - - postgres-data: - mariadb-data: -# mongodb-data: -# mongodb-data-auth: + # postgres-data: + # mariadb-data: + # mongodb-data: + # mongodb-data-auth: + sqlite-data: networks: portabase: diff --git a/docker/Dockerfile b/docker/Dockerfile index d149b6d..5ae11de 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,6 +15,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ zlib1g \ curl \ mariadb-client \ + sqlite3 \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -92,6 +93,7 @@ RUN apt-get update && apt-get install -y \ libncurses6 \ zlib1g \ mariadb-client \ + sqlite3 \ && rm -rf /var/lib/apt/lists/* diff --git a/scripts/README.md b/scripts/README.md index 4160bc3..9045b17 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,6 +1,5 @@ # Seed instructions -## MongoDB ```bash make seed-mongo @@ -9,5 +8,19 @@ make seed-mysql make seed-mysql-1gb make seed-postgres make seed-postgres-1gb -make seed-all +make seed-all +make seed-sqlite +make seed-sqlite SEED=big +``` + +## Verify commands + +### Sqlite + +```bash +docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT * FROM users LIMIT 10;" +``` + +```bash +docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT name FROM sqlite_master WHERE type='table';" ``` diff --git a/scripts/sqlite/seed-big.sql b/scripts/sqlite/seed-big.sql new file mode 100644 index 0000000..ad0ff4e --- /dev/null +++ b/scripts/sqlite/seed-big.sql @@ -0,0 +1,74 @@ +PRAGMA foreign_keys = ON; +BEGIN TRANSACTION; + +-- Drop tables +DROP TABLE IF EXISTS users; +DROP TABLE IF EXISTS projects; +DROP TABLE IF EXISTS tasks; + +-- Users +CREATE TABLE users ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + email TEXT NOT NULL UNIQUE, + full_name TEXT NOT NULL, + role TEXT NOT NULL CHECK (role IN ('admin','manager','user')), + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +-- Projects +CREATE TABLE projects ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + description TEXT, + owner_id INTEGER NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + FOREIGN KEY (owner_id) REFERENCES users(id) ON DELETE CASCADE +); + +-- Tasks +CREATE TABLE tasks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + project_id INTEGER NOT NULL, + title TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('todo','in_progress','done')), + priority INTEGER NOT NULL DEFAULT 3, + due_date TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE +); + +-- Seed minimal users +INSERT INTO users (email, full_name, role) VALUES + ('admin@example.com', 'System Admin', 'admin'), + ('manager@example.com', 'Project Manager', 'manager'), + ('user@example.com', 'Standard User', 'user'); + +-- Generate 10_000 projects +WITH RECURSIVE numbers(x) AS ( + SELECT 1 + UNION ALL + SELECT x+1 FROM numbers WHERE x<10000 +) +INSERT INTO projects (name, description, owner_id) +SELECT + 'Project #' || x, + 'Auto-generated project description for project #' || x, + (1 + (x % 3)) -- cycle users 1..3 +FROM numbers; + +-- Generate 100_000 tasks +WITH RECURSIVE numbers(x) AS ( + SELECT 1 + UNION ALL + SELECT x+1 FROM numbers WHERE x<100000 +) +INSERT INTO tasks (project_id, title, status, priority, due_date) +SELECT + (1 + (x % 10000)), -- project id cycle + 'Task #' || x, + CASE (x % 3) WHEN 0 THEN 'todo' WHEN 1 THEN 'in_progress' ELSE 'done' END, + 1 + (x % 5), + date('now', '+' || (x % 30) || ' days') +FROM numbers; + +COMMIT; \ No newline at end of file diff --git a/scripts/sqlite/seed.sql b/scripts/sqlite/seed.sql new file mode 100644 index 0000000..73b5f08 --- /dev/null +++ b/scripts/sqlite/seed.sql @@ -0,0 +1,59 @@ +PRAGMA foreign_keys = ON; + +BEGIN TRANSACTION; + +-- Drop existing tables (idempotent reset) +DROP TABLE IF EXISTS users; +DROP TABLE IF EXISTS projects; +DROP TABLE IF EXISTS tasks; + +-- Users +CREATE TABLE users ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + email TEXT NOT NULL UNIQUE, + full_name TEXT NOT NULL, + role TEXT NOT NULL CHECK (role IN ('admin','manager','user')), + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +-- Projects +CREATE TABLE projects ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + description TEXT, + owner_id INTEGER NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + FOREIGN KEY (owner_id) REFERENCES users(id) ON DELETE CASCADE +); + +-- Tasks +CREATE TABLE tasks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + project_id INTEGER NOT NULL, + title TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('todo','in_progress','done')), + priority INTEGER NOT NULL DEFAULT 3, + due_date TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE +); + +-- Seed users +INSERT INTO users (email, full_name, role) VALUES + ('admin@example.com', 'System Admin', 'admin'), + ('manager@example.com', 'Project Manager', 'manager'), + ('user@example.com', 'Standard User', 'user'); + +-- Seed projects +INSERT INTO projects (name, description, owner_id) VALUES + ('Internal Tooling', 'Backoffice automation platform', 2), + ('Client Portal', 'Customer-facing SaaS interface', 2); + +-- Seed tasks +INSERT INTO tasks (project_id, title, status, priority, due_date) VALUES + (1, 'Define architecture', 'done', 1, date('now', '+3 days')), + (1, 'Implement authentication', 'in_progress', 1, date('now', '+7 days')), + (2, 'Design landing page', 'todo', 2, date('now', '+5 days')), + (2, 'Setup CI/CD', 'todo', 2, date('now', '+10 days')); + +COMMIT; \ No newline at end of file diff --git a/src/domain/factory.rs b/src/domain/factory.rs index 8a3b2fe..46ff74e 100644 --- a/src/domain/factory.rs +++ b/src/domain/factory.rs @@ -6,6 +6,7 @@ use crate::services::config::{DatabaseConfig, DbType}; use anyhow::Result; use std::path::{Path, PathBuf}; use std::sync::Arc; +use crate::domain::sqlite::database::SqliteDatabase; #[async_trait::async_trait] pub trait Database: Send + Sync { @@ -27,6 +28,7 @@ impl DatabaseFactory { DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)), DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)), DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)), + DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)), } } @@ -39,6 +41,7 @@ impl DatabaseFactory { DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)), DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)), DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)), + DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)), } } } diff --git a/src/domain/mod.rs b/src/domain/mod.rs index 1ecf7ba..12cc3a0 100644 --- a/src/domain/mod.rs +++ b/src/domain/mod.rs @@ -2,4 +2,5 @@ pub mod factory; pub mod postgres; pub mod mysql; mod mongodb; +mod sqlite; diff --git a/src/domain/sqlite/backup.rs b/src/domain/sqlite/backup.rs new file mode 100644 index 0000000..86e894d --- /dev/null +++ b/src/domain/sqlite/backup.rs @@ -0,0 +1,43 @@ +use crate::services::config::DatabaseConfig; +use anyhow::{Context, Result}; +use std::path::PathBuf; +use std::process::Command; +use tracing::{debug, error, info}; + +pub async fn run( + cfg: DatabaseConfig, + backup_dir: PathBuf, + file_extension: &'static str, +) -> Result { + tokio::task::spawn_blocking(move || -> Result { + debug!("Starting SQLite backup for database {}", cfg.name); + + let db_path_str = cfg.path + .as_ref() + .ok_or_else(|| anyhow::anyhow!("Database path not configured"))?; + + let db_path = PathBuf::from(db_path_str); + + if !db_path.exists() { + anyhow::bail!("SQLite database file not found: {}", db_path.display()); + } + + let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension)); + + let output = Command::new("sqlite3") + .arg(db_path.as_os_str()) + .arg(format!(".backup '{}'", file_path.display())) + .output() + .context("SQLite backup command failed to start")?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + error!("SQLite backup failed for {}: {}", cfg.name, stderr); + anyhow::bail!("SQLite backup failed for {}: {}", cfg.name, stderr); + } + + info!("SQLite backup completed for {}", cfg.name); + Ok(file_path) + }) + .await? +} \ No newline at end of file diff --git a/src/domain/sqlite/database.rs b/src/domain/sqlite/database.rs new file mode 100644 index 0000000..cbc575a --- /dev/null +++ b/src/domain/sqlite/database.rs @@ -0,0 +1,43 @@ +use anyhow::Result; +use async_trait::async_trait; +use std::path::{Path, PathBuf}; + +use super::{backup, ping, restore}; +use crate::domain::factory::Database; +use crate::services::config::DatabaseConfig; +use crate::utils::locks::{DbOpLock, FileLock}; + +pub struct SqliteDatabase { + cfg: DatabaseConfig, +} + +impl SqliteDatabase { + pub fn new(cfg: DatabaseConfig) -> Self { + Self { cfg } + } +} + +#[async_trait] +impl Database for SqliteDatabase { + fn file_extension(&self) -> &'static str { + ".backup" + } + + async fn ping(&self) -> Result { + ping::run(self.cfg.clone()).await + } + + async fn backup(&self, dir: &Path) -> Result { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; + let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await; + FileLock::release(&self.cfg.generated_id).await?; + res + } + + async fn restore(&self, file: &Path) -> Result<()> { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?; + let res = restore::run(self.cfg.clone(), file.to_path_buf()).await; + FileLock::release(&self.cfg.generated_id).await?; + res + } +} diff --git a/src/domain/sqlite/mod.rs b/src/domain/sqlite/mod.rs new file mode 100644 index 0000000..5efda1c --- /dev/null +++ b/src/domain/sqlite/mod.rs @@ -0,0 +1,4 @@ +mod backup; +mod restore; +mod ping; +pub mod database; \ No newline at end of file diff --git a/src/domain/sqlite/ping.rs b/src/domain/sqlite/ping.rs new file mode 100644 index 0000000..51791c9 --- /dev/null +++ b/src/domain/sqlite/ping.rs @@ -0,0 +1,5 @@ +use crate::services::config::DatabaseConfig; + +pub async fn run(_cfg: DatabaseConfig) -> anyhow::Result { + Ok(true) +} diff --git a/src/domain/sqlite/restore.rs b/src/domain/sqlite/restore.rs new file mode 100644 index 0000000..147a8fb --- /dev/null +++ b/src/domain/sqlite/restore.rs @@ -0,0 +1,47 @@ +use crate::services::config::DatabaseConfig; +use anyhow::{Context, Result}; +use std::path::PathBuf; +use std::process::Command; +use tracing::{debug, error, info}; + +pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> { + tokio::task::spawn_blocking(move || -> Result<()> { + debug!("Starting SQLite restore for database {}", cfg.name); + + + let db_path_str = cfg.path + .as_ref() + .ok_or_else(|| anyhow::anyhow!("Database path not configured"))?; + + let db_path = PathBuf::from(db_path_str); + + + if !restore_file.exists() { + anyhow::bail!( + "Restore file not found: {}", + restore_file.display() + ); + } + + if db_path.exists() { + std::fs::remove_file(&db_path) + .with_context(|| format!("Failed to remove existing DB {}", db_path.display()))?; + } + + let output = Command::new("sqlite3") + .arg(db_path.as_os_str()) + .arg(format!(".restore '{}'", restore_file.display())) + .output() + .with_context(|| format!("Failed to run sqlite3 restore for {}", cfg.name))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + error!("SQLite restore failed for {}: {}", cfg.name, stderr); + anyhow::bail!("SQLite restore failed for {}", cfg.name); + } + + info!("SQLite restore completed for {}", cfg.name); + Ok(()) + }) + .await? +} \ No newline at end of file diff --git a/src/services/config.rs b/src/services/config.rs index 92ddba2..dab2834 100644 --- a/src/services/config.rs +++ b/src/services/config.rs @@ -16,8 +16,8 @@ pub enum DbType { Mysql, Mariadb, Postgresql, - MongoDB - // Sqlite, + MongoDB, + Sqlite // Add other DB types if needed } @@ -28,7 +28,7 @@ impl DbType { DbType::Mariadb => "mysql", DbType::Postgresql => "postgresql", DbType::MongoDB => "mongodb", - // DbType::Sqlite => "sqlite", + DbType::Sqlite => "sqlite", } } } @@ -45,6 +45,7 @@ pub struct DatabaseConfig { pub port: u16, pub host: String, pub generated_id: String, + pub path: Option } #[allow(dead_code)] From 35933aae5261277394b00a420ad3464da61705a4 Mon Sep 17 00:00:00 2001 From: charlesgauthereau Date: Mon, 23 Feb 2026 21:47:18 +0100 Subject: [PATCH 2/2] fix: refactoring config check on startup agent. --- databases.json | 7 -- docker-compose.yml | 140 +++++++++++------------ src/domain/mongodb/backup.rs | 2 +- src/domain/mongodb/connection.rs | 14 ++- src/domain/mongodb/ping.rs | 12 +- src/domain/mongodb/restore.rs | 2 +- src/domain/mysql/backup.rs | 2 +- src/domain/mysql/connection.rs | 2 + src/domain/mysql/database.rs | 2 +- src/domain/mysql/ping.rs | 1 + src/domain/mysql/restore.rs | 13 +-- src/domain/postgres/backup.rs | 2 +- src/domain/postgres/connection.rs | 9 +- src/domain/postgres/database.rs | 1 - src/domain/postgres/restore.rs | 8 +- src/domain/sqlite/backup.rs | 8 +- src/domain/sqlite/restore.rs | 19 ++- src/services/config.rs | 97 +++++++++++++++- src/services/storage/providers/s3/mod.rs | 15 ++- 19 files changed, 222 insertions(+), 134 deletions(-) diff --git a/databases.json b/databases.json index 4f42d3a..2e15f61 100644 --- a/databases.json +++ b/databases.json @@ -34,19 +34,12 @@ "name": "Test database 5 - MongoDB", "database": "testdb", "type": "mongodb", - "username": "", - "password": "", "port": 27017, "host": "db-mongodb", "generated_id": "16678147-ff7e-4c97-8c83-0adeff214681" }, { "name": "Test database 6 - SQLite DB", - "database": "", - "username": "", - "password": "", - "port": 0, - "host": "", "type": "sqlite", "path": "/sqlite-data/workspace/data/app.db", "generated_id": "16678178-ff7e-4c97-8c83-0adeff214681" diff --git a/docker-compose.yml b/docker-compose.yml index 6a8a992..e0c7a38 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,73 +25,72 @@ services: networks: - portabase - # db-postgres: - # container_name: db-postgres - # image: postgres:17-alpine - # ports: - # - "5436:5432" - # volumes: - # - postgres-data:/var/lib/postgresql/data - # environment: - # - POSTGRES_DB=devdb - # - POSTGRES_USER=devuser - # - POSTGRES_PASSWORD=changeme - # networks: - # - portabase - # - # db-mariadb: - # container_name: db-mariadb - # image: mariadb:latest - # ports: - # - "3311:3306" - # environment: - # - MYSQL_DATABASE=mariadb - # - MYSQL_USER=mariadb - # - MYSQL_PASSWORD=changeme - # - MYSQL_RANDOM_ROOT_PASSWORD=yes - # volumes: - # - mariadb-data:/var/lib/mysql - # networks: - # - portabase - # - # - # db-mongodb-auth: - # container_name: db-mongodb-auth - # image: mongo:latest - # ports: - # - "27082:27017" - # environment: - # MONGO_INITDB_ROOT_USERNAME: root - # MONGO_INITDB_ROOT_PASSWORD: rootpassword - # MONGO_INITDB_DATABASE: testdbauth - # command: mongod --auth - # networks: - # - portabase - # volumes: - # - mongodb-data-auth:/data/db - # healthcheck: - # test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] - # interval: 5s - # timeout: 5s - # retries: 10 - # - # db-mongodb: - # container_name: db-mongodb - # image: mongo:latest - # ports: - # - "27083:27017" - # volumes: - # - mongodb-data:/data/db - # healthcheck: - # test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ] - # interval: 5s - # timeout: 5s - # retries: 10 - # environment: - # MONGO_INITDB_DATABASE: testdb - # networks: - # - portabase + db-postgres: + container_name: db-postgres + image: postgres:17-alpine + ports: + - "5436:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + environment: + - POSTGRES_DB=devdb + - POSTGRES_USER=devuser + - POSTGRES_PASSWORD=changeme + networks: + - portabase + db-mariadb: + container_name: db-mariadb + image: mariadb:latest + ports: + - "3311:3306" + environment: + - MYSQL_DATABASE=mariadb + - MYSQL_USER=mariadb + - MYSQL_PASSWORD=changeme + - MYSQL_RANDOM_ROOT_PASSWORD=yes + volumes: + - mariadb-data:/var/lib/mysql + networks: + - portabase + + + db-mongodb-auth: + container_name: db-mongodb-auth + image: mongo:latest + ports: + - "27082:27017" + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: rootpassword + MONGO_INITDB_DATABASE: testdbauth + command: mongod --auth + networks: + - portabase + volumes: + - mongodb-data-auth:/data/db + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + timeout: 5s + retries: 10 + + db-mongodb: + container_name: db-mongodb + image: mongo:latest + ports: + - "27083:27017" + volumes: + - mongodb-data:/data/db + healthcheck: + test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ] + interval: 5s + timeout: 5s + retries: 10 + environment: + MONGO_INITDB_DATABASE: testdb + networks: + - portabase sqlite: container_name: db-sqlite @@ -103,15 +102,16 @@ services: stdin_open: true tty: true + volumes: cargo-registry: cargo-git: # cargo-target: - # postgres-data: - # mariadb-data: - # mongodb-data: - # mongodb-data-auth: + postgres-data: + mariadb-data: + mongodb-data: + mongodb-data-auth: sqlite-data: networks: diff --git a/src/domain/mongodb/backup.rs b/src/domain/mongodb/backup.rs index 1737ab2..d4724b3 100644 --- a/src/domain/mongodb/backup.rs +++ b/src/domain/mongodb/backup.rs @@ -15,7 +15,7 @@ pub async fn run( let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension)); let mongodump = select_mongo_path().join("mongodump"); - let uri = get_mongo_uri(cfg.clone()); + let uri = get_mongo_uri(cfg.clone())?; let output = Command::new(mongodump) .arg(format!("--uri={}", uri)) diff --git a/src/domain/mongodb/connection.rs b/src/domain/mongodb/connection.rs index ef760f2..dcc48d4 100644 --- a/src/domain/mongodb/connection.rs +++ b/src/domain/mongodb/connection.rs @@ -3,7 +3,7 @@ use anyhow::Result; use mongodb::Client; pub async fn connect(cfg: DatabaseConfig) -> Result { - let uri = get_mongo_uri(cfg); + let uri = get_mongo_uri(cfg)?; let mut options = mongodb::options::ClientOptions::parse(&uri).await?; options.server_selection_timeout = Some(std::time::Duration::from_secs(3)); options.connect_timeout = Some(std::time::Duration::from_secs(3)); @@ -15,14 +15,16 @@ pub fn select_mongo_path() -> std::path::PathBuf { "/usr/local/mongodb/bin".to_string().into() } -pub fn get_mongo_uri(cfg: DatabaseConfig) -> String { - if cfg.username.is_empty() { - format!("mongodb://{}:{}/{}", cfg.host, cfg.port, cfg.database) +pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result { + if cfg.username.is_empty() && cfg.password.is_empty() { + Ok(format!("mongodb://{}:{}/{}", cfg.host, cfg.port, cfg.database)) } else { - format!( + Ok(format!( "mongodb://{}:{}@{}:{}/{}?authSource=admin", cfg.username, cfg.password, cfg.host, cfg.port, cfg.database - ) + )) } } + + diff --git a/src/domain/mongodb/ping.rs b/src/domain/mongodb/ping.rs index 0d288ad..6a3478b 100644 --- a/src/domain/mongodb/ping.rs +++ b/src/domain/mongodb/ping.rs @@ -1,14 +1,20 @@ #![allow(dead_code)] +use crate::domain::mongodb::connection::connect; use crate::services::config::DatabaseConfig; use anyhow::Result; use mongodb::bson::doc; -use tracing::{error}; -use crate::domain::mongodb::connection::connect; +use tracing::error; pub async fn run(cfg: DatabaseConfig) -> Result { let client = connect(cfg.clone()).await?; - let db_name = if cfg.username.is_empty() { &cfg.database } else { "admin" }; + + let db_name = if cfg.username.is_empty() && cfg.password.is_empty() { + &cfg.database + } else { + "admin" + }; + match client.database(db_name).run_command(doc! {"ping": 1}).await { Ok(_) => Ok(true), Err(e) => { diff --git a/src/domain/mongodb/restore.rs b/src/domain/mongodb/restore.rs index cb37a46..617834d 100644 --- a/src/domain/mongodb/restore.rs +++ b/src/domain/mongodb/restore.rs @@ -10,7 +10,7 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> { debug!("Starting MongoDB restore for database {}", cfg.name); let mongorestore = select_mongo_path().join("mongorestore"); - let uri = get_mongo_uri(cfg.clone()); + let uri = get_mongo_uri(cfg.clone())?; let output = Command::new(mongorestore) .arg(format!("--uri={}", uri)) diff --git a/src/domain/mysql/backup.rs b/src/domain/mysql/backup.rs index 2ec7531..1413bd4 100644 --- a/src/domain/mysql/backup.rs +++ b/src/domain/mysql/backup.rs @@ -1,4 +1,4 @@ -use crate::domain::mysql::connection::server_version; +use crate::domain::mysql::connection::{server_version}; use crate::services::config::DatabaseConfig; use anyhow::{Context, Result}; use std::collections::HashMap; diff --git a/src/domain/mysql/connection.rs b/src/domain/mysql/connection.rs index b89a0f8..db26bbb 100644 --- a/src/domain/mysql/connection.rs +++ b/src/domain/mysql/connection.rs @@ -3,6 +3,7 @@ use std::process::Command; use anyhow::Result; pub async fn server_version(cfg: &DatabaseConfig) -> Result { + let output = Command::new("mysql") .arg("--host").arg(&cfg.host) .arg("--port").arg(cfg.port.to_string()) @@ -25,3 +26,4 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result { Ok(version) } + diff --git a/src/domain/mysql/database.rs b/src/domain/mysql/database.rs index 4c129ed..13fa06f 100644 --- a/src/domain/mysql/database.rs +++ b/src/domain/mysql/database.rs @@ -21,7 +21,7 @@ impl MySQLDatabase { fn build_env(&self) -> HashMap { let mut envs = std::env::vars().collect::>(); - envs.insert("MYSQL_PWD".to_string(), self.cfg.password.clone()); + envs.insert("MYSQL_PWD".to_string(), self.cfg.password.to_string()); envs } } diff --git a/src/domain/mysql/ping.rs b/src/domain/mysql/ping.rs index fde0407..dd2491f 100644 --- a/src/domain/mysql/ping.rs +++ b/src/domain/mysql/ping.rs @@ -4,6 +4,7 @@ use tokio::process::Command; use tokio::time::{Duration, timeout}; pub async fn run(cfg: DatabaseConfig, env: HashMap) -> anyhow::Result { + let mut cmd = Command::new("mysqladmin"); cmd.arg("--host") .arg(cfg.host) diff --git a/src/domain/mysql/restore.rs b/src/domain/mysql/restore.rs index 2039b1a..f662a50 100644 --- a/src/domain/mysql/restore.rs +++ b/src/domain/mysql/restore.rs @@ -1,11 +1,10 @@ +use crate::services::config::DatabaseConfig; +use anyhow::{Context, Result}; use std::fs::File; use std::io::{Read, Write}; -use anyhow::{Context, Result}; -use tracing::{debug, error, info}; use std::path::PathBuf; use std::process::Command; - -use crate::services::config::DatabaseConfig; +use tracing::{debug, error, info}; pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> { let handle = tokio::task::spawn_blocking(move || -> Result<()> { @@ -55,7 +54,8 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> { .with_context(|| format!("Failed to start mysql restore for {}", cfg.name))?; let mut stdin = child.stdin.take().context("Failed to open child stdin")?; - stdin.write_all(sql_content.as_bytes()) + stdin + .write_all(sql_content.as_bytes()) .context("Failed to write SQL content to mysql stdin")?; stdin.flush()?; drop(stdin); @@ -74,8 +74,7 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> { Ok(()) }); - handle - .await??; + handle.await??; Ok(()) } diff --git a/src/domain/postgres/backup.rs b/src/domain/postgres/backup.rs index 107f30d..c7dbfcb 100644 --- a/src/domain/postgres/backup.rs +++ b/src/domain/postgres/backup.rs @@ -1,7 +1,7 @@ use anyhow::Result; -use tracing::{debug, error, info}; use std::path::PathBuf; use std::process::Command; +use tracing::{debug, error, info}; use super::connection::{select_pg_path, server_version}; use super::format::PostgresDumpFormat; diff --git a/src/domain/postgres/connection.rs b/src/domain/postgres/connection.rs index e14fcda..92648f3 100644 --- a/src/domain/postgres/connection.rs +++ b/src/domain/postgres/connection.rs @@ -1,11 +1,12 @@ -use std::path::Path; use crate::domain::postgres::format::PostgresDumpFormat; use crate::services::config::DatabaseConfig; use anyhow::Result; +use std::path::Path; use tokio_postgres::{Client, NoTls}; -use tracing::info; +use tracing::{error, info}; pub async fn connect(cfg: &DatabaseConfig) -> Result { + info!("Connecting to postgres database {}:{}", cfg.host, cfg.port); let dsn = format!( "host={} port={} user={} password={} dbname={}", cfg.host, cfg.port, cfg.username, cfg.password, cfg.database @@ -14,7 +15,7 @@ pub async fn connect(cfg: &DatabaseConfig) -> Result { let (client, connection) = tokio_postgres::connect(&dsn, NoTls).await?; tokio::spawn(async move { if let Err(e) = connection.await { - tracing::error!("Postgres connection error: {}", e); + error!("Postgres connection error: {}", e); } }); Ok(client) @@ -34,7 +35,7 @@ pub fn select_pg_path(version: &str) -> std::path::PathBuf { pub async fn terminate_connections(cfg: &DatabaseConfig) -> Result<()> { let mut admin = cfg.clone(); - admin.database = "postgres".into(); + admin.database = "postgres".to_string().into(); let client = connect(&admin).await?; diff --git a/src/domain/postgres/database.rs b/src/domain/postgres/database.rs index f2ac8cb..618c1c3 100644 --- a/src/domain/postgres/database.rs +++ b/src/domain/postgres/database.rs @@ -39,7 +39,6 @@ impl Database for PostgresDatabase { FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; let res = backup::run(self.cfg.clone(), self.format, dir.to_path_buf()).await; FileLock::release(&self.cfg.generated_id).await?; - res } diff --git a/src/domain/postgres/restore.rs b/src/domain/postgres/restore.rs index cb3d130..0458d82 100644 --- a/src/domain/postgres/restore.rs +++ b/src/domain/postgres/restore.rs @@ -1,7 +1,7 @@ use anyhow::Result; -use tracing::{debug, error, info}; use std::path::PathBuf; use std::process::Command; +use tracing::{debug, error, info}; use super::connection::{select_pg_path, server_version, terminate_connections}; use super::format::PostgresDumpFormat; @@ -14,6 +14,7 @@ pub async fn run( ) -> Result<()> { tokio::task::spawn_blocking(move || -> Result<()> { debug!("Starting restore for database {}", cfg.name); + let version = match futures::executor::block_on(server_version(&cfg)) { Ok(v) => { debug!("Postgres version detected: {}", v); @@ -91,8 +92,6 @@ pub async fn run( let dec = flate2::read::GzDecoder::new(tar_gz); let mut archive = tar::Archive::new(dec); - - let tmp_dir = match tempfile::TempDir::new() { Ok(d) => d, Err(e) => { @@ -104,8 +103,6 @@ pub async fn run( } }; - - if let Err(e) = archive.unpack(tmp_dir.path()) { error!("Failed to unpack FD archive for {}: {:?}", cfg.name, e); return Err(e.into()); @@ -113,7 +110,6 @@ pub async fn run( debug!("Listing contents of temp dir: {}", tmp_dir.path().display()); - for entry in std::fs::read_dir(tmp_dir.path())? { if let Ok(entry) = entry { let path = entry.path(); diff --git a/src/domain/sqlite/backup.rs b/src/domain/sqlite/backup.rs index 86e894d..578cfd6 100644 --- a/src/domain/sqlite/backup.rs +++ b/src/domain/sqlite/backup.rs @@ -12,9 +12,11 @@ pub async fn run( tokio::task::spawn_blocking(move || -> Result { debug!("Starting SQLite backup for database {}", cfg.name); - let db_path_str = cfg.path - .as_ref() - .ok_or_else(|| anyhow::anyhow!("Database path not configured"))?; + let db_path_str = if cfg.path.is_empty() { + anyhow::bail!("Database path not configured"); + } else { + cfg.path.as_str().to_string() + }; let db_path = PathBuf::from(db_path_str); diff --git a/src/domain/sqlite/restore.rs b/src/domain/sqlite/restore.rs index 147a8fb..a386a5d 100644 --- a/src/domain/sqlite/restore.rs +++ b/src/domain/sqlite/restore.rs @@ -8,19 +8,16 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> { tokio::task::spawn_blocking(move || -> Result<()> { debug!("Starting SQLite restore for database {}", cfg.name); - - let db_path_str = cfg.path - .as_ref() - .ok_or_else(|| anyhow::anyhow!("Database path not configured"))?; + let db_path_str = if cfg.path.is_empty() { + anyhow::bail!("Database path not configured"); + } else { + cfg.path.as_str().to_string() + }; let db_path = PathBuf::from(db_path_str); - if !restore_file.exists() { - anyhow::bail!( - "Restore file not found: {}", - restore_file.display() - ); + anyhow::bail!("Restore file not found: {}", restore_file.display()); } if db_path.exists() { @@ -43,5 +40,5 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> { info!("SQLite restore completed for {}", cfg.name); Ok(()) }) - .await? -} \ No newline at end of file + .await? +} diff --git a/src/services/config.rs b/src/services/config.rs index dab2834..af98f07 100644 --- a/src/services/config.rs +++ b/src/services/config.rs @@ -9,6 +9,7 @@ use std::path::Path; use std::sync::Arc; use toml; use tracing::info; +use uuid::Uuid; #[derive(Debug, Deserialize, Clone)] #[serde(rename_all = "lowercase")] @@ -17,7 +18,7 @@ pub enum DbType { Mariadb, Postgresql, MongoDB, - Sqlite + Sqlite, // Add other DB types if needed } @@ -45,7 +46,7 @@ pub struct DatabaseConfig { pub port: u16, pub host: String, pub generated_id: String, - pub path: Option + pub path: String, } #[allow(dead_code)] @@ -54,6 +55,29 @@ pub struct DatabasesConfig { pub databases: Vec, } + +#[allow(dead_code)] +#[derive(Debug, Deserialize, Clone)] +pub struct InputDatabaseConfig { + pub name: String, + pub database: Option, + #[serde(rename = "type")] + pub db_type: DbType, + pub username: Option, + pub password: Option, + pub port: Option, + pub host: Option, + pub generated_id: String, + pub path: Option, +} + +#[allow(dead_code)] +#[derive(Debug, Deserialize, Clone)] +pub struct InputDatabasesConfig { + pub databases: Vec, +} + + pub struct ConfigService { ctx: Arc, } @@ -96,7 +120,7 @@ impl ConfigService { file.read_to_string(&mut contents) .map_err(|e| format!("Failed to read config file: {}", e))?; - let config: DatabasesConfig = match extension { + let input_config: InputDatabasesConfig = match extension { "json" => { serde_json::from_str(&contents).map_err(|e| format!("JSON parsing error: {}", e))? } @@ -106,8 +130,71 @@ impl ConfigService { _ => return Err("Unsupported config file format. Use .json or .toml".to_string()), }; - info!("Databases : {:?} instances loaded", config.databases.len()); + fn required(opt: &Option, db_name: &str, field_name: &str) -> Result { + match opt { + Some(v) => Ok(v.clone()), + None => { + let msg = format!("Missing required field '{}' for database '{}'", field_name, db_name); + Err(msg) + } + } + } - Ok(config) + fn optional(opt: &Option) -> T where T: Default { + opt.clone().unwrap_or_default() + } + + let mut databases = Vec::with_capacity(input_config.databases.len()); + + for db in input_config.databases { + if Uuid::parse_str(&db.generated_id).is_err() { + return Err(format!("Invalid UUID for database '{}'", db.name)); + } + + let username = match db.db_type { + DbType::Postgresql | DbType::Mysql | DbType::Mariadb => required(&db.username, &db.name, "username")?, + _ => optional(&db.username), + }; + + let password = match db.db_type { + DbType::Postgresql | DbType::Mysql | DbType::Mariadb => required(&db.password, &db.name, "password")?, + _ => optional(&db.password), + }; + + let host = match db.db_type { + DbType::Postgresql | DbType::Mysql | DbType::Mariadb | DbType::MongoDB => required(&db.host, &db.name, "host")?, + DbType::Sqlite => optional(&db.host), + }; + + let port = match db.db_type { + DbType::Postgresql | DbType::Mysql | DbType::Mariadb | DbType::MongoDB => required(&db.port, &db.name, "port")?, + DbType::Sqlite => db.port.unwrap_or(0), + }; + + let database_name = match db.db_type { + DbType::Sqlite => optional(&db.database), + _ => required(&db.database, &db.name, "database")? + }; + + let path_val = match db.db_type { + DbType::Sqlite => required(&db.path, &db.name, "path")?, + _ => optional(&db.path), + }; + + databases.push(DatabaseConfig { + name: db.name, + database: database_name, + db_type: db.db_type, + username, + password, + host, + port, + generated_id: db.generated_id, + path: path_val, + }); + } + + info!("Databases: {} instances loaded", databases.len()); + Ok(DatabasesConfig { databases }) } } diff --git a/src/services/storage/providers/s3/mod.rs b/src/services/storage/providers/s3/mod.rs index fb23c63..bc1430d 100644 --- a/src/services/storage/providers/s3/mod.rs +++ b/src/services/storage/providers/s3/mod.rs @@ -94,12 +94,15 @@ impl StorageProvider for S3Provider { ); let region = Region::new(config.region.clone().unwrap_or("us-east-1".to_string())); - - let scheme = if config.ssl { "https" } else { "http" }; - - let endpoint = match config.port { - Some(port) => format!("{scheme}://{}:{port}", config.end_point_url), - None => format!("{scheme}://{}", config.end_point_url), + + let endpoint = if let Some(port) = &config.port { + if port.trim().is_empty() { + format!("{}://{}", if config.ssl { "https" } else { "http" }, config.end_point_url) + } else { + format!("{}://{}:{}", if config.ssl { "https" } else { "http" }, config.end_point_url, port) + } + } else { + format!("{}://{}", if config.ssl { "https" } else { "http" }, config.end_point_url) }; info!("S3 endpoint to {}", &endpoint);