From 178628cb1cb69234190586aa657ff0f82d74c705 Mon Sep 17 00:00:00 2001 From: charlesgauthereau Date: Sun, 22 Feb 2026 21:46:16 +0100 Subject: [PATCH] feat: add sqlite support, first working version --- Makefile | 17 ++++ databases.json | 11 +++ docker-compose.yml | 154 ++++++++++++++++++---------------- docker/Dockerfile | 2 + scripts/README.md | 17 +++- scripts/sqlite/seed-big.sql | 74 ++++++++++++++++ scripts/sqlite/seed.sql | 59 +++++++++++++ src/domain/factory.rs | 3 + src/domain/mod.rs | 1 + src/domain/sqlite/backup.rs | 43 ++++++++++ src/domain/sqlite/database.rs | 43 ++++++++++ src/domain/sqlite/mod.rs | 4 + src/domain/sqlite/ping.rs | 5 ++ src/domain/sqlite/restore.rs | 47 +++++++++++ src/services/config.rs | 7 +- 15 files changed, 411 insertions(+), 76 deletions(-) create mode 100644 scripts/sqlite/seed-big.sql create mode 100644 scripts/sqlite/seed.sql create mode 100644 src/domain/sqlite/backup.rs create mode 100644 src/domain/sqlite/database.rs create mode 100644 src/domain/sqlite/mod.rs create mode 100644 src/domain/sqlite/ping.rs create mode 100644 src/domain/sqlite/restore.rs diff --git a/Makefile b/Makefile index 56a9060..a363fa2 100644 --- a/Makefile +++ b/Makefile @@ -30,4 +30,21 @@ seed-postgres-1gb: docker exec -i -e PGPASSWORD=$$PG_PASSWORD $$PG_CONTAINER \ psql -U $$PG_USER -d $$PG_DB < ./scripts/postgres/seed-1gb.sql + +SQLITE_SEED_FILE := $(if $(filter big,$(SEED)),./scripts/sqlite/seed-big.sql,./scripts/sqlite/seed.sql) + +seed-sqlite: + @echo "Seeding Sqlite..." + @echo "Run as root to fix permissions inside the volume" + docker exec -u 0 -it db-sqlite sh -c "chmod -R 777 /workspace/data" + @echo "Create the database file (if it doesn’t exist)" + docker exec -u 0 -it db-sqlite sh -c "touch /workspace/data/app.db" + @echo "Seed the database" + docker exec -i db-sqlite sh -c "sqlite3 /workspace/data/app.db" < $(SQLITE_SEED_FILE) + @echo "Verify" + docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT name FROM sqlite_master WHERE type='table';" + @echo "Done" + + + seed-all: seed-mongo seed-mysql seed-postgres seed-postgres-1gb diff --git a/databases.json b/databases.json index 7b83c99..4f42d3a 100644 --- a/databases.json +++ b/databases.json @@ -39,6 +39,17 @@ "port": 27017, "host": "db-mongodb", "generated_id": "16678147-ff7e-4c97-8c83-0adeff214681" + }, + { + "name": "Test database 6 - SQLite DB", + "database": "", + "username": "", + "password": "", + "port": 0, + "host": "", + "type": "sqlite", + "path": "/sqlite-data/workspace/data/app.db", + "generated_id": "16678178-ff7e-4c97-8c83-0adeff214681" } ] } diff --git a/docker-compose.yml b/docker-compose.yml index 066a66a..6a8a992 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,8 @@ services: # - ./databases.toml:/config/config.toml - cargo-registry:/usr/local/cargo/registry - cargo-git:/usr/local/cargo/git -# - cargo-target:/app/target + # - cargo-target:/app/target + - sqlite-data:/sqlite-data/workspace/data environment: APP_ENV: development LOG: debug @@ -24,83 +25,94 @@ services: networks: - portabase - db-postgres: - container_name: db-postgres - image: postgres:17-alpine - ports: - - "5436:5432" - volumes: - - postgres-data:/var/lib/postgresql/data - environment: - - POSTGRES_DB=devdb - - POSTGRES_USER=devuser - - POSTGRES_PASSWORD=changeme - networks: - - portabase + # db-postgres: + # container_name: db-postgres + # image: postgres:17-alpine + # ports: + # - "5436:5432" + # volumes: + # - postgres-data:/var/lib/postgresql/data + # environment: + # - POSTGRES_DB=devdb + # - POSTGRES_USER=devuser + # - POSTGRES_PASSWORD=changeme + # networks: + # - portabase + # + # db-mariadb: + # container_name: db-mariadb + # image: mariadb:latest + # ports: + # - "3311:3306" + # environment: + # - MYSQL_DATABASE=mariadb + # - MYSQL_USER=mariadb + # - MYSQL_PASSWORD=changeme + # - MYSQL_RANDOM_ROOT_PASSWORD=yes + # volumes: + # - mariadb-data:/var/lib/mysql + # networks: + # - portabase + # + # + # db-mongodb-auth: + # container_name: db-mongodb-auth + # image: mongo:latest + # ports: + # - "27082:27017" + # environment: + # MONGO_INITDB_ROOT_USERNAME: root + # MONGO_INITDB_ROOT_PASSWORD: rootpassword + # MONGO_INITDB_DATABASE: testdbauth + # command: mongod --auth + # networks: + # - portabase + # volumes: + # - mongodb-data-auth:/data/db + # healthcheck: + # test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + # interval: 5s + # timeout: 5s + # retries: 10 + # + # db-mongodb: + # container_name: db-mongodb + # image: mongo:latest + # ports: + # - "27083:27017" + # volumes: + # - mongodb-data:/data/db + # healthcheck: + # test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ] + # interval: 5s + # timeout: 5s + # retries: 10 + # environment: + # MONGO_INITDB_DATABASE: testdb + # networks: + # - portabase - db-mariadb: - container_name: db-mariadb - image: mariadb:latest - ports: - - "3311:3306" - environment: - - MYSQL_DATABASE=mariadb - - MYSQL_USER=mariadb - - MYSQL_PASSWORD=changeme - - MYSQL_RANDOM_ROOT_PASSWORD=yes - volumes: - - mariadb-data:/var/lib/mysql - networks: - - portabase -# -# db-mongodb-auth: -# container_name: db-mongodb-auth -# image: mongo:latest -# ports: -# - "27082:27017" -# environment: -# MONGO_INITDB_ROOT_USERNAME: root -# MONGO_INITDB_ROOT_PASSWORD: rootpassword -# MONGO_INITDB_DATABASE: testdbauth -# command: mongod --auth -# networks: -# - portabase -# volumes: -# - mongodb-data-auth:/data/db -# healthcheck: -# test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] -# interval: 5s -# timeout: 5s -# retries: 10 -# -# db-mongodb: -# container_name: db-mongodb -# image: mongo:latest -# ports: -# - "27083:27017" -# volumes: -# - mongodb-data:/data/db -# healthcheck: -# test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ] -# interval: 5s -# timeout: 5s -# retries: 10 -# environment: -# MONGO_INITDB_DATABASE: testdb -# networks: -# - portabase + sqlite: + container_name: db-sqlite + image: keinos/sqlite3 + volumes: + - sqlite-data:/workspace/data + working_dir: /workspace + command: tail -f /dev/null + stdin_open: true + tty: true volumes: cargo-registry: cargo-git: -# cargo-target: + # cargo-target: - - postgres-data: - mariadb-data: -# mongodb-data: -# mongodb-data-auth: + # postgres-data: + # mariadb-data: + # mongodb-data: + # mongodb-data-auth: + sqlite-data: networks: portabase: diff --git a/docker/Dockerfile b/docker/Dockerfile index d149b6d..5ae11de 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,6 +15,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ zlib1g \ curl \ mariadb-client \ + sqlite3 \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -92,6 +93,7 @@ RUN apt-get update && apt-get install -y \ libncurses6 \ zlib1g \ mariadb-client \ + sqlite3 \ && rm -rf /var/lib/apt/lists/* diff --git a/scripts/README.md b/scripts/README.md index 4160bc3..9045b17 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,6 +1,5 @@ # Seed instructions -## MongoDB ```bash make seed-mongo @@ -9,5 +8,19 @@ make seed-mysql make seed-mysql-1gb make seed-postgres make seed-postgres-1gb -make seed-all +make seed-all +make seed-sqlite +make seed-sqlite SEED=big +``` + +## Verify commands + +### Sqlite + +```bash +docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT * FROM users LIMIT 10;" +``` + +```bash +docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT name FROM sqlite_master WHERE type='table';" ``` diff --git a/scripts/sqlite/seed-big.sql b/scripts/sqlite/seed-big.sql new file mode 100644 index 0000000..ad0ff4e --- /dev/null +++ b/scripts/sqlite/seed-big.sql @@ -0,0 +1,74 @@ +PRAGMA foreign_keys = ON; +BEGIN TRANSACTION; + +-- Drop tables +DROP TABLE IF EXISTS users; +DROP TABLE IF EXISTS projects; +DROP TABLE IF EXISTS tasks; + +-- Users +CREATE TABLE users ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + email TEXT NOT NULL UNIQUE, + full_name TEXT NOT NULL, + role TEXT NOT NULL CHECK (role IN ('admin','manager','user')), + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +-- Projects +CREATE TABLE projects ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + description TEXT, + owner_id INTEGER NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + FOREIGN KEY (owner_id) REFERENCES users(id) ON DELETE CASCADE +); + +-- Tasks +CREATE TABLE tasks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + project_id INTEGER NOT NULL, + title TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('todo','in_progress','done')), + priority INTEGER NOT NULL DEFAULT 3, + due_date TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE +); + +-- Seed minimal users +INSERT INTO users (email, full_name, role) VALUES + ('admin@example.com', 'System Admin', 'admin'), + ('manager@example.com', 'Project Manager', 'manager'), + ('user@example.com', 'Standard User', 'user'); + +-- Generate 10_000 projects +WITH RECURSIVE numbers(x) AS ( + SELECT 1 + UNION ALL + SELECT x+1 FROM numbers WHERE x<10000 +) +INSERT INTO projects (name, description, owner_id) +SELECT + 'Project #' || x, + 'Auto-generated project description for project #' || x, + (1 + (x % 3)) -- cycle users 1..3 +FROM numbers; + +-- Generate 100_000 tasks +WITH RECURSIVE numbers(x) AS ( + SELECT 1 + UNION ALL + SELECT x+1 FROM numbers WHERE x<100000 +) +INSERT INTO tasks (project_id, title, status, priority, due_date) +SELECT + (1 + (x % 10000)), -- project id cycle + 'Task #' || x, + CASE (x % 3) WHEN 0 THEN 'todo' WHEN 1 THEN 'in_progress' ELSE 'done' END, + 1 + (x % 5), + date('now', '+' || (x % 30) || ' days') +FROM numbers; + +COMMIT; \ No newline at end of file diff --git a/scripts/sqlite/seed.sql b/scripts/sqlite/seed.sql new file mode 100644 index 0000000..73b5f08 --- /dev/null +++ b/scripts/sqlite/seed.sql @@ -0,0 +1,59 @@ +PRAGMA foreign_keys = ON; + +BEGIN TRANSACTION; + +-- Drop existing tables (idempotent reset) +DROP TABLE IF EXISTS users; +DROP TABLE IF EXISTS projects; +DROP TABLE IF EXISTS tasks; + +-- Users +CREATE TABLE users ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + email TEXT NOT NULL UNIQUE, + full_name TEXT NOT NULL, + role TEXT NOT NULL CHECK (role IN ('admin','manager','user')), + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +-- Projects +CREATE TABLE projects ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + description TEXT, + owner_id INTEGER NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + FOREIGN KEY (owner_id) REFERENCES users(id) ON DELETE CASCADE +); + +-- Tasks +CREATE TABLE tasks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + project_id INTEGER NOT NULL, + title TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('todo','in_progress','done')), + priority INTEGER NOT NULL DEFAULT 3, + due_date TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE +); + +-- Seed users +INSERT INTO users (email, full_name, role) VALUES + ('admin@example.com', 'System Admin', 'admin'), + ('manager@example.com', 'Project Manager', 'manager'), + ('user@example.com', 'Standard User', 'user'); + +-- Seed projects +INSERT INTO projects (name, description, owner_id) VALUES + ('Internal Tooling', 'Backoffice automation platform', 2), + ('Client Portal', 'Customer-facing SaaS interface', 2); + +-- Seed tasks +INSERT INTO tasks (project_id, title, status, priority, due_date) VALUES + (1, 'Define architecture', 'done', 1, date('now', '+3 days')), + (1, 'Implement authentication', 'in_progress', 1, date('now', '+7 days')), + (2, 'Design landing page', 'todo', 2, date('now', '+5 days')), + (2, 'Setup CI/CD', 'todo', 2, date('now', '+10 days')); + +COMMIT; \ No newline at end of file diff --git a/src/domain/factory.rs b/src/domain/factory.rs index 8a3b2fe..46ff74e 100644 --- a/src/domain/factory.rs +++ b/src/domain/factory.rs @@ -6,6 +6,7 @@ use crate::services::config::{DatabaseConfig, DbType}; use anyhow::Result; use std::path::{Path, PathBuf}; use std::sync::Arc; +use crate::domain::sqlite::database::SqliteDatabase; #[async_trait::async_trait] pub trait Database: Send + Sync { @@ -27,6 +28,7 @@ impl DatabaseFactory { DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)), DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)), DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)), + DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)), } } @@ -39,6 +41,7 @@ impl DatabaseFactory { DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)), DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)), DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)), + DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)), } } } diff --git a/src/domain/mod.rs b/src/domain/mod.rs index 1ecf7ba..12cc3a0 100644 --- a/src/domain/mod.rs +++ b/src/domain/mod.rs @@ -2,4 +2,5 @@ pub mod factory; pub mod postgres; pub mod mysql; mod mongodb; +mod sqlite; diff --git a/src/domain/sqlite/backup.rs b/src/domain/sqlite/backup.rs new file mode 100644 index 0000000..86e894d --- /dev/null +++ b/src/domain/sqlite/backup.rs @@ -0,0 +1,43 @@ +use crate::services::config::DatabaseConfig; +use anyhow::{Context, Result}; +use std::path::PathBuf; +use std::process::Command; +use tracing::{debug, error, info}; + +pub async fn run( + cfg: DatabaseConfig, + backup_dir: PathBuf, + file_extension: &'static str, +) -> Result { + tokio::task::spawn_blocking(move || -> Result { + debug!("Starting SQLite backup for database {}", cfg.name); + + let db_path_str = cfg.path + .as_ref() + .ok_or_else(|| anyhow::anyhow!("Database path not configured"))?; + + let db_path = PathBuf::from(db_path_str); + + if !db_path.exists() { + anyhow::bail!("SQLite database file not found: {}", db_path.display()); + } + + let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension)); + + let output = Command::new("sqlite3") + .arg(db_path.as_os_str()) + .arg(format!(".backup '{}'", file_path.display())) + .output() + .context("SQLite backup command failed to start")?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + error!("SQLite backup failed for {}: {}", cfg.name, stderr); + anyhow::bail!("SQLite backup failed for {}: {}", cfg.name, stderr); + } + + info!("SQLite backup completed for {}", cfg.name); + Ok(file_path) + }) + .await? +} \ No newline at end of file diff --git a/src/domain/sqlite/database.rs b/src/domain/sqlite/database.rs new file mode 100644 index 0000000..cbc575a --- /dev/null +++ b/src/domain/sqlite/database.rs @@ -0,0 +1,43 @@ +use anyhow::Result; +use async_trait::async_trait; +use std::path::{Path, PathBuf}; + +use super::{backup, ping, restore}; +use crate::domain::factory::Database; +use crate::services::config::DatabaseConfig; +use crate::utils::locks::{DbOpLock, FileLock}; + +pub struct SqliteDatabase { + cfg: DatabaseConfig, +} + +impl SqliteDatabase { + pub fn new(cfg: DatabaseConfig) -> Self { + Self { cfg } + } +} + +#[async_trait] +impl Database for SqliteDatabase { + fn file_extension(&self) -> &'static str { + ".backup" + } + + async fn ping(&self) -> Result { + ping::run(self.cfg.clone()).await + } + + async fn backup(&self, dir: &Path) -> Result { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; + let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await; + FileLock::release(&self.cfg.generated_id).await?; + res + } + + async fn restore(&self, file: &Path) -> Result<()> { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?; + let res = restore::run(self.cfg.clone(), file.to_path_buf()).await; + FileLock::release(&self.cfg.generated_id).await?; + res + } +} diff --git a/src/domain/sqlite/mod.rs b/src/domain/sqlite/mod.rs new file mode 100644 index 0000000..5efda1c --- /dev/null +++ b/src/domain/sqlite/mod.rs @@ -0,0 +1,4 @@ +mod backup; +mod restore; +mod ping; +pub mod database; \ No newline at end of file diff --git a/src/domain/sqlite/ping.rs b/src/domain/sqlite/ping.rs new file mode 100644 index 0000000..51791c9 --- /dev/null +++ b/src/domain/sqlite/ping.rs @@ -0,0 +1,5 @@ +use crate::services::config::DatabaseConfig; + +pub async fn run(_cfg: DatabaseConfig) -> anyhow::Result { + Ok(true) +} diff --git a/src/domain/sqlite/restore.rs b/src/domain/sqlite/restore.rs new file mode 100644 index 0000000..147a8fb --- /dev/null +++ b/src/domain/sqlite/restore.rs @@ -0,0 +1,47 @@ +use crate::services::config::DatabaseConfig; +use anyhow::{Context, Result}; +use std::path::PathBuf; +use std::process::Command; +use tracing::{debug, error, info}; + +pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> { + tokio::task::spawn_blocking(move || -> Result<()> { + debug!("Starting SQLite restore for database {}", cfg.name); + + + let db_path_str = cfg.path + .as_ref() + .ok_or_else(|| anyhow::anyhow!("Database path not configured"))?; + + let db_path = PathBuf::from(db_path_str); + + + if !restore_file.exists() { + anyhow::bail!( + "Restore file not found: {}", + restore_file.display() + ); + } + + if db_path.exists() { + std::fs::remove_file(&db_path) + .with_context(|| format!("Failed to remove existing DB {}", db_path.display()))?; + } + + let output = Command::new("sqlite3") + .arg(db_path.as_os_str()) + .arg(format!(".restore '{}'", restore_file.display())) + .output() + .with_context(|| format!("Failed to run sqlite3 restore for {}", cfg.name))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + error!("SQLite restore failed for {}: {}", cfg.name, stderr); + anyhow::bail!("SQLite restore failed for {}", cfg.name); + } + + info!("SQLite restore completed for {}", cfg.name); + Ok(()) + }) + .await? +} \ No newline at end of file diff --git a/src/services/config.rs b/src/services/config.rs index 92ddba2..dab2834 100644 --- a/src/services/config.rs +++ b/src/services/config.rs @@ -16,8 +16,8 @@ pub enum DbType { Mysql, Mariadb, Postgresql, - MongoDB - // Sqlite, + MongoDB, + Sqlite // Add other DB types if needed } @@ -28,7 +28,7 @@ impl DbType { DbType::Mariadb => "mysql", DbType::Postgresql => "postgresql", DbType::MongoDB => "mongodb", - // DbType::Sqlite => "sqlite", + DbType::Sqlite => "sqlite", } } } @@ -45,6 +45,7 @@ pub struct DatabaseConfig { pub port: u16, pub host: String, pub generated_id: String, + pub path: Option } #[allow(dead_code)]