mirror of
https://github.com/Portabase/agent.git
synced 2026-09-10 01:57:10 +00:00
feat: separate mysql and mariadb
This commit is contained in:
+11
-1
@@ -13,7 +13,7 @@
|
||||
{
|
||||
"name": "Test database 2 - MariaDB",
|
||||
"database": "mariadb",
|
||||
"type": "mysql",
|
||||
"type": "mariadb",
|
||||
"username": "mariadb",
|
||||
"password": "changeme",
|
||||
"port": 3306,
|
||||
@@ -81,6 +81,16 @@
|
||||
"username": "default",
|
||||
"host": "db-valkey-auth",
|
||||
"generated_id": "16678561-ff7e-4c97-8c83-0adeff214681"
|
||||
},
|
||||
{
|
||||
"name": "Test database 12 - Mysql",
|
||||
"database": "mysqldb",
|
||||
"type": "mysql",
|
||||
"username": "mysqldb",
|
||||
"password": "changeme",
|
||||
"port": 3306,
|
||||
"host": "db-mysql",
|
||||
"generated_id": "16656124-ff7e-4c97-8c83-0adeff214681"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
+97
-82
@@ -11,8 +11,7 @@ services:
|
||||
# - ./databases.toml:/config/config.toml
|
||||
- cargo-registry:/usr/local/cargo/registry
|
||||
- cargo-git:/usr/local/cargo/git
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
|
||||
# - /var/run/docker.sock:/var/run/docker.sock
|
||||
# - cargo-target:/app/target
|
||||
# - sqlite-data:/sqlite-data/workspace/data
|
||||
# - ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data
|
||||
@@ -28,34 +27,49 @@ services:
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
db-postgres:
|
||||
container_name: db-postgres
|
||||
image: postgres:17-alpine
|
||||
# db-postgres:
|
||||
# container_name: db-postgres
|
||||
# image: postgres:17-alpine
|
||||
# ports:
|
||||
# - "5436:5432"
|
||||
# volumes:
|
||||
# - postgres-data:/var/lib/postgresql/data
|
||||
# environment:
|
||||
# - POSTGRES_DB=devdb
|
||||
# - POSTGRES_USER=devuser
|
||||
# - POSTGRES_PASSWORD=changeme
|
||||
# networks:
|
||||
# - portabase
|
||||
|
||||
db-mariadb:
|
||||
container_name: db-mariadb
|
||||
image: mariadb:latest
|
||||
ports:
|
||||
- "5436:5432"
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
- "3311:3306"
|
||||
environment:
|
||||
- POSTGRES_DB=devdb
|
||||
- POSTGRES_USER=devuser
|
||||
- POSTGRES_PASSWORD=changeme
|
||||
- MYSQL_DATABASE=mariadb
|
||||
- MYSQL_USER=mariadb
|
||||
- MYSQL_PASSWORD=changeme
|
||||
- MYSQL_RANDOM_ROOT_PASSWORD=yes
|
||||
volumes:
|
||||
- mariadb-data:/var/lib/mysql
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
db-mysql:
|
||||
container_name: db-mysql
|
||||
image: mysql:9.5
|
||||
ports:
|
||||
- "3312:3306"
|
||||
environment:
|
||||
- MYSQL_DATABASE=mysqldb
|
||||
- MYSQL_USER=mysqldb
|
||||
- MYSQL_PASSWORD=changeme
|
||||
- MYSQL_RANDOM_ROOT_PASSWORD=yes
|
||||
volumes:
|
||||
- mysql-data:/var/lib/mysql
|
||||
networks:
|
||||
- portabase
|
||||
#
|
||||
# db-mariadb:
|
||||
# container_name: db-mariadb
|
||||
# image: mariadb:latest
|
||||
# ports:
|
||||
# - "3311:3306"
|
||||
# environment:
|
||||
# - MYSQL_DATABASE=mariadb
|
||||
# - MYSQL_USER=mariadb
|
||||
# - MYSQL_PASSWORD=changeme
|
||||
# - MYSQL_RANDOM_ROOT_PASSWORD=yes
|
||||
# volumes:
|
||||
# - mariadb-data:/var/lib/mysql
|
||||
# networks:
|
||||
# - portabase
|
||||
#
|
||||
#
|
||||
# db-mongodb-auth:
|
||||
@@ -105,67 +119,68 @@ services:
|
||||
# stdin_open: true
|
||||
# tty: true
|
||||
|
||||
db-redis:
|
||||
image: redis:latest
|
||||
container_name: db-redis
|
||||
ports:
|
||||
- "6379:6379"
|
||||
volumes:
|
||||
- redis-data:/data
|
||||
command: [ "redis-server", "--appendonly", "yes" ]
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
db-redis-auth:
|
||||
image: redis:latest
|
||||
container_name: db-redis-auth
|
||||
ports:
|
||||
- "6380:6379"
|
||||
volumes:
|
||||
- redis-data-auth:/data
|
||||
environment:
|
||||
- REDIS_PASSWORD=supersecurepassword
|
||||
command: [ "redis-server", "--requirepass", "supersecurepassword", "--appendonly", "yes" ]
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
db-valkey:
|
||||
image: valkey/valkey
|
||||
container_name: db-valkey
|
||||
environment:
|
||||
- ALLOW_EMPTY_PASSWORD=yes
|
||||
ports:
|
||||
- '6381:6379'
|
||||
volumes:
|
||||
- valkey-data:/data
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
db-valkey-auth:
|
||||
image: valkey/valkey
|
||||
container_name: db-valkey-auth
|
||||
command: >
|
||||
--requirepass "supersecurepassword"
|
||||
ports:
|
||||
- '6382:6379'
|
||||
volumes:
|
||||
- valkey-data-auth:/data
|
||||
networks:
|
||||
- portabase
|
||||
# db-redis:
|
||||
# image: redis:latest
|
||||
# container_name: db-redis
|
||||
# ports:
|
||||
# - "6379:6379"
|
||||
# volumes:
|
||||
# - redis-data:/data
|
||||
# command: [ "redis-server", "--appendonly", "yes" ]
|
||||
# networks:
|
||||
# - portabase
|
||||
#
|
||||
# db-redis-auth:
|
||||
# image: redis:latest
|
||||
# container_name: db-redis-auth
|
||||
# ports:
|
||||
# - "6380:6379"
|
||||
# volumes:
|
||||
# - redis-data-auth:/data
|
||||
# environment:
|
||||
# - REDIS_PASSWORD=supersecurepassword
|
||||
# command: [ "redis-server", "--requirepass", "supersecurepassword", "--appendonly", "yes" ]
|
||||
# networks:
|
||||
# - portabase
|
||||
#
|
||||
# db-valkey:
|
||||
# image: valkey/valkey
|
||||
# container_name: db-valkey
|
||||
# environment:
|
||||
# - ALLOW_EMPTY_PASSWORD=yes
|
||||
# ports:
|
||||
# - '6381:6379'
|
||||
# volumes:
|
||||
# - valkey-data:/data
|
||||
# networks:
|
||||
# - portabase
|
||||
#
|
||||
# db-valkey-auth:
|
||||
# image: valkey/valkey
|
||||
# container_name: db-valkey-auth
|
||||
# command: >
|
||||
# --requirepass "supersecurepassword"
|
||||
# ports:
|
||||
# - '6382:6379'
|
||||
# volumes:
|
||||
# - valkey-data-auth:/data
|
||||
# networks:
|
||||
# - portabase
|
||||
|
||||
volumes:
|
||||
cargo-registry:
|
||||
cargo-git:
|
||||
# cargo-target:
|
||||
postgres-data:
|
||||
# mariadb-data:
|
||||
# mongodb-data:
|
||||
# mongodb-data-auth:
|
||||
# sqlite-data:
|
||||
redis-data:
|
||||
redis-data-auth:
|
||||
valkey-data:
|
||||
valkey-data-auth:
|
||||
# postgres-data:
|
||||
mariadb-data:
|
||||
mysql-data:
|
||||
# mongodb-data:
|
||||
# mongodb-data-auth:
|
||||
# sqlite-data:
|
||||
# redis-data:
|
||||
# redis-data-auth:
|
||||
# valkey-data:
|
||||
# valkey-data-auth:
|
||||
|
||||
networks:
|
||||
portabase:
|
||||
|
||||
@@ -46,6 +46,7 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \
|
||||
rm -rf /tmp/pg-x64 /tmp/pg-arm && \
|
||||
chmod +x /usr/lib/postgresql/*/bin/*
|
||||
|
||||
|
||||
# =========================
|
||||
# MongoDB client binaries
|
||||
# =========================
|
||||
|
||||
@@ -9,6 +9,7 @@ use crate::services::config::{DatabaseConfig, DbType};
|
||||
use anyhow::Result;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use crate::domain::mariadb::database::MariaDBDatabase;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait Database: Send + Sync {
|
||||
@@ -28,7 +29,7 @@ impl DatabaseFactory {
|
||||
Arc::new(PostgresDatabase::new(cfg, format))
|
||||
}
|
||||
DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)),
|
||||
DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)),
|
||||
DbType::Mariadb => Arc::new(MariaDBDatabase::new(cfg)),
|
||||
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
|
||||
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
|
||||
DbType::Redis => Arc::new(RedisDatabase::new(cfg)),
|
||||
@@ -43,7 +44,7 @@ impl DatabaseFactory {
|
||||
Arc::new(PostgresDatabase::new(cfg, format))
|
||||
}
|
||||
DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)),
|
||||
DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)),
|
||||
DbType::Mariadb => Arc::new(MariaDBDatabase::new(cfg)),
|
||||
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
|
||||
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
|
||||
DbType::Redis => Arc::new(RedisDatabase::new(cfg)),
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
use crate::domain::mariadb::connection::{select_mariadb_path, server_version};
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
backup_dir: PathBuf,
|
||||
env: HashMap<String, String>,
|
||||
file_extension: &'static str,
|
||||
) -> Result<PathBuf> {
|
||||
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||
debug!("Starting backup for database {}", cfg.name);
|
||||
|
||||
let version = match futures::executor::block_on(server_version(&cfg)) {
|
||||
Ok(v) => {
|
||||
debug!("Mariadb version detected: {}", v);
|
||||
v
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to get server version for {}: {:?}", cfg.name, e);
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
|
||||
info!("Mariadb version found: {}", version);
|
||||
|
||||
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
|
||||
|
||||
let mariadb_dump = select_mariadb_path(&version).join("mariadb-dump");
|
||||
info!("Mariadb dump found: {}", mariadb_dump.display());
|
||||
|
||||
let output = Command::new("mariadb-dump")
|
||||
.arg("--host").arg(&cfg.host)
|
||||
.arg("--port").arg(cfg.port.to_string())
|
||||
.arg("--user").arg(&cfg.username)
|
||||
.arg("--routines")
|
||||
.arg("--events")
|
||||
.arg("--triggers")
|
||||
.arg("--single-transaction")
|
||||
.arg("--quick")
|
||||
.arg("--skip-lock-tables")
|
||||
.arg("--add-drop-database")
|
||||
.arg("--databases").arg(&cfg.database)
|
||||
.arg("--compress")
|
||||
.arg("--max-allowed-packet=512M")
|
||||
.arg("--net-buffer-length=16K")
|
||||
.arg("--default-character-set=utf8mb4")
|
||||
.arg("-r").arg(&file_path)
|
||||
.envs(env)
|
||||
.output()
|
||||
.with_context(|| format!("Failed to run mariadb-dump for {}", cfg.name))?;
|
||||
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
anyhow::bail!("Mariadb backup failed for {}: {}", cfg.name, stderr);
|
||||
}
|
||||
|
||||
Ok(file_path)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
use std::path::PathBuf;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::Result;
|
||||
use std::process::Command;
|
||||
|
||||
pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
|
||||
let output = Command::new("mariadb")
|
||||
.arg("--host")
|
||||
.arg(&cfg.host)
|
||||
.arg("--port")
|
||||
.arg(cfg.port.to_string())
|
||||
.arg("--user")
|
||||
.arg(&cfg.username)
|
||||
.arg("-e")
|
||||
.arg("SELECT VERSION();")
|
||||
.env("MYSQL_PWD", &cfg.password)
|
||||
.output()?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
anyhow::bail!("Version query failed: {}", stderr);
|
||||
}
|
||||
|
||||
let version = String::from_utf8_lossy(&output.stdout)
|
||||
.lines()
|
||||
.nth(1)
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.to_string();
|
||||
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
|
||||
pub fn select_mariadb_path(version: &str) -> PathBuf {
|
||||
let mut parts = version.split('.');
|
||||
let major = parts.next().and_then(|v| v.parse::<u32>().ok()).unwrap_or(10);
|
||||
let minor = parts.next().and_then(|v| v.parse::<u32>().ok()).unwrap_or(0);
|
||||
|
||||
if major < 10 || (major == 10 && minor <= 6) {
|
||||
"/usr/local/mariadb-10.6/bin".into()
|
||||
} else {
|
||||
"/usr/local/mariadb-12.1/bin".into()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
use super::{backup, ping, restore};
|
||||
use crate::domain::factory::Database;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::utils::locks::{DbOpLock, FileLock};
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
pub struct MariaDBDatabase {
|
||||
cfg: DatabaseConfig,
|
||||
}
|
||||
|
||||
impl MariaDBDatabase {
|
||||
pub fn new(cfg: DatabaseConfig) -> Self {
|
||||
Self { cfg }
|
||||
}
|
||||
|
||||
fn build_env(&self) -> HashMap<String, String> {
|
||||
let mut envs = std::env::vars().collect::<HashMap<_, _>>();
|
||||
envs.insert("MYSQL_PWD".to_string(), self.cfg.password.to_string());
|
||||
envs
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Database for MariaDBDatabase {
|
||||
fn file_extension(&self) -> &'static str {
|
||||
".sql"
|
||||
}
|
||||
|
||||
async fn ping(&self) -> Result<bool> {
|
||||
ping::run(self.cfg.clone(), self.build_env().clone()).await
|
||||
}
|
||||
|
||||
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
let res = backup::run(
|
||||
self.cfg.clone(),
|
||||
dir.to_path_buf(),
|
||||
self.build_env().clone(),
|
||||
self.file_extension(),
|
||||
)
|
||||
.await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
|
||||
async fn restore(&self, file: &Path) -> Result<()> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
pub mod backup;
|
||||
mod connection;
|
||||
pub mod database;
|
||||
mod ping;
|
||||
mod restore;
|
||||
@@ -0,0 +1,28 @@
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use std::collections::HashMap;
|
||||
use tokio::process::Command;
|
||||
use tokio::time::{Duration, timeout};
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::Result<bool> {
|
||||
let mut cmd = Command::new("mysqladmin");
|
||||
cmd.arg("--host")
|
||||
.arg(cfg.host)
|
||||
.arg("--port")
|
||||
.arg(cfg.port.to_string())
|
||||
.arg("--user")
|
||||
.arg(cfg.username)
|
||||
.arg("ping")
|
||||
.envs(env);
|
||||
|
||||
let result = timeout(Duration::from_secs(10), cmd.output()).await;
|
||||
|
||||
match result {
|
||||
Ok(output) => {
|
||||
let output = output?;
|
||||
Ok(output.status.success())
|
||||
}
|
||||
Err(_) => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::fs::File;
|
||||
use std::io::{Read, Write};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
|
||||
let handle = tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
debug!("Starting restore for database {}", cfg.name);
|
||||
|
||||
let mut sql_content = String::new();
|
||||
let mut file = File::open(&restore_file)
|
||||
.with_context(|| format!("Failed to open restore file {}", restore_file.display()))?;
|
||||
file.read_to_string(&mut sql_content)
|
||||
.with_context(|| format!("Failed to read restore file {}", restore_file.display()))?;
|
||||
|
||||
let drop_create_cmd = format!(
|
||||
"DROP DATABASE IF EXISTS {0}; CREATE DATABASE {0};",
|
||||
cfg.database
|
||||
);
|
||||
|
||||
let drop_status = Command::new("mariadb")
|
||||
.arg("--host")
|
||||
.arg(&cfg.host)
|
||||
.arg("--port")
|
||||
.arg(cfg.port.to_string())
|
||||
.arg("--user")
|
||||
.arg(&cfg.username)
|
||||
.arg("-e")
|
||||
.arg(&drop_create_cmd)
|
||||
.env("MYSQL_PWD", &cfg.password)
|
||||
.status()
|
||||
.with_context(|| format!("Failed to drop/recreate database {}", cfg.name))?;
|
||||
|
||||
if !drop_status.success() {
|
||||
error!("Drop/create database failed for {}", cfg.name);
|
||||
anyhow::bail!("Failed to drop/recreate database {}", cfg.name);
|
||||
}
|
||||
info!("Database {} dropped and recreated", cfg.name);
|
||||
|
||||
let mut child = Command::new("mariadb")
|
||||
.arg("--host")
|
||||
.arg(&cfg.host)
|
||||
.arg("--port")
|
||||
.arg(cfg.port.to_string())
|
||||
.arg("--user")
|
||||
.arg(&cfg.username)
|
||||
.arg(&cfg.database)
|
||||
.env("MYSQL_PWD", &cfg.password)
|
||||
.stdin(std::process::Stdio::piped())
|
||||
.spawn()
|
||||
.with_context(|| format!("Failed to start MariaDB restore for {}", cfg.name))?;
|
||||
|
||||
let mut stdin = child.stdin.take().context("Failed to open child stdin")?;
|
||||
stdin
|
||||
.write_all(sql_content.as_bytes())
|
||||
.context("Failed to write SQL content to MariaDB stdin")?;
|
||||
stdin.flush()?;
|
||||
drop(stdin);
|
||||
|
||||
let output = child
|
||||
.wait_with_output()
|
||||
.with_context(|| format!("Failed to complete MariaDB restore for {}", cfg.name))?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
error!("MariaDB restore failed for {}: {}", cfg.name, stderr);
|
||||
anyhow::bail!("MariaDB restore failed for {}", cfg.name);
|
||||
}
|
||||
|
||||
info!("Restore finished successfully for database {}", cfg.name);
|
||||
Ok(())
|
||||
});
|
||||
|
||||
handle.await??;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -5,3 +5,4 @@ pub mod postgres;
|
||||
mod redis;
|
||||
mod sqlite;
|
||||
mod valkey;
|
||||
mod mariadb;
|
||||
|
||||
@@ -30,6 +30,9 @@ pub async fn run(
|
||||
|
||||
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
|
||||
|
||||
// let mysql_dump = select_mysql_path(&version).join("mysqldump");
|
||||
// info!("MySQL dump found: {}", mysql_dump.display());
|
||||
|
||||
let output = Command::new("mysqldump")
|
||||
.arg("--host")
|
||||
.arg(cfg.host)
|
||||
@@ -54,9 +57,12 @@ pub async fn run(
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
info!("mysqldump stderr: {}", stderr);
|
||||
anyhow::bail!("MySQL backup failed for {}: {}", cfg.name, stderr);
|
||||
}
|
||||
|
||||
info!("Output {}", String::from_utf8_lossy(&output.stdout));
|
||||
|
||||
Ok(file_path)
|
||||
})
|
||||
.await?
|
||||
|
||||
@@ -4,7 +4,7 @@ use tokio::process::Command;
|
||||
use tokio::time::{Duration, timeout};
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::Result<bool> {
|
||||
let mut cmd = Command::new("mysqladmin");
|
||||
let mut cmd = Command::new("mariadb-admin");
|
||||
cmd.arg("--host")
|
||||
.arg(cfg.host)
|
||||
.arg("--port")
|
||||
|
||||
Reference in New Issue
Block a user