First Commit.

This commit is contained in:
charlesgauthereau
2026-01-03 23:55:47 +01:00
parent 50214f391e
commit 24ab764a6f
22 changed files with 3265 additions and 73 deletions
+5
View File
@@ -0,0 +1,5 @@
/target
.idea
/src/data/
Generated
+2415
View File
File diff suppressed because it is too large Load Diff
+32
View File
@@ -0,0 +1,32 @@
[package]
name = "portabase-agent"
version = "0.1.0"
edition = "2024"
[dependencies]
redis = { version = "1.0.2", features = ["aio", "tokio-comp"] }
cron = "0.15.0"
chrono = { version = "0.4", features = ["serde"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
tracing = "0.1"
tracing-subscriber = "0.3"
dotenvy = "0.15"
once_cell = "1.17"
base64 = "0.22.1"
thiserror = "2.0.17"
log = "0.4.29"
toml = "0.9.10"
reqwest = { version = "0.13.1", features = ["json", "blocking", "multipart"] }
anyhow = "1.0.100"
tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "fs"] }
async-trait = "0.1.89"
tempfile = "3.24.0"
openssl = "0.10.75"
hex = "0.4.3"
flate2 = "1.1.5"
tar = "0.4.44"
[[bin]]
name = "app"
path = "src/main.rs"
+13 -1
View File
@@ -30,10 +30,22 @@ services:
- POSTGRES_PASSWORD=changeme
network_mode: host
db2:
image: mariadb:latest
ports:
- "3306:3306"
environment:
- MYSQL_DATABASE=mariadb
- MYSQL_USER=mariadb
- MYSQL_PASSWORD=changeme
- MYSQL_RANDOM_ROOT_PASSWORD=yes
volumes:
- mariadb-data:/var/lib/mysql
network_mode: host
volumes:
cargo-registry:
cargo-git:
postgres-data:
# mariadb-data:
mariadb-data:
+1 -7
View File
@@ -44,11 +44,7 @@ FROM base AS builder
COPY . .
RUN cargo build --release
#RUN echo "APP_VERSION=$(cargo pkgid | awk -F# '{print $2}')" > /app/version.env
# Extract package version from Cargo.toml
ARG APP_VERSION
RUN export APP_VERSION=$(cargo pkgid | awk -F# '{print $2}') && echo "Built version: $APP_VERSION"
RUN echo "APP_VERSION=$(cargo pkgid | awk -F# '{print $2}')" > /app/version.env
# =========================
# Runtime (production)
@@ -72,7 +68,5 @@ COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh
ENV APP_ENV=production
ARG APP_VERSION
ENV APP_VERSION=$APP_VERSION
CMD ["/entrypoint.sh"]
+1 -1
View File
@@ -11,7 +11,7 @@ echo " /____/
if [ "$APP_ENV" = "production" ]; then
if [ -f /app/version.env ]; then
# . /app/version.env
. /app/version.env
PROJECT_NAME_VERSION=${APP_VERSION:-production}
else
PROJECT_NAME_VERSION="development"
+69
View File
@@ -0,0 +1,69 @@
-- ============================================================
-- HARD RESET
-- ============================================================
DROP TABLE IF EXISTS posts;
DROP TABLE IF EXISTS users;
-- ============================================================
-- SCHEMA
-- ============================================================
CREATE TABLE users (
id BIGSERIAL PRIMARY KEY,
username VARCHAR(50) NOT NULL,
email VARCHAR(100) NOT NULL,
password_hash VARCHAR(255) NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE posts (
id BIGSERIAL PRIMARY KEY,
user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
title VARCHAR(255) NOT NULL,
content TEXT NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX idx_posts_user_id ON posts(user_id);
-- ============================================================
-- USERS (~1 million rows)
-- ============================================================
INSERT INTO users (username, email, password_hash)
SELECT
'user_' || gs,
'user_' || gs || '@example.com',
md5(random()::text)
FROM generate_series(1, 1000000) AS gs;
-- ============================================================
-- POSTS
-- ============================================================
-- Each post content ≈ 4 KB
-- 300k users × 100 posts = 30 million rows
-- 30M × 4 KB ≈ 120 GB logical
-- After TOAST + compression ≈ 12 GB on disk
--
-- Adjust numbers if needed
-- ============================================================
INSERT INTO posts (user_id, title, content)
SELECT
u.id,
'Post #' || p.post_no || ' by user ' || u.id,
repeat(
'Lorem ipsum dolor sit amet, consectetur adipiscing elit. '
|| 'Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. ',
40
)
FROM users u
JOIN generate_series(1, 30) AS p(post_no)
ON u.id <= 300000;
-- ============================================================
-- OPTIONAL: FORCE DISK MATERIALIZATION
-- ============================================================
VACUUM ANALYZE;
+36
View File
@@ -0,0 +1,36 @@
-- MariaDB seed file for database "mariadb"
-- Drop tables if they exist
DROP TABLE IF EXISTS users;
DROP TABLE IF EXISTS products;
-- Create a users table
CREATE TABLE users (
id INT AUTO_INCREMENT PRIMARY KEY,
username VARCHAR(50) NOT NULL UNIQUE,
email VARCHAR(100) NOT NULL UNIQUE,
password VARCHAR(255) NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
-- Insert sample users
INSERT INTO users (username, email, password) VALUES
('alice', 'alice@example.com', 'changeme'),
('bob', 'bob@example.com', 'changeme');
-- Create a products table
CREATE TABLE products (
id INT AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(100) NOT NULL,
description TEXT,
price DECIMAL(10,2) NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
-- Insert sample products
INSERT INTO products (name, description, price) VALUES
('Laptop', 'High performance laptop', 1299.99),
('Phone', 'Smartphone with OLED display', 799.99),
('Headphones', 'Noise-cancelling headphones', 199.99);
-- Done
+32
View File
@@ -0,0 +1,32 @@
-- Drop tables if they exist
DROP TABLE IF EXISTS posts;
DROP TABLE IF EXISTS users;
-- Create users table
CREATE TABLE users (
id SERIAL PRIMARY KEY,
username VARCHAR(50) UNIQUE NOT NULL,
email VARCHAR(100) UNIQUE NOT NULL,
password_hash VARCHAR(255) NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
-- Create posts table
CREATE TABLE posts (
id SERIAL PRIMARY KEY,
user_id INT REFERENCES users(id) ON DELETE CASCADE,
title VARCHAR(255) NOT NULL,
content TEXT NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
-- Insert sample users
INSERT INTO users (username, email, password_hash) VALUES
('alice', 'alice@example.com', 'hashedpassword1'),
('bob', 'bob@example.com', 'hashedpassword2');
-- Insert sample posts
INSERT INTO posts (user_id, title, content) VALUES
(1, 'Hello World', 'This is the first post by Alice.'),
(2, 'Greetings', 'This is a post by Bob.'),
(1, 'Another Post', 'Alice writes again.');
+8 -5
View File
@@ -8,6 +8,7 @@ use crate::services::status::StatusService;
use crate::utils::common::BackupMethod;
use std::sync::Arc;
use tracing::info;
use crate::services::restore::RestoreService;
pub struct Agent {
ctx: Arc<Context>,
@@ -15,7 +16,7 @@ pub struct Agent {
status_service: StatusService,
cron_service: CronService,
backup_service: BackupService,
// restore_service: RestoreService,
restore_service: RestoreService,
}
impl Agent {
@@ -23,10 +24,9 @@ impl Agent {
let config_service = ConfigService::new(ctx.clone());
let status_service = StatusService::new(ctx.clone());
let cron_service = CronService::new(ctx.clone()).await;
// let backup_service = BackupService::new(ctx.clone()).await;
let backup_service = BackupService::new(ctx.clone());
// let restore_service = RestoreService::new(&ctx);
let restore_service = RestoreService::new(ctx.clone());
Agent {
ctx,
@@ -34,7 +34,7 @@ impl Agent {
status_service,
cron_service,
backup_service,
// restore_service,
restore_service,
}
}
@@ -55,7 +55,10 @@ impl Agent {
.dispatch(&db.generated_id, &config, method.clone())
.await;
} else if db.data.restore.action {
// handle restore
let _ = self
.restore_service
.dispatch(db, &config)
.await;
}
}
+18 -3
View File
@@ -5,6 +5,7 @@ use crate::services::config::DatabaseConfig;
use anyhow::Result;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use crate::domain::mysql::MySQLDatabase;
#[async_trait::async_trait]
pub trait Database: Send + Sync {
@@ -18,10 +19,24 @@ pub trait Database: Send + Sync {
pub struct DatabaseFactory;
impl DatabaseFactory {
pub fn create(cfg: DatabaseConfig) -> Arc<dyn Database> {
pub async fn create_for_backup(cfg: DatabaseConfig) -> Arc<dyn Database> {
match cfg.db_type.as_str() {
"postgresql" => Arc::new(PostgresDatabase::new(cfg)),
// "mysql" => Arc::new(MySQLDatabase::new(cfg)),
"postgresql" => {
let format = PostgresDatabase::detect_format_from_size(&cfg).await;
Arc::new(PostgresDatabase::new(cfg, format))
}
"mysql" => Arc::new(MySQLDatabase::new(cfg)),
_ => panic!("Unsupported DB type: {}", cfg.db_type),
}
}
pub async fn create_for_restore(cfg: DatabaseConfig, restore_file: &Path) -> Arc<dyn Database> {
match cfg.db_type.as_str() {
"postgresql" => {
let format = PostgresDatabase::detect_format_from_file(restore_file);
Arc::new(PostgresDatabase::new(cfg, format))
}
"mysql" => Arc::new(MySQLDatabase::new(cfg)),
_ => panic!("Unsupported DB type: {}", cfg.db_type),
}
}
+1
View File
@@ -1,2 +1,3 @@
pub mod factory;
pub mod postgres;
pub mod mysql;
+140
View File
@@ -0,0 +1,140 @@
use crate::domain::factory::Database;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::collections::HashMap;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::Command;
pub struct MySQLDatabase {
cfg: DatabaseConfig,
}
impl MySQLDatabase {
pub fn new(cfg: DatabaseConfig) -> Self {
Self { cfg }
}
fn build_env(&self) -> HashMap<String, String> {
let mut envs = std::env::vars().collect::<HashMap<_, _>>();
envs.insert("MYSQL_PWD".to_string(), self.cfg.password.clone());
envs
}
}
#[async_trait::async_trait]
impl Database for MySQLDatabase {
fn file_extension(&self) -> &'static str {
".sql"
}
async fn ping(&self) -> Result<bool> {
let output = Command::new("mysqladmin")
.arg("--host")
.arg(&self.cfg.host)
.arg("--port")
.arg(self.cfg.port.to_string())
.arg("--user")
.arg(&self.cfg.username)
.arg("ping")
.envs(self.build_env())
.output()
.with_context(|| format!("Failed to ping MySQL server {}", self.cfg.name))?;
Ok(output.status.success())
}
async fn backup(&self, backup_dir: &Path) -> Result<PathBuf> {
let file_path = backup_dir.join(format!(
"{}{}",
self.cfg.generated_id,
self.file_extension()
));
let output = Command::new("mysqldump")
.arg("--host")
.arg(&self.cfg.host)
.arg("--port")
.arg(self.cfg.port.to_string())
.arg("--user")
.arg(&self.cfg.username)
.arg("--routines")
.arg("--events")
.arg("--triggers")
.arg("--verbose")
.arg("--single-transaction")
.arg("--quick")
.arg("--add-drop-database")
.arg("--databases")
.arg(&self.cfg.database)
.arg("-r")
.arg(&file_path)
.envs(self.build_env())
.output()
.with_context(|| format!("Failed to run mysqldump for {}", self.cfg.name))?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
anyhow::bail!("MySQL backup failed for {}: {}", self.cfg.name, stderr);
}
Ok(file_path)
}
async fn restore(&self, restore_file: &Path) -> Result<()> {
let sql_content = tokio::fs::read_to_string(restore_file)
.await
.with_context(|| format!("Failed to read restore file {}", restore_file.display()))?;
let drop_create_cmd = format!(
"DROP DATABASE IF EXISTS {0}; CREATE DATABASE {0};",
self.cfg.database
);
let drop_status = Command::new("mysql")
.arg("--host")
.arg(&self.cfg.host)
.arg("--port")
.arg(self.cfg.port.to_string())
.arg("--user")
.arg(&self.cfg.username)
.arg("-e")
.arg(&drop_create_cmd)
.env("MYSQL_PWD", &self.cfg.password)
.status()
.with_context(|| format!("Failed to drop/recreate database {}", self.cfg.name))?;
if !drop_status.success() {
anyhow::bail!("Failed to drop/recreate database {}", self.cfg.name);
}
let mut child = Command::new("mysql")
.arg("--host")
.arg(&self.cfg.host)
.arg("--port")
.arg(self.cfg.port.to_string())
.arg("--user")
.arg(&self.cfg.username)
.arg(&self.cfg.database)
.env("MYSQL_PWD", &self.cfg.password)
.stdin(std::process::Stdio::piped())
.spawn()
.with_context(|| format!("Failed to start mysql restore for {}", self.cfg.name))?;
let mut stdin = child.stdin.take().context("Failed to open child stdin")?;
stdin.write_all(sql_content.as_bytes())?;
stdin.flush()?;
drop(stdin);
let output = child
.wait_with_output()
.with_context(|| format!("Failed to complete mysql restore for {}", self.cfg.name))?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
anyhow::bail!("MySQL restore failed for {}: {}", self.cfg.name, stderr);
}
Ok(())
}
}
+283 -50
View File
@@ -1,58 +1,258 @@
use std::path::{Path, PathBuf};
use std::process::Command;
// use std::path::{Path, PathBuf};
// use std::process::Command;
// use crate::domain::factory::Database;
// use crate::services::config::DatabaseConfig;
//
// pub struct PostgresDatabase {
// cfg: DatabaseConfig,
// }
//
// impl PostgresDatabase {
// pub fn new(cfg: DatabaseConfig) -> Self {
// Self { cfg }
// }
// }
//
// #[async_trait::async_trait]
// impl Database for PostgresDatabase {
// fn file_extension(&self) -> &'static str {
// ".dump"
// }
//
// async fn ping(&self) -> anyhow::Result<bool> {
// let url = format!(
// "postgresql://{}:{}@{}:{}/{}",
// self.cfg.username, self.cfg.password, self.cfg.host, self.cfg.port, self.cfg.database
// );
// let status = Command::new("pg_isready").arg("--dbname").arg(&url).status()?;
// Ok(status.success())
// }
//
// async fn backup(&self, backup_dir: &Path) -> anyhow::Result<PathBuf> {
// let file_path = backup_dir.join(format!("{}{}", self.cfg.generated_id, self.file_extension()));
// let url = format!(
// "postgresql://{}:{}@{}:{}/{}",
// self.cfg.username, self.cfg.password, self.cfg.host, self.cfg.port, self.cfg.database
// );
//
// let status = Command::new("pg_dump")
// .arg("--dbname")
// .arg(url)
// .arg("-Fc")
// .arg("-f")
// .arg(&file_path)
// .arg("-v")
// .arg("--compress=3")
// .status()?;
//
// if !status.success() {
// anyhow::bail!("Postgres backup failed for {}", self.cfg.name);
// }
//
// Ok(file_path)
// }
//
// async fn restore(&self, restore_file: &Path) -> anyhow::Result<()> {
// let url = format!(
// "postgresql://{}:{}@{}:{}/{}",
// self.cfg.username, self.cfg.password, self.cfg.host, self.cfg.port, "postgres"
// );
//
// // Terminate connections
// let terminate_cmd = format!(
// "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname='{}' AND pid<>pg_backend_pid();",
// self.cfg.database
// );
// Command::new("psql")
// .arg("-U").arg(&self.cfg.username)
// .arg("-d").arg("postgres")
// .arg("-h").arg(&self.cfg.host)
// .arg("-p").arg(self.cfg.port.to_string())
// .arg("-c").arg(terminate_cmd)
// .env("PGPASSWORD", &self.cfg.password)
// .status()?;
//
// // Restore
// let status = Command::new("pg_restore")
// .arg("--no-owner")
// .arg("--no-privileges")
// .arg("--clean")
// .arg("--if-exists")
// .arg("--create")
// .arg("--dbname").arg(url)
// .arg("-v")
// .arg(restore_file)
// .env("PGPASSWORD", &self.cfg.password)
// .status()?;
//
// if !status.success() {
// anyhow::bail!("Postgres restore failed for {}", self.cfg.name);
// }
//
// Ok(())
// }
// }
//
#![allow(dead_code)]
use crate::domain::factory::Database;
use crate::services::config::DatabaseConfig;
use anyhow::{Context as AnyhowContext, Result};
use async_trait::async_trait;
use flate2::Compression;
use flate2::write::GzEncoder;
use std::path::{Path, PathBuf};
use std::process::Command;
use log::info;
#[derive(Clone, Copy)]
pub enum PostgresDumpFormat {
Fc, // legacy
Fd, // directory format
}
pub struct PostgresDatabase {
cfg: DatabaseConfig,
format: PostgresDumpFormat,
}
impl PostgresDatabase {
pub fn new(cfg: DatabaseConfig) -> Self {
Self { cfg }
pub fn new(cfg: DatabaseConfig, format: PostgresDumpFormat) -> Self {
Self { cfg, format }
}
pub fn detect_format_from_file(restore_file: &Path) -> PostgresDumpFormat {
match restore_file.extension().and_then(|e| e.to_str()) {
Some("dump") => PostgresDumpFormat::Fc,
Some("gz") => PostgresDumpFormat::Fd,
// Some("tar.gz") => PostgresDumpFormat::Fd,
_ => PostgresDumpFormat::Fc,
}
}
pub async fn detect_format_from_size(cfg: &DatabaseConfig) -> PostgresDumpFormat {
let url = format!(
"host={} port={} user={} password={} dbname={}",
cfg.host, cfg.port, cfg.username, cfg.password, cfg.database
);
let output = std::process::Command::new("psql")
.arg(&url)
.arg("-t")
.arg("-c")
.arg("SELECT pg_database_size(current_database());")
.output();
match output {
Ok(out) if out.status.success() => {
let size_bytes: i64 = String::from_utf8_lossy(&out.stdout)
.trim()
.parse()
.unwrap_or(0);
// > 1 Go
if size_bytes > 1_000_000_000 {
PostgresDumpFormat::Fd
} else {
PostgresDumpFormat::Fc
}
}
_ => PostgresDumpFormat::Fc, // fallback legacy
}
}
}
#[async_trait::async_trait]
#[async_trait]
impl Database for PostgresDatabase {
fn file_extension(&self) -> &'static str {
".dump"
match self.format {
PostgresDumpFormat::Fc => ".dump",
PostgresDumpFormat::Fd => ".tar.gz",
}
}
async fn ping(&self) -> anyhow::Result<bool> {
async fn ping(&self) -> Result<bool> {
let url = format!(
"postgresql://{}:{}@{}:{}/{}",
self.cfg.username, self.cfg.password, self.cfg.host, self.cfg.port, self.cfg.database
);
let status = Command::new("pg_isready").arg("--dbname").arg(&url).status()?;
let status = Command::new("pg_isready")
.arg("--dbname")
.arg(url)
.status()
.context("Failed to ping Postgres")?;
Ok(status.success())
}
async fn backup(&self, backup_dir: &Path) -> anyhow::Result<PathBuf> {
let file_path = backup_dir.join(format!("{}{}", self.cfg.generated_id, self.file_extension()));
let url = format!(
"postgresql://{}:{}@{}:{}/{}",
self.cfg.username, self.cfg.password, self.cfg.host, self.cfg.port, self.cfg.database
);
async fn backup(&self, backup_dir: &Path) -> Result<PathBuf> {
match self.format {
PostgresDumpFormat::Fc => {
let file_path = backup_dir.join(format!(
"{}{}",
self.cfg.generated_id,
self.file_extension()
));
let url = format!(
"postgresql://{}:{}@{}:{}/{}",
self.cfg.username,
self.cfg.password,
self.cfg.host,
self.cfg.port,
self.cfg.database
);
let status = Command::new("pg_dump")
.arg("--dbname")
.arg(url)
.arg("-Fc")
.arg("-f")
.arg(&file_path)
.arg("-v")
.arg("--compress=3")
.status()?;
if !status.success() {
anyhow::bail!("Postgres backup failed for {}", self.cfg.name);
}
Ok(file_path)
}
PostgresDumpFormat::Fd => {
// directory dump -> tar.gz
let dump_dir = backup_dir.join(format!("{}_dir", self.cfg.generated_id));
let tar_file = backup_dir.join(format!("{}.tar.gz", self.cfg.generated_id));
std::fs::create_dir_all(&dump_dir)?;
let url = format!(
"postgresql://{}:{}@{}:{}/{}",
self.cfg.username,
self.cfg.password,
self.cfg.host,
self.cfg.port,
self.cfg.database
);
let status = Command::new("pg_dump")
.arg("--dbname")
.arg(url)
.arg("-Fd")
.arg("-j")
.arg("4")
.arg("-f")
.arg(&dump_dir)
.arg("-v")
.status()?;
if !status.success() {
anyhow::bail!("Postgres Fd backup failed for {}", self.cfg.name);
}
let status = Command::new("pg_dump")
.arg("--dbname")
.arg(url)
.arg("-Fc")
.arg("-f")
.arg(&file_path)
.arg("-v")
.arg("--compress=3")
.status()?;
if !status.success() {
anyhow::bail!("Postgres backup failed for {}", self.cfg.name);
// Compression tar.gz
let tar_gz = std::fs::File::create(&tar_file)?;
let enc = GzEncoder::new(tar_gz, Compression::default());
let mut tar = tar::Builder::new(enc);
tar.append_dir_all(".", &dump_dir)?;
tar.finish()?;
Ok(tar_file)
}
}
Ok(file_path)
}
async fn restore(&self, restore_file: &Path) -> anyhow::Result<()> {
async fn restore(&self, restore_file: &Path) -> Result<()> {
let url = format!(
"postgresql://{}:{}@{}:{}/{}",
self.cfg.username, self.cfg.password, self.cfg.host, self.cfg.port, "postgres"
@@ -64,32 +264,65 @@ impl Database for PostgresDatabase {
self.cfg.database
);
Command::new("psql")
.arg("-U").arg(&self.cfg.username)
.arg("-d").arg("postgres")
.arg("-h").arg(&self.cfg.host)
.arg("-p").arg(self.cfg.port.to_string())
.arg("-c").arg(terminate_cmd)
.arg("-U")
.arg(&self.cfg.username)
.arg("-d")
.arg("postgres")
.arg("-h")
.arg(&self.cfg.host)
.arg("-p")
.arg(self.cfg.port.to_string())
.arg("-c")
.arg(&terminate_cmd)
.env("PGPASSWORD", &self.cfg.password)
.status()?;
// Restore
let status = Command::new("pg_restore")
.arg("--no-owner")
.arg("--no-privileges")
.arg("--clean")
.arg("--if-exists")
.arg("--create")
.arg("--dbname").arg(url)
.arg("-v")
.arg(restore_file)
.env("PGPASSWORD", &self.cfg.password)
.status()?;
match self.format {
PostgresDumpFormat::Fc => {
let status = Command::new("pg_restore")
.arg("--no-owner")
.arg("--no-privileges")
.arg("--clean")
.arg("--if-exists")
.arg("--create")
.arg("--dbname")
.arg(url)
.arg("-v")
.arg(restore_file)
.env("PGPASSWORD", &self.cfg.password)
.status()?;
if !status.success() {
anyhow::bail!("Postgres restore failed for {}", self.cfg.name);
}
}
PostgresDumpFormat::Fd => {
let tar_gz = std::fs::File::open(restore_file)?;
let dec = flate2::read::GzDecoder::new(tar_gz);
let mut archive = tar::Archive::new(dec);
let tmp_dir = tempfile::TempDir::new()?;
archive.unpack(tmp_dir.path())?;
if !status.success() {
anyhow::bail!("Postgres restore failed for {}", self.cfg.name);
let dump_dir = tmp_dir.path();
info!("Restoring dump from {}", dump_dir.display());
let status = Command::new("pg_restore")
.arg("--no-owner")
.arg("--no-privileges")
.arg("--clean")
.arg("--if-exists")
.arg("--create")
.arg("--dbname")
.arg(url)
.arg("-v")
.arg(dump_dir)
.env("PGPASSWORD", &self.cfg.password)
.status()?;
if !status.success() {
anyhow::bail!("Postgres Fd restore failed for {}", self.cfg.name);
}
}
}
Ok(())
}
}
+24
View File
@@ -0,0 +1,24 @@
mod settings;
mod tasks;
mod utils;
mod core;
mod services;
mod domain;
use tracing_subscriber;
use utils::redis_client;
use utils::task_manager::scheduler;
use crate::tasks::ping::ping_server;
#[tokio::main]
async fn main() {
tracing_subscriber::fmt::init();
tokio::join!(
ping_server(),
async {
let conn = redis_client::redis_connection().await;
scheduler::scheduler_loop(conn).await;
}
);
}
+6 -3
View File
@@ -4,6 +4,7 @@ use crate::core::context::Context;
use crate::domain::factory::DatabaseFactory;
use crate::services::config::{DatabaseConfig, DatabasesConfig};
use crate::utils::common::BackupMethod;
use crate::utils::file::full_extension;
use anyhow::Result;
use hex;
use log::{error, info};
@@ -71,9 +72,8 @@ impl BackupService {
}
}
pub async fn run(cfg: DatabaseConfig, tmp_path: &Path) -> Result<BackupResult> {
let db_instance = DatabaseFactory::create(cfg.clone());
let db_instance = DatabaseFactory::create_for_backup(cfg.clone()).await;
let generated_id = cfg.generated_id.clone();
let db_type = cfg.db_type.clone();
@@ -156,6 +156,8 @@ impl BackupService {
let encrypted_len = encrypter.encrypt(&aes_key, &mut encrypted_key).unwrap();
encrypted_key.truncate(encrypted_len);
let extension = full_extension(&file_path);
// Attach file and AES info to multipart form
form = form
.part(
@@ -164,7 +166,8 @@ impl BackupService {
.file_name(format!("{}.enc", result.generated_id)),
)
.text("aes_key", hex::encode(encrypted_key))
.text("iv", hex::encode(iv));
.text("iv", hex::encode(iv))
.text("extension", extension);
}
Err(e) => {
error!("Failed to read backup file: {}", e);
+2 -1
View File
@@ -1,4 +1,5 @@
pub mod config;
pub mod status;
pub mod cron;
pub mod backup;
pub mod backup;
pub mod restore;
+159
View File
@@ -0,0 +1,159 @@
#![allow(dead_code)]
use crate::core::context::Context;
use crate::domain::factory::DatabaseFactory;
use crate::services::config::{DatabaseConfig, DatabasesConfig};
use crate::services::status::DatabaseStatus;
use anyhow::Result;
use log::{error, info};
use serde::Serialize;
use std::path::Path;
use std::sync::Arc;
use tempfile::TempDir;
#[derive(Debug, Serialize)]
pub struct RestoreResult {
#[serde(rename = "generatedId")]
pub generated_id: String,
pub status: String,
}
pub struct RestoreService {
ctx: Arc<Context>,
}
impl RestoreService {
pub fn new(ctx: Arc<Context>) -> Self {
Self { ctx }
}
pub async fn dispatch(&self, db: &DatabaseStatus, config: &DatabasesConfig) {
if let Some(cfg) = config
.databases
.iter()
.find(|c| c.generated_id == db.generated_id)
{
let db_cfg = cfg.clone();
let ctx_clone = self.ctx.clone();
let file_to_restore = db.data.restore.file.clone();
tokio::spawn(async move {
match TempDir::new() {
Ok(temp_dir) => {
let tmp_path = temp_dir.path().to_path_buf();
info!("Created temp directory {}", tmp_path.display());
match RestoreService::run(db_cfg, &tmp_path, &file_to_restore).await {
Ok(result) => {
let service = RestoreService { ctx: ctx_clone };
service.send_result(result).await;
}
Err(e) => error!("Restoration error {}", e),
}
// TempDir is automatically deleted when dropped here
}
Err(e) => error!("Failed to create temp dir: {}", e),
}
});
}
}
pub async fn run(
cfg: DatabaseConfig,
tmp_path: &Path,
file_url: &str,
) -> Result<RestoreResult> {
let generated_id = cfg.generated_id.clone();
let client = reqwest::Client::new();
let response = client.get(file_url).send().await?;
if !response.status().is_success() {
error!("Backup download failed with status {}", response.status());
return Ok(RestoreResult {
generated_id,
status: "failed".into(),
});
}
let bytes = response.bytes().await?;
let ext = if bytes.starts_with(b"PGDMP") {
// Postgres custom format
"dump"
} else if bytes.starts_with(&[0x1F, 0x8B]) {
// gzip compressed -> could be Postgres directory dump or MySQL gzipped SQL
"tar.gz"
} else if bytes.starts_with(b"--") || bytes.starts_with(b"/*") {
// Plain MySQL SQL dump
"sql"
} else {
// Fallback generic
"dump"
};
info!("Backup dump from {} to {}", tmp_path.display(), ext);
let backup_file_path = tmp_path.join(format!("backup_file_tmp.{}", ext));
tokio::fs::write(&backup_file_path, &bytes).await?;
info!("Backup downloaded to {}", backup_file_path.display());
let db_instance = DatabaseFactory::create_for_restore(cfg.clone(), &backup_file_path).await;
let reachable = db_instance.ping().await.unwrap_or(false);
if !reachable {
return Ok(RestoreResult {
generated_id,
status: "failed".into(),
});
}
match db_instance.restore(&backup_file_path).await {
Ok(_) => Ok(RestoreResult {
generated_id,
status: "success".into(),
}),
Err(e) => {
log::error!("Restore failed: {:?}", e);
Ok(RestoreResult {
generated_id,
status: "failed".into(),
})
}
}
}
pub async fn send_result(&self, result: RestoreResult) {
info!(
"[RestoreService] DB: {} | Status: {}",
result.generated_id, result.status,
);
let client = reqwest::Client::new();
let url = format!(
"{}/api/agent/{}/restore",
self.ctx.edge_key.server_url, self.ctx.edge_key.agent_id
);
let body = RestoreResult {
generated_id: result.generated_id,
status: result.status,
};
match client.post(&url).json(&body).send().await {
Ok(resp) => {
let status = resp.status();
if status.is_success() {
info!("Restoration result sent successfully");
} else {
let text = resp.text().await.unwrap_or_default(); // consumes resp
error!(
"Restoration result failed, status: {}, body: {}",
status, text
);
}
}
Err(e) => {
error!("Failed to send restoration result: {}", e);
}
}
}
}
+3 -1
View File
@@ -93,8 +93,10 @@ impl StatusService {
})
.collect();
let version_str = format!("{}-rust", CONFIG.app_version);
let body = StatusRequestBody {
version: CONFIG.app_version.as_str(),
version: &version_str,
databases: databases_payload,
};
+1 -1
View File
@@ -18,7 +18,7 @@ impl Settings {
dotenv().ok();
Self {
app_version: env::var("APP_VERSION").unwrap_or_else(|_| "unknown".into()),
app_version: env::var("CARGO_PKG_VERSION").unwrap_or_else(|_| "unknown".into()),
app_env: env::var("APP_ENV").unwrap_or_else(|_| "development".into()),
redis_url: env::var("CELERY_BROKER_URL")
.unwrap_or_else(|_| "redis://localhost:6379/".into()),
+15
View File
@@ -0,0 +1,15 @@
use std::path::Path;
pub fn full_extension(path: &Path) -> String {
path.file_name()
.and_then(|name| name.to_str())
.map(|name| {
match name.find('.') {
Some(idx) => &name[idx..],
None => "",
}
})
.unwrap_or_default()
.to_string()
}
+1
View File
@@ -3,3 +3,4 @@ pub mod edge_key;
pub mod redis_client;
pub mod task_manager;
pub mod text;
pub mod file;