mirror of
https://github.com/Portabase/agent.git
synced 2026-09-09 17:47:25 +00:00
First Commit.
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
/target
|
||||
|
||||
.idea
|
||||
|
||||
/src/data/
|
||||
Generated
+2415
File diff suppressed because it is too large
Load Diff
+32
@@ -0,0 +1,32 @@
|
||||
[package]
|
||||
name = "portabase-agent"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
redis = { version = "1.0.2", features = ["aio", "tokio-comp"] }
|
||||
cron = "0.15.0"
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = "0.3"
|
||||
dotenvy = "0.15"
|
||||
once_cell = "1.17"
|
||||
base64 = "0.22.1"
|
||||
thiserror = "2.0.17"
|
||||
log = "0.4.29"
|
||||
toml = "0.9.10"
|
||||
reqwest = { version = "0.13.1", features = ["json", "blocking", "multipart"] }
|
||||
anyhow = "1.0.100"
|
||||
tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "fs"] }
|
||||
async-trait = "0.1.89"
|
||||
tempfile = "3.24.0"
|
||||
openssl = "0.10.75"
|
||||
hex = "0.4.3"
|
||||
flate2 = "1.1.5"
|
||||
tar = "0.4.44"
|
||||
|
||||
[[bin]]
|
||||
name = "app"
|
||||
path = "src/main.rs"
|
||||
+13
-1
@@ -30,10 +30,22 @@ services:
|
||||
- POSTGRES_PASSWORD=changeme
|
||||
network_mode: host
|
||||
|
||||
db2:
|
||||
image: mariadb:latest
|
||||
ports:
|
||||
- "3306:3306"
|
||||
environment:
|
||||
- MYSQL_DATABASE=mariadb
|
||||
- MYSQL_USER=mariadb
|
||||
- MYSQL_PASSWORD=changeme
|
||||
- MYSQL_RANDOM_ROOT_PASSWORD=yes
|
||||
volumes:
|
||||
- mariadb-data:/var/lib/mysql
|
||||
network_mode: host
|
||||
|
||||
volumes:
|
||||
cargo-registry:
|
||||
cargo-git:
|
||||
|
||||
postgres-data:
|
||||
# mariadb-data:
|
||||
mariadb-data:
|
||||
+1
-7
@@ -44,11 +44,7 @@ FROM base AS builder
|
||||
COPY . .
|
||||
RUN cargo build --release
|
||||
|
||||
#RUN echo "APP_VERSION=$(cargo pkgid | awk -F# '{print $2}')" > /app/version.env
|
||||
# Extract package version from Cargo.toml
|
||||
ARG APP_VERSION
|
||||
RUN export APP_VERSION=$(cargo pkgid | awk -F# '{print $2}') && echo "Built version: $APP_VERSION"
|
||||
|
||||
RUN echo "APP_VERSION=$(cargo pkgid | awk -F# '{print $2}')" > /app/version.env
|
||||
|
||||
# =========================
|
||||
# Runtime (production)
|
||||
@@ -72,7 +68,5 @@ COPY entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /entrypoint.sh
|
||||
|
||||
ENV APP_ENV=production
|
||||
ARG APP_VERSION
|
||||
ENV APP_VERSION=$APP_VERSION
|
||||
|
||||
CMD ["/entrypoint.sh"]
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@ echo " /____/
|
||||
|
||||
if [ "$APP_ENV" = "production" ]; then
|
||||
if [ -f /app/version.env ]; then
|
||||
# . /app/version.env
|
||||
. /app/version.env
|
||||
PROJECT_NAME_VERSION=${APP_VERSION:-production}
|
||||
else
|
||||
PROJECT_NAME_VERSION="development"
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
-- ============================================================
|
||||
-- HARD RESET
|
||||
-- ============================================================
|
||||
|
||||
DROP TABLE IF EXISTS posts;
|
||||
DROP TABLE IF EXISTS users;
|
||||
|
||||
-- ============================================================
|
||||
-- SCHEMA
|
||||
-- ============================================================
|
||||
|
||||
CREATE TABLE users (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
username VARCHAR(50) NOT NULL,
|
||||
email VARCHAR(100) NOT NULL,
|
||||
password_hash VARCHAR(255) NOT NULL,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE posts (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
title VARCHAR(255) NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX idx_posts_user_id ON posts(user_id);
|
||||
|
||||
-- ============================================================
|
||||
-- USERS (~1 million rows)
|
||||
-- ============================================================
|
||||
|
||||
INSERT INTO users (username, email, password_hash)
|
||||
SELECT
|
||||
'user_' || gs,
|
||||
'user_' || gs || '@example.com',
|
||||
md5(random()::text)
|
||||
FROM generate_series(1, 1000000) AS gs;
|
||||
|
||||
-- ============================================================
|
||||
-- POSTS
|
||||
-- ============================================================
|
||||
-- Each post content ≈ 4 KB
|
||||
-- 300k users × 100 posts = 30 million rows
|
||||
-- 30M × 4 KB ≈ 120 GB logical
|
||||
-- After TOAST + compression ≈ 1–2 GB on disk
|
||||
--
|
||||
-- Adjust numbers if needed
|
||||
-- ============================================================
|
||||
|
||||
INSERT INTO posts (user_id, title, content)
|
||||
SELECT
|
||||
u.id,
|
||||
'Post #' || p.post_no || ' by user ' || u.id,
|
||||
repeat(
|
||||
'Lorem ipsum dolor sit amet, consectetur adipiscing elit. '
|
||||
|| 'Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. ',
|
||||
40
|
||||
)
|
||||
FROM users u
|
||||
JOIN generate_series(1, 30) AS p(post_no)
|
||||
ON u.id <= 300000;
|
||||
|
||||
-- ============================================================
|
||||
-- OPTIONAL: FORCE DISK MATERIALIZATION
|
||||
-- ============================================================
|
||||
|
||||
VACUUM ANALYZE;
|
||||
@@ -0,0 +1,36 @@
|
||||
-- MariaDB seed file for database "mariadb"
|
||||
|
||||
-- Drop tables if they exist
|
||||
DROP TABLE IF EXISTS users;
|
||||
DROP TABLE IF EXISTS products;
|
||||
|
||||
-- Create a users table
|
||||
CREATE TABLE users (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
username VARCHAR(50) NOT NULL UNIQUE,
|
||||
email VARCHAR(100) NOT NULL UNIQUE,
|
||||
password VARCHAR(255) NOT NULL,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Insert sample users
|
||||
INSERT INTO users (username, email, password) VALUES
|
||||
('alice', 'alice@example.com', 'changeme'),
|
||||
('bob', 'bob@example.com', 'changeme');
|
||||
|
||||
-- Create a products table
|
||||
CREATE TABLE products (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
name VARCHAR(100) NOT NULL,
|
||||
description TEXT,
|
||||
price DECIMAL(10,2) NOT NULL,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Insert sample products
|
||||
INSERT INTO products (name, description, price) VALUES
|
||||
('Laptop', 'High performance laptop', 1299.99),
|
||||
('Phone', 'Smartphone with OLED display', 799.99),
|
||||
('Headphones', 'Noise-cancelling headphones', 199.99);
|
||||
|
||||
-- Done
|
||||
@@ -0,0 +1,32 @@
|
||||
-- Drop tables if they exist
|
||||
DROP TABLE IF EXISTS posts;
|
||||
DROP TABLE IF EXISTS users;
|
||||
|
||||
-- Create users table
|
||||
CREATE TABLE users (
|
||||
id SERIAL PRIMARY KEY,
|
||||
username VARCHAR(50) UNIQUE NOT NULL,
|
||||
email VARCHAR(100) UNIQUE NOT NULL,
|
||||
password_hash VARCHAR(255) NOT NULL,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Create posts table
|
||||
CREATE TABLE posts (
|
||||
id SERIAL PRIMARY KEY,
|
||||
user_id INT REFERENCES users(id) ON DELETE CASCADE,
|
||||
title VARCHAR(255) NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Insert sample users
|
||||
INSERT INTO users (username, email, password_hash) VALUES
|
||||
('alice', 'alice@example.com', 'hashedpassword1'),
|
||||
('bob', 'bob@example.com', 'hashedpassword2');
|
||||
|
||||
-- Insert sample posts
|
||||
INSERT INTO posts (user_id, title, content) VALUES
|
||||
(1, 'Hello World', 'This is the first post by Alice.'),
|
||||
(2, 'Greetings', 'This is a post by Bob.'),
|
||||
(1, 'Another Post', 'Alice writes again.');
|
||||
+8
-5
@@ -8,6 +8,7 @@ use crate::services::status::StatusService;
|
||||
use crate::utils::common::BackupMethod;
|
||||
use std::sync::Arc;
|
||||
use tracing::info;
|
||||
use crate::services::restore::RestoreService;
|
||||
|
||||
pub struct Agent {
|
||||
ctx: Arc<Context>,
|
||||
@@ -15,7 +16,7 @@ pub struct Agent {
|
||||
status_service: StatusService,
|
||||
cron_service: CronService,
|
||||
backup_service: BackupService,
|
||||
// restore_service: RestoreService,
|
||||
restore_service: RestoreService,
|
||||
}
|
||||
|
||||
impl Agent {
|
||||
@@ -23,10 +24,9 @@ impl Agent {
|
||||
let config_service = ConfigService::new(ctx.clone());
|
||||
let status_service = StatusService::new(ctx.clone());
|
||||
let cron_service = CronService::new(ctx.clone()).await;
|
||||
// let backup_service = BackupService::new(ctx.clone()).await;
|
||||
|
||||
let backup_service = BackupService::new(ctx.clone());
|
||||
// let restore_service = RestoreService::new(&ctx);
|
||||
let restore_service = RestoreService::new(ctx.clone());
|
||||
|
||||
Agent {
|
||||
ctx,
|
||||
@@ -34,7 +34,7 @@ impl Agent {
|
||||
status_service,
|
||||
cron_service,
|
||||
backup_service,
|
||||
// restore_service,
|
||||
restore_service,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,7 +55,10 @@ impl Agent {
|
||||
.dispatch(&db.generated_id, &config, method.clone())
|
||||
.await;
|
||||
} else if db.data.restore.action {
|
||||
// handle restore
|
||||
let _ = self
|
||||
.restore_service
|
||||
.dispatch(db, &config)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+18
-3
@@ -5,6 +5,7 @@ use crate::services::config::DatabaseConfig;
|
||||
use anyhow::Result;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use crate::domain::mysql::MySQLDatabase;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait Database: Send + Sync {
|
||||
@@ -18,10 +19,24 @@ pub trait Database: Send + Sync {
|
||||
pub struct DatabaseFactory;
|
||||
|
||||
impl DatabaseFactory {
|
||||
pub fn create(cfg: DatabaseConfig) -> Arc<dyn Database> {
|
||||
pub async fn create_for_backup(cfg: DatabaseConfig) -> Arc<dyn Database> {
|
||||
match cfg.db_type.as_str() {
|
||||
"postgresql" => Arc::new(PostgresDatabase::new(cfg)),
|
||||
// "mysql" => Arc::new(MySQLDatabase::new(cfg)),
|
||||
"postgresql" => {
|
||||
let format = PostgresDatabase::detect_format_from_size(&cfg).await;
|
||||
Arc::new(PostgresDatabase::new(cfg, format))
|
||||
}
|
||||
"mysql" => Arc::new(MySQLDatabase::new(cfg)),
|
||||
_ => panic!("Unsupported DB type: {}", cfg.db_type),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn create_for_restore(cfg: DatabaseConfig, restore_file: &Path) -> Arc<dyn Database> {
|
||||
match cfg.db_type.as_str() {
|
||||
"postgresql" => {
|
||||
let format = PostgresDatabase::detect_format_from_file(restore_file);
|
||||
Arc::new(PostgresDatabase::new(cfg, format))
|
||||
}
|
||||
"mysql" => Arc::new(MySQLDatabase::new(cfg)),
|
||||
_ => panic!("Unsupported DB type: {}", cfg.db_type),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,2 +1,3 @@
|
||||
pub mod factory;
|
||||
pub mod postgres;
|
||||
pub mod mysql;
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
use crate::domain::factory::Database;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::collections::HashMap;
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
|
||||
pub struct MySQLDatabase {
|
||||
cfg: DatabaseConfig,
|
||||
}
|
||||
|
||||
impl MySQLDatabase {
|
||||
pub fn new(cfg: DatabaseConfig) -> Self {
|
||||
Self { cfg }
|
||||
}
|
||||
|
||||
fn build_env(&self) -> HashMap<String, String> {
|
||||
let mut envs = std::env::vars().collect::<HashMap<_, _>>();
|
||||
envs.insert("MYSQL_PWD".to_string(), self.cfg.password.clone());
|
||||
envs
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Database for MySQLDatabase {
|
||||
fn file_extension(&self) -> &'static str {
|
||||
".sql"
|
||||
}
|
||||
|
||||
async fn ping(&self) -> Result<bool> {
|
||||
let output = Command::new("mysqladmin")
|
||||
.arg("--host")
|
||||
.arg(&self.cfg.host)
|
||||
.arg("--port")
|
||||
.arg(self.cfg.port.to_string())
|
||||
.arg("--user")
|
||||
.arg(&self.cfg.username)
|
||||
.arg("ping")
|
||||
.envs(self.build_env())
|
||||
.output()
|
||||
.with_context(|| format!("Failed to ping MySQL server {}", self.cfg.name))?;
|
||||
|
||||
Ok(output.status.success())
|
||||
}
|
||||
|
||||
async fn backup(&self, backup_dir: &Path) -> Result<PathBuf> {
|
||||
let file_path = backup_dir.join(format!(
|
||||
"{}{}",
|
||||
self.cfg.generated_id,
|
||||
self.file_extension()
|
||||
));
|
||||
|
||||
let output = Command::new("mysqldump")
|
||||
.arg("--host")
|
||||
.arg(&self.cfg.host)
|
||||
.arg("--port")
|
||||
.arg(self.cfg.port.to_string())
|
||||
.arg("--user")
|
||||
.arg(&self.cfg.username)
|
||||
.arg("--routines")
|
||||
.arg("--events")
|
||||
.arg("--triggers")
|
||||
.arg("--verbose")
|
||||
.arg("--single-transaction")
|
||||
.arg("--quick")
|
||||
.arg("--add-drop-database")
|
||||
.arg("--databases")
|
||||
.arg(&self.cfg.database)
|
||||
.arg("-r")
|
||||
.arg(&file_path)
|
||||
.envs(self.build_env())
|
||||
.output()
|
||||
.with_context(|| format!("Failed to run mysqldump for {}", self.cfg.name))?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
anyhow::bail!("MySQL backup failed for {}: {}", self.cfg.name, stderr);
|
||||
}
|
||||
|
||||
Ok(file_path)
|
||||
}
|
||||
|
||||
async fn restore(&self, restore_file: &Path) -> Result<()> {
|
||||
let sql_content = tokio::fs::read_to_string(restore_file)
|
||||
.await
|
||||
.with_context(|| format!("Failed to read restore file {}", restore_file.display()))?;
|
||||
|
||||
let drop_create_cmd = format!(
|
||||
"DROP DATABASE IF EXISTS {0}; CREATE DATABASE {0};",
|
||||
self.cfg.database
|
||||
);
|
||||
|
||||
let drop_status = Command::new("mysql")
|
||||
.arg("--host")
|
||||
.arg(&self.cfg.host)
|
||||
.arg("--port")
|
||||
.arg(self.cfg.port.to_string())
|
||||
.arg("--user")
|
||||
.arg(&self.cfg.username)
|
||||
.arg("-e")
|
||||
.arg(&drop_create_cmd)
|
||||
.env("MYSQL_PWD", &self.cfg.password)
|
||||
.status()
|
||||
.with_context(|| format!("Failed to drop/recreate database {}", self.cfg.name))?;
|
||||
|
||||
if !drop_status.success() {
|
||||
anyhow::bail!("Failed to drop/recreate database {}", self.cfg.name);
|
||||
}
|
||||
|
||||
let mut child = Command::new("mysql")
|
||||
.arg("--host")
|
||||
.arg(&self.cfg.host)
|
||||
.arg("--port")
|
||||
.arg(self.cfg.port.to_string())
|
||||
.arg("--user")
|
||||
.arg(&self.cfg.username)
|
||||
.arg(&self.cfg.database)
|
||||
.env("MYSQL_PWD", &self.cfg.password)
|
||||
.stdin(std::process::Stdio::piped())
|
||||
.spawn()
|
||||
.with_context(|| format!("Failed to start mysql restore for {}", self.cfg.name))?;
|
||||
|
||||
let mut stdin = child.stdin.take().context("Failed to open child stdin")?;
|
||||
stdin.write_all(sql_content.as_bytes())?;
|
||||
stdin.flush()?;
|
||||
drop(stdin);
|
||||
|
||||
let output = child
|
||||
.wait_with_output()
|
||||
.with_context(|| format!("Failed to complete mysql restore for {}", self.cfg.name))?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
anyhow::bail!("MySQL restore failed for {}: {}", self.cfg.name, stderr);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
+283
-50
@@ -1,58 +1,258 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
// use std::path::{Path, PathBuf};
|
||||
// use std::process::Command;
|
||||
// use crate::domain::factory::Database;
|
||||
// use crate::services::config::DatabaseConfig;
|
||||
//
|
||||
// pub struct PostgresDatabase {
|
||||
// cfg: DatabaseConfig,
|
||||
// }
|
||||
//
|
||||
// impl PostgresDatabase {
|
||||
// pub fn new(cfg: DatabaseConfig) -> Self {
|
||||
// Self { cfg }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// #[async_trait::async_trait]
|
||||
// impl Database for PostgresDatabase {
|
||||
// fn file_extension(&self) -> &'static str {
|
||||
// ".dump"
|
||||
// }
|
||||
//
|
||||
// async fn ping(&self) -> anyhow::Result<bool> {
|
||||
// let url = format!(
|
||||
// "postgresql://{}:{}@{}:{}/{}",
|
||||
// self.cfg.username, self.cfg.password, self.cfg.host, self.cfg.port, self.cfg.database
|
||||
// );
|
||||
// let status = Command::new("pg_isready").arg("--dbname").arg(&url).status()?;
|
||||
// Ok(status.success())
|
||||
// }
|
||||
//
|
||||
// async fn backup(&self, backup_dir: &Path) -> anyhow::Result<PathBuf> {
|
||||
// let file_path = backup_dir.join(format!("{}{}", self.cfg.generated_id, self.file_extension()));
|
||||
// let url = format!(
|
||||
// "postgresql://{}:{}@{}:{}/{}",
|
||||
// self.cfg.username, self.cfg.password, self.cfg.host, self.cfg.port, self.cfg.database
|
||||
// );
|
||||
//
|
||||
// let status = Command::new("pg_dump")
|
||||
// .arg("--dbname")
|
||||
// .arg(url)
|
||||
// .arg("-Fc")
|
||||
// .arg("-f")
|
||||
// .arg(&file_path)
|
||||
// .arg("-v")
|
||||
// .arg("--compress=3")
|
||||
// .status()?;
|
||||
//
|
||||
// if !status.success() {
|
||||
// anyhow::bail!("Postgres backup failed for {}", self.cfg.name);
|
||||
// }
|
||||
//
|
||||
// Ok(file_path)
|
||||
// }
|
||||
//
|
||||
// async fn restore(&self, restore_file: &Path) -> anyhow::Result<()> {
|
||||
// let url = format!(
|
||||
// "postgresql://{}:{}@{}:{}/{}",
|
||||
// self.cfg.username, self.cfg.password, self.cfg.host, self.cfg.port, "postgres"
|
||||
// );
|
||||
//
|
||||
// // Terminate connections
|
||||
// let terminate_cmd = format!(
|
||||
// "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname='{}' AND pid<>pg_backend_pid();",
|
||||
// self.cfg.database
|
||||
// );
|
||||
// Command::new("psql")
|
||||
// .arg("-U").arg(&self.cfg.username)
|
||||
// .arg("-d").arg("postgres")
|
||||
// .arg("-h").arg(&self.cfg.host)
|
||||
// .arg("-p").arg(self.cfg.port.to_string())
|
||||
// .arg("-c").arg(terminate_cmd)
|
||||
// .env("PGPASSWORD", &self.cfg.password)
|
||||
// .status()?;
|
||||
//
|
||||
// // Restore
|
||||
// let status = Command::new("pg_restore")
|
||||
// .arg("--no-owner")
|
||||
// .arg("--no-privileges")
|
||||
// .arg("--clean")
|
||||
// .arg("--if-exists")
|
||||
// .arg("--create")
|
||||
// .arg("--dbname").arg(url)
|
||||
// .arg("-v")
|
||||
// .arg(restore_file)
|
||||
// .env("PGPASSWORD", &self.cfg.password)
|
||||
// .status()?;
|
||||
//
|
||||
// if !status.success() {
|
||||
// anyhow::bail!("Postgres restore failed for {}", self.cfg.name);
|
||||
// }
|
||||
//
|
||||
// Ok(())
|
||||
// }
|
||||
// }
|
||||
//
|
||||
#![allow(dead_code)]
|
||||
|
||||
use crate::domain::factory::Database;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context as AnyhowContext, Result};
|
||||
use async_trait::async_trait;
|
||||
use flate2::Compression;
|
||||
use flate2::write::GzEncoder;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
use log::info;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub enum PostgresDumpFormat {
|
||||
Fc, // legacy
|
||||
Fd, // directory format
|
||||
}
|
||||
|
||||
pub struct PostgresDatabase {
|
||||
cfg: DatabaseConfig,
|
||||
format: PostgresDumpFormat,
|
||||
}
|
||||
|
||||
impl PostgresDatabase {
|
||||
pub fn new(cfg: DatabaseConfig) -> Self {
|
||||
Self { cfg }
|
||||
pub fn new(cfg: DatabaseConfig, format: PostgresDumpFormat) -> Self {
|
||||
Self { cfg, format }
|
||||
}
|
||||
|
||||
pub fn detect_format_from_file(restore_file: &Path) -> PostgresDumpFormat {
|
||||
match restore_file.extension().and_then(|e| e.to_str()) {
|
||||
Some("dump") => PostgresDumpFormat::Fc,
|
||||
Some("gz") => PostgresDumpFormat::Fd,
|
||||
// Some("tar.gz") => PostgresDumpFormat::Fd,
|
||||
_ => PostgresDumpFormat::Fc,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn detect_format_from_size(cfg: &DatabaseConfig) -> PostgresDumpFormat {
|
||||
let url = format!(
|
||||
"host={} port={} user={} password={} dbname={}",
|
||||
cfg.host, cfg.port, cfg.username, cfg.password, cfg.database
|
||||
);
|
||||
|
||||
let output = std::process::Command::new("psql")
|
||||
.arg(&url)
|
||||
.arg("-t")
|
||||
.arg("-c")
|
||||
.arg("SELECT pg_database_size(current_database());")
|
||||
.output();
|
||||
|
||||
match output {
|
||||
Ok(out) if out.status.success() => {
|
||||
let size_bytes: i64 = String::from_utf8_lossy(&out.stdout)
|
||||
.trim()
|
||||
.parse()
|
||||
.unwrap_or(0);
|
||||
|
||||
// > 1 Go
|
||||
if size_bytes > 1_000_000_000 {
|
||||
PostgresDumpFormat::Fd
|
||||
} else {
|
||||
PostgresDumpFormat::Fc
|
||||
}
|
||||
}
|
||||
_ => PostgresDumpFormat::Fc, // fallback legacy
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
#[async_trait]
|
||||
impl Database for PostgresDatabase {
|
||||
fn file_extension(&self) -> &'static str {
|
||||
".dump"
|
||||
match self.format {
|
||||
PostgresDumpFormat::Fc => ".dump",
|
||||
PostgresDumpFormat::Fd => ".tar.gz",
|
||||
}
|
||||
}
|
||||
|
||||
async fn ping(&self) -> anyhow::Result<bool> {
|
||||
async fn ping(&self) -> Result<bool> {
|
||||
let url = format!(
|
||||
"postgresql://{}:{}@{}:{}/{}",
|
||||
self.cfg.username, self.cfg.password, self.cfg.host, self.cfg.port, self.cfg.database
|
||||
);
|
||||
let status = Command::new("pg_isready").arg("--dbname").arg(&url).status()?;
|
||||
let status = Command::new("pg_isready")
|
||||
.arg("--dbname")
|
||||
.arg(url)
|
||||
.status()
|
||||
.context("Failed to ping Postgres")?;
|
||||
Ok(status.success())
|
||||
}
|
||||
|
||||
async fn backup(&self, backup_dir: &Path) -> anyhow::Result<PathBuf> {
|
||||
let file_path = backup_dir.join(format!("{}{}", self.cfg.generated_id, self.file_extension()));
|
||||
let url = format!(
|
||||
"postgresql://{}:{}@{}:{}/{}",
|
||||
self.cfg.username, self.cfg.password, self.cfg.host, self.cfg.port, self.cfg.database
|
||||
);
|
||||
async fn backup(&self, backup_dir: &Path) -> Result<PathBuf> {
|
||||
match self.format {
|
||||
PostgresDumpFormat::Fc => {
|
||||
let file_path = backup_dir.join(format!(
|
||||
"{}{}",
|
||||
self.cfg.generated_id,
|
||||
self.file_extension()
|
||||
));
|
||||
let url = format!(
|
||||
"postgresql://{}:{}@{}:{}/{}",
|
||||
self.cfg.username,
|
||||
self.cfg.password,
|
||||
self.cfg.host,
|
||||
self.cfg.port,
|
||||
self.cfg.database
|
||||
);
|
||||
let status = Command::new("pg_dump")
|
||||
.arg("--dbname")
|
||||
.arg(url)
|
||||
.arg("-Fc")
|
||||
.arg("-f")
|
||||
.arg(&file_path)
|
||||
.arg("-v")
|
||||
.arg("--compress=3")
|
||||
.status()?;
|
||||
if !status.success() {
|
||||
anyhow::bail!("Postgres backup failed for {}", self.cfg.name);
|
||||
}
|
||||
Ok(file_path)
|
||||
}
|
||||
PostgresDumpFormat::Fd => {
|
||||
// directory dump -> tar.gz
|
||||
let dump_dir = backup_dir.join(format!("{}_dir", self.cfg.generated_id));
|
||||
let tar_file = backup_dir.join(format!("{}.tar.gz", self.cfg.generated_id));
|
||||
std::fs::create_dir_all(&dump_dir)?;
|
||||
let url = format!(
|
||||
"postgresql://{}:{}@{}:{}/{}",
|
||||
self.cfg.username,
|
||||
self.cfg.password,
|
||||
self.cfg.host,
|
||||
self.cfg.port,
|
||||
self.cfg.database
|
||||
);
|
||||
let status = Command::new("pg_dump")
|
||||
.arg("--dbname")
|
||||
.arg(url)
|
||||
.arg("-Fd")
|
||||
.arg("-j")
|
||||
.arg("4")
|
||||
.arg("-f")
|
||||
.arg(&dump_dir)
|
||||
.arg("-v")
|
||||
.status()?;
|
||||
if !status.success() {
|
||||
anyhow::bail!("Postgres Fd backup failed for {}", self.cfg.name);
|
||||
}
|
||||
|
||||
let status = Command::new("pg_dump")
|
||||
.arg("--dbname")
|
||||
.arg(url)
|
||||
.arg("-Fc")
|
||||
.arg("-f")
|
||||
.arg(&file_path)
|
||||
.arg("-v")
|
||||
.arg("--compress=3")
|
||||
.status()?;
|
||||
|
||||
if !status.success() {
|
||||
anyhow::bail!("Postgres backup failed for {}", self.cfg.name);
|
||||
// Compression tar.gz
|
||||
let tar_gz = std::fs::File::create(&tar_file)?;
|
||||
let enc = GzEncoder::new(tar_gz, Compression::default());
|
||||
let mut tar = tar::Builder::new(enc);
|
||||
tar.append_dir_all(".", &dump_dir)?;
|
||||
tar.finish()?;
|
||||
Ok(tar_file)
|
||||
}
|
||||
}
|
||||
|
||||
Ok(file_path)
|
||||
}
|
||||
|
||||
async fn restore(&self, restore_file: &Path) -> anyhow::Result<()> {
|
||||
async fn restore(&self, restore_file: &Path) -> Result<()> {
|
||||
let url = format!(
|
||||
"postgresql://{}:{}@{}:{}/{}",
|
||||
self.cfg.username, self.cfg.password, self.cfg.host, self.cfg.port, "postgres"
|
||||
@@ -64,32 +264,65 @@ impl Database for PostgresDatabase {
|
||||
self.cfg.database
|
||||
);
|
||||
Command::new("psql")
|
||||
.arg("-U").arg(&self.cfg.username)
|
||||
.arg("-d").arg("postgres")
|
||||
.arg("-h").arg(&self.cfg.host)
|
||||
.arg("-p").arg(self.cfg.port.to_string())
|
||||
.arg("-c").arg(terminate_cmd)
|
||||
.arg("-U")
|
||||
.arg(&self.cfg.username)
|
||||
.arg("-d")
|
||||
.arg("postgres")
|
||||
.arg("-h")
|
||||
.arg(&self.cfg.host)
|
||||
.arg("-p")
|
||||
.arg(self.cfg.port.to_string())
|
||||
.arg("-c")
|
||||
.arg(&terminate_cmd)
|
||||
.env("PGPASSWORD", &self.cfg.password)
|
||||
.status()?;
|
||||
|
||||
// Restore
|
||||
let status = Command::new("pg_restore")
|
||||
.arg("--no-owner")
|
||||
.arg("--no-privileges")
|
||||
.arg("--clean")
|
||||
.arg("--if-exists")
|
||||
.arg("--create")
|
||||
.arg("--dbname").arg(url)
|
||||
.arg("-v")
|
||||
.arg(restore_file)
|
||||
.env("PGPASSWORD", &self.cfg.password)
|
||||
.status()?;
|
||||
match self.format {
|
||||
PostgresDumpFormat::Fc => {
|
||||
let status = Command::new("pg_restore")
|
||||
.arg("--no-owner")
|
||||
.arg("--no-privileges")
|
||||
.arg("--clean")
|
||||
.arg("--if-exists")
|
||||
.arg("--create")
|
||||
.arg("--dbname")
|
||||
.arg(url)
|
||||
.arg("-v")
|
||||
.arg(restore_file)
|
||||
.env("PGPASSWORD", &self.cfg.password)
|
||||
.status()?;
|
||||
if !status.success() {
|
||||
anyhow::bail!("Postgres restore failed for {}", self.cfg.name);
|
||||
}
|
||||
}
|
||||
PostgresDumpFormat::Fd => {
|
||||
let tar_gz = std::fs::File::open(restore_file)?;
|
||||
let dec = flate2::read::GzDecoder::new(tar_gz);
|
||||
let mut archive = tar::Archive::new(dec);
|
||||
let tmp_dir = tempfile::TempDir::new()?;
|
||||
archive.unpack(tmp_dir.path())?;
|
||||
|
||||
if !status.success() {
|
||||
anyhow::bail!("Postgres restore failed for {}", self.cfg.name);
|
||||
|
||||
let dump_dir = tmp_dir.path();
|
||||
|
||||
info!("Restoring dump from {}", dump_dir.display());
|
||||
let status = Command::new("pg_restore")
|
||||
.arg("--no-owner")
|
||||
.arg("--no-privileges")
|
||||
.arg("--clean")
|
||||
.arg("--if-exists")
|
||||
.arg("--create")
|
||||
.arg("--dbname")
|
||||
.arg(url)
|
||||
.arg("-v")
|
||||
.arg(dump_dir)
|
||||
.env("PGPASSWORD", &self.cfg.password)
|
||||
.status()?;
|
||||
if !status.success() {
|
||||
anyhow::bail!("Postgres Fd restore failed for {}", self.cfg.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
mod settings;
|
||||
mod tasks;
|
||||
mod utils;
|
||||
mod core;
|
||||
mod services;
|
||||
mod domain;
|
||||
|
||||
use tracing_subscriber;
|
||||
use utils::redis_client;
|
||||
use utils::task_manager::scheduler;
|
||||
use crate::tasks::ping::ping_server;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
tracing_subscriber::fmt::init();
|
||||
|
||||
tokio::join!(
|
||||
ping_server(),
|
||||
async {
|
||||
let conn = redis_client::redis_connection().await;
|
||||
scheduler::scheduler_loop(conn).await;
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -4,6 +4,7 @@ use crate::core::context::Context;
|
||||
use crate::domain::factory::DatabaseFactory;
|
||||
use crate::services::config::{DatabaseConfig, DatabasesConfig};
|
||||
use crate::utils::common::BackupMethod;
|
||||
use crate::utils::file::full_extension;
|
||||
use anyhow::Result;
|
||||
use hex;
|
||||
use log::{error, info};
|
||||
@@ -71,9 +72,8 @@ impl BackupService {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, tmp_path: &Path) -> Result<BackupResult> {
|
||||
let db_instance = DatabaseFactory::create(cfg.clone());
|
||||
let db_instance = DatabaseFactory::create_for_backup(cfg.clone()).await;
|
||||
let generated_id = cfg.generated_id.clone();
|
||||
let db_type = cfg.db_type.clone();
|
||||
|
||||
@@ -156,6 +156,8 @@ impl BackupService {
|
||||
let encrypted_len = encrypter.encrypt(&aes_key, &mut encrypted_key).unwrap();
|
||||
encrypted_key.truncate(encrypted_len);
|
||||
|
||||
let extension = full_extension(&file_path);
|
||||
|
||||
// Attach file and AES info to multipart form
|
||||
form = form
|
||||
.part(
|
||||
@@ -164,7 +166,8 @@ impl BackupService {
|
||||
.file_name(format!("{}.enc", result.generated_id)),
|
||||
)
|
||||
.text("aes_key", hex::encode(encrypted_key))
|
||||
.text("iv", hex::encode(iv));
|
||||
.text("iv", hex::encode(iv))
|
||||
.text("extension", extension);
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to read backup file: {}", e);
|
||||
|
||||
+2
-1
@@ -1,4 +1,5 @@
|
||||
pub mod config;
|
||||
pub mod status;
|
||||
pub mod cron;
|
||||
pub mod backup;
|
||||
pub mod backup;
|
||||
pub mod restore;
|
||||
@@ -0,0 +1,159 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use crate::core::context::Context;
|
||||
use crate::domain::factory::DatabaseFactory;
|
||||
use crate::services::config::{DatabaseConfig, DatabasesConfig};
|
||||
use crate::services::status::DatabaseStatus;
|
||||
use anyhow::Result;
|
||||
use log::{error, info};
|
||||
use serde::Serialize;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct RestoreResult {
|
||||
#[serde(rename = "generatedId")]
|
||||
pub generated_id: String,
|
||||
pub status: String,
|
||||
}
|
||||
|
||||
pub struct RestoreService {
|
||||
ctx: Arc<Context>,
|
||||
}
|
||||
|
||||
impl RestoreService {
|
||||
pub fn new(ctx: Arc<Context>) -> Self {
|
||||
Self { ctx }
|
||||
}
|
||||
|
||||
pub async fn dispatch(&self, db: &DatabaseStatus, config: &DatabasesConfig) {
|
||||
if let Some(cfg) = config
|
||||
.databases
|
||||
.iter()
|
||||
.find(|c| c.generated_id == db.generated_id)
|
||||
{
|
||||
let db_cfg = cfg.clone();
|
||||
let ctx_clone = self.ctx.clone();
|
||||
let file_to_restore = db.data.restore.file.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
match TempDir::new() {
|
||||
Ok(temp_dir) => {
|
||||
let tmp_path = temp_dir.path().to_path_buf();
|
||||
info!("Created temp directory {}", tmp_path.display());
|
||||
|
||||
match RestoreService::run(db_cfg, &tmp_path, &file_to_restore).await {
|
||||
Ok(result) => {
|
||||
let service = RestoreService { ctx: ctx_clone };
|
||||
service.send_result(result).await;
|
||||
}
|
||||
Err(e) => error!("Restoration error {}", e),
|
||||
}
|
||||
// TempDir is automatically deleted when dropped here
|
||||
}
|
||||
Err(e) => error!("Failed to create temp dir: {}", e),
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
tmp_path: &Path,
|
||||
file_url: &str,
|
||||
) -> Result<RestoreResult> {
|
||||
let generated_id = cfg.generated_id.clone();
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let response = client.get(file_url).send().await?;
|
||||
if !response.status().is_success() {
|
||||
error!("Backup download failed with status {}", response.status());
|
||||
return Ok(RestoreResult {
|
||||
generated_id,
|
||||
status: "failed".into(),
|
||||
});
|
||||
}
|
||||
|
||||
let bytes = response.bytes().await?;
|
||||
|
||||
let ext = if bytes.starts_with(b"PGDMP") {
|
||||
// Postgres custom format
|
||||
"dump"
|
||||
} else if bytes.starts_with(&[0x1F, 0x8B]) {
|
||||
// gzip compressed -> could be Postgres directory dump or MySQL gzipped SQL
|
||||
"tar.gz"
|
||||
} else if bytes.starts_with(b"--") || bytes.starts_with(b"/*") {
|
||||
// Plain MySQL SQL dump
|
||||
"sql"
|
||||
} else {
|
||||
// Fallback generic
|
||||
"dump"
|
||||
};
|
||||
|
||||
info!("Backup dump from {} to {}", tmp_path.display(), ext);
|
||||
|
||||
let backup_file_path = tmp_path.join(format!("backup_file_tmp.{}", ext));
|
||||
tokio::fs::write(&backup_file_path, &bytes).await?;
|
||||
info!("Backup downloaded to {}", backup_file_path.display());
|
||||
|
||||
let db_instance = DatabaseFactory::create_for_restore(cfg.clone(), &backup_file_path).await;
|
||||
let reachable = db_instance.ping().await.unwrap_or(false);
|
||||
if !reachable {
|
||||
return Ok(RestoreResult {
|
||||
generated_id,
|
||||
status: "failed".into(),
|
||||
});
|
||||
}
|
||||
|
||||
match db_instance.restore(&backup_file_path).await {
|
||||
Ok(_) => Ok(RestoreResult {
|
||||
generated_id,
|
||||
status: "success".into(),
|
||||
}),
|
||||
Err(e) => {
|
||||
log::error!("Restore failed: {:?}", e);
|
||||
Ok(RestoreResult {
|
||||
generated_id,
|
||||
status: "failed".into(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn send_result(&self, result: RestoreResult) {
|
||||
info!(
|
||||
"[RestoreService] DB: {} | Status: {}",
|
||||
result.generated_id, result.status,
|
||||
);
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let url = format!(
|
||||
"{}/api/agent/{}/restore",
|
||||
self.ctx.edge_key.server_url, self.ctx.edge_key.agent_id
|
||||
);
|
||||
|
||||
let body = RestoreResult {
|
||||
generated_id: result.generated_id,
|
||||
status: result.status,
|
||||
};
|
||||
|
||||
match client.post(&url).json(&body).send().await {
|
||||
Ok(resp) => {
|
||||
let status = resp.status();
|
||||
if status.is_success() {
|
||||
info!("Restoration result sent successfully");
|
||||
} else {
|
||||
let text = resp.text().await.unwrap_or_default(); // consumes resp
|
||||
error!(
|
||||
"Restoration result failed, status: {}, body: {}",
|
||||
status, text
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to send restoration result: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -93,8 +93,10 @@ impl StatusService {
|
||||
})
|
||||
.collect();
|
||||
|
||||
let version_str = format!("{}-rust", CONFIG.app_version);
|
||||
|
||||
let body = StatusRequestBody {
|
||||
version: CONFIG.app_version.as_str(),
|
||||
version: &version_str,
|
||||
databases: databases_payload,
|
||||
};
|
||||
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ impl Settings {
|
||||
dotenv().ok();
|
||||
|
||||
Self {
|
||||
app_version: env::var("APP_VERSION").unwrap_or_else(|_| "unknown".into()),
|
||||
app_version: env::var("CARGO_PKG_VERSION").unwrap_or_else(|_| "unknown".into()),
|
||||
app_env: env::var("APP_ENV").unwrap_or_else(|_| "development".into()),
|
||||
redis_url: env::var("CELERY_BROKER_URL")
|
||||
.unwrap_or_else(|_| "redis://localhost:6379/".into()),
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
use std::path::Path;
|
||||
|
||||
pub fn full_extension(path: &Path) -> String {
|
||||
path.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.map(|name| {
|
||||
match name.find('.') {
|
||||
Some(idx) => &name[idx..],
|
||||
None => "",
|
||||
}
|
||||
})
|
||||
.unwrap_or_default()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
@@ -3,3 +3,4 @@ pub mod edge_key;
|
||||
pub mod redis_client;
|
||||
pub mod task_manager;
|
||||
pub mod text;
|
||||
pub mod file;
|
||||
Reference in New Issue
Block a user