fix: tests (#47)

* fix: s3 port type mismatch, add string_or_number_to_string deserializer

* fix: adding some tests

* fix: adding some tests

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
This commit is contained in:
Charles GTE
2026-03-27 22:24:59 +01:00
committed by GitHub
parent 8e8ebb5920
commit 725312e1d5
127 changed files with 1157 additions and 622 deletions
+145 -31
View File
@@ -1,3 +1,86 @@
#name: Codecov Rust
#
#on:
# push:
# branches: ["main"]
# pull_request:
# branches: ["main"]
#
#env:
# CARGO_TERM_COLOR: always
#
#jobs:
# coverage:
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v4
#
# - name: Install Rust toolchain
# uses: dtolnay/rust-toolchain@stable
# with:
# components: llvm-tools-preview
#
# - name: Build test image (with grcov included)
# run: docker compose -f docker-compose.test.yml build agent-test
#
# - name: Run tests in container
# env:
# CARGO_TARGET_DIR: /app/target
# CARGO_INCREMENTAL: 0
# RUSTFLAGS: "-C instrument-coverage -C link-dead-code"
# LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw"
# run: |
# docker compose -f docker-compose.test.yml run \
# -e CARGO_TARGET_DIR \
# -e CARGO_INCREMENTAL \
# -e RUSTFLAGS \
# -e LLVM_PROFILE_FILE \
# agent-test bash -c "cargo clean && cargo test --verbose && sync"
#
# - name: Verify profraw files exist
# run: |
# docker compose -f docker-compose.test.yml run agent-test \
# find /app/coverage -type f -name "*.profraw" | wc -l || true
#
# - name: Generate coverage report inside container
# run: |
# docker compose -f docker-compose.test.yml run agent-test bash -c "
# rustup component add llvm-tools &&
# grcov /app/coverage \
# --binary-path /app/target/debug \
# -s /app \
# --llvm \
# -t lcov \
# --branch \
# --ignore-not-existing \
# --ignore '/app/target/*' \
# --ignore '/*' \
# -o /app/lcov.info
# "
#
# - name: Copy lcov.info from container to host
# run: |
# docker compose -f docker-compose.test.yml cp agent-test:/app/lcov.info ./lcov.info
#
# - name: Remove container
# run: |
# docker rm agent-test-run
#
# - name: Show basic coverage report info (debug)
# run: |
# echo "lcov.info size:" $(wc -c ./lcov.info | awk '{print $1}')
# head -n 30 ./lcov.info || true
#
# - name: Upload coverage to Codecov
# uses: codecov/codecov-action@v5
# with:
# files: ./lcov.info
# flags: unittests
# name: rust-unit-coverage
# verbose: true
# fail_ci_if_error: true
# env:
# CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
name: Codecov Rust
on:
@@ -8,51 +91,82 @@ on:
env:
CARGO_TERM_COLOR: always
CARGO_TARGET_DIR: /app/target
CARGO_INCREMENTAL: 0
RUSTFLAGS: "-C instrument-coverage -C link-dead-code"
LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw"
jobs:
build:
coverage:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: actions-rs/toolchain@v1
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: stable
override: true
components: llvm-tools-preview
- name: Install grcov
run: cargo install grcov
- name: Build test image (with grcov included)
run: docker compose -f docker-compose.test.yml build agent-test
- name: Install test requirements
run: bash scripts/tests/requirements.sh
- name: Start agent-test container
run: docker compose -f docker-compose.test.yml up -d agent-test
- name: Build
run: cargo build --verbose
- name: Run tests
env:
CARGO_INCREMENTAL: 0
RUSTFLAGS: "-C instrument-coverage"
LLVM_PROFILE_FILE: "cargo-test-%p-%m.profraw"
run: cargo test --verbose
- name: Generate coverage
- name: Run tests inside container
run: |
grcov . \
--binary-path ./target/debug/ \
-s . \
-t lcov \
--branch \
--ignore-not-existing \
-o lcov.info
docker compose -f docker-compose.test.yml exec \
-e CARGO_TARGET_DIR \
-e CARGO_INCREMENTAL \
-e RUSTFLAGS \
-e LLVM_PROFILE_FILE \
agent-test bash -c "
cargo clean &&
cargo test --verbose &&
sync
"
- name: Upload to Codecov
- name: Verify profraw files exist
run: |
docker compose -f docker-compose.test.yml exec \
-e LLVM_PROFILE_FILE \
agent-test find /app/coverage -type f -name "*.profraw" | wc -l || true
- name: Generate coverage report inside container
run: |
docker compose -f docker-compose.test.yml exec \
agent-test bash -c "
rustup component add llvm-tools &&
grcov /app/coverage \
--binary-path /app/target/debug \
-s /app \
--llvm \
-t lcov \
--branch \
--ignore-not-existing \
--ignore '/app/target/*' \
--ignore '/*' \
-o /app/lcov.info
"
- name: Copy lcov.info from container to host
run: docker compose -f docker-compose.test.yml cp agent-test:/app/lcov.info ./lcov.info
- name: Show basic coverage report info (debug)
run: |
echo "lcov.info size:" $(wc -c ./lcov.info | awk '{print $1}')
head -n 30 ./lcov.info || true
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v5
with:
files: lcov.info
files: ./lcov.info
flags: unittests
name: rust-unit-coverage
verbose: true
fail_ci_if_error: true
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
- name: Stop and remove container
run: docker compose -f docker-compose.test.yml down
+2 -2
View File
@@ -49,7 +49,7 @@ tokio-stream = "0.1.18"
aes = "0.9.0-rc.4"
typenum = "1.19.0"
testcontainers = "0.27.1"
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "valkey"] }
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "valkey", "mysql", "mariadb", "mongo"] }
postgres = "0.19.12"
url = "2.5.8"
@@ -57,7 +57,7 @@ url = "2.5.8"
tokio = { version = "1", features = ["full"] }
mockall = "0.13"
testcontainers = "0.27.1"
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis"] }
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "mysql", "mariadb", "mongo"] }
wiremock = "0.6"
[profile.release]
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
+22
View File
@@ -0,0 +1,22 @@
services:
agent-test:
build:
context: .
dockerfile: docker/Dockerfile
target: dev
working_dir: /app
container_name: agent-test
volumes:
- .:/app
- cargo-registry:/usr/local/cargo/registry
- cargo-git:/usr/local/cargo/git
- /var/run/docker.sock:/var/run/docker.sock
environment:
APP_ENV: test
LOG: debug
TZ: Europe/Paris
EDGE_KEY: ""
volumes:
cargo-registry:
cargo-git:
+4 -2
View File
@@ -11,14 +11,16 @@ services:
# - ./databases.toml:/config/config.toml
- cargo-registry:/usr/local/cargo/registry
- cargo-git:/usr/local/cargo/git
# - cargo-target:/app/target
- /var/run/docker.sock:/var/run/docker.sock
# - cargo-target:/app/target
# - sqlite-data:/sqlite-data/workspace/data
# - ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data
environment:
APP_ENV: development
LOG: debug
TZ: "Europe/Paris"
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiNWU1OGU2MGEtODhiMy00YTBjLWI0NDktNTQ3OWZhOTQzZDBkIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ=="
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiNGI1OTM2MGItNTNkMi00ZTZmLWE1ODctODcyMmQ1NDc1MTNmIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ=="
#POOLING: 1
#DATABASES_CONFIG_FILE: "config.toml"
extra_hosts:
+1
View File
@@ -60,6 +60,7 @@ WORKDIR /app
FROM base AS dev
RUN cargo install cargo-watch
RUN cargo install grcov --locked
COPY Cargo.toml Cargo.lock ./
RUN mkdir src && echo "fn main() {}" > src/main.rs
+4
View File
@@ -24,3 +24,7 @@ docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT * FROM users LI
```bash
docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT name FROM sqlite_master WHERE type='table';"
```
```bash
docker compose -f docker-compose.test.yml run agent-test bash -c "cargo clean && cargo test"
```
-88
View File
@@ -1,88 +0,0 @@
#!/usr/bin/env bash
set -e
POSTGRES_BASE="/usr/local/postgresql"
echo "Detecting OS and architecture..."
OS_TYPE="$(uname -s)"
ARCH="$(uname -m)"
install_pg_binaries() {
echo "Installing PostgreSQL binaries for versions 12-18..."
for v in 12 13 14 15 16 17 18; do
TARGET_DIR="$POSTGRES_BASE/$v/bin"
sudo mkdir -p "$TARGET_DIR"
if [[ "$OS_TYPE" == "Linux" ]]; then
if [[ "$ARCH" == "x86_64" ]]; then
SRC_DIR="./assets/tools/amd64/postgresql/postgresql-$v/bin"
elif [[ "$ARCH" == "aarch64" ]]; then
SRC_DIR="./assets/tools/arm64/postgresql/postgresql-$v/bin"
else
echo "Unsupported architecture: $ARCH"
continue
fi
if [[ -d "$SRC_DIR" ]]; then
echo "Copying PostgreSQL $v binaries from $SRC_DIR to $TARGET_DIR"
sudo cp -r "$SRC_DIR"/* "$TARGET_DIR/"
else
echo "Binaries for PostgreSQL $v not found for Linux, skipping..."
continue
fi
elif [[ "$OS_TYPE" == "Darwin" ]]; then
PG_SRC="$(brew --prefix postgresql@$v)/bin" 2>/dev/null || true
if [[ ! -d "$PG_SRC" ]]; then
echo "PostgreSQL $v not installed via Homebrew. Trying to install..."
if ! brew install postgresql@$v; then
echo "PostgreSQL $v not available, skipping..."
continue
fi
PG_SRC="$(brew --prefix postgresql@$v)/bin"
fi
echo "Copying PostgreSQL $v binaries from $PG_SRC to $TARGET_DIR"
sudo cp -r "$PG_SRC"/* "$TARGET_DIR/"
fi
sudo chown -R "$(whoami)" "$TARGET_DIR"
chmod +x "$TARGET_DIR"/*
done
echo "PostgreSQL binaries installed under $POSTGRES_BASE"
}
if [[ "$OS_TYPE" == "Linux" ]]; then
if command -v apt >/dev/null 2>&1; then
echo "Linux detected with apt. Installing prerequisites..."
sudo apt update
sudo apt install -y wget gnupg lsb-release redis-tools valkey
install_pg_binaries
else
echo "Unsupported Linux distribution. Only apt-based distros are supported."
exit 1
fi
elif [[ "$OS_TYPE" == "Darwin" ]]; then
if command -v brew >/dev/null 2>&1; then
echo "macOS detected. Installing prerequisites..."
brew install redis
brew install valkey
sudo mkdir -p "$POSTGRES_BASE"
sudo chown -R "$(whoami)" "$POSTGRES_BASE"
install_pg_binaries
else
echo "Homebrew not found. Please install Homebrew first: https://brew.sh/"
exit 1
fi
else
echo "Unsupported OS: $OS_TYPE"
exit 1
fi
echo "Tools installation completed successfully."
+15 -8
View File
@@ -4,11 +4,11 @@ use crate::core::context::Context;
use crate::services::backup::BackupService;
use crate::services::config::ConfigService;
use crate::services::cron::CronService;
use crate::services::restore::RestoreService;
use crate::services::status::StatusService;
use crate::utils::common::BackupMethod;
use std::sync::Arc;
use tracing::info;
use crate::services::restore::RestoreService;
pub struct Agent {
ctx: Arc<Context>,
@@ -43,23 +43,30 @@ impl Agent {
let ping_result = self.status_service.ping(&config.databases).await?;
for db in ping_result.databases.iter() {
let database = config.databases.iter().find(|cfg_db|cfg_db.generated_id == db.generated_id).unwrap();
let database = config
.databases
.iter()
.find(|cfg_db| cfg_db.generated_id == db.generated_id)
.unwrap();
info!(
"Generated Id: {} | backup action: {} | restore action: {} | Database Name: {}",
db.generated_id, db.data.backup.action, db.data.restore.action, database.name,
db.generated_id, db.data.backup.action, db.data.restore.action, database.name,
);
let _ = self.cron_service.sync(db).await;
if db.data.backup.action {
let _ = self
.backup_service
.dispatch(&db.generated_id, &config, method.clone(), &db.storages, db.encrypt)
.dispatch(
&db.generated_id,
&config,
method.clone(),
&db.storages,
db.encrypt,
)
.await;
} else if db.data.restore.action {
let _ = self
.restore_service
.dispatch(db, &config)
.await;
let _ = self.restore_service.dispatch(db, &config).await;
}
}
+1 -1
View File
@@ -35,7 +35,7 @@ impl Context {
panic!("Cannot initialize AgentContext due to invalid EDGE_KEY");
}
};
let server_url = format!("{}/api", edge_key.server_url);
let api_client = ApiClient::new(server_url);
+1 -1
View File
@@ -1,2 +1,2 @@
pub mod context;
pub mod agent;
pub mod context;
+5 -6
View File
@@ -4,18 +4,18 @@ use crate::domain::postgres::database::PostgresDatabase;
use crate::domain::postgres::{detect_format_from_file, detect_format_from_size};
use crate::domain::redis::database::RedisDatabase;
use crate::domain::sqlite::database::SqliteDatabase;
use crate::domain::valkey::database::ValkeyDatabase;
use crate::services::config::{DatabaseConfig, DbType};
use anyhow::Result;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use crate::domain::valkey::database::ValkeyDatabase;
#[async_trait::async_trait]
pub trait Database: Send + Sync {
fn file_extension(&self) -> &'static str;
async fn ping(&self) -> Result<bool>;
async fn backup(&self, backup_dir: &Path, is_test: Option<bool>) -> Result<PathBuf>;
async fn restore(&self, restore_file: &Path, is_test: Option<bool>) -> Result<()>;
async fn backup(&self, backup_dir: &Path) -> Result<PathBuf>;
async fn restore(&self, restore_file: &Path) -> Result<()>;
}
pub struct DatabaseFactory;
@@ -32,7 +32,7 @@ impl DatabaseFactory {
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
DbType::Redis => Arc::new(RedisDatabase::new(cfg)),
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg))
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)),
}
}
@@ -47,8 +47,7 @@ impl DatabaseFactory {
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
DbType::Redis => Arc::new(RedisDatabase::new(cfg)),
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg))
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)),
}
}
}
+3 -4
View File
@@ -1,8 +1,7 @@
pub mod factory;
pub mod postgres;
pub mod mysql;
mod mongodb;
mod sqlite;
pub mod mysql;
pub mod postgres;
mod redis;
mod sqlite;
mod valkey;
+1
View File
@@ -15,6 +15,7 @@ pub async fn run(
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
let mongodump = select_mongo_path().join("mongodump");
info!("{:?}", mongodump);
let uri = get_mongo_uri(cfg.clone())?;
let output = Command::new(mongodump)
+4 -4
View File
@@ -16,9 +16,11 @@ pub fn select_mongo_path() -> std::path::PathBuf {
}
pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result<String> {
if cfg.username.is_empty() || cfg.password.is_empty() {
Ok(format!("mongodb://{}:{}/{}", cfg.host, cfg.port, cfg.database))
Ok(format!(
"mongodb://{}:{}/{}",
cfg.host, cfg.port, cfg.database
))
} else {
Ok(format!(
"mongodb://{}:{}@{}:{}/{}?authSource=admin",
@@ -26,5 +28,3 @@ pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result<String> {
))
}
}
+6 -16
View File
@@ -27,27 +27,17 @@ impl Database for MongoDatabase {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
}
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, is_test: Option<bool>) -> Result<()> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
}
async fn restore(&self, file: &Path) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+2 -2
View File
@@ -1,5 +1,5 @@
mod backup;
mod restore;
mod connection;
pub mod database;
mod ping;
mod connection;
mod restore;
+1 -1
View File
@@ -24,7 +24,7 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
error!("Full Error: {}", e);
error!("Check you database network connectivity");
error!("----------------------------------------");
Err(anyhow::anyhow!("Ping failed for {}: {}", cfg.name, e))
Ok(false)
}
}
}
+1 -1
View File
@@ -1,4 +1,4 @@
use crate::domain::mysql::connection::{server_version};
use crate::domain::mysql::connection::server_version;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::collections::HashMap;
+10 -8
View File
@@ -1,14 +1,17 @@
use crate::services::config::DatabaseConfig;
use std::process::Command;
use anyhow::Result;
use std::process::Command;
pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
let output = Command::new("mysql")
.arg("--host").arg(&cfg.host)
.arg("--port").arg(cfg.port.to_string())
.arg("--user").arg(&cfg.username)
.arg("-e").arg("SELECT VERSION();")
.arg("--host")
.arg(&cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(&cfg.username)
.arg("-e")
.arg("SELECT VERSION();")
.env("MYSQL_PWD", &cfg.password)
.output()?;
@@ -19,11 +22,10 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
let version = String::from_utf8_lossy(&output.stdout)
.lines()
.nth(1) // skip column header
.nth(1)
.unwrap_or_default()
.trim()
.to_string();
Ok(version)
}
+18 -26
View File
@@ -1,14 +1,11 @@
use std::collections::HashMap;
use anyhow::Result;
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use super::{
backup,
ping, restore,
};
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
use anyhow::Result;
use async_trait::async_trait;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
pub struct MySQLDatabase {
cfg: DatabaseConfig,
@@ -36,28 +33,23 @@ impl Database for MySQLDatabase {
ping::run(self.cfg.clone(), self.build_env().clone()).await
}
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
}
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.build_env().clone(), self.file_extension()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(
self.cfg.clone(),
dir.to_path_buf(),
self.build_env().clone(),
self.file_extension(),
)
.await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, is_test: Option<bool>) -> Result<()> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
}
async fn restore(&self, file: &Path) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+2 -2
View File
@@ -1,5 +1,5 @@
pub mod backup;
mod connection;
pub mod database;
mod restore;
mod ping;
mod connection;
mod restore;
-1
View File
@@ -4,7 +4,6 @@ use tokio::process::Command;
use tokio::time::{Duration, timeout};
pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::Result<bool> {
let mut cmd = Command::new("mysqladmin");
cmd.arg("--host")
.arg(cfg.host)
+2 -3
View File
@@ -11,7 +11,6 @@ pub async fn run(
cfg: DatabaseConfig,
format: PostgresDumpFormat,
backup_dir: PathBuf,
is_test: Option<bool>
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
debug!("Starting backup for database {}", cfg.name);
@@ -27,8 +26,8 @@ pub async fn run(
}
};
let pg_dump = select_pg_path(&version, is_test).join("pg_dump");
let pg_dump = select_pg_path(&version).join("pg_dump");
debug!("Using pg_dump at {:?}", pg_dump);
match format {
+2 -7
View File
@@ -28,14 +28,9 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
Ok(version)
}
pub fn select_pg_path(version: &str, is_test: Option<bool>) -> std::path::PathBuf {
pub fn select_pg_path(version: &str) -> std::path::PathBuf {
let major = version.split('.').next().unwrap_or("17");
if is_test.unwrap_or(false) {
format!("/usr/local/postgresql/{}/bin", major).into()
} else {
format!("/usr/lib/postgresql/{}/bin", major).into()
}
format!("/usr/lib/postgresql/{}/bin", major).into()
}
pub async fn terminate_connections(cfg: &DatabaseConfig) -> Result<()> {
+8 -18
View File
@@ -31,27 +31,17 @@ impl Database for PostgresDatabase {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
}
let res = backup::run(self.cfg.clone(), self.format, dir.to_path_buf(), is_test).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), self.format, dir.to_path_buf()).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, is_test: Option<bool>) -> Result<()> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
}
let res = restore::run(self.cfg.clone(), self.format, file.to_path_buf(), is_test).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
async fn restore(&self, file: &Path) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), self.format, file.to_path_buf()).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+1 -1
View File
@@ -2,4 +2,4 @@
pub enum PostgresDumpFormat {
Fc,
Fd,
}
}
+3 -3
View File
@@ -1,8 +1,8 @@
pub mod backup;
pub mod database;
mod restore;
mod connection;
pub mod database;
mod format;
mod ping;
mod restore;
pub use connection::{detect_format_from_size, detect_format_from_file};
pub use connection::{detect_format_from_file, detect_format_from_size};
+2 -4
View File
@@ -1,8 +1,6 @@
use super::connection::connect;
use crate::services::config::DatabaseConfig;
pub async fn run(
cfg: DatabaseConfig,
) -> anyhow::Result<bool> {
pub async fn run(cfg: DatabaseConfig) -> anyhow::Result<bool> {
Ok(connect(&cfg).await.is_ok())
}
}
+1 -2
View File
@@ -11,7 +11,6 @@ pub async fn run(
cfg: DatabaseConfig,
format: PostgresDumpFormat,
restore_file: PathBuf,
is_test: Option<bool>,
) -> Result<()> {
tokio::task::spawn_blocking(move || -> Result<()> {
debug!("Starting restore for database {}", cfg.name);
@@ -27,7 +26,7 @@ pub async fn run(
}
};
let pg_restore = select_pg_path(&version, is_test).join("pg_restore");
let pg_restore = select_pg_path(&version).join("pg_restore");
debug!("Using pg_restore at {:?}", pg_restore);
+4 -9
View File
@@ -27,19 +27,14 @@ impl Database for RedisDatabase {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
}
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, _file: &Path, _is_test: Option<bool>) -> Result<()> {
async fn restore(&self, _file: &Path) -> Result<()> {
bail!("Restore not supported for Redis databases")
}
}
+4 -5
View File
@@ -1,8 +1,8 @@
use tracing::{debug, info};
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use tokio::process::Command;
use tokio::time::{timeout, Duration};
use anyhow::{Result, Context};
use tokio::time::{Duration, timeout};
use tracing::{debug, info};
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
let mut cmd = Command::new("redis-cli");
@@ -23,7 +23,6 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
debug!("Command Ping: {:?}", cmd);
let result = timeout(Duration::from_secs(10), cmd.output()).await;
match result {
@@ -52,4 +51,4 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
Ok(false)
}
}
}
}
+2 -2
View File
@@ -41,5 +41,5 @@ pub async fn run(
info!("SQLite backup completed for {}", cfg.name);
Ok(file_path)
})
.await?
}
.await?
}
+6 -16
View File
@@ -27,26 +27,16 @@ impl Database for SqliteDatabase {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
}
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, is_test: Option<bool>) -> Result<()> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
}
async fn restore(&self, file: &Path) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+2 -2
View File
@@ -1,4 +1,4 @@
mod backup;
mod restore;
pub mod database;
mod ping;
pub mod database;
mod restore;
+4 -9
View File
@@ -27,19 +27,14 @@ impl Database for ValkeyDatabase {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
}
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, _file: &Path, _is_test: Option<bool>) -> Result<()> {
async fn restore(&self, _file: &Path) -> Result<()> {
bail!("Restore not supported for Valkey databases")
}
}
+4 -5
View File
@@ -1,8 +1,8 @@
use tracing::{debug, info};
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use tokio::process::Command;
use tokio::time::{timeout, Duration};
use anyhow::{Result, Context};
use tokio::time::{Duration, timeout};
use tracing::{debug, info};
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
let mut cmd = Command::new("valkey-cli");
@@ -23,7 +23,6 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
debug!("Command Ping: {:?}", cmd);
let result = timeout(Duration::from_secs(10), cmd.output()).await;
match result {
@@ -52,4 +51,4 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
Ok(false)
}
}
}
}
+2 -3
View File
@@ -3,19 +3,18 @@ mod domain;
mod services;
mod settings;
mod tasks;
mod utils;
#[cfg(test)]
mod tests;
mod utils;
use crate::tasks::ping::ping_server;
use crate::utils::locks::FileLock;
use crate::utils::logging;
use utils::redis_client;
use utils::task_manager::scheduler;
use crate::utils::logging;
#[tokio::main]
async fn main() {
logging::init_logger();
// Remove all locks on startup
+2 -7
View File
@@ -1,9 +1,9 @@
#![allow(dead_code)]
use crate::services::api::ApiError;
use reqwest::{Client, Method};
use serde::de::DeserializeOwned;
use std::time::Duration;
use crate::services::api::ApiError;
#[derive(Clone, Debug)]
pub struct ApiClient {
@@ -57,12 +57,7 @@ impl ApiClient {
{
let url = format!("{}{}", self.base_url, path);
let res = self
.http
.request(method, &url)
.json(body)
.send()
.await?;
let res = self.http.request(method, &url).json(body).send().await?;
let status = res.status();
let body_text = res.text().await.unwrap_or_default();
@@ -23,7 +23,6 @@ pub struct BackupUpdateRequest {
pub generated_id: String,
}
impl ApiClient {
pub async fn backup_create(
&self,
@@ -1,8 +1,8 @@
use crate::services::api::models::agent::backup::BackupUploadResponse;
use crate::services::api::{ApiClient, ApiError};
use anyhow::Result;
use reqwest::Method;
use serde::Serialize;
use crate::services::api::models::agent::backup::BackupUploadResponse;
#[derive(Serialize)]
pub struct InitUploadRequest {
@@ -1,2 +1,2 @@
pub mod init;
pub mod status;
pub mod status;
+2 -2
View File
@@ -1,3 +1,3 @@
pub mod status;
pub mod backup;
pub mod restore;
pub mod restore;
pub mod status;
+3 -2
View File
@@ -31,6 +31,7 @@ impl ApiClient {
let agent_id = agent_id.into();
let path = format!("/agent/{}/status", agent_id);
self.request_with_body(Method::POST, path.as_str(), &body).await
self.request_with_body(Method::POST, path.as_str(), &body)
.await
}
}
}
+1 -1
View File
@@ -1,3 +1,3 @@
pub mod agent;
pub use agent::status;
pub use agent::status;
+1 -4
View File
@@ -11,10 +11,7 @@ pub enum ApiError {
Serialization(#[from] serde_json::Error),
#[error("api error: status={status}, body={body}")]
HttpResponse {
status: StatusCode,
body: String,
},
HttpResponse { status: StatusCode, body: String },
#[error("api returned unexpected response")]
UnexpectedResponse,
+1 -2
View File
@@ -1,8 +1,7 @@
pub mod client;
pub mod endpoints;
pub mod error;
pub mod models;
pub mod endpoints;
pub use client::ApiClient;
pub use error::ApiError;
+2 -2
View File
@@ -1,3 +1,3 @@
pub mod status;
pub mod backup;
pub mod restore;
pub mod restore;
pub mod status;
+1 -1
View File
@@ -4,4 +4,4 @@ use serde::{Deserialize, Serialize};
pub struct ResultRestoreResponse {
pub message: String,
pub status: bool,
}
}
+1 -1
View File
@@ -1,8 +1,8 @@
#![allow(dead_code)]
use crate::utils::deserializer::deserialize_snake_case;
use serde::{Deserialize, Serialize};
use toml::Value;
use crate::utils::deserializer::deserialize_snake_case;
#[derive(Debug, Deserialize)]
pub struct PingResult {
-1
View File
@@ -1,2 +1 @@
pub mod agent;
+4 -10
View File
@@ -1,20 +1,14 @@
use super::service::BackupService;
use crate::utils::compress::compress_to_tar_gz_large;
use std::path::PathBuf;
use anyhow::Result;
use std::path::PathBuf;
impl BackupService {
pub async fn compress_backup(
&self,
backup_file: Option<PathBuf>,
) -> Result<PathBuf> {
let file = backup_file
.ok_or_else(|| anyhow::anyhow!("No backup file generated"))?;
pub async fn compress_backup(&self, backup_file: Option<PathBuf>) -> Result<PathBuf> {
let file = backup_file.ok_or_else(|| anyhow::anyhow!("No backup file generated"))?;
let compression = compress_to_tar_gz_large(&file).await?;
Ok(compression.compressed_path)
}
}
}
+2 -4
View File
@@ -1,11 +1,10 @@
use super::service::BackupService;
use crate::services::config::DatabasesConfig;
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::config::DatabasesConfig;
use crate::utils::common::BackupMethod;
use tracing::error;
impl BackupService {
pub async fn dispatch(
&self,
generated_id: &String,
@@ -14,7 +13,6 @@ impl BackupService {
storages: &Vec<DatabaseStorage>,
encrypt: bool,
) {
let Some(cfg) = config
.databases
.iter()
@@ -41,4 +39,4 @@ impl BackupService {
}
});
}
}
}
+4 -6
View File
@@ -1,14 +1,13 @@
use super::service::BackupService;
use crate::services::config::DatabaseConfig;
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::config::DatabaseConfig;
use crate::utils::common::BackupMethod;
use crate::utils::locks::FileLock;
use tempfile::TempDir;
use anyhow::Result;
use tempfile::TempDir;
impl BackupService {
pub async fn execute_backup(
&self,
generated_id: String,
@@ -17,11 +16,10 @@ impl BackupService {
storages: Vec<DatabaseStorage>,
encrypt: bool,
) -> Result<()> {
if FileLock::is_locked(&generated_id).await? {
anyhow::bail!("backup already running");
}
let backup = self.create_backup_record(&generated_id, &method).await?;
let backup_id = backup.backup.id;
@@ -46,4 +44,4 @@ impl BackupService {
Ok(())
}
}
}
+2 -4
View File
@@ -1,16 +1,14 @@
use super::service::BackupService;
use crate::services::api::models::agent::backup::BackupResponse;
use crate::utils::common::BackupMethod;
use anyhow::{Result, anyhow};
use crate::services::api::models::agent::backup::BackupResponse;
impl BackupService {
pub async fn create_backup_record(
&self,
generated_id: &str,
method: &BackupMethod,
) -> Result<BackupResponse> {
let response = self
.ctx
.api
@@ -23,4 +21,4 @@ impl BackupService {
response.ok_or_else(|| anyhow!("backup_create returned empty response"))
}
}
}
+6 -6
View File
@@ -1,11 +1,11 @@
pub mod service;
pub mod compressor;
pub mod dispatcher;
pub mod executor;
pub mod compressor;
pub mod uploader;
pub mod result;
pub mod models;
pub mod helpers;
pub mod models;
pub mod result;
pub mod runner;
pub mod service;
pub mod uploader;
pub use service::BackupService;
pub use service::BackupService;
+2 -2
View File
@@ -1,7 +1,7 @@
#![allow(dead_code)]
use std::path::PathBuf;
use crate::services::config::DbType;
use std::path::PathBuf;
#[derive(Debug, Clone)]
pub struct BackupResult {
@@ -19,4 +19,4 @@ pub struct UploadResult {
pub error: Option<String>,
pub remote_file_path: Option<String>,
pub total_size: Option<u64>,
}
}
+3 -9
View File
@@ -9,12 +9,7 @@ use std::path::Path;
use tracing::{error, info};
impl BackupService {
pub async fn run(
cfg: DatabaseConfig,
tmp_path: &Path,
) -> Result<BackupResult> {
pub async fn run(cfg: DatabaseConfig, tmp_path: &Path) -> Result<BackupResult> {
let db = DatabaseFactory::create_for_backup(cfg.clone()).await;
let generated_id = cfg.generated_id.clone();
@@ -40,8 +35,7 @@ impl BackupService {
});
}
match db.backup(tmp_path, Some(false)).await {
match db.backup(tmp_path).await {
Ok(file) => Ok(BackupResult {
generated_id,
db_type,
@@ -67,4 +61,4 @@ impl BackupService {
}),
}
}
}
}
+2 -2
View File
@@ -1,5 +1,5 @@
use std::sync::Arc;
use crate::core::context::Context as CoreContext;
use std::sync::Arc;
pub struct BackupService {
pub ctx: Arc<CoreContext>,
@@ -9,4 +9,4 @@ impl BackupService {
pub fn new(ctx: Arc<CoreContext>) -> Self {
Self { ctx }
}
}
}
+44 -41
View File
@@ -1,16 +1,15 @@
use super::service::BackupService;
use super::models::{BackupResult, UploadResult};
use super::service::BackupService;
use crate::services::storage;
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::storage;
use crate::utils::common::BackupMethod;
use futures::future::join_all;
use anyhow::{Result, bail};
use tracing::{info, error};
use futures::future::join_all;
use tracing::{error, info};
impl BackupService {
pub async fn upload(
&self,
result: BackupResult,
@@ -19,7 +18,6 @@ impl BackupService {
encrypt: bool,
backup_id: &String,
) -> Result<Vec<UploadResult>> {
if result.code.as_deref() == Some("backup_already_in_progress") {
info!("Skipping send: backup already in progress");
bail!("backup_already_in_progress");
@@ -28,7 +26,6 @@ impl BackupService {
let ctx = self.ctx.clone();
let futures = storages.into_iter().map(|storage| {
let ctx_clone = ctx.clone();
let result_clone = result.clone();
let provider = storage::get_provider(&storage);
@@ -37,13 +34,16 @@ impl BackupService {
let generated_id = result_clone.generated_id.clone();
async move {
info!("Uploading storage -> {:?} for {:?}", storage.provider, storage_id);
info!(
"Uploading storage -> {:?} for {:?}",
storage.provider, storage_id
);
/*
INIT STEP
*/
let init = match ctx_clone.api
let init = match ctx_clone
.api
.backup_upload_init(
ctx_clone.edge_key.agent_id.clone(),
generated_id.clone(),
@@ -107,7 +107,11 @@ impl BackupService {
)
.await;
let status = if upload_result.success { "success" } else { "failed" };
let status = if upload_result.success {
"success"
} else {
"failed"
};
if status != "success" {
return upload_result;
@@ -115,49 +119,48 @@ impl BackupService {
info!(
"Storage {} uploaded to remote path {:?}",
storage_id,
upload_result.remote_file_path
storage_id, upload_result.remote_file_path
);
/*
METADATA VALIDATION
*/
let (remote_path, total_size) = match (
&upload_result.remote_file_path,
upload_result.total_size,
) {
(Some(path), Some(size)) => (path.clone(), size),
_ => {
return UploadResult {
storage_id,
success: false,
error: Some("remote_file_path or total_size missing".into()),
remote_file_path: None,
total_size: None,
};
}
};
let (remote_path, total_size) =
match (&upload_result.remote_file_path, upload_result.total_size) {
(Some(path), Some(size)) => (path.clone(), size),
_ => {
return UploadResult {
storage_id,
success: false,
error: Some("remote_file_path or total_size missing".into()),
remote_file_path: None,
total_size: None,
};
}
};
/*
STATUS UPDATE
*/
match ctx_clone.api.backup_upload_status(
ctx_clone.edge_key.agent_id.clone(),
generated_id,
backup_storage_id,
status,
remote_path,
total_size,
backup_id,
).await {
match ctx_clone
.api
.backup_upload_status(
ctx_clone.edge_key.agent_id.clone(),
generated_id,
backup_storage_id,
status,
remote_path,
total_size,
backup_id,
)
.await
{
Ok(_) => upload_result,
Err(err) => {
error!(
"backup_upload_status failed (storage_id={}): {}",
storage_id,
err
storage_id, err
);
UploadResult {
@@ -178,4 +181,4 @@ impl BackupService {
Ok(results)
}
}
}
+33 -11
View File
@@ -20,7 +20,7 @@ pub enum DbType {
MongoDB,
Sqlite,
Redis,
Valkey
Valkey,
}
impl DbType {
@@ -58,7 +58,6 @@ pub struct DatabasesConfig {
pub databases: Vec<DatabaseConfig>,
}
#[allow(dead_code)]
#[derive(Debug, Deserialize, Clone)]
pub struct InputDatabaseConfig {
@@ -80,7 +79,6 @@ pub struct InputDatabasesConfig {
pub databases: Vec<InputDatabaseConfig>,
}
pub struct ConfigService {
ctx: Arc<Context>,
}
@@ -133,17 +131,27 @@ impl ConfigService {
_ => return Err("Unsupported config file format. Use .json or .toml".to_string()),
};
fn required<T: Clone>(opt: &Option<T>, db_name: &str, field_name: &str) -> Result<T, String> {
fn required<T: Clone>(
opt: &Option<T>,
db_name: &str,
field_name: &str,
) -> Result<T, String> {
match opt {
Some(v) => Ok(v.clone()),
None => {
let msg = format!("Missing required field '{}' for database '{}'", field_name, db_name);
let msg = format!(
"Missing required field '{}' for database '{}'",
field_name, db_name
);
Err(msg)
}
}
}
fn optional<T: Clone>(opt: &Option<T>) -> T where T: Default {
fn optional<T: Clone>(opt: &Option<T>) -> T
where
T: Default,
{
opt.clone().unwrap_or_default()
}
@@ -155,28 +163,42 @@ impl ConfigService {
}
let username = match db.db_type {
DbType::Postgresql | DbType::Mysql | DbType::Mariadb => required(&db.username, &db.name, "username")?,
DbType::Postgresql | DbType::Mysql | DbType::Mariadb => {
required(&db.username, &db.name, "username")?
}
_ => optional(&db.username),
};
let password = match db.db_type {
DbType::Postgresql | DbType::Mysql | DbType::Mariadb => required(&db.password, &db.name, "password")?,
DbType::Postgresql | DbType::Mysql | DbType::Mariadb => {
required(&db.password, &db.name, "password")?
}
_ => optional(&db.password),
};
let host = match db.db_type {
DbType::Postgresql | DbType::Mysql | DbType::Mariadb | DbType::MongoDB | DbType::Redis | DbType::Valkey => required(&db.host, &db.name, "host")?,
DbType::Postgresql
| DbType::Mysql
| DbType::Mariadb
| DbType::MongoDB
| DbType::Redis
| DbType::Valkey => required(&db.host, &db.name, "host")?,
DbType::Sqlite => optional(&db.host),
};
let port = match db.db_type {
DbType::Postgresql | DbType::Mysql | DbType::Mariadb | DbType::MongoDB | DbType::Redis | DbType::Valkey => required(&db.port, &db.name, "port")?,
DbType::Postgresql
| DbType::Mysql
| DbType::Mariadb
| DbType::MongoDB
| DbType::Redis
| DbType::Valkey => required(&db.port, &db.name, "port")?,
DbType::Sqlite => db.port.unwrap_or(0),
};
let database_name = match db.db_type {
DbType::Sqlite | DbType::Redis | DbType::Valkey => optional(&db.database),
_ => required(&db.database, &db.name, "database")?
_ => required(&db.database, &db.name, "database")?,
};
let path_val = match db.db_type {
+1 -1
View File
@@ -1,13 +1,13 @@
#![allow(dead_code)]
use crate::core::context::Context;
use crate::services::api::models::agent::status::DatabaseStatus;
use crate::utils::common::vec_to_option_json;
use crate::utils::redis_client;
use crate::utils::task_manager::cron::check_and_update_cron;
use redis::aio::MultiplexedConnection;
use serde_json::{Value, json};
use std::sync::Arc;
use crate::services::api::models::agent::status::DatabaseStatus;
pub struct CronService {
ctx: Arc<Context>,
+3 -3
View File
@@ -1,7 +1,7 @@
pub mod api;
pub mod backup;
pub mod config;
pub mod cron;
pub mod backup;
pub mod restore;
pub mod status;
mod storage;
pub mod api;
pub mod status;
+2 -5
View File
@@ -7,13 +7,11 @@ use anyhow::Result;
use std::path::{Path, PathBuf};
impl RestoreService {
pub async fn prepare_archive(
&self,
downloaded_file: PathBuf,
tmp_path: &Path,
) -> Result<PathBuf> {
let filename = downloaded_file
.file_name()
.unwrap()
@@ -31,7 +29,6 @@ impl RestoreService {
let mut archive = downloaded_file.clone();
if encrypted {
let new_name = filename.strip_suffix(".enc").unwrap();
let decrypted = tmp_path.join(new_name);
@@ -41,7 +38,7 @@ impl RestoreService {
decrypted.clone(),
self.ctx.edge_key.master_key_b64.clone(),
)
.await?;
.await?;
archive = decrypted;
}
@@ -58,4 +55,4 @@ impl RestoreService {
Ok(archive)
}
}
}
}
+3 -10
View File
@@ -1,13 +1,11 @@
use super::service::RestoreService;
use crate::services::config::DatabasesConfig;
use crate::services::api::models::agent::status::DatabaseStatus;
use crate::services::config::DatabasesConfig;
use tracing::error;
impl RestoreService {
pub async fn dispatch(&self, db: &DatabaseStatus, config: &DatabasesConfig) {
let Some(cfg) = config
.databases
.iter()
@@ -29,14 +27,9 @@ impl RestoreService {
let db_cfg = cfg.clone();
tokio::spawn(async move {
if let Err(e) = service
.execute_restore(db_cfg, file_to_restore)
.await
{
if let Err(e) = service.execute_restore(db_cfg, file_to_restore).await {
error!("Restore failed: {}", e);
}
});
}
}
}
+3 -10
View File
@@ -1,18 +1,12 @@
use super::service::RestoreService;
use reqwest::{Client, Url};
use anyhow::Result;
use reqwest::{Client, Url};
use std::path::{Path, PathBuf};
use tracing::info;
impl RestoreService {
pub async fn download_backup(
&self,
file_url: &str,
tmp_path: &Path,
) -> Result<PathBuf> {
pub async fn download_backup(&self, file_url: &str, tmp_path: &Path) -> Result<PathBuf> {
let client = Client::new();
let response = client.get(file_url).send().await?;
@@ -28,7 +22,6 @@ impl RestoreService {
.and_then(|s| s.split("filename=").nth(1))
.map(|f| f.trim_matches('"').to_string());
let filename_from_url = Url::parse(file_url).ok().and_then(|u| {
u.path_segments()?
.last()
@@ -50,4 +43,4 @@ impl RestoreService {
Ok(path)
}
}
}
+3 -9
View File
@@ -1,17 +1,11 @@
use super::service::RestoreService;
use crate::services::config::DatabaseConfig;
use tempfile::TempDir;
use anyhow::Result;
use tempfile::TempDir;
use tracing::info;
impl RestoreService {
pub async fn execute_restore(
&self,
cfg: DatabaseConfig,
file_url: String,
) -> Result<()> {
pub async fn execute_restore(&self, cfg: DatabaseConfig, file_url: String) -> Result<()> {
let temp_dir = TempDir::new()?;
let tmp_path = temp_dir.path();
@@ -27,4 +21,4 @@ impl RestoreService {
Ok(())
}
}
}
+7 -7
View File
@@ -1,10 +1,10 @@
pub mod service;
pub mod dispatcher;
pub mod executor;
pub mod downloader;
pub mod archive;
pub mod runner;
pub mod result;
pub mod dispatcher;
pub mod downloader;
pub mod executor;
pub mod models;
pub mod result;
pub mod runner;
pub mod service;
pub use service::RestoreService;
pub use service::RestoreService;
+1 -1
View File
@@ -5,4 +5,4 @@ pub struct RestoreResult {
#[serde(rename = "generatedId")]
pub generated_id: String,
pub status: String,
}
}
+7 -6
View File
@@ -1,24 +1,25 @@
use super::service::RestoreService;
use super::models::RestoreResult;
use super::service::RestoreService;
use tracing::{info, error};
use tracing::{error, info};
impl RestoreService {
pub async fn send_result(&self, result: RestoreResult) {
info!(
"[RestoreService] DB: {} | Status: {}",
result.generated_id, result.status
);
match self.ctx
match self
.ctx
.api
.restore_result(
self.ctx.edge_key.agent_id.clone(),
&result.generated_id,
&result.status,
)
.await {
.await
{
Ok(_) => {
info!("Restoration result sent successfully");
}
@@ -27,4 +28,4 @@ impl RestoreService {
}
}
}
}
}
+4 -7
View File
@@ -1,21 +1,19 @@
use super::service::RestoreService;
use super::models::RestoreResult;
use super::service::RestoreService;
use crate::domain::factory::DatabaseFactory;
use crate::services::config::DatabaseConfig;
use anyhow::Result;
use std::path::PathBuf;
use tracing::{info, error};
use tracing::{error, info};
impl RestoreService {
pub async fn run_restore(
&self,
cfg: DatabaseConfig,
backup_file: PathBuf,
) -> Result<RestoreResult> {
let generated_id = cfg.generated_id.clone();
let db = DatabaseFactory::create_for_restore(cfg.clone(), &backup_file).await;
@@ -31,8 +29,7 @@ impl RestoreService {
});
}
match db.restore(&backup_file, Some(false)).await {
match db.restore(&backup_file).await {
Ok(_) => Ok(RestoreResult {
generated_id,
status: "success".into(),
@@ -48,4 +45,4 @@ impl RestoreService {
}
}
}
}
}
+2 -2
View File
@@ -1,5 +1,5 @@
use std::sync::Arc;
use crate::core::context::Context;
use std::sync::Arc;
pub struct RestoreService {
pub ctx: Arc<Context>,
@@ -9,4 +9,4 @@ impl RestoreService {
pub fn new(ctx: Arc<Context>) -> Self {
Self { ctx }
}
}
}
+8 -3
View File
@@ -1,13 +1,13 @@
#![allow(dead_code)]
use crate::core::context::Context;
use crate::services::api::endpoints::status::DatabasePayload;
use crate::services::api::models::agent::status::PingResult;
use crate::services::config::DatabaseConfig;
use crate::settings::CONFIG;
use reqwest::Client;
use std::error::Error;
use std::sync::Arc;
use crate::services::api::endpoints::status::DatabasePayload;
use crate::services::api::models::agent::status::PingResult;
pub struct StatusService {
ctx: Arc<Context>,
@@ -35,7 +35,12 @@ impl StatusService {
.collect();
let version_str = CONFIG.app_version.as_str();
let result = self.ctx.api.agent_status(&edge_key.agent_id, &version_str, databases_payload).await?.unwrap();
let result = self
.ctx
.api
.agent_status(&edge_key.agent_id, &version_str, databases_payload)
.await?
.unwrap();
Ok(result)
}
}
+7 -7
View File
@@ -1,15 +1,15 @@
pub mod providers;
use crate::core::context::Context;
use crate::utils::common::BackupMethod;
use async_trait::async_trait;
use providers::local;
use providers::s3;
use providers::google_drive;
use std::sync::Arc;
use tracing::{error, info};
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::backup::models::{BackupResult, UploadResult};
use crate::utils::common::BackupMethod;
use async_trait::async_trait;
use providers::google_drive;
use providers::local;
use providers::s3;
use std::sync::Arc;
use tracing::{error, info};
#[async_trait]
pub trait StorageProvider: Send + Sync {
@@ -1,15 +1,15 @@
use crate::services::storage::providers::google_drive::models::GoogleDriveProviderConfig;
use anyhow::{Context, Result, anyhow};
use bytes::Bytes;
use futures::Stream;
use futures::StreamExt;
use oauth2::{
AuthUrl, ClientId, ClientSecret, RefreshToken, TokenResponse, TokenUrl, basic::BasicClient,
reqwest::Client as OAuth2ReqwestClient,
};
use reqwest::{Client as ReqwestClient, StatusCode};
use serde_json::{Value, json};
use futures::{Stream};
use bytes::Bytes;
use reqwest::{Client, header};
use crate::services::storage::providers::google_drive::models::GoogleDriveProviderConfig;
use serde_json::{Value, json};
pub async fn get_google_drive_token(config: &GoogleDriveProviderConfig) -> Result<String> {
let http_client = OAuth2ReqwestClient::new();
@@ -34,7 +34,10 @@ pub async fn get_google_drive_token(config: &GoogleDriveProviderConfig) -> Resul
Ok(token_result.access_token().secret().clone())
}
pub async fn ensure_folder_path(config: &GoogleDriveProviderConfig, path_parts: &[&str]) -> Result<String> {
pub async fn ensure_folder_path(
config: &GoogleDriveProviderConfig,
path_parts: &[&str],
) -> Result<String> {
if path_parts.is_empty() {
return Ok(config.folder_id.clone());
}
@@ -152,7 +155,10 @@ pub async fn upload_stream_to_google_drive(
let folder_id = ensure_folder_path(config, folder_path).await?;
if find_file_by_name(config, file_name, &folder_id).await?.is_some() {
if find_file_by_name(config, file_name, &folder_id)
.await?
.is_some()
{
return Err(anyhow::anyhow!("File already exists: {}", full_path));
}
@@ -172,7 +178,7 @@ pub async fn upload_stream_to_google_drive(
.post("https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable")
.bearer_auth(&token)
.header("X-Upload-Content-Type", mime)
.header("X-Upload-Content-Length", total_size.to_string()) // Helps a lot
.header("X-Upload-Content-Length", total_size.to_string()) // Helps a lot
.json(&metadata)
.send()
.await
@@ -237,12 +243,15 @@ pub async fn upload_stream_to_google_drive(
.put(&upload_url)
.header("Content-Range", &content_range)
.header("Content-Length", chunk_bytes.len().to_string())
.body(chunk_bytes.clone()) // clone is cheap if small; optimize later if needed
.body(chunk_bytes.clone()) // clone is cheap if small; optimize later if needed
.send()
.await;
match res {
Ok(resp) if resp.status().is_success() || resp.status() == StatusCode::PERMANENT_REDIRECT => {
Ok(resp)
if resp.status().is_success()
|| resp.status() == StatusCode::PERMANENT_REDIRECT =>
{
// 200 or 308 = good
uploaded += chunk_bytes.len() as u64;
tracing::info!("Uploaded {}/{} bytes", uploaded, total_size);
@@ -1,9 +1,12 @@
mod helpers;
mod models;
mod models;
use crate::core::context::Context;
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::backup::models::{BackupResult, UploadResult};
use crate::services::storage::StorageProvider;
use crate::services::storage::providers::google_drive::helpers::upload_stream_to_google_drive;
use crate::services::storage::providers::google_drive::models::GoogleDriveProviderConfig;
use crate::utils::common::BackupMethod;
use crate::utils::file::{full_file_name, full_file_path};
use crate::utils::stream::build_stream;
@@ -11,9 +14,6 @@ use async_trait::async_trait;
use std::sync::Arc;
use tokio::fs;
use tracing::{error, info};
use crate::services::backup::models::{BackupResult, UploadResult};
use crate::services::storage::providers::google_drive::helpers::{upload_stream_to_google_drive};
use crate::services::storage::providers::google_drive::models::GoogleDriveProviderConfig;
pub struct GoogleDriveProvider {}
@@ -53,13 +53,7 @@ impl StorageProvider for GoogleDriveProvider {
let encrypt = encrypt.unwrap_or(false);
let upload = match build_stream(
&file_path,
encrypt,
&ctx.edge_key.master_key_b64
)
.await
{
let upload = match build_stream(&file_path, encrypt, &ctx.edge_key.master_key_b64).await {
Ok(u) => u,
Err(e) => {
error!("Stream build failed: {}", e);
@@ -86,7 +80,6 @@ impl StorageProvider for GoogleDriveProvider {
}
};
let file_name = full_file_name(encrypt);
info!("Uploading file {}", file_name);
@@ -96,12 +89,13 @@ impl StorageProvider for GoogleDriveProvider {
match upload_stream_to_google_drive(
&config,
&remote_file_path,
upload.stream,
upload.stream,
total_size,
Some("application/octet-stream"),
).await {
)
.await
{
Ok(_file_id) => {
info!("Google Drive upload successful");
UploadResult {
@@ -124,6 +118,5 @@ impl StorageProvider for GoogleDriveProvider {
}
}
}
}
}
@@ -7,5 +7,3 @@ pub struct GoogleDriveProviderConfig {
pub refresh_token: String,
pub folder_id: String,
}
+16 -12
View File
@@ -1,5 +1,6 @@
use crate::core::context::Context;
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::backup::models::{BackupResult, UploadResult};
use crate::services::storage::StorageProvider;
use crate::utils::common::BackupMethod;
use crate::utils::file::{full_file_name, full_file_path};
@@ -10,7 +11,6 @@ use reqwest::header::{HeaderMap, HeaderValue};
use std::sync::Arc;
use tokio::fs;
use tracing::error;
use crate::services::backup::models::{BackupResult, UploadResult};
pub struct LocalProvider;
@@ -53,13 +53,7 @@ impl StorageProvider for LocalProvider {
}
};
let upload = match build_stream(
&file_path,
encrypt,
&ctx.edge_key.master_key_b64
)
.await
{
let upload = match build_stream(&file_path, encrypt, &ctx.edge_key.master_key_b64).await {
Ok(u) => u,
Err(e) => {
error!("Stream build failed: {}", e);
@@ -68,7 +62,7 @@ impl StorageProvider for LocalProvider {
success: false,
error: Some(e.to_string()),
remote_file_path: None,
total_size: None
total_size: None,
};
}
};
@@ -76,7 +70,10 @@ impl StorageProvider for LocalProvider {
let mut extra_headers = HeaderMap::new();
extra_headers.insert("X-File-Name", HeaderValue::from_str(&file_name).unwrap());
extra_headers.insert("X-File-Size", HeaderValue::from_str(&total_size.to_string()).unwrap());
extra_headers.insert(
"X-File-Size",
HeaderValue::from_str(&total_size.to_string()).unwrap(),
);
extra_headers.insert(
"X-File-Path",
HeaderValue::from_str(&remote_file_path).unwrap(),
@@ -90,10 +87,17 @@ impl StorageProvider for LocalProvider {
"X-Method",
HeaderValue::from_str(&method.to_string()).unwrap(),
);
let tus_endpoint = format!("{}/tus/files", ctx.edge_key.server_url);
match upload_to_tus_stream_with_headers(upload.stream, &tus_endpoint, extra_headers, total_size).await {
match upload_to_tus_stream_with_headers(
upload.stream,
&tus_endpoint,
extra_headers,
total_size,
)
.await
{
Ok(_) => UploadResult {
storage_id: storage.id.clone(),
success: true,
+1 -1
View File
@@ -1,3 +1,3 @@
pub mod google_drive;
pub mod local;
pub mod s3;
pub mod google_drive;
+20 -7
View File
@@ -2,26 +2,26 @@ mod models;
use crate::core::context::Context;
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::backup::models::{BackupResult, UploadResult};
use crate::services::storage::StorageProvider;
use crate::services::storage::providers::s3::models::S3ProviderConfig;
use crate::utils::common::BackupMethod;
use crate::utils::file::{full_file_name, full_file_path};
use crate::utils::stream::build_stream;
use async_trait::async_trait;
use aws_config::retry::RetryConfig;
use aws_sdk_s3 as s3;
use aws_sdk_s3::config::BehaviorVersion;
use aws_sdk_s3::config::Region;
use aws_sdk_s3::config::retry::ReconnectMode;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use futures::StreamExt;
use std::pin::Pin;
use std::sync::Arc;
use std::time::Duration;
use aws_config::retry::RetryConfig;
use aws_sdk_s3::config::retry::ReconnectMode;
use tokio::fs;
use tracing::{error, info};
use crate::services::backup::models::{BackupResult, UploadResult};
pub struct S3Provider {}
@@ -97,15 +97,28 @@ impl StorageProvider for S3Provider {
);
let region = Region::new(config.region.clone().unwrap_or("us-east-1".to_string()));
let endpoint = if let Some(port) = &config.port {
if port.trim().is_empty() {
format!("{}://{}", if config.ssl { "https" } else { "http" }, config.end_point_url)
format!(
"{}://{}",
if config.ssl { "https" } else { "http" },
config.end_point_url
)
} else {
format!("{}://{}:{}", if config.ssl { "https" } else { "http" }, config.end_point_url, port)
format!(
"{}://{}:{}",
if config.ssl { "https" } else { "http" },
config.end_point_url,
port
)
}
} else {
format!("{}://{}", if config.ssl { "https" } else { "http" }, config.end_point_url)
format!(
"{}://{}",
if config.ssl { "https" } else { "http" },
config.end_point_url
)
};
info!("S3 endpoint to {}", &endpoint);
+1 -2
View File
@@ -1,5 +1,5 @@
use serde::{Deserialize, Serialize};
use crate::utils::deserializer::string_or_number_to_string;
use serde::{Deserialize, Serialize};
#[derive(Debug, Deserialize, Serialize)]
pub struct S3ProviderConfig {
@@ -12,4 +12,3 @@ pub struct S3ProviderConfig {
#[serde(default, deserialize_with = "string_or_number_to_string")]
pub port: Option<String>,
}
+1 -1
View File
@@ -1 +1 @@
pub mod ping;
pub mod ping;
+1 -1
View File
@@ -21,4 +21,4 @@ pub async fn ping_server() {
}
tokio::time::sleep(Duration::from_secs(CONFIG.pooling as u64)).await;
}
}
}
+94
View File
@@ -0,0 +1,94 @@
use crate::domain::factory::DatabaseFactory;
use crate::services::config::{DatabaseConfig, DbType};
use crate::tests::init_tracing_for_test;
use crate::utils::compress::{compress_to_tar_gz_large, decompress_large_tar_gz};
use oauth2::url;
use std::path::PathBuf;
use tempfile::TempDir;
use testcontainers::runners::AsyncRunner;
use testcontainers::{ContainerAsync, ImageExt};
use testcontainers_modules::mariadb::Mariadb;
use tracing::{error, info};
use url::Host;
async fn create_config() -> (ContainerAsync<Mariadb>, DatabaseConfig) {
let container = Mariadb::default().with_tag("11.3").start().await.unwrap();
let host = container
.get_host()
.await
.unwrap_or(Host::parse("127.0.0.1").unwrap());
let port = container.get_host_port_ipv4(3306).await.unwrap_or(3306);
let config = DatabaseConfig {
name: "Test MariaDB".to_string(),
database: "test".to_string(),
db_type: DbType::Mariadb,
username: "root".to_string(),
password: "".to_string(),
port,
host: host.to_string(),
generated_id: "3c4b4eb4-c2c6-4bde-a423-ee1385dcf6d2".to_string(),
path: "".to_string(),
};
(container, config)
}
#[tokio::test]
async fn mariadb_ping_test() {
init_tracing_for_test();
let (_container, config) = create_config().await;
let db = DatabaseFactory::create_for_backup(config.clone()).await;
let reachable = db.ping().await.unwrap_or(false);
assert!(reachable);
}
#[tokio::test]
async fn mariadb_backup_restore_test() {
init_tracing_for_test();
let (_container, config) = create_config().await;
let temp_dir = TempDir::new().unwrap();
let backup_path = temp_dir.path();
let db = DatabaseFactory::create_for_backup(config.clone()).await;
let file_path = db.backup(backup_path).await.unwrap();
assert!(file_path.is_file());
let compression = compress_to_tar_gz_large(&file_path).await.unwrap();
assert!(compression.compressed_path.is_file());
let files = decompress_large_tar_gz(compression.compressed_path.as_path(), temp_dir.path())
.await
.unwrap();
let backup_file: PathBuf = if files.len() == 1 {
files[0].clone()
} else {
"".into()
};
let db = DatabaseFactory::create_for_restore(config.clone(), &backup_file).await;
let reachable = db.ping().await.unwrap_or(false);
info!("Reachable: {}", reachable);
assert!(reachable);
match db.restore(&backup_file).await {
Ok(_) => {
info!("Restore succeeded for {}", config.generated_id);
assert!(true)
}
Err(e) => {
error!("Restore failed for {}: {:?}", config.generated_id, e);
assert!(false)
}
}
}
+4 -1
View File
@@ -1,3 +1,6 @@
mod mariadb;
mod mongodb;
mod mysql;
mod postgres;
mod redis;
mod valkey;
mod valkey;

Some files were not shown because too many files have changed in this diff Show More