mirror of
https://github.com/Portabase/agent.git
synced 2026-09-09 17:47:25 +00:00
fix: tests (#47)
* fix: s3 port type mismatch, add string_or_number_to_string deserializer * fix: adding some tests * fix: adding some tests * fix: working on pipeline * fix: working on pipeline * fix: working on pipeline * fix: working on pipeline * fix: working on pipeline * fix: working on pipeline * fix: working on pipeline * fix: working on pipeline * fix: working on pipeline * fix: working on pipeline * fix: working on pipeline * fix: working on pipeline * fix: working on pipeline * fix: working on pipeline * fix: working on pipeline * fix: working on pipeline * fix: working on pipeline * fix: working on pipeline * fix: working on pipeline * fix: working on pipeline * fix: working on pipeline * fix: working on pipeline --------- Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
This commit is contained in:
+145
-31
@@ -1,3 +1,86 @@
|
||||
#name: Codecov Rust
|
||||
#
|
||||
#on:
|
||||
# push:
|
||||
# branches: ["main"]
|
||||
# pull_request:
|
||||
# branches: ["main"]
|
||||
#
|
||||
#env:
|
||||
# CARGO_TERM_COLOR: always
|
||||
#
|
||||
#jobs:
|
||||
# coverage:
|
||||
# runs-on: ubuntu-latest
|
||||
# steps:
|
||||
# - uses: actions/checkout@v4
|
||||
#
|
||||
# - name: Install Rust toolchain
|
||||
# uses: dtolnay/rust-toolchain@stable
|
||||
# with:
|
||||
# components: llvm-tools-preview
|
||||
#
|
||||
# - name: Build test image (with grcov included)
|
||||
# run: docker compose -f docker-compose.test.yml build agent-test
|
||||
#
|
||||
# - name: Run tests in container
|
||||
# env:
|
||||
# CARGO_TARGET_DIR: /app/target
|
||||
# CARGO_INCREMENTAL: 0
|
||||
# RUSTFLAGS: "-C instrument-coverage -C link-dead-code"
|
||||
# LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw"
|
||||
# run: |
|
||||
# docker compose -f docker-compose.test.yml run \
|
||||
# -e CARGO_TARGET_DIR \
|
||||
# -e CARGO_INCREMENTAL \
|
||||
# -e RUSTFLAGS \
|
||||
# -e LLVM_PROFILE_FILE \
|
||||
# agent-test bash -c "cargo clean && cargo test --verbose && sync"
|
||||
#
|
||||
# - name: Verify profraw files exist
|
||||
# run: |
|
||||
# docker compose -f docker-compose.test.yml run agent-test \
|
||||
# find /app/coverage -type f -name "*.profraw" | wc -l || true
|
||||
#
|
||||
# - name: Generate coverage report inside container
|
||||
# run: |
|
||||
# docker compose -f docker-compose.test.yml run agent-test bash -c "
|
||||
# rustup component add llvm-tools &&
|
||||
# grcov /app/coverage \
|
||||
# --binary-path /app/target/debug \
|
||||
# -s /app \
|
||||
# --llvm \
|
||||
# -t lcov \
|
||||
# --branch \
|
||||
# --ignore-not-existing \
|
||||
# --ignore '/app/target/*' \
|
||||
# --ignore '/*' \
|
||||
# -o /app/lcov.info
|
||||
# "
|
||||
#
|
||||
# - name: Copy lcov.info from container to host
|
||||
# run: |
|
||||
# docker compose -f docker-compose.test.yml cp agent-test:/app/lcov.info ./lcov.info
|
||||
#
|
||||
# - name: Remove container
|
||||
# run: |
|
||||
# docker rm agent-test-run
|
||||
#
|
||||
# - name: Show basic coverage report info (debug)
|
||||
# run: |
|
||||
# echo "lcov.info size:" $(wc -c ./lcov.info | awk '{print $1}')
|
||||
# head -n 30 ./lcov.info || true
|
||||
#
|
||||
# - name: Upload coverage to Codecov
|
||||
# uses: codecov/codecov-action@v5
|
||||
# with:
|
||||
# files: ./lcov.info
|
||||
# flags: unittests
|
||||
# name: rust-unit-coverage
|
||||
# verbose: true
|
||||
# fail_ci_if_error: true
|
||||
# env:
|
||||
# CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
name: Codecov Rust
|
||||
|
||||
on:
|
||||
@@ -8,51 +91,82 @@ on:
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
CARGO_TARGET_DIR: /app/target
|
||||
CARGO_INCREMENTAL: 0
|
||||
RUSTFLAGS: "-C instrument-coverage -C link-dead-code"
|
||||
LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw"
|
||||
|
||||
jobs:
|
||||
build:
|
||||
coverage:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions-rs/toolchain@v1
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: stable
|
||||
override: true
|
||||
components: llvm-tools-preview
|
||||
|
||||
- name: Install grcov
|
||||
run: cargo install grcov
|
||||
- name: Build test image (with grcov included)
|
||||
run: docker compose -f docker-compose.test.yml build agent-test
|
||||
|
||||
- name: Install test requirements
|
||||
run: bash scripts/tests/requirements.sh
|
||||
- name: Start agent-test container
|
||||
run: docker compose -f docker-compose.test.yml up -d agent-test
|
||||
|
||||
- name: Build
|
||||
run: cargo build --verbose
|
||||
|
||||
- name: Run tests
|
||||
env:
|
||||
CARGO_INCREMENTAL: 0
|
||||
RUSTFLAGS: "-C instrument-coverage"
|
||||
LLVM_PROFILE_FILE: "cargo-test-%p-%m.profraw"
|
||||
run: cargo test --verbose
|
||||
|
||||
- name: Generate coverage
|
||||
- name: Run tests inside container
|
||||
run: |
|
||||
grcov . \
|
||||
--binary-path ./target/debug/ \
|
||||
-s . \
|
||||
-t lcov \
|
||||
--branch \
|
||||
--ignore-not-existing \
|
||||
-o lcov.info
|
||||
docker compose -f docker-compose.test.yml exec \
|
||||
-e CARGO_TARGET_DIR \
|
||||
-e CARGO_INCREMENTAL \
|
||||
-e RUSTFLAGS \
|
||||
-e LLVM_PROFILE_FILE \
|
||||
agent-test bash -c "
|
||||
cargo clean &&
|
||||
cargo test --verbose &&
|
||||
sync
|
||||
"
|
||||
|
||||
- name: Upload to Codecov
|
||||
- name: Verify profraw files exist
|
||||
run: |
|
||||
docker compose -f docker-compose.test.yml exec \
|
||||
-e LLVM_PROFILE_FILE \
|
||||
agent-test find /app/coverage -type f -name "*.profraw" | wc -l || true
|
||||
|
||||
- name: Generate coverage report inside container
|
||||
run: |
|
||||
docker compose -f docker-compose.test.yml exec \
|
||||
agent-test bash -c "
|
||||
rustup component add llvm-tools &&
|
||||
grcov /app/coverage \
|
||||
--binary-path /app/target/debug \
|
||||
-s /app \
|
||||
--llvm \
|
||||
-t lcov \
|
||||
--branch \
|
||||
--ignore-not-existing \
|
||||
--ignore '/app/target/*' \
|
||||
--ignore '/*' \
|
||||
-o /app/lcov.info
|
||||
"
|
||||
|
||||
- name: Copy lcov.info from container to host
|
||||
run: docker compose -f docker-compose.test.yml cp agent-test:/app/lcov.info ./lcov.info
|
||||
|
||||
- name: Show basic coverage report info (debug)
|
||||
run: |
|
||||
echo "lcov.info size:" $(wc -c ./lcov.info | awk '{print $1}')
|
||||
head -n 30 ./lcov.info || true
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v5
|
||||
with:
|
||||
files: lcov.info
|
||||
files: ./lcov.info
|
||||
flags: unittests
|
||||
name: rust-unit-coverage
|
||||
verbose: true
|
||||
fail_ci_if_error: true
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
|
||||
- name: Stop and remove container
|
||||
run: docker compose -f docker-compose.test.yml down
|
||||
+2
-2
@@ -49,7 +49,7 @@ tokio-stream = "0.1.18"
|
||||
aes = "0.9.0-rc.4"
|
||||
typenum = "1.19.0"
|
||||
testcontainers = "0.27.1"
|
||||
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "valkey"] }
|
||||
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "valkey", "mysql", "mariadb", "mongo"] }
|
||||
postgres = "0.19.12"
|
||||
url = "2.5.8"
|
||||
|
||||
@@ -57,7 +57,7 @@ url = "2.5.8"
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
mockall = "0.13"
|
||||
testcontainers = "0.27.1"
|
||||
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis"] }
|
||||
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "mysql", "mariadb", "mongo"] }
|
||||
wiremock = "0.6"
|
||||
|
||||
[profile.release]
|
||||
|
||||
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
@@ -0,0 +1,22 @@
|
||||
services:
|
||||
agent-test:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/Dockerfile
|
||||
target: dev
|
||||
working_dir: /app
|
||||
container_name: agent-test
|
||||
volumes:
|
||||
- .:/app
|
||||
- cargo-registry:/usr/local/cargo/registry
|
||||
- cargo-git:/usr/local/cargo/git
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
environment:
|
||||
APP_ENV: test
|
||||
LOG: debug
|
||||
TZ: Europe/Paris
|
||||
EDGE_KEY: ""
|
||||
|
||||
volumes:
|
||||
cargo-registry:
|
||||
cargo-git:
|
||||
+4
-2
@@ -11,14 +11,16 @@ services:
|
||||
# - ./databases.toml:/config/config.toml
|
||||
- cargo-registry:/usr/local/cargo/registry
|
||||
- cargo-git:/usr/local/cargo/git
|
||||
# - cargo-target:/app/target
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
|
||||
# - cargo-target:/app/target
|
||||
# - sqlite-data:/sqlite-data/workspace/data
|
||||
# - ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data
|
||||
environment:
|
||||
APP_ENV: development
|
||||
LOG: debug
|
||||
TZ: "Europe/Paris"
|
||||
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiNWU1OGU2MGEtODhiMy00YTBjLWI0NDktNTQ3OWZhOTQzZDBkIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ=="
|
||||
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiNGI1OTM2MGItNTNkMi00ZTZmLWE1ODctODcyMmQ1NDc1MTNmIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ=="
|
||||
#POOLING: 1
|
||||
#DATABASES_CONFIG_FILE: "config.toml"
|
||||
extra_hosts:
|
||||
|
||||
@@ -60,6 +60,7 @@ WORKDIR /app
|
||||
FROM base AS dev
|
||||
|
||||
RUN cargo install cargo-watch
|
||||
RUN cargo install grcov --locked
|
||||
|
||||
COPY Cargo.toml Cargo.lock ./
|
||||
RUN mkdir src && echo "fn main() {}" > src/main.rs
|
||||
|
||||
@@ -24,3 +24,7 @@ docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT * FROM users LI
|
||||
```bash
|
||||
docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT name FROM sqlite_master WHERE type='table';"
|
||||
```
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.test.yml run agent-test bash -c "cargo clean && cargo test"
|
||||
```
|
||||
@@ -1,88 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
POSTGRES_BASE="/usr/local/postgresql"
|
||||
echo "Detecting OS and architecture..."
|
||||
OS_TYPE="$(uname -s)"
|
||||
ARCH="$(uname -m)"
|
||||
|
||||
install_pg_binaries() {
|
||||
echo "Installing PostgreSQL binaries for versions 12-18..."
|
||||
|
||||
for v in 12 13 14 15 16 17 18; do
|
||||
TARGET_DIR="$POSTGRES_BASE/$v/bin"
|
||||
sudo mkdir -p "$TARGET_DIR"
|
||||
|
||||
if [[ "$OS_TYPE" == "Linux" ]]; then
|
||||
if [[ "$ARCH" == "x86_64" ]]; then
|
||||
SRC_DIR="./assets/tools/amd64/postgresql/postgresql-$v/bin"
|
||||
elif [[ "$ARCH" == "aarch64" ]]; then
|
||||
SRC_DIR="./assets/tools/arm64/postgresql/postgresql-$v/bin"
|
||||
else
|
||||
echo "Unsupported architecture: $ARCH"
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ -d "$SRC_DIR" ]]; then
|
||||
echo "Copying PostgreSQL $v binaries from $SRC_DIR to $TARGET_DIR"
|
||||
sudo cp -r "$SRC_DIR"/* "$TARGET_DIR/"
|
||||
else
|
||||
echo "Binaries for PostgreSQL $v not found for Linux, skipping..."
|
||||
continue
|
||||
fi
|
||||
|
||||
elif [[ "$OS_TYPE" == "Darwin" ]]; then
|
||||
PG_SRC="$(brew --prefix postgresql@$v)/bin" 2>/dev/null || true
|
||||
|
||||
if [[ ! -d "$PG_SRC" ]]; then
|
||||
echo "PostgreSQL $v not installed via Homebrew. Trying to install..."
|
||||
if ! brew install postgresql@$v; then
|
||||
echo "PostgreSQL $v not available, skipping..."
|
||||
continue
|
||||
fi
|
||||
PG_SRC="$(brew --prefix postgresql@$v)/bin"
|
||||
fi
|
||||
|
||||
echo "Copying PostgreSQL $v binaries from $PG_SRC to $TARGET_DIR"
|
||||
sudo cp -r "$PG_SRC"/* "$TARGET_DIR/"
|
||||
fi
|
||||
|
||||
sudo chown -R "$(whoami)" "$TARGET_DIR"
|
||||
chmod +x "$TARGET_DIR"/*
|
||||
done
|
||||
|
||||
echo "PostgreSQL binaries installed under $POSTGRES_BASE"
|
||||
}
|
||||
|
||||
if [[ "$OS_TYPE" == "Linux" ]]; then
|
||||
if command -v apt >/dev/null 2>&1; then
|
||||
echo "Linux detected with apt. Installing prerequisites..."
|
||||
sudo apt update
|
||||
sudo apt install -y wget gnupg lsb-release redis-tools valkey
|
||||
install_pg_binaries
|
||||
else
|
||||
echo "Unsupported Linux distribution. Only apt-based distros are supported."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
elif [[ "$OS_TYPE" == "Darwin" ]]; then
|
||||
if command -v brew >/dev/null 2>&1; then
|
||||
echo "macOS detected. Installing prerequisites..."
|
||||
brew install redis
|
||||
brew install valkey
|
||||
|
||||
sudo mkdir -p "$POSTGRES_BASE"
|
||||
sudo chown -R "$(whoami)" "$POSTGRES_BASE"
|
||||
|
||||
install_pg_binaries
|
||||
else
|
||||
echo "Homebrew not found. Please install Homebrew first: https://brew.sh/"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
else
|
||||
echo "Unsupported OS: $OS_TYPE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Tools installation completed successfully."
|
||||
+15
-8
@@ -4,11 +4,11 @@ use crate::core::context::Context;
|
||||
use crate::services::backup::BackupService;
|
||||
use crate::services::config::ConfigService;
|
||||
use crate::services::cron::CronService;
|
||||
use crate::services::restore::RestoreService;
|
||||
use crate::services::status::StatusService;
|
||||
use crate::utils::common::BackupMethod;
|
||||
use std::sync::Arc;
|
||||
use tracing::info;
|
||||
use crate::services::restore::RestoreService;
|
||||
|
||||
pub struct Agent {
|
||||
ctx: Arc<Context>,
|
||||
@@ -43,23 +43,30 @@ impl Agent {
|
||||
let ping_result = self.status_service.ping(&config.databases).await?;
|
||||
|
||||
for db in ping_result.databases.iter() {
|
||||
let database = config.databases.iter().find(|cfg_db|cfg_db.generated_id == db.generated_id).unwrap();
|
||||
let database = config
|
||||
.databases
|
||||
.iter()
|
||||
.find(|cfg_db| cfg_db.generated_id == db.generated_id)
|
||||
.unwrap();
|
||||
info!(
|
||||
"Generated Id: {} | backup action: {} | restore action: {} | Database Name: {}",
|
||||
db.generated_id, db.data.backup.action, db.data.restore.action, database.name,
|
||||
db.generated_id, db.data.backup.action, db.data.restore.action, database.name,
|
||||
);
|
||||
let _ = self.cron_service.sync(db).await;
|
||||
|
||||
if db.data.backup.action {
|
||||
let _ = self
|
||||
.backup_service
|
||||
.dispatch(&db.generated_id, &config, method.clone(), &db.storages, db.encrypt)
|
||||
.dispatch(
|
||||
&db.generated_id,
|
||||
&config,
|
||||
method.clone(),
|
||||
&db.storages,
|
||||
db.encrypt,
|
||||
)
|
||||
.await;
|
||||
} else if db.data.restore.action {
|
||||
let _ = self
|
||||
.restore_service
|
||||
.dispatch(db, &config)
|
||||
.await;
|
||||
let _ = self.restore_service.dispatch(db, &config).await;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -35,7 +35,7 @@ impl Context {
|
||||
panic!("Cannot initialize AgentContext due to invalid EDGE_KEY");
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
let server_url = format!("{}/api", edge_key.server_url);
|
||||
let api_client = ApiClient::new(server_url);
|
||||
|
||||
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
pub mod context;
|
||||
pub mod agent;
|
||||
pub mod context;
|
||||
|
||||
@@ -4,18 +4,18 @@ use crate::domain::postgres::database::PostgresDatabase;
|
||||
use crate::domain::postgres::{detect_format_from_file, detect_format_from_size};
|
||||
use crate::domain::redis::database::RedisDatabase;
|
||||
use crate::domain::sqlite::database::SqliteDatabase;
|
||||
use crate::domain::valkey::database::ValkeyDatabase;
|
||||
use crate::services::config::{DatabaseConfig, DbType};
|
||||
use anyhow::Result;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use crate::domain::valkey::database::ValkeyDatabase;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait Database: Send + Sync {
|
||||
fn file_extension(&self) -> &'static str;
|
||||
async fn ping(&self) -> Result<bool>;
|
||||
async fn backup(&self, backup_dir: &Path, is_test: Option<bool>) -> Result<PathBuf>;
|
||||
async fn restore(&self, restore_file: &Path, is_test: Option<bool>) -> Result<()>;
|
||||
async fn backup(&self, backup_dir: &Path) -> Result<PathBuf>;
|
||||
async fn restore(&self, restore_file: &Path) -> Result<()>;
|
||||
}
|
||||
|
||||
pub struct DatabaseFactory;
|
||||
@@ -32,7 +32,7 @@ impl DatabaseFactory {
|
||||
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
|
||||
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
|
||||
DbType::Redis => Arc::new(RedisDatabase::new(cfg)),
|
||||
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg))
|
||||
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,8 +47,7 @@ impl DatabaseFactory {
|
||||
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
|
||||
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
|
||||
DbType::Redis => Arc::new(RedisDatabase::new(cfg)),
|
||||
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg))
|
||||
|
||||
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+3
-4
@@ -1,8 +1,7 @@
|
||||
pub mod factory;
|
||||
pub mod postgres;
|
||||
pub mod mysql;
|
||||
mod mongodb;
|
||||
mod sqlite;
|
||||
pub mod mysql;
|
||||
pub mod postgres;
|
||||
mod redis;
|
||||
mod sqlite;
|
||||
mod valkey;
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ pub async fn run(
|
||||
|
||||
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
|
||||
let mongodump = select_mongo_path().join("mongodump");
|
||||
info!("{:?}", mongodump);
|
||||
let uri = get_mongo_uri(cfg.clone())?;
|
||||
|
||||
let output = Command::new(mongodump)
|
||||
|
||||
@@ -16,9 +16,11 @@ pub fn select_mongo_path() -> std::path::PathBuf {
|
||||
}
|
||||
|
||||
pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result<String> {
|
||||
|
||||
if cfg.username.is_empty() || cfg.password.is_empty() {
|
||||
Ok(format!("mongodb://{}:{}/{}", cfg.host, cfg.port, cfg.database))
|
||||
Ok(format!(
|
||||
"mongodb://{}:{}/{}",
|
||||
cfg.host, cfg.port, cfg.database
|
||||
))
|
||||
} else {
|
||||
Ok(format!(
|
||||
"mongodb://{}:{}@{}:{}/{}?authSource=admin",
|
||||
@@ -26,5 +28,3 @@ pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result<String> {
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -27,27 +27,17 @@ impl Database for MongoDatabase {
|
||||
ping::run(self.cfg.clone()).await
|
||||
}
|
||||
|
||||
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
|
||||
let test_mode = is_test.unwrap_or(false);
|
||||
if !test_mode {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
}
|
||||
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
|
||||
if !test_mode {
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
}
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
|
||||
async fn restore(&self, file: &Path, is_test: Option<bool>) -> Result<()> {
|
||||
let test_mode = is_test.unwrap_or(false);
|
||||
if !test_mode {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||
}
|
||||
async fn restore(&self, file: &Path) -> Result<()> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
|
||||
if !test_mode {
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
}
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
mod backup;
|
||||
mod restore;
|
||||
mod connection;
|
||||
pub mod database;
|
||||
mod ping;
|
||||
mod connection;
|
||||
mod restore;
|
||||
|
||||
@@ -24,7 +24,7 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
|
||||
error!("Full Error: {}", e);
|
||||
error!("Check you database network connectivity");
|
||||
error!("----------------------------------------");
|
||||
Err(anyhow::anyhow!("Ping failed for {}: {}", cfg.name, e))
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use crate::domain::mysql::connection::{server_version};
|
||||
use crate::domain::mysql::connection::server_version;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -1,14 +1,17 @@
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use std::process::Command;
|
||||
use anyhow::Result;
|
||||
use std::process::Command;
|
||||
|
||||
pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
|
||||
|
||||
let output = Command::new("mysql")
|
||||
.arg("--host").arg(&cfg.host)
|
||||
.arg("--port").arg(cfg.port.to_string())
|
||||
.arg("--user").arg(&cfg.username)
|
||||
.arg("-e").arg("SELECT VERSION();")
|
||||
.arg("--host")
|
||||
.arg(&cfg.host)
|
||||
.arg("--port")
|
||||
.arg(cfg.port.to_string())
|
||||
.arg("--user")
|
||||
.arg(&cfg.username)
|
||||
.arg("-e")
|
||||
.arg("SELECT VERSION();")
|
||||
.env("MYSQL_PWD", &cfg.password)
|
||||
.output()?;
|
||||
|
||||
@@ -19,11 +22,10 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
|
||||
|
||||
let version = String::from_utf8_lossy(&output.stdout)
|
||||
.lines()
|
||||
.nth(1) // skip column header
|
||||
.nth(1)
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.to_string();
|
||||
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,14 +1,11 @@
|
||||
use std::collections::HashMap;
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use std::path::{Path, PathBuf};
|
||||
use super::{
|
||||
backup,
|
||||
ping, restore,
|
||||
};
|
||||
use super::{backup, ping, restore};
|
||||
use crate::domain::factory::Database;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::utils::locks::{DbOpLock, FileLock};
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
pub struct MySQLDatabase {
|
||||
cfg: DatabaseConfig,
|
||||
@@ -36,28 +33,23 @@ impl Database for MySQLDatabase {
|
||||
ping::run(self.cfg.clone(), self.build_env().clone()).await
|
||||
}
|
||||
|
||||
|
||||
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
|
||||
let test_mode = is_test.unwrap_or(false);
|
||||
if !test_mode {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
}
|
||||
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.build_env().clone(), self.file_extension()).await;
|
||||
if !test_mode {
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
}
|
||||
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
let res = backup::run(
|
||||
self.cfg.clone(),
|
||||
dir.to_path_buf(),
|
||||
self.build_env().clone(),
|
||||
self.file_extension(),
|
||||
)
|
||||
.await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
|
||||
async fn restore(&self, file: &Path, is_test: Option<bool>) -> Result<()> {
|
||||
let test_mode = is_test.unwrap_or(false);
|
||||
if !test_mode {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||
}
|
||||
async fn restore(&self, file: &Path) -> Result<()> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
|
||||
if !test_mode {
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
}
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
pub mod backup;
|
||||
mod connection;
|
||||
pub mod database;
|
||||
mod restore;
|
||||
mod ping;
|
||||
mod connection;
|
||||
mod restore;
|
||||
|
||||
@@ -4,7 +4,6 @@ use tokio::process::Command;
|
||||
use tokio::time::{Duration, timeout};
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::Result<bool> {
|
||||
|
||||
let mut cmd = Command::new("mysqladmin");
|
||||
cmd.arg("--host")
|
||||
.arg(cfg.host)
|
||||
|
||||
@@ -11,7 +11,6 @@ pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
format: PostgresDumpFormat,
|
||||
backup_dir: PathBuf,
|
||||
is_test: Option<bool>
|
||||
) -> Result<PathBuf> {
|
||||
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||
debug!("Starting backup for database {}", cfg.name);
|
||||
@@ -27,8 +26,8 @@ pub async fn run(
|
||||
}
|
||||
};
|
||||
|
||||
let pg_dump = select_pg_path(&version, is_test).join("pg_dump");
|
||||
|
||||
let pg_dump = select_pg_path(&version).join("pg_dump");
|
||||
|
||||
debug!("Using pg_dump at {:?}", pg_dump);
|
||||
|
||||
match format {
|
||||
|
||||
@@ -28,14 +28,9 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
pub fn select_pg_path(version: &str, is_test: Option<bool>) -> std::path::PathBuf {
|
||||
pub fn select_pg_path(version: &str) -> std::path::PathBuf {
|
||||
let major = version.split('.').next().unwrap_or("17");
|
||||
|
||||
if is_test.unwrap_or(false) {
|
||||
format!("/usr/local/postgresql/{}/bin", major).into()
|
||||
} else {
|
||||
format!("/usr/lib/postgresql/{}/bin", major).into()
|
||||
}
|
||||
format!("/usr/lib/postgresql/{}/bin", major).into()
|
||||
}
|
||||
|
||||
pub async fn terminate_connections(cfg: &DatabaseConfig) -> Result<()> {
|
||||
|
||||
@@ -31,27 +31,17 @@ impl Database for PostgresDatabase {
|
||||
ping::run(self.cfg.clone()).await
|
||||
}
|
||||
|
||||
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
|
||||
let test_mode = is_test.unwrap_or(false);
|
||||
if !test_mode {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
}
|
||||
let res = backup::run(self.cfg.clone(), self.format, dir.to_path_buf(), is_test).await;
|
||||
if !test_mode {
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
}
|
||||
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
let res = backup::run(self.cfg.clone(), self.format, dir.to_path_buf()).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
|
||||
async fn restore(&self, file: &Path, is_test: Option<bool>) -> Result<()> {
|
||||
let test_mode = is_test.unwrap_or(false);
|
||||
if !test_mode {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||
}
|
||||
let res = restore::run(self.cfg.clone(), self.format, file.to_path_buf(), is_test).await;
|
||||
if !test_mode {
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
}
|
||||
async fn restore(&self, file: &Path) -> Result<()> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||
let res = restore::run(self.cfg.clone(), self.format, file.to_path_buf()).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,4 +2,4 @@
|
||||
pub enum PostgresDumpFormat {
|
||||
Fc,
|
||||
Fd,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
pub mod backup;
|
||||
pub mod database;
|
||||
mod restore;
|
||||
mod connection;
|
||||
pub mod database;
|
||||
mod format;
|
||||
mod ping;
|
||||
mod restore;
|
||||
|
||||
pub use connection::{detect_format_from_size, detect_format_from_file};
|
||||
pub use connection::{detect_format_from_file, detect_format_from_size};
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
use super::connection::connect;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
) -> anyhow::Result<bool> {
|
||||
pub async fn run(cfg: DatabaseConfig) -> anyhow::Result<bool> {
|
||||
Ok(connect(&cfg).await.is_ok())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,7 +11,6 @@ pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
format: PostgresDumpFormat,
|
||||
restore_file: PathBuf,
|
||||
is_test: Option<bool>,
|
||||
) -> Result<()> {
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
debug!("Starting restore for database {}", cfg.name);
|
||||
@@ -27,7 +26,7 @@ pub async fn run(
|
||||
}
|
||||
};
|
||||
|
||||
let pg_restore = select_pg_path(&version, is_test).join("pg_restore");
|
||||
let pg_restore = select_pg_path(&version).join("pg_restore");
|
||||
|
||||
debug!("Using pg_restore at {:?}", pg_restore);
|
||||
|
||||
|
||||
@@ -27,19 +27,14 @@ impl Database for RedisDatabase {
|
||||
ping::run(self.cfg.clone()).await
|
||||
}
|
||||
|
||||
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
|
||||
let test_mode = is_test.unwrap_or(false);
|
||||
if !test_mode {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
}
|
||||
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
|
||||
if !test_mode {
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
}
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
|
||||
async fn restore(&self, _file: &Path, _is_test: Option<bool>) -> Result<()> {
|
||||
async fn restore(&self, _file: &Path) -> Result<()> {
|
||||
bail!("Restore not supported for Redis databases")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
use tracing::{debug, info};
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use tokio::process::Command;
|
||||
use tokio::time::{timeout, Duration};
|
||||
use anyhow::{Result, Context};
|
||||
use tokio::time::{Duration, timeout};
|
||||
use tracing::{debug, info};
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
|
||||
let mut cmd = Command::new("redis-cli");
|
||||
@@ -23,7 +23,6 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
|
||||
|
||||
debug!("Command Ping: {:?}", cmd);
|
||||
|
||||
|
||||
let result = timeout(Duration::from_secs(10), cmd.output()).await;
|
||||
|
||||
match result {
|
||||
@@ -52,4 +51,4 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,5 +41,5 @@ pub async fn run(
|
||||
info!("SQLite backup completed for {}", cfg.name);
|
||||
Ok(file_path)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
.await?
|
||||
}
|
||||
|
||||
@@ -27,26 +27,16 @@ impl Database for SqliteDatabase {
|
||||
ping::run(self.cfg.clone()).await
|
||||
}
|
||||
|
||||
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
|
||||
let test_mode = is_test.unwrap_or(false);
|
||||
if !test_mode {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
}
|
||||
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
|
||||
if !test_mode {
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
}
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
async fn restore(&self, file: &Path, is_test: Option<bool>) -> Result<()> {
|
||||
let test_mode = is_test.unwrap_or(false);
|
||||
if !test_mode {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||
}
|
||||
async fn restore(&self, file: &Path) -> Result<()> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
|
||||
if !test_mode {
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
}
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
mod backup;
|
||||
mod restore;
|
||||
pub mod database;
|
||||
mod ping;
|
||||
pub mod database;
|
||||
mod restore;
|
||||
|
||||
@@ -27,19 +27,14 @@ impl Database for ValkeyDatabase {
|
||||
ping::run(self.cfg.clone()).await
|
||||
}
|
||||
|
||||
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
|
||||
let test_mode = is_test.unwrap_or(false);
|
||||
if !test_mode {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
}
|
||||
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
|
||||
if !test_mode {
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
}
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
|
||||
async fn restore(&self, _file: &Path, _is_test: Option<bool>) -> Result<()> {
|
||||
async fn restore(&self, _file: &Path) -> Result<()> {
|
||||
bail!("Restore not supported for Valkey databases")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
use tracing::{debug, info};
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use tokio::process::Command;
|
||||
use tokio::time::{timeout, Duration};
|
||||
use anyhow::{Result, Context};
|
||||
use tokio::time::{Duration, timeout};
|
||||
use tracing::{debug, info};
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
|
||||
let mut cmd = Command::new("valkey-cli");
|
||||
@@ -23,7 +23,6 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
|
||||
|
||||
debug!("Command Ping: {:?}", cmd);
|
||||
|
||||
|
||||
let result = timeout(Duration::from_secs(10), cmd.output()).await;
|
||||
|
||||
match result {
|
||||
@@ -52,4 +51,4 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+2
-3
@@ -3,19 +3,18 @@ mod domain;
|
||||
mod services;
|
||||
mod settings;
|
||||
mod tasks;
|
||||
mod utils;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
mod utils;
|
||||
|
||||
use crate::tasks::ping::ping_server;
|
||||
use crate::utils::locks::FileLock;
|
||||
use crate::utils::logging;
|
||||
use utils::redis_client;
|
||||
use utils::task_manager::scheduler;
|
||||
use crate::utils::logging;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
|
||||
logging::init_logger();
|
||||
|
||||
// Remove all locks on startup
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use crate::services::api::ApiError;
|
||||
use reqwest::{Client, Method};
|
||||
use serde::de::DeserializeOwned;
|
||||
use std::time::Duration;
|
||||
use crate::services::api::ApiError;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ApiClient {
|
||||
@@ -57,12 +57,7 @@ impl ApiClient {
|
||||
{
|
||||
let url = format!("{}{}", self.base_url, path);
|
||||
|
||||
let res = self
|
||||
.http
|
||||
.request(method, &url)
|
||||
.json(body)
|
||||
.send()
|
||||
.await?;
|
||||
let res = self.http.request(method, &url).json(body).send().await?;
|
||||
|
||||
let status = res.status();
|
||||
let body_text = res.text().await.unwrap_or_default();
|
||||
|
||||
@@ -23,7 +23,6 @@ pub struct BackupUpdateRequest {
|
||||
pub generated_id: String,
|
||||
}
|
||||
|
||||
|
||||
impl ApiClient {
|
||||
pub async fn backup_create(
|
||||
&self,
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
use crate::services::api::models::agent::backup::BackupUploadResponse;
|
||||
use crate::services::api::{ApiClient, ApiError};
|
||||
use anyhow::Result;
|
||||
use reqwest::Method;
|
||||
use serde::Serialize;
|
||||
use crate::services::api::models::agent::backup::BackupUploadResponse;
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct InitUploadRequest {
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
pub mod init;
|
||||
pub mod status;
|
||||
pub mod status;
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
pub mod status;
|
||||
pub mod backup;
|
||||
pub mod restore;
|
||||
pub mod restore;
|
||||
pub mod status;
|
||||
|
||||
@@ -31,6 +31,7 @@ impl ApiClient {
|
||||
let agent_id = agent_id.into();
|
||||
let path = format!("/agent/{}/status", agent_id);
|
||||
|
||||
self.request_with_body(Method::POST, path.as_str(), &body).await
|
||||
self.request_with_body(Method::POST, path.as_str(), &body)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
pub mod agent;
|
||||
|
||||
pub use agent::status;
|
||||
pub use agent::status;
|
||||
|
||||
@@ -11,10 +11,7 @@ pub enum ApiError {
|
||||
Serialization(#[from] serde_json::Error),
|
||||
|
||||
#[error("api error: status={status}, body={body}")]
|
||||
HttpResponse {
|
||||
status: StatusCode,
|
||||
body: String,
|
||||
},
|
||||
HttpResponse { status: StatusCode, body: String },
|
||||
|
||||
#[error("api returned unexpected response")]
|
||||
UnexpectedResponse,
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
pub mod client;
|
||||
pub mod endpoints;
|
||||
pub mod error;
|
||||
pub mod models;
|
||||
pub mod endpoints;
|
||||
|
||||
pub use client::ApiClient;
|
||||
pub use error::ApiError;
|
||||
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
pub mod status;
|
||||
pub mod backup;
|
||||
pub mod restore;
|
||||
pub mod restore;
|
||||
pub mod status;
|
||||
|
||||
@@ -4,4 +4,4 @@ use serde::{Deserialize, Serialize};
|
||||
pub struct ResultRestoreResponse {
|
||||
pub message: String,
|
||||
pub status: bool,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use crate::utils::deserializer::deserialize_snake_case;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use toml::Value;
|
||||
use crate::utils::deserializer::deserialize_snake_case;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct PingResult {
|
||||
|
||||
@@ -1,2 +1 @@
|
||||
pub mod agent;
|
||||
|
||||
|
||||
@@ -1,20 +1,14 @@
|
||||
use super::service::BackupService;
|
||||
use crate::utils::compress::compress_to_tar_gz_large;
|
||||
use std::path::PathBuf;
|
||||
use anyhow::Result;
|
||||
use std::path::PathBuf;
|
||||
|
||||
impl BackupService {
|
||||
|
||||
pub async fn compress_backup(
|
||||
&self,
|
||||
backup_file: Option<PathBuf>,
|
||||
) -> Result<PathBuf> {
|
||||
|
||||
let file = backup_file
|
||||
.ok_or_else(|| anyhow::anyhow!("No backup file generated"))?;
|
||||
pub async fn compress_backup(&self, backup_file: Option<PathBuf>) -> Result<PathBuf> {
|
||||
let file = backup_file.ok_or_else(|| anyhow::anyhow!("No backup file generated"))?;
|
||||
|
||||
let compression = compress_to_tar_gz_large(&file).await?;
|
||||
|
||||
Ok(compression.compressed_path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
use super::service::BackupService;
|
||||
use crate::services::config::DatabasesConfig;
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::services::config::DatabasesConfig;
|
||||
use crate::utils::common::BackupMethod;
|
||||
use tracing::error;
|
||||
|
||||
impl BackupService {
|
||||
|
||||
pub async fn dispatch(
|
||||
&self,
|
||||
generated_id: &String,
|
||||
@@ -14,7 +13,6 @@ impl BackupService {
|
||||
storages: &Vec<DatabaseStorage>,
|
||||
encrypt: bool,
|
||||
) {
|
||||
|
||||
let Some(cfg) = config
|
||||
.databases
|
||||
.iter()
|
||||
@@ -41,4 +39,4 @@ impl BackupService {
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,14 +1,13 @@
|
||||
use super::service::BackupService;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::utils::common::BackupMethod;
|
||||
use crate::utils::locks::FileLock;
|
||||
|
||||
use tempfile::TempDir;
|
||||
use anyhow::Result;
|
||||
use tempfile::TempDir;
|
||||
|
||||
impl BackupService {
|
||||
|
||||
pub async fn execute_backup(
|
||||
&self,
|
||||
generated_id: String,
|
||||
@@ -17,11 +16,10 @@ impl BackupService {
|
||||
storages: Vec<DatabaseStorage>,
|
||||
encrypt: bool,
|
||||
) -> Result<()> {
|
||||
|
||||
if FileLock::is_locked(&generated_id).await? {
|
||||
anyhow::bail!("backup already running");
|
||||
}
|
||||
|
||||
|
||||
let backup = self.create_backup_record(&generated_id, &method).await?;
|
||||
let backup_id = backup.backup.id;
|
||||
|
||||
@@ -46,4 +44,4 @@ impl BackupService {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,14 @@
|
||||
use super::service::BackupService;
|
||||
use crate::services::api::models::agent::backup::BackupResponse;
|
||||
use crate::utils::common::BackupMethod;
|
||||
use anyhow::{Result, anyhow};
|
||||
use crate::services::api::models::agent::backup::BackupResponse;
|
||||
|
||||
impl BackupService {
|
||||
|
||||
pub async fn create_backup_record(
|
||||
&self,
|
||||
generated_id: &str,
|
||||
method: &BackupMethod,
|
||||
) -> Result<BackupResponse> {
|
||||
|
||||
let response = self
|
||||
.ctx
|
||||
.api
|
||||
@@ -23,4 +21,4 @@ impl BackupService {
|
||||
|
||||
response.ok_or_else(|| anyhow!("backup_create returned empty response"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
pub mod service;
|
||||
pub mod compressor;
|
||||
pub mod dispatcher;
|
||||
pub mod executor;
|
||||
pub mod compressor;
|
||||
pub mod uploader;
|
||||
pub mod result;
|
||||
pub mod models;
|
||||
pub mod helpers;
|
||||
pub mod models;
|
||||
pub mod result;
|
||||
pub mod runner;
|
||||
pub mod service;
|
||||
pub mod uploader;
|
||||
|
||||
pub use service::BackupService;
|
||||
pub use service::BackupService;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::path::PathBuf;
|
||||
use crate::services::config::DbType;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BackupResult {
|
||||
@@ -19,4 +19,4 @@ pub struct UploadResult {
|
||||
pub error: Option<String>,
|
||||
pub remote_file_path: Option<String>,
|
||||
pub total_size: Option<u64>,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,12 +9,7 @@ use std::path::Path;
|
||||
use tracing::{error, info};
|
||||
|
||||
impl BackupService {
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
tmp_path: &Path,
|
||||
) -> Result<BackupResult> {
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, tmp_path: &Path) -> Result<BackupResult> {
|
||||
let db = DatabaseFactory::create_for_backup(cfg.clone()).await;
|
||||
|
||||
let generated_id = cfg.generated_id.clone();
|
||||
@@ -40,8 +35,7 @@ impl BackupService {
|
||||
});
|
||||
}
|
||||
|
||||
match db.backup(tmp_path, Some(false)).await {
|
||||
|
||||
match db.backup(tmp_path).await {
|
||||
Ok(file) => Ok(BackupResult {
|
||||
generated_id,
|
||||
db_type,
|
||||
@@ -67,4 +61,4 @@ impl BackupService {
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use std::sync::Arc;
|
||||
use crate::core::context::Context as CoreContext;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub struct BackupService {
|
||||
pub ctx: Arc<CoreContext>,
|
||||
@@ -9,4 +9,4 @@ impl BackupService {
|
||||
pub fn new(ctx: Arc<CoreContext>) -> Self {
|
||||
Self { ctx }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,15 @@
|
||||
use super::service::BackupService;
|
||||
use super::models::{BackupResult, UploadResult};
|
||||
use super::service::BackupService;
|
||||
|
||||
use crate::services::storage;
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::services::storage;
|
||||
use crate::utils::common::BackupMethod;
|
||||
|
||||
use futures::future::join_all;
|
||||
use anyhow::{Result, bail};
|
||||
use tracing::{info, error};
|
||||
use futures::future::join_all;
|
||||
use tracing::{error, info};
|
||||
|
||||
impl BackupService {
|
||||
|
||||
pub async fn upload(
|
||||
&self,
|
||||
result: BackupResult,
|
||||
@@ -19,7 +18,6 @@ impl BackupService {
|
||||
encrypt: bool,
|
||||
backup_id: &String,
|
||||
) -> Result<Vec<UploadResult>> {
|
||||
|
||||
if result.code.as_deref() == Some("backup_already_in_progress") {
|
||||
info!("Skipping send: backup already in progress");
|
||||
bail!("backup_already_in_progress");
|
||||
@@ -28,7 +26,6 @@ impl BackupService {
|
||||
let ctx = self.ctx.clone();
|
||||
|
||||
let futures = storages.into_iter().map(|storage| {
|
||||
|
||||
let ctx_clone = ctx.clone();
|
||||
let result_clone = result.clone();
|
||||
let provider = storage::get_provider(&storage);
|
||||
@@ -37,13 +34,16 @@ impl BackupService {
|
||||
let generated_id = result_clone.generated_id.clone();
|
||||
|
||||
async move {
|
||||
|
||||
info!("Uploading storage -> {:?} for {:?}", storage.provider, storage_id);
|
||||
info!(
|
||||
"Uploading storage -> {:?} for {:?}",
|
||||
storage.provider, storage_id
|
||||
);
|
||||
|
||||
/*
|
||||
INIT STEP
|
||||
*/
|
||||
let init = match ctx_clone.api
|
||||
let init = match ctx_clone
|
||||
.api
|
||||
.backup_upload_init(
|
||||
ctx_clone.edge_key.agent_id.clone(),
|
||||
generated_id.clone(),
|
||||
@@ -107,7 +107,11 @@ impl BackupService {
|
||||
)
|
||||
.await;
|
||||
|
||||
let status = if upload_result.success { "success" } else { "failed" };
|
||||
let status = if upload_result.success {
|
||||
"success"
|
||||
} else {
|
||||
"failed"
|
||||
};
|
||||
|
||||
if status != "success" {
|
||||
return upload_result;
|
||||
@@ -115,49 +119,48 @@ impl BackupService {
|
||||
|
||||
info!(
|
||||
"Storage {} uploaded to remote path {:?}",
|
||||
storage_id,
|
||||
upload_result.remote_file_path
|
||||
storage_id, upload_result.remote_file_path
|
||||
);
|
||||
|
||||
/*
|
||||
METADATA VALIDATION
|
||||
*/
|
||||
let (remote_path, total_size) = match (
|
||||
&upload_result.remote_file_path,
|
||||
upload_result.total_size,
|
||||
) {
|
||||
(Some(path), Some(size)) => (path.clone(), size),
|
||||
_ => {
|
||||
return UploadResult {
|
||||
storage_id,
|
||||
success: false,
|
||||
error: Some("remote_file_path or total_size missing".into()),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
};
|
||||
}
|
||||
};
|
||||
let (remote_path, total_size) =
|
||||
match (&upload_result.remote_file_path, upload_result.total_size) {
|
||||
(Some(path), Some(size)) => (path.clone(), size),
|
||||
_ => {
|
||||
return UploadResult {
|
||||
storage_id,
|
||||
success: false,
|
||||
error: Some("remote_file_path or total_size missing".into()),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
/*
|
||||
STATUS UPDATE
|
||||
*/
|
||||
match ctx_clone.api.backup_upload_status(
|
||||
ctx_clone.edge_key.agent_id.clone(),
|
||||
generated_id,
|
||||
backup_storage_id,
|
||||
status,
|
||||
remote_path,
|
||||
total_size,
|
||||
backup_id,
|
||||
).await {
|
||||
|
||||
match ctx_clone
|
||||
.api
|
||||
.backup_upload_status(
|
||||
ctx_clone.edge_key.agent_id.clone(),
|
||||
generated_id,
|
||||
backup_storage_id,
|
||||
status,
|
||||
remote_path,
|
||||
total_size,
|
||||
backup_id,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => upload_result,
|
||||
|
||||
Err(err) => {
|
||||
error!(
|
||||
"backup_upload_status failed (storage_id={}): {}",
|
||||
storage_id,
|
||||
err
|
||||
storage_id, err
|
||||
);
|
||||
|
||||
UploadResult {
|
||||
@@ -178,4 +181,4 @@ impl BackupService {
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+33
-11
@@ -20,7 +20,7 @@ pub enum DbType {
|
||||
MongoDB,
|
||||
Sqlite,
|
||||
Redis,
|
||||
Valkey
|
||||
Valkey,
|
||||
}
|
||||
|
||||
impl DbType {
|
||||
@@ -58,7 +58,6 @@ pub struct DatabasesConfig {
|
||||
pub databases: Vec<DatabaseConfig>,
|
||||
}
|
||||
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct InputDatabaseConfig {
|
||||
@@ -80,7 +79,6 @@ pub struct InputDatabasesConfig {
|
||||
pub databases: Vec<InputDatabaseConfig>,
|
||||
}
|
||||
|
||||
|
||||
pub struct ConfigService {
|
||||
ctx: Arc<Context>,
|
||||
}
|
||||
@@ -133,17 +131,27 @@ impl ConfigService {
|
||||
_ => return Err("Unsupported config file format. Use .json or .toml".to_string()),
|
||||
};
|
||||
|
||||
fn required<T: Clone>(opt: &Option<T>, db_name: &str, field_name: &str) -> Result<T, String> {
|
||||
fn required<T: Clone>(
|
||||
opt: &Option<T>,
|
||||
db_name: &str,
|
||||
field_name: &str,
|
||||
) -> Result<T, String> {
|
||||
match opt {
|
||||
Some(v) => Ok(v.clone()),
|
||||
None => {
|
||||
let msg = format!("Missing required field '{}' for database '{}'", field_name, db_name);
|
||||
let msg = format!(
|
||||
"Missing required field '{}' for database '{}'",
|
||||
field_name, db_name
|
||||
);
|
||||
Err(msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn optional<T: Clone>(opt: &Option<T>) -> T where T: Default {
|
||||
fn optional<T: Clone>(opt: &Option<T>) -> T
|
||||
where
|
||||
T: Default,
|
||||
{
|
||||
opt.clone().unwrap_or_default()
|
||||
}
|
||||
|
||||
@@ -155,28 +163,42 @@ impl ConfigService {
|
||||
}
|
||||
|
||||
let username = match db.db_type {
|
||||
DbType::Postgresql | DbType::Mysql | DbType::Mariadb => required(&db.username, &db.name, "username")?,
|
||||
DbType::Postgresql | DbType::Mysql | DbType::Mariadb => {
|
||||
required(&db.username, &db.name, "username")?
|
||||
}
|
||||
_ => optional(&db.username),
|
||||
};
|
||||
|
||||
let password = match db.db_type {
|
||||
DbType::Postgresql | DbType::Mysql | DbType::Mariadb => required(&db.password, &db.name, "password")?,
|
||||
DbType::Postgresql | DbType::Mysql | DbType::Mariadb => {
|
||||
required(&db.password, &db.name, "password")?
|
||||
}
|
||||
_ => optional(&db.password),
|
||||
};
|
||||
|
||||
let host = match db.db_type {
|
||||
DbType::Postgresql | DbType::Mysql | DbType::Mariadb | DbType::MongoDB | DbType::Redis | DbType::Valkey => required(&db.host, &db.name, "host")?,
|
||||
DbType::Postgresql
|
||||
| DbType::Mysql
|
||||
| DbType::Mariadb
|
||||
| DbType::MongoDB
|
||||
| DbType::Redis
|
||||
| DbType::Valkey => required(&db.host, &db.name, "host")?,
|
||||
DbType::Sqlite => optional(&db.host),
|
||||
};
|
||||
|
||||
let port = match db.db_type {
|
||||
DbType::Postgresql | DbType::Mysql | DbType::Mariadb | DbType::MongoDB | DbType::Redis | DbType::Valkey => required(&db.port, &db.name, "port")?,
|
||||
DbType::Postgresql
|
||||
| DbType::Mysql
|
||||
| DbType::Mariadb
|
||||
| DbType::MongoDB
|
||||
| DbType::Redis
|
||||
| DbType::Valkey => required(&db.port, &db.name, "port")?,
|
||||
DbType::Sqlite => db.port.unwrap_or(0),
|
||||
};
|
||||
|
||||
let database_name = match db.db_type {
|
||||
DbType::Sqlite | DbType::Redis | DbType::Valkey => optional(&db.database),
|
||||
_ => required(&db.database, &db.name, "database")?
|
||||
_ => required(&db.database, &db.name, "database")?,
|
||||
};
|
||||
|
||||
let path_val = match db.db_type {
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use crate::core::context::Context;
|
||||
use crate::services::api::models::agent::status::DatabaseStatus;
|
||||
use crate::utils::common::vec_to_option_json;
|
||||
use crate::utils::redis_client;
|
||||
use crate::utils::task_manager::cron::check_and_update_cron;
|
||||
use redis::aio::MultiplexedConnection;
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::Arc;
|
||||
use crate::services::api::models::agent::status::DatabaseStatus;
|
||||
|
||||
pub struct CronService {
|
||||
ctx: Arc<Context>,
|
||||
|
||||
+3
-3
@@ -1,7 +1,7 @@
|
||||
pub mod api;
|
||||
pub mod backup;
|
||||
pub mod config;
|
||||
pub mod cron;
|
||||
pub mod backup;
|
||||
pub mod restore;
|
||||
pub mod status;
|
||||
mod storage;
|
||||
pub mod api;
|
||||
pub mod status;
|
||||
@@ -7,13 +7,11 @@ use anyhow::Result;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
impl RestoreService {
|
||||
|
||||
pub async fn prepare_archive(
|
||||
&self,
|
||||
downloaded_file: PathBuf,
|
||||
tmp_path: &Path,
|
||||
) -> Result<PathBuf> {
|
||||
|
||||
let filename = downloaded_file
|
||||
.file_name()
|
||||
.unwrap()
|
||||
@@ -31,7 +29,6 @@ impl RestoreService {
|
||||
let mut archive = downloaded_file.clone();
|
||||
|
||||
if encrypted {
|
||||
|
||||
let new_name = filename.strip_suffix(".enc").unwrap();
|
||||
|
||||
let decrypted = tmp_path.join(new_name);
|
||||
@@ -41,7 +38,7 @@ impl RestoreService {
|
||||
decrypted.clone(),
|
||||
self.ctx.edge_key.master_key_b64.clone(),
|
||||
)
|
||||
.await?;
|
||||
.await?;
|
||||
|
||||
archive = decrypted;
|
||||
}
|
||||
@@ -58,4 +55,4 @@ impl RestoreService {
|
||||
Ok(archive)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
use super::service::RestoreService;
|
||||
use crate::services::config::DatabasesConfig;
|
||||
use crate::services::api::models::agent::status::DatabaseStatus;
|
||||
use crate::services::config::DatabasesConfig;
|
||||
|
||||
use tracing::error;
|
||||
|
||||
impl RestoreService {
|
||||
|
||||
pub async fn dispatch(&self, db: &DatabaseStatus, config: &DatabasesConfig) {
|
||||
|
||||
let Some(cfg) = config
|
||||
.databases
|
||||
.iter()
|
||||
@@ -29,14 +27,9 @@ impl RestoreService {
|
||||
let db_cfg = cfg.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
|
||||
if let Err(e) = service
|
||||
.execute_restore(db_cfg, file_to_restore)
|
||||
.await
|
||||
{
|
||||
if let Err(e) = service.execute_restore(db_cfg, file_to_restore).await {
|
||||
error!("Restore failed: {}", e);
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,18 +1,12 @@
|
||||
use super::service::RestoreService;
|
||||
|
||||
use reqwest::{Client, Url};
|
||||
use anyhow::Result;
|
||||
use reqwest::{Client, Url};
|
||||
use std::path::{Path, PathBuf};
|
||||
use tracing::info;
|
||||
|
||||
impl RestoreService {
|
||||
|
||||
pub async fn download_backup(
|
||||
&self,
|
||||
file_url: &str,
|
||||
tmp_path: &Path,
|
||||
) -> Result<PathBuf> {
|
||||
|
||||
pub async fn download_backup(&self, file_url: &str, tmp_path: &Path) -> Result<PathBuf> {
|
||||
let client = Client::new();
|
||||
|
||||
let response = client.get(file_url).send().await?;
|
||||
@@ -28,7 +22,6 @@ impl RestoreService {
|
||||
.and_then(|s| s.split("filename=").nth(1))
|
||||
.map(|f| f.trim_matches('"').to_string());
|
||||
|
||||
|
||||
let filename_from_url = Url::parse(file_url).ok().and_then(|u| {
|
||||
u.path_segments()?
|
||||
.last()
|
||||
@@ -50,4 +43,4 @@ impl RestoreService {
|
||||
|
||||
Ok(path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,17 +1,11 @@
|
||||
use super::service::RestoreService;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use tempfile::TempDir;
|
||||
use anyhow::Result;
|
||||
use tempfile::TempDir;
|
||||
use tracing::info;
|
||||
|
||||
impl RestoreService {
|
||||
|
||||
pub async fn execute_restore(
|
||||
&self,
|
||||
cfg: DatabaseConfig,
|
||||
file_url: String,
|
||||
) -> Result<()> {
|
||||
|
||||
pub async fn execute_restore(&self, cfg: DatabaseConfig, file_url: String) -> Result<()> {
|
||||
let temp_dir = TempDir::new()?;
|
||||
let tmp_path = temp_dir.path();
|
||||
|
||||
@@ -27,4 +21,4 @@ impl RestoreService {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
pub mod service;
|
||||
pub mod dispatcher;
|
||||
pub mod executor;
|
||||
pub mod downloader;
|
||||
pub mod archive;
|
||||
pub mod runner;
|
||||
pub mod result;
|
||||
pub mod dispatcher;
|
||||
pub mod downloader;
|
||||
pub mod executor;
|
||||
pub mod models;
|
||||
pub mod result;
|
||||
pub mod runner;
|
||||
pub mod service;
|
||||
|
||||
pub use service::RestoreService;
|
||||
pub use service::RestoreService;
|
||||
|
||||
@@ -5,4 +5,4 @@ pub struct RestoreResult {
|
||||
#[serde(rename = "generatedId")]
|
||||
pub generated_id: String,
|
||||
pub status: String,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,24 +1,25 @@
|
||||
use super::service::RestoreService;
|
||||
use super::models::RestoreResult;
|
||||
use super::service::RestoreService;
|
||||
|
||||
use tracing::{info, error};
|
||||
use tracing::{error, info};
|
||||
|
||||
impl RestoreService {
|
||||
pub async fn send_result(&self, result: RestoreResult) {
|
||||
|
||||
info!(
|
||||
"[RestoreService] DB: {} | Status: {}",
|
||||
result.generated_id, result.status
|
||||
);
|
||||
|
||||
match self.ctx
|
||||
match self
|
||||
.ctx
|
||||
.api
|
||||
.restore_result(
|
||||
self.ctx.edge_key.agent_id.clone(),
|
||||
&result.generated_id,
|
||||
&result.status,
|
||||
)
|
||||
.await {
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
info!("Restoration result sent successfully");
|
||||
}
|
||||
@@ -27,4 +28,4 @@ impl RestoreService {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,21 +1,19 @@
|
||||
use super::service::RestoreService;
|
||||
use super::models::RestoreResult;
|
||||
use super::service::RestoreService;
|
||||
|
||||
use crate::domain::factory::DatabaseFactory;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
|
||||
use anyhow::Result;
|
||||
use std::path::PathBuf;
|
||||
use tracing::{info, error};
|
||||
use tracing::{error, info};
|
||||
|
||||
impl RestoreService {
|
||||
|
||||
pub async fn run_restore(
|
||||
&self,
|
||||
cfg: DatabaseConfig,
|
||||
backup_file: PathBuf,
|
||||
) -> Result<RestoreResult> {
|
||||
|
||||
let generated_id = cfg.generated_id.clone();
|
||||
|
||||
let db = DatabaseFactory::create_for_restore(cfg.clone(), &backup_file).await;
|
||||
@@ -31,8 +29,7 @@ impl RestoreService {
|
||||
});
|
||||
}
|
||||
|
||||
match db.restore(&backup_file, Some(false)).await {
|
||||
|
||||
match db.restore(&backup_file).await {
|
||||
Ok(_) => Ok(RestoreResult {
|
||||
generated_id,
|
||||
status: "success".into(),
|
||||
@@ -48,4 +45,4 @@ impl RestoreService {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use std::sync::Arc;
|
||||
use crate::core::context::Context;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub struct RestoreService {
|
||||
pub ctx: Arc<Context>,
|
||||
@@ -9,4 +9,4 @@ impl RestoreService {
|
||||
pub fn new(ctx: Arc<Context>) -> Self {
|
||||
Self { ctx }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use crate::core::context::Context;
|
||||
use crate::services::api::endpoints::status::DatabasePayload;
|
||||
use crate::services::api::models::agent::status::PingResult;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::settings::CONFIG;
|
||||
use reqwest::Client;
|
||||
use std::error::Error;
|
||||
use std::sync::Arc;
|
||||
use crate::services::api::endpoints::status::DatabasePayload;
|
||||
use crate::services::api::models::agent::status::PingResult;
|
||||
|
||||
pub struct StatusService {
|
||||
ctx: Arc<Context>,
|
||||
@@ -35,7 +35,12 @@ impl StatusService {
|
||||
.collect();
|
||||
|
||||
let version_str = CONFIG.app_version.as_str();
|
||||
let result = self.ctx.api.agent_status(&edge_key.agent_id, &version_str, databases_payload).await?.unwrap();
|
||||
let result = self
|
||||
.ctx
|
||||
.api
|
||||
.agent_status(&edge_key.agent_id, &version_str, databases_payload)
|
||||
.await?
|
||||
.unwrap();
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
pub mod providers;
|
||||
|
||||
use crate::core::context::Context;
|
||||
use crate::utils::common::BackupMethod;
|
||||
use async_trait::async_trait;
|
||||
use providers::local;
|
||||
use providers::s3;
|
||||
use providers::google_drive;
|
||||
use std::sync::Arc;
|
||||
use tracing::{error, info};
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::services::backup::models::{BackupResult, UploadResult};
|
||||
use crate::utils::common::BackupMethod;
|
||||
use async_trait::async_trait;
|
||||
use providers::google_drive;
|
||||
use providers::local;
|
||||
use providers::s3;
|
||||
use std::sync::Arc;
|
||||
use tracing::{error, info};
|
||||
|
||||
#[async_trait]
|
||||
pub trait StorageProvider: Send + Sync {
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
use crate::services::storage::providers::google_drive::models::GoogleDriveProviderConfig;
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use bytes::Bytes;
|
||||
use futures::Stream;
|
||||
use futures::StreamExt;
|
||||
use oauth2::{
|
||||
AuthUrl, ClientId, ClientSecret, RefreshToken, TokenResponse, TokenUrl, basic::BasicClient,
|
||||
reqwest::Client as OAuth2ReqwestClient,
|
||||
};
|
||||
use reqwest::{Client as ReqwestClient, StatusCode};
|
||||
use serde_json::{Value, json};
|
||||
use futures::{Stream};
|
||||
use bytes::Bytes;
|
||||
use reqwest::{Client, header};
|
||||
use crate::services::storage::providers::google_drive::models::GoogleDriveProviderConfig;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
pub async fn get_google_drive_token(config: &GoogleDriveProviderConfig) -> Result<String> {
|
||||
let http_client = OAuth2ReqwestClient::new();
|
||||
@@ -34,7 +34,10 @@ pub async fn get_google_drive_token(config: &GoogleDriveProviderConfig) -> Resul
|
||||
Ok(token_result.access_token().secret().clone())
|
||||
}
|
||||
|
||||
pub async fn ensure_folder_path(config: &GoogleDriveProviderConfig, path_parts: &[&str]) -> Result<String> {
|
||||
pub async fn ensure_folder_path(
|
||||
config: &GoogleDriveProviderConfig,
|
||||
path_parts: &[&str],
|
||||
) -> Result<String> {
|
||||
if path_parts.is_empty() {
|
||||
return Ok(config.folder_id.clone());
|
||||
}
|
||||
@@ -152,7 +155,10 @@ pub async fn upload_stream_to_google_drive(
|
||||
|
||||
let folder_id = ensure_folder_path(config, folder_path).await?;
|
||||
|
||||
if find_file_by_name(config, file_name, &folder_id).await?.is_some() {
|
||||
if find_file_by_name(config, file_name, &folder_id)
|
||||
.await?
|
||||
.is_some()
|
||||
{
|
||||
return Err(anyhow::anyhow!("File already exists: {}", full_path));
|
||||
}
|
||||
|
||||
@@ -172,7 +178,7 @@ pub async fn upload_stream_to_google_drive(
|
||||
.post("https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable")
|
||||
.bearer_auth(&token)
|
||||
.header("X-Upload-Content-Type", mime)
|
||||
.header("X-Upload-Content-Length", total_size.to_string()) // Helps a lot
|
||||
.header("X-Upload-Content-Length", total_size.to_string()) // Helps a lot
|
||||
.json(&metadata)
|
||||
.send()
|
||||
.await
|
||||
@@ -237,12 +243,15 @@ pub async fn upload_stream_to_google_drive(
|
||||
.put(&upload_url)
|
||||
.header("Content-Range", &content_range)
|
||||
.header("Content-Length", chunk_bytes.len().to_string())
|
||||
.body(chunk_bytes.clone()) // clone is cheap if small; optimize later if needed
|
||||
.body(chunk_bytes.clone()) // clone is cheap if small; optimize later if needed
|
||||
.send()
|
||||
.await;
|
||||
|
||||
match res {
|
||||
Ok(resp) if resp.status().is_success() || resp.status() == StatusCode::PERMANENT_REDIRECT => {
|
||||
Ok(resp)
|
||||
if resp.status().is_success()
|
||||
|| resp.status() == StatusCode::PERMANENT_REDIRECT =>
|
||||
{
|
||||
// 200 or 308 = good
|
||||
uploaded += chunk_bytes.len() as u64;
|
||||
tracing::info!("Uploaded {}/{} bytes", uploaded, total_size);
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
mod helpers;
|
||||
mod models;
|
||||
mod models;
|
||||
|
||||
use crate::core::context::Context;
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::services::backup::models::{BackupResult, UploadResult};
|
||||
use crate::services::storage::StorageProvider;
|
||||
use crate::services::storage::providers::google_drive::helpers::upload_stream_to_google_drive;
|
||||
use crate::services::storage::providers::google_drive::models::GoogleDriveProviderConfig;
|
||||
use crate::utils::common::BackupMethod;
|
||||
use crate::utils::file::{full_file_name, full_file_path};
|
||||
use crate::utils::stream::build_stream;
|
||||
@@ -11,9 +14,6 @@ use async_trait::async_trait;
|
||||
use std::sync::Arc;
|
||||
use tokio::fs;
|
||||
use tracing::{error, info};
|
||||
use crate::services::backup::models::{BackupResult, UploadResult};
|
||||
use crate::services::storage::providers::google_drive::helpers::{upload_stream_to_google_drive};
|
||||
use crate::services::storage::providers::google_drive::models::GoogleDriveProviderConfig;
|
||||
|
||||
pub struct GoogleDriveProvider {}
|
||||
|
||||
@@ -53,13 +53,7 @@ impl StorageProvider for GoogleDriveProvider {
|
||||
|
||||
let encrypt = encrypt.unwrap_or(false);
|
||||
|
||||
let upload = match build_stream(
|
||||
&file_path,
|
||||
encrypt,
|
||||
&ctx.edge_key.master_key_b64
|
||||
)
|
||||
.await
|
||||
{
|
||||
let upload = match build_stream(&file_path, encrypt, &ctx.edge_key.master_key_b64).await {
|
||||
Ok(u) => u,
|
||||
Err(e) => {
|
||||
error!("Stream build failed: {}", e);
|
||||
@@ -86,7 +80,6 @@ impl StorageProvider for GoogleDriveProvider {
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
let file_name = full_file_name(encrypt);
|
||||
|
||||
info!("Uploading file {}", file_name);
|
||||
@@ -96,12 +89,13 @@ impl StorageProvider for GoogleDriveProvider {
|
||||
match upload_stream_to_google_drive(
|
||||
&config,
|
||||
&remote_file_path,
|
||||
upload.stream,
|
||||
upload.stream,
|
||||
total_size,
|
||||
Some("application/octet-stream"),
|
||||
).await {
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_file_id) => {
|
||||
|
||||
info!("Google Drive upload successful");
|
||||
|
||||
UploadResult {
|
||||
@@ -124,6 +118,5 @@ impl StorageProvider for GoogleDriveProvider {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,5 +7,3 @@ pub struct GoogleDriveProviderConfig {
|
||||
pub refresh_token: String,
|
||||
pub folder_id: String,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use crate::core::context::Context;
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::services::backup::models::{BackupResult, UploadResult};
|
||||
use crate::services::storage::StorageProvider;
|
||||
use crate::utils::common::BackupMethod;
|
||||
use crate::utils::file::{full_file_name, full_file_path};
|
||||
@@ -10,7 +11,6 @@ use reqwest::header::{HeaderMap, HeaderValue};
|
||||
use std::sync::Arc;
|
||||
use tokio::fs;
|
||||
use tracing::error;
|
||||
use crate::services::backup::models::{BackupResult, UploadResult};
|
||||
|
||||
pub struct LocalProvider;
|
||||
|
||||
@@ -53,13 +53,7 @@ impl StorageProvider for LocalProvider {
|
||||
}
|
||||
};
|
||||
|
||||
let upload = match build_stream(
|
||||
&file_path,
|
||||
encrypt,
|
||||
&ctx.edge_key.master_key_b64
|
||||
)
|
||||
.await
|
||||
{
|
||||
let upload = match build_stream(&file_path, encrypt, &ctx.edge_key.master_key_b64).await {
|
||||
Ok(u) => u,
|
||||
Err(e) => {
|
||||
error!("Stream build failed: {}", e);
|
||||
@@ -68,7 +62,7 @@ impl StorageProvider for LocalProvider {
|
||||
success: false,
|
||||
error: Some(e.to_string()),
|
||||
remote_file_path: None,
|
||||
total_size: None
|
||||
total_size: None,
|
||||
};
|
||||
}
|
||||
};
|
||||
@@ -76,7 +70,10 @@ impl StorageProvider for LocalProvider {
|
||||
let mut extra_headers = HeaderMap::new();
|
||||
|
||||
extra_headers.insert("X-File-Name", HeaderValue::from_str(&file_name).unwrap());
|
||||
extra_headers.insert("X-File-Size", HeaderValue::from_str(&total_size.to_string()).unwrap());
|
||||
extra_headers.insert(
|
||||
"X-File-Size",
|
||||
HeaderValue::from_str(&total_size.to_string()).unwrap(),
|
||||
);
|
||||
extra_headers.insert(
|
||||
"X-File-Path",
|
||||
HeaderValue::from_str(&remote_file_path).unwrap(),
|
||||
@@ -90,10 +87,17 @@ impl StorageProvider for LocalProvider {
|
||||
"X-Method",
|
||||
HeaderValue::from_str(&method.to_string()).unwrap(),
|
||||
);
|
||||
|
||||
|
||||
let tus_endpoint = format!("{}/tus/files", ctx.edge_key.server_url);
|
||||
|
||||
match upload_to_tus_stream_with_headers(upload.stream, &tus_endpoint, extra_headers, total_size).await {
|
||||
match upload_to_tus_stream_with_headers(
|
||||
upload.stream,
|
||||
&tus_endpoint,
|
||||
extra_headers,
|
||||
total_size,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => UploadResult {
|
||||
storage_id: storage.id.clone(),
|
||||
success: true,
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
pub mod google_drive;
|
||||
pub mod local;
|
||||
pub mod s3;
|
||||
pub mod google_drive;
|
||||
@@ -2,26 +2,26 @@ mod models;
|
||||
|
||||
use crate::core::context::Context;
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::services::backup::models::{BackupResult, UploadResult};
|
||||
use crate::services::storage::StorageProvider;
|
||||
use crate::services::storage::providers::s3::models::S3ProviderConfig;
|
||||
use crate::utils::common::BackupMethod;
|
||||
use crate::utils::file::{full_file_name, full_file_path};
|
||||
use crate::utils::stream::build_stream;
|
||||
use async_trait::async_trait;
|
||||
use aws_config::retry::RetryConfig;
|
||||
use aws_sdk_s3 as s3;
|
||||
use aws_sdk_s3::config::BehaviorVersion;
|
||||
use aws_sdk_s3::config::Region;
|
||||
use aws_sdk_s3::config::retry::ReconnectMode;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use futures::StreamExt;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use aws_config::retry::RetryConfig;
|
||||
use aws_sdk_s3::config::retry::ReconnectMode;
|
||||
use tokio::fs;
|
||||
use tracing::{error, info};
|
||||
use crate::services::backup::models::{BackupResult, UploadResult};
|
||||
|
||||
pub struct S3Provider {}
|
||||
|
||||
@@ -97,15 +97,28 @@ impl StorageProvider for S3Provider {
|
||||
);
|
||||
|
||||
let region = Region::new(config.region.clone().unwrap_or("us-east-1".to_string()));
|
||||
|
||||
|
||||
let endpoint = if let Some(port) = &config.port {
|
||||
if port.trim().is_empty() {
|
||||
format!("{}://{}", if config.ssl { "https" } else { "http" }, config.end_point_url)
|
||||
format!(
|
||||
"{}://{}",
|
||||
if config.ssl { "https" } else { "http" },
|
||||
config.end_point_url
|
||||
)
|
||||
} else {
|
||||
format!("{}://{}:{}", if config.ssl { "https" } else { "http" }, config.end_point_url, port)
|
||||
format!(
|
||||
"{}://{}:{}",
|
||||
if config.ssl { "https" } else { "http" },
|
||||
config.end_point_url,
|
||||
port
|
||||
)
|
||||
}
|
||||
} else {
|
||||
format!("{}://{}", if config.ssl { "https" } else { "http" }, config.end_point_url)
|
||||
format!(
|
||||
"{}://{}",
|
||||
if config.ssl { "https" } else { "http" },
|
||||
config.end_point_url
|
||||
)
|
||||
};
|
||||
|
||||
info!("S3 endpoint to {}", &endpoint);
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use crate::utils::deserializer::string_or_number_to_string;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
pub struct S3ProviderConfig {
|
||||
@@ -12,4 +12,3 @@ pub struct S3ProviderConfig {
|
||||
#[serde(default, deserialize_with = "string_or_number_to_string")]
|
||||
pub port: Option<String>,
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
pub mod ping;
|
||||
pub mod ping;
|
||||
|
||||
+1
-1
@@ -21,4 +21,4 @@ pub async fn ping_server() {
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(CONFIG.pooling as u64)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
use crate::domain::factory::DatabaseFactory;
|
||||
use crate::services::config::{DatabaseConfig, DbType};
|
||||
use crate::tests::init_tracing_for_test;
|
||||
use crate::utils::compress::{compress_to_tar_gz_large, decompress_large_tar_gz};
|
||||
use oauth2::url;
|
||||
use std::path::PathBuf;
|
||||
use tempfile::TempDir;
|
||||
use testcontainers::runners::AsyncRunner;
|
||||
use testcontainers::{ContainerAsync, ImageExt};
|
||||
use testcontainers_modules::mariadb::Mariadb;
|
||||
use tracing::{error, info};
|
||||
use url::Host;
|
||||
|
||||
async fn create_config() -> (ContainerAsync<Mariadb>, DatabaseConfig) {
|
||||
let container = Mariadb::default().with_tag("11.3").start().await.unwrap();
|
||||
|
||||
let host = container
|
||||
.get_host()
|
||||
.await
|
||||
.unwrap_or(Host::parse("127.0.0.1").unwrap());
|
||||
|
||||
let port = container.get_host_port_ipv4(3306).await.unwrap_or(3306);
|
||||
|
||||
let config = DatabaseConfig {
|
||||
name: "Test MariaDB".to_string(),
|
||||
database: "test".to_string(),
|
||||
db_type: DbType::Mariadb,
|
||||
username: "root".to_string(),
|
||||
password: "".to_string(),
|
||||
port,
|
||||
host: host.to_string(),
|
||||
generated_id: "3c4b4eb4-c2c6-4bde-a423-ee1385dcf6d2".to_string(),
|
||||
path: "".to_string(),
|
||||
};
|
||||
|
||||
(container, config)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mariadb_ping_test() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let db = DatabaseFactory::create_for_backup(config.clone()).await;
|
||||
let reachable = db.ping().await.unwrap_or(false);
|
||||
|
||||
assert!(reachable);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mariadb_backup_restore_test() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let backup_path = temp_dir.path();
|
||||
|
||||
let db = DatabaseFactory::create_for_backup(config.clone()).await;
|
||||
let file_path = db.backup(backup_path).await.unwrap();
|
||||
|
||||
assert!(file_path.is_file());
|
||||
|
||||
let compression = compress_to_tar_gz_large(&file_path).await.unwrap();
|
||||
assert!(compression.compressed_path.is_file());
|
||||
|
||||
let files = decompress_large_tar_gz(compression.compressed_path.as_path(), temp_dir.path())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let backup_file: PathBuf = if files.len() == 1 {
|
||||
files[0].clone()
|
||||
} else {
|
||||
"".into()
|
||||
};
|
||||
|
||||
let db = DatabaseFactory::create_for_restore(config.clone(), &backup_file).await;
|
||||
let reachable = db.ping().await.unwrap_or(false);
|
||||
|
||||
info!("Reachable: {}", reachable);
|
||||
assert!(reachable);
|
||||
|
||||
match db.restore(&backup_file).await {
|
||||
Ok(_) => {
|
||||
info!("Restore succeeded for {}", config.generated_id);
|
||||
assert!(true)
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Restore failed for {}: {:?}", config.generated_id, e);
|
||||
assert!(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,6 @@
|
||||
mod mariadb;
|
||||
mod mongodb;
|
||||
mod mysql;
|
||||
mod postgres;
|
||||
mod redis;
|
||||
mod valkey;
|
||||
mod valkey;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user