mirror of
https://github.com/Portabase/agent.git
synced 2026-09-11 02:27:10 +00:00
Compare commits
103 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4df3fe2d0c | |||
| ec7715ca26 | |||
| 1d15f40662 | |||
| 324a2ea3f2 | |||
| 9ec92af6c7 | |||
| 1dafdca2a0 | |||
| d12817a960 | |||
| 608b82a254 | |||
| bbecca92f9 | |||
| b2b2733c4d | |||
| 2c257c5a58 | |||
| c24c0d7058 | |||
| 34f353b68f | |||
| ee51996401 | |||
| 694b463787 | |||
| aed1e86956 | |||
| 6388aff1e3 | |||
| 39a77b18a4 | |||
| e62167182e | |||
| 4be54a614e | |||
| c5eaa5086c | |||
| 610d443afc | |||
| 16a37e033e | |||
| 29ca5a317b | |||
| 01f5d34c96 | |||
| 0b7b7ef59b | |||
| 038c5523b7 | |||
| ea010ce713 | |||
| 8897568281 | |||
| 66cad4e12e | |||
| 221ed4e7e1 | |||
| 84735ac399 | |||
| f7639de096 | |||
| e504d09cb3 | |||
| 0363b300a4 | |||
| bf6e7d41ab | |||
| 2c7065ae95 | |||
| 1fece577cc | |||
| 273475fa3b | |||
| 628c017584 | |||
| f02218708a | |||
| 16152328b0 | |||
| 7328827435 | |||
| 348eaac81b | |||
| 0f6c93ecd0 | |||
| befec0deae | |||
| 2052ab0ff5 | |||
| 461e92d67e | |||
| 23678bf2d6 | |||
| 02105a8171 | |||
| f94656a39b | |||
| b94ff4e987 | |||
| dc32c442f3 | |||
| 1453851555 | |||
| 185a7ee612 | |||
| d8328f60ca | |||
| 82c7122790 | |||
| fba72f5c26 | |||
| b537980c71 | |||
| 0261a07fc2 | |||
| c840f91680 | |||
| 2d52eccbb2 | |||
| f9bf481c79 | |||
| 3c853ee7ca | |||
| 45d791358c | |||
| 0b915d87c9 | |||
| 531a25f292 | |||
| 41e2b197cb | |||
| f149363b12 | |||
| aa3ff39bb3 | |||
| 740ee43038 | |||
| 1fccc0bc19 | |||
| 2c15ea64ae | |||
| 18bfdf3b7e | |||
| db26910679 | |||
| eac2853b83 | |||
| 8690a15888 | |||
| a4bbb37a4e | |||
| ac8f7fd8d8 | |||
| c6baa8fdc3 | |||
| 14068846b9 | |||
| 6edf2890f1 | |||
| 5f579690ff | |||
| bfe63d7fd3 | |||
| 0a422448b4 | |||
| 14a6c9ab0a | |||
| 5c35375df7 | |||
| 86027fd6bd | |||
| 61c7224104 | |||
| d7958e03b9 | |||
| 8eab97a3d6 | |||
| 25c95108f0 | |||
| f37e4d2de9 | |||
| b269d98d3c | |||
| a1aa66ff3f | |||
| d4abcbcebc | |||
| 2d44b62844 | |||
| 092f760431 | |||
| 2c5c805308 | |||
| 38a3274c43 | |||
| bb45c1961f | |||
| ee7016d1fa | |||
| c46c301371 |
@@ -60,7 +60,7 @@ representative at an online or offline event.
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
||||
reported to the community leaders responsible for enforcement at
|
||||
contact@soluce-technologies.com.
|
||||
contact@portabase.io.
|
||||
All complaints will be reviewed and investigated promptly and fairly.
|
||||
|
||||
All community leaders are obligated to respect the privacy and security of the
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ We take security seriously and aim to support the following versions of the proj
|
||||
If you discover a security vulnerability in this project, we appreciate your help in disclosing it responsibly.
|
||||
|
||||
1. **Contact Us**
|
||||
Please report the vulnerability by emailing **[contact@soluce-technologies.com](mailto:contact@soluce-technologies.com)**. Include the following details:
|
||||
Please report the vulnerability by emailing **[contact@portabase.io](mailto:contact@portabase.io)**. Include the following details:
|
||||
- A detailed description of the issue.
|
||||
- Steps to reproduce the vulnerability (if applicable).
|
||||
- Any potential impacts or risks.
|
||||
|
||||
+20
-102
@@ -1,86 +1,3 @@
|
||||
#name: Codecov Rust
|
||||
#
|
||||
#on:
|
||||
# push:
|
||||
# branches: ["main"]
|
||||
# pull_request:
|
||||
# branches: ["main"]
|
||||
#
|
||||
#env:
|
||||
# CARGO_TERM_COLOR: always
|
||||
#
|
||||
#jobs:
|
||||
# coverage:
|
||||
# runs-on: ubuntu-latest
|
||||
# steps:
|
||||
# - uses: actions/checkout@v4
|
||||
#
|
||||
# - name: Install Rust toolchain
|
||||
# uses: dtolnay/rust-toolchain@stable
|
||||
# with:
|
||||
# components: llvm-tools-preview
|
||||
#
|
||||
# - name: Build test image (with grcov included)
|
||||
# run: docker compose -f docker-compose.test.yml build agent-test
|
||||
#
|
||||
# - name: Run tests in container
|
||||
# env:
|
||||
# CARGO_TARGET_DIR: /app/target
|
||||
# CARGO_INCREMENTAL: 0
|
||||
# RUSTFLAGS: "-C instrument-coverage -C link-dead-code"
|
||||
# LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw"
|
||||
# run: |
|
||||
# docker compose -f docker-compose.test.yml run \
|
||||
# -e CARGO_TARGET_DIR \
|
||||
# -e CARGO_INCREMENTAL \
|
||||
# -e RUSTFLAGS \
|
||||
# -e LLVM_PROFILE_FILE \
|
||||
# agent-test bash -c "cargo clean && cargo test --verbose && sync"
|
||||
#
|
||||
# - name: Verify profraw files exist
|
||||
# run: |
|
||||
# docker compose -f docker-compose.test.yml run agent-test \
|
||||
# find /app/coverage -type f -name "*.profraw" | wc -l || true
|
||||
#
|
||||
# - name: Generate coverage report inside container
|
||||
# run: |
|
||||
# docker compose -f docker-compose.test.yml run agent-test bash -c "
|
||||
# rustup component add llvm-tools &&
|
||||
# grcov /app/coverage \
|
||||
# --binary-path /app/target/debug \
|
||||
# -s /app \
|
||||
# --llvm \
|
||||
# -t lcov \
|
||||
# --branch \
|
||||
# --ignore-not-existing \
|
||||
# --ignore '/app/target/*' \
|
||||
# --ignore '/*' \
|
||||
# -o /app/lcov.info
|
||||
# "
|
||||
#
|
||||
# - name: Copy lcov.info from container to host
|
||||
# run: |
|
||||
# docker compose -f docker-compose.test.yml cp agent-test:/app/lcov.info ./lcov.info
|
||||
#
|
||||
# - name: Remove container
|
||||
# run: |
|
||||
# docker rm agent-test-run
|
||||
#
|
||||
# - name: Show basic coverage report info (debug)
|
||||
# run: |
|
||||
# echo "lcov.info size:" $(wc -c ./lcov.info | awk '{print $1}')
|
||||
# head -n 30 ./lcov.info || true
|
||||
#
|
||||
# - name: Upload coverage to Codecov
|
||||
# uses: codecov/codecov-action@v5
|
||||
# with:
|
||||
# files: ./lcov.info
|
||||
# flags: unittests
|
||||
# name: rust-unit-coverage
|
||||
# verbose: true
|
||||
# fail_ci_if_error: true
|
||||
# env:
|
||||
# CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
name: Codecov Rust
|
||||
|
||||
on:
|
||||
@@ -99,15 +16,11 @@ env:
|
||||
jobs:
|
||||
coverage:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
components: llvm-tools-preview
|
||||
|
||||
- name: Build test image (with grcov included)
|
||||
- name: Build test image
|
||||
run: docker compose -f docker-compose.test.yml build agent-test
|
||||
|
||||
- name: Start agent-test container
|
||||
@@ -115,28 +28,32 @@ jobs:
|
||||
|
||||
- name: Run tests inside container
|
||||
run: |
|
||||
docker compose -f docker-compose.test.yml exec \
|
||||
docker compose -f docker-compose.test.yml exec -T \
|
||||
-e CARGO_TARGET_DIR \
|
||||
-e CARGO_INCREMENTAL \
|
||||
-e RUSTFLAGS \
|
||||
-e LLVM_PROFILE_FILE \
|
||||
agent-test bash -c "
|
||||
cargo clean &&
|
||||
cargo test --verbose &&
|
||||
mkdir -p /app/coverage &&
|
||||
rm -rf /app/target/* /app/coverage/* &&
|
||||
cargo test --verbose -- --test-threads=2 &&
|
||||
sync
|
||||
"
|
||||
|
||||
- name: Verify profraw files exist
|
||||
run: |
|
||||
docker compose -f docker-compose.test.yml exec \
|
||||
-e LLVM_PROFILE_FILE \
|
||||
agent-test find /app/coverage -type f -name "*.profraw" | wc -l || true
|
||||
docker compose -f docker-compose.test.yml exec -T \
|
||||
agent-test bash -c '
|
||||
count=$(find /app/coverage -type f -name "*.profraw" | wc -l)
|
||||
echo "profraw files: $count"
|
||||
test "$count" -gt 0
|
||||
'
|
||||
|
||||
- name: Generate coverage report inside container
|
||||
run: |
|
||||
docker compose -f docker-compose.test.yml exec \
|
||||
docker compose -f docker-compose.test.yml exec -T \
|
||||
agent-test bash -c "
|
||||
rustup component add llvm-tools &&
|
||||
rustup component add llvm-tools-preview &&
|
||||
grcov /app/coverage \
|
||||
--binary-path /app/target/debug \
|
||||
-s /app \
|
||||
@@ -145,17 +62,17 @@ jobs:
|
||||
--branch \
|
||||
--ignore-not-existing \
|
||||
--ignore '/app/target/*' \
|
||||
--ignore '/*' \
|
||||
-o /app/lcov.info
|
||||
"
|
||||
|
||||
- name: Copy lcov.info from container to host
|
||||
run: docker compose -f docker-compose.test.yml cp agent-test:/app/lcov.info ./lcov.info
|
||||
|
||||
- name: Show basic coverage report info (debug)
|
||||
- name: Show basic coverage report info
|
||||
run: |
|
||||
echo "lcov.info size:" $(wc -c ./lcov.info | awk '{print $1}')
|
||||
head -n 30 ./lcov.info || true
|
||||
test -s ./lcov.info
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v5
|
||||
@@ -165,8 +82,9 @@ jobs:
|
||||
name: rust-unit-coverage
|
||||
verbose: true
|
||||
fail_ci_if_error: true
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
use_pypi: true
|
||||
|
||||
- name: Stop and remove container
|
||||
run: docker compose -f docker-compose.test.yml down
|
||||
if: always()
|
||||
run: docker compose -f docker-compose.test.yml down --volumes
|
||||
@@ -0,0 +1,96 @@
|
||||
name: Build Windows release
|
||||
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
tags:
|
||||
- '[0-9]+.[0-9]+.[0-9]+'
|
||||
branches:
|
||||
- main
|
||||
- master
|
||||
|
||||
jobs:
|
||||
build-windows:
|
||||
runs-on: windows-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Rust toolchain (MSVC)
|
||||
uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
toolchain: stable-x86_64-pc-windows-msvc
|
||||
profile: minimal
|
||||
override: true
|
||||
|
||||
- name: Install vcpkg and OpenSSL (x64)
|
||||
shell: pwsh
|
||||
run: |
|
||||
# Install vcpkg and the prebuilt OpenSSL package
|
||||
git clone https://github.com/microsoft/vcpkg C:\vcpkg
|
||||
C:\vcpkg\bootstrap-vcpkg.bat
|
||||
C:\vcpkg\vcpkg install openssl:x64-windows
|
||||
# Export variables for subsequent steps
|
||||
'VCPKG_ROOT=C:\vcpkg' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
'OPENSSL_DIR=C:\vcpkg\installed\x64-windows' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
|
||||
- name: Build (cargo release)
|
||||
shell: pwsh
|
||||
env:
|
||||
# Cargo / openssl-sys will pick up OPENSSL_DIR from the environment
|
||||
OPENSSL_DIR: ${{ env.OPENSSL_DIR }}
|
||||
run: |
|
||||
# Ensure the environment variable is present for this step
|
||||
if (-Not $env:OPENSSL_DIR) { Write-Host "OPENSSL_DIR not set, printing env for debugging"; Get-ChildItem Env: | ForEach-Object { Write-Host $_ } }
|
||||
# Build the declared bin target explicitly (Cargo.toml [[bin]] name = "app")
|
||||
cargo build --release --bin app
|
||||
|
||||
- name: Prepare artifact zip
|
||||
id: prepare_artifact
|
||||
shell: pwsh
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.ref_name }}
|
||||
run: |
|
||||
$tag = $env:RELEASE_TAG
|
||||
if (-not $tag) { $tag = $env:GITHUB_SHA }
|
||||
|
||||
# Package the declared bin target deterministically (Cargo.toml [[bin]] name = "app")
|
||||
$exe = "target\release\app.exe"
|
||||
if (-not (Test-Path $exe)) { Write-Error "Built binary $exe not found in target/release"; exit 1 }
|
||||
|
||||
$outDir = "artifact"
|
||||
New-Item -ItemType Directory -Path $outDir -Force | Out-Null
|
||||
# Ship under the package name, not the internal bin name "app"
|
||||
Copy-Item -Path $exe -Destination "$outDir\portabase-agent.exe"
|
||||
|
||||
$zipName = "windows-release-$tag.zip"
|
||||
if (Test-Path $zipName) { Remove-Item $zipName }
|
||||
Compress-Archive -Path "$outDir\*" -DestinationPath $zipName -Force
|
||||
Write-Host "ZIP=$zipName"
|
||||
Write-Output "zip=$zipName" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
|
||||
|
||||
- name: Upload build artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: windows-release
|
||||
path: windows-release-*.zip
|
||||
|
||||
- name: Create GitHub Release
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
id: create_release
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
tag_name: ${{ github.ref_name }}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload release asset
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: actions/upload-release-asset@v1
|
||||
with:
|
||||
upload_url: ${{ steps.create_release.outputs.upload_url }}
|
||||
asset_path: windows-release-${{ github.ref_name }}.zip
|
||||
asset_name: windows-release-${{ github.ref_name }}.zip
|
||||
asset_content_type: application/zip
|
||||
+5
-1
@@ -3,4 +3,8 @@
|
||||
/src/data/
|
||||
|
||||
.DS_Store
|
||||
.env
|
||||
.env
|
||||
|
||||
.claude
|
||||
|
||||
/docs
|
||||
|
||||
+1
-1
@@ -27,5 +27,5 @@ keywords:
|
||||
- self-hosted
|
||||
- portabase
|
||||
license: Apache-2.0
|
||||
version: 1.7.0
|
||||
version: 1.16.2
|
||||
date-released: '2026-02-24'
|
||||
|
||||
Generated
+762
-88
File diff suppressed because it is too large
Load Diff
+8
-2
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "portabase-agent"
|
||||
version = "1.7.0"
|
||||
version = "1.16.2"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
@@ -35,9 +35,14 @@ rand = "0.9.2"
|
||||
bytes = "1.11.0"
|
||||
async-stream = "0.3.6"
|
||||
uuid = { version = "1.20.0", features = ["v4"] }
|
||||
tokio-util = "0.7.18"
|
||||
tokio-util = { version = "0.7.18", features = ["compat", "io"] }
|
||||
tiberius = { version = "0.12", default-features = false, features = ["rustls", "chrono"] }
|
||||
aws-config = "1.8.13"
|
||||
aws-sdk-s3 = { version = "1.122.0", features = ["behavior-version-latest"] }
|
||||
azure_core = "1.0.0"
|
||||
azure_storage_blob = "1.0.0"
|
||||
google-cloud-storage = "1.15"
|
||||
google-cloud-auth = "1.13"
|
||||
async-compression = { version = "0.4.37", features = ["tokio", "gzip"] }
|
||||
tokio-tar = "0.3.1"
|
||||
oauth2 = "5.0.0"
|
||||
@@ -52,6 +57,7 @@ testcontainers = "0.27.1"
|
||||
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "valkey", "mysql", "mariadb", "mongo"] }
|
||||
postgres = "0.19.12"
|
||||
url = "2.5.8"
|
||||
bollard = "0.20.0"
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
include .env
|
||||
export $(shell sed 's/=.*//' .env)
|
||||
CLUSTER_SCRIPT=docker/entrypoints/app-dev-entrypoint.sh
|
||||
|
||||
.PHONY: seed-mongo seed-mysql seed-postgres
|
||||
|
||||
up:
|
||||
@bash $(CLUSTER_SCRIPT)
|
||||
|
||||
seed-mongo:
|
||||
@echo "Seeding MongoDB..."
|
||||
bash ./scripts/mongo/seed-mongo.sh
|
||||
|
||||
seed-mongo-auth:
|
||||
@echo "Seeding MongoDB with auth..."
|
||||
bash ./scripts/mongo/seed-mongo.sh auth
|
||||
|
||||
seed-mysql:
|
||||
@echo "Seeding MySQL..."
|
||||
mysql -h 127.0.0.1 -P "$$MYSQL_PORT" -u "$$MYSQL_USER" -p"$$MYSQL_PASSWORD" "$$MYSQL_DB" < ./scripts/mysql/seed-mysql.sql
|
||||
|
||||
seed-mysql-1gb:
|
||||
@echo "Seeding MySQL..."
|
||||
mysql -h 127.0.0.1 -P "$$MYSQL_PORT" -u "$$MYSQL_USER" -p"$$MYSQL_PASSWORD" "$$MYSQL_DB" < ./scripts/mysql/seed-1gb.sql
|
||||
|
||||
|
||||
seed-postgres:
|
||||
@echo "Seeding Postgres..."
|
||||
docker exec -i -e PGPASSWORD=$$PG_PASSWORD $$PG_CONTAINER \
|
||||
psql -U $$PG_USER -d $$PG_DB < ./scripts/postgres/seed.sql
|
||||
|
||||
seed-postgres-1gb:
|
||||
@echo "Seeding Postgres..."
|
||||
docker exec -i -e PGPASSWORD=$$PG_PASSWORD $$PG_CONTAINER \
|
||||
psql -U $$PG_USER -d $$PG_DB < ./scripts/postgres/seed-1gb.sql
|
||||
|
||||
|
||||
SQLITE_SEED_FILE := $(if $(filter big,$(SEED)),./scripts/sqlite/seed-big.sql,./scripts/sqlite/seed.sql)
|
||||
|
||||
seed-sqlite:
|
||||
@echo "Seeding Sqlite..."
|
||||
@echo "Run as root to fix permissions inside the volume"
|
||||
docker exec -u 0 -it db-sqlite sh -c "chmod -R 777 /workspace/data"
|
||||
@echo "Create the database file (if it doesn’t exist)"
|
||||
docker exec -u 0 -it db-sqlite sh -c "touch /workspace/data/app.db"
|
||||
@echo "Seed the database"
|
||||
docker exec -i db-sqlite sh -c "sqlite3 /workspace/data/app.db" < $(SQLITE_SEED_FILE)
|
||||
@echo "Verify"
|
||||
docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT name FROM sqlite_master WHERE type='table';"
|
||||
@echo "Done"
|
||||
|
||||
|
||||
|
||||
seed-all: seed-mongo seed-mysql seed-postgres seed-postgres-1gb
|
||||
@@ -13,6 +13,7 @@
|
||||
|
||||
[](LICENSE)
|
||||
[](https://hub.docker.com/r/portabase/agent)
|
||||
[](https://github.com/Portabase/agent/pkgs/container/charts%2Fportabase-agent)
|
||||
[](https://github.com/Portabase/portabase)
|
||||
[](https://www.buymeacoffee.com/portabase)
|
||||
|
||||
@@ -20,9 +21,12 @@
|
||||
[](https://www.mysql.com/)
|
||||
[](https://sqlite.org/)
|
||||
[](https://redis.io/)
|
||||
[](https://valkey.io/)
|
||||
[](https://mariadb.org/)
|
||||
[](https://www.mongodb.com/)
|
||||
[](https://valkey.io/)
|
||||
[](https://firebirdsql.org/)
|
||||
|
||||
|
||||
[](https://github.com/Portabase/portabase)
|
||||
[](https://github.com/Portabase/portabase)
|
||||
|
||||
@@ -50,7 +54,7 @@ You have 4 ways to install Portabase Agent:
|
||||
|
||||
- Automated CLI (recommended) - [details](https://portabase.io/docs/agent/setup#cli)
|
||||
- Docker Compose setup - [details](https://portabase.io/docs/agent/setup#docker)
|
||||
- Kubernetes with Helm (soon)
|
||||
- Kubernetes with Helm [details](https://portabase.io/docs/agent/setup#helm)
|
||||
- Development setup - [details](https://portabase.io/docs/agent/setup#development)
|
||||
|
||||
**Ensure Docker is installed on your machine before getting started.**
|
||||
|
||||
+59
-9
@@ -10,10 +10,23 @@
|
||||
"host": "db-postgres",
|
||||
"generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"
|
||||
},
|
||||
{
|
||||
"name": "Test database 1 - PostgreSQL - BIS",
|
||||
"database": "devdb2",
|
||||
"type": "postgresql",
|
||||
"username": "devuser2",
|
||||
"password": "changeme2",
|
||||
"port": 5432,
|
||||
"host": "db-postgres-2",
|
||||
"generated_id": "16678159-ff7e-5697-8c83-0adeff214681",
|
||||
"options": {
|
||||
"keep_ownership": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Test database 2 - MariaDB",
|
||||
"database": "mariadb",
|
||||
"type": "mysql",
|
||||
"type": "mariadb",
|
||||
"username": "mariadb",
|
||||
"password": "changeme",
|
||||
"port": 3306,
|
||||
@@ -21,7 +34,7 @@
|
||||
"generated_id": "16678124-ff7e-4c97-8c83-0adeff214681"
|
||||
},
|
||||
{
|
||||
"name": "Test database 4 - MongoDB Auth",
|
||||
"name": "Test database 3 - MongoDB Auth",
|
||||
"database": "testdbauth",
|
||||
"type": "mongodb",
|
||||
"username": "root",
|
||||
@@ -31,7 +44,7 @@
|
||||
"generated_id": "16678145-ff7e-4c97-8c83-0adeff214681"
|
||||
},
|
||||
{
|
||||
"name": "Test database 5 - MongoDB",
|
||||
"name": "Test database 4 - MongoDB",
|
||||
"database": "testdb",
|
||||
"type": "mongodb",
|
||||
"port": 27017,
|
||||
@@ -39,26 +52,26 @@
|
||||
"generated_id": "16678147-ff7e-4c97-8c83-0adeff214681"
|
||||
},
|
||||
{
|
||||
"name": "Test database 6 - SQLite DB",
|
||||
"name": "Test database 5 - SQLite DB",
|
||||
"type": "sqlite",
|
||||
"path": "/sqlite-data/workspace/data/app.db",
|
||||
"generated_id": "16678178-ff7e-4c97-8c83-0adeff214681"
|
||||
},
|
||||
{
|
||||
"name": "Test database 7 - SQLite DB",
|
||||
"name": "Test database 6 - SQLite DB",
|
||||
"type": "sqlite",
|
||||
"path": "/sqlite-data-2/workspace/data/app.db",
|
||||
"generated_id": "16678179-ff7e-4c97-8c83-0adeff214681"
|
||||
},
|
||||
{
|
||||
"name": "Test database 8 - Redis",
|
||||
"name": "Test database 7 - Redis",
|
||||
"type": "redis",
|
||||
"port": 6379,
|
||||
"host": "db-redis",
|
||||
"generated_id": "16678166-ff7e-4c97-8c83-0adeff214681"
|
||||
},
|
||||
{
|
||||
"name": "Test database 9 - Redis Auth",
|
||||
"name": "Test database 8 - Redis Auth",
|
||||
"type": "redis",
|
||||
"password": "supersecurepassword",
|
||||
"port": 6379,
|
||||
@@ -67,20 +80,57 @@
|
||||
"generated_id": "16678160-ff7e-4c97-8c83-0adeff214681"
|
||||
},
|
||||
{
|
||||
"name": "Test database 10 - Valkey",
|
||||
"name": "Test database 9 - Valkey",
|
||||
"type": "valkey",
|
||||
"port": 6379,
|
||||
"host": "db-valkey",
|
||||
"generated_id": "16678560-ff7e-4c97-8c83-0adeff214681"
|
||||
},
|
||||
{
|
||||
"name": "Test database 11 - Valkey Auth",
|
||||
"name": "Test database 10 - Valkey Auth",
|
||||
"type": "valkey",
|
||||
"password": "supersecurepassword",
|
||||
"port": 6379,
|
||||
"username": "default",
|
||||
"host": "db-valkey-auth",
|
||||
"generated_id": "16678561-ff7e-4c97-8c83-0adeff214681"
|
||||
},
|
||||
{
|
||||
"name": "Test database 11 - Mysql",
|
||||
"database": "mysqldb",
|
||||
"type": "mysql",
|
||||
"username": "mysqldb",
|
||||
"password": "changeme",
|
||||
"port": 3306,
|
||||
"host": "db-mysql",
|
||||
"generated_id": "16656124-ff7e-4c97-8c83-0adeff214681"
|
||||
},
|
||||
{
|
||||
"name": "Test database 12 - Firebird",
|
||||
"database": "/var/lib/firebird/data/mirror.fdb",
|
||||
"type": "firebird",
|
||||
"username": "alice",
|
||||
"password": "fake_password",
|
||||
"port": 3050,
|
||||
"host": "db-firebird",
|
||||
"generated_id": "16706124-ff7e-4c97-8c83-0adeff214681"
|
||||
},
|
||||
{
|
||||
"name": "Test database 13 - MsSQL",
|
||||
"database": "myappdb",
|
||||
"type": "mssql",
|
||||
"username": "sa",
|
||||
"password": "Portabase!Strong1",
|
||||
"port": 1433,
|
||||
"host": "db-mssql",
|
||||
"generated_id": "16706125-ff7e-4c97-8c83-0adeff214682"
|
||||
},
|
||||
{
|
||||
"name": "Test database 14 - Docker Volume",
|
||||
"type": "docker-volume",
|
||||
"volume_name": "databases_sqlite-data",
|
||||
"generated_id": "16706126-ff7e-4c97-8c83-0adeff214690",
|
||||
"container_name": "db-sqlite"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
+77
-3
@@ -29,11 +29,85 @@ host = "db-mongodb-auth"
|
||||
generated_id = "16678145-ff7e-4c97-8c83-0adeff214681"
|
||||
|
||||
[[databases]]
|
||||
name = "Test database 3 - MongoDB"
|
||||
name = "Test database 4 - MongoDB"
|
||||
database = "testdb"
|
||||
type = "mongodb"
|
||||
username = ""
|
||||
password = ""
|
||||
port = 27017
|
||||
host = "db-mongodb"
|
||||
generated_id = "16678147-ff7e-4c97-8c83-0adeff214681"
|
||||
|
||||
[[databases]]
|
||||
name = "Test database 5 - SQLite DB"
|
||||
type = "sqlite"
|
||||
path = "/sqlite-data/workspace/data/app.db"
|
||||
generated_id = "16678178-ff7e-4c97-8c83-0adeff214681"
|
||||
|
||||
[[databases]]
|
||||
name = "Test database 6 - SQLite DB 2"
|
||||
type = "sqlite"
|
||||
path = "/sqlite-data-2/workspace/data/app.db"
|
||||
generated_id = "16678179-ff7e-4c97-8c83-0adeff214681"
|
||||
|
||||
[[databases]]
|
||||
name = "Test database 7 - Redis"
|
||||
type = "redis"
|
||||
port = 6379
|
||||
host = "db-redis"
|
||||
generated_id = "16678166-ff7e-4c97-8c83-0adeff214681"
|
||||
|
||||
[[databases]]
|
||||
name = "Test database 8 - Redis Auth"
|
||||
type = "redis"
|
||||
password = "supersecurepassword"
|
||||
username = "default"
|
||||
port = 6379
|
||||
host = "db-redis-auth"
|
||||
generated_id = "16678160-ff7e-4c97-8c83-0adeff214681"
|
||||
|
||||
[[databases]]
|
||||
name = "Test database 9 - Valkey"
|
||||
type = "valkey"
|
||||
port = 6379
|
||||
host = "db-valkey"
|
||||
generated_id = "16678560-ff7e-4c97-8c83-0adeff214681"
|
||||
|
||||
[[databases]]
|
||||
name = "Test database 10 - Valkey Auth"
|
||||
type = "valkey"
|
||||
password = "supersecurepassword"
|
||||
username = "default"
|
||||
port = 6379
|
||||
host = "db-valkey-auth"
|
||||
generated_id = "16678561-ff7e-4c97-8c83-0adeff214681"
|
||||
|
||||
[[databases]]
|
||||
name = "Test database 11 - Mysql"
|
||||
database = "mysqldb"
|
||||
type = "mysql"
|
||||
password = "changeme"
|
||||
username = "mysqldb"
|
||||
port = 3306
|
||||
host = "db-mysql"
|
||||
generated_id = "16656124-ff7e-4c97-8c83-0adeff214681"
|
||||
|
||||
[[databases]]
|
||||
name = "Test database 12 - Firebird"
|
||||
database = "/var/lib/firebird/data/mirror.fdb"
|
||||
type = "firebird"
|
||||
password = "fake_password"
|
||||
username = "alice"
|
||||
port = 3050
|
||||
host = "db-firebird"
|
||||
generated_id = "16706124-ff7e-4c97-8c83-0adeff214681"
|
||||
|
||||
[[databases]]
|
||||
name = "Test database 13 - MSSQL"
|
||||
database = "myappdb"
|
||||
type = "mssql"
|
||||
username = "sa"
|
||||
password = "Portabase!Strong1"
|
||||
port = 1433
|
||||
host = "db-mssql"
|
||||
generated_id = "16706125-ff7e-4c97-8c83-0adeff214682"
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
name: databases
|
||||
|
||||
services:
|
||||
db-postgres:
|
||||
container_name: db-postgres
|
||||
image: postgres:17-alpine
|
||||
ports:
|
||||
- "5436:5432"
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
environment:
|
||||
- POSTGRES_DB=devdb
|
||||
- POSTGRES_USER=devuser
|
||||
- POSTGRES_PASSWORD=changeme
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
db-postgres-2:
|
||||
container_name: db-postgres-2
|
||||
image: postgres:17-alpine
|
||||
ports:
|
||||
- "5438:5432"
|
||||
volumes:
|
||||
- postgres-data-2:/var/lib/postgresql/data
|
||||
environment:
|
||||
- POSTGRES_DB=devdb2
|
||||
- POSTGRES_USER=devuser2
|
||||
- POSTGRES_PASSWORD=changeme2
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
db-mariadb:
|
||||
container_name: db-mariadb
|
||||
image: mariadb:latest
|
||||
ports:
|
||||
- "3311:3306"
|
||||
environment:
|
||||
- MYSQL_DATABASE=mariadb
|
||||
- MYSQL_USER=mariadb
|
||||
- MYSQL_PASSWORD=changeme
|
||||
- MYSQL_RANDOM_ROOT_PASSWORD=yes
|
||||
volumes:
|
||||
- mariadb-data:/var/lib/mysql
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
db-mysql:
|
||||
container_name: db-mysql
|
||||
image: mysql:9.5
|
||||
ports:
|
||||
- "3312:3306"
|
||||
environment:
|
||||
- MYSQL_DATABASE=mysqldb
|
||||
- MYSQL_USER=mysqldb
|
||||
- MYSQL_PASSWORD=changeme
|
||||
- MYSQL_RANDOM_ROOT_PASSWORD=yes
|
||||
volumes:
|
||||
- mysql-data:/var/lib/mysql
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
|
||||
db-mongodb-auth:
|
||||
container_name: db-mongodb-auth
|
||||
image: mongo:latest
|
||||
ports:
|
||||
- "27082:27017"
|
||||
environment:
|
||||
MONGO_INITDB_ROOT_USERNAME: root
|
||||
MONGO_INITDB_ROOT_PASSWORD: rootpassword
|
||||
MONGO_INITDB_DATABASE: testdbauth
|
||||
command: mongod --auth
|
||||
networks:
|
||||
- portabase
|
||||
volumes:
|
||||
- mongodb-data-auth:/data/db
|
||||
healthcheck:
|
||||
test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
db-mongodb:
|
||||
container_name: db-mongodb
|
||||
image: mongo:latest
|
||||
ports:
|
||||
- "27083:27017"
|
||||
volumes:
|
||||
- mongodb-data:/data/db
|
||||
healthcheck:
|
||||
test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
environment:
|
||||
MONGO_INITDB_DATABASE: testdb
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
sqlite:
|
||||
container_name: db-sqlite
|
||||
image: keinos/sqlite3
|
||||
volumes:
|
||||
- sqlite-data:/workspace/data
|
||||
working_dir: /workspace
|
||||
command: tail -f /dev/null
|
||||
stdin_open: true
|
||||
tty: true
|
||||
|
||||
db-redis:
|
||||
image: redis:latest
|
||||
container_name: db-redis
|
||||
ports:
|
||||
- "6378:6379"
|
||||
volumes:
|
||||
- redis-data:/data
|
||||
command: [ "redis-server", "--appendonly", "yes" ]
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
db-redis-auth:
|
||||
image: redis:latest
|
||||
container_name: db-redis-auth
|
||||
ports:
|
||||
- "6380:6379"
|
||||
volumes:
|
||||
- redis-data-auth:/data
|
||||
environment:
|
||||
- REDIS_PASSWORD=supersecurepassword
|
||||
command: [ "redis-server", "--requirepass", "supersecurepassword", "--appendonly", "yes" ]
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
db-valkey:
|
||||
image: valkey/valkey
|
||||
container_name: db-valkey
|
||||
environment:
|
||||
- ALLOW_EMPTY_PASSWORD=yes
|
||||
ports:
|
||||
- '6381:6379'
|
||||
volumes:
|
||||
- valkey-data:/data
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
db-valkey-auth:
|
||||
image: valkey/valkey
|
||||
container_name: db-valkey-auth
|
||||
command: >
|
||||
--requirepass "supersecurepassword"
|
||||
ports:
|
||||
- '6382:6379'
|
||||
volumes:
|
||||
- valkey-data-auth:/data
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
db-firebird:
|
||||
image: firebirdsql/firebird
|
||||
container_name: db-firebird
|
||||
restart: always
|
||||
environment:
|
||||
- FIREBIRD_ROOT_PASSWORD=fake_root_password
|
||||
- FIREBIRD_USER=alice
|
||||
- FIREBIRD_PASSWORD=fake_password
|
||||
- FIREBIRD_DATABASE=mirror.fdb
|
||||
- FIREBIRD_DATABASE_DEFAULT_CHARSET=UTF8
|
||||
volumes:
|
||||
- firebird-data:/var/lib/firebird/data
|
||||
ports:
|
||||
- "3060:3050"
|
||||
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
db-mssql:
|
||||
container_name: db-mssql
|
||||
image: mcr.microsoft.com/azure-sql-edge:latest
|
||||
ports:
|
||||
- "1433:1433"
|
||||
environment:
|
||||
ACCEPT_EULA: "Y"
|
||||
MSSQL_SA_PASSWORD: "Portabase!Strong1"
|
||||
volumes:
|
||||
- mssql-data:/var/opt/mssql
|
||||
networks:
|
||||
- portabase
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "cat /proc/net/tcp6 | grep -q '059901' || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
|
||||
volumes:
|
||||
postgres-data:
|
||||
postgres-data-2:
|
||||
mariadb-data:
|
||||
mysql-data:
|
||||
mongodb-data:
|
||||
mongodb-data-auth:
|
||||
sqlite-data:
|
||||
redis-data:
|
||||
redis-data-auth:
|
||||
valkey-data:
|
||||
valkey-data-auth:
|
||||
firebird-data:
|
||||
mssql-data:
|
||||
|
||||
networks:
|
||||
portabase:
|
||||
name: portabase_network
|
||||
external: true
|
||||
|
||||
+13
-140
@@ -7,165 +7,38 @@ services:
|
||||
container_name: rust-dev
|
||||
volumes:
|
||||
- .:/app
|
||||
- ./databases.json:/config/config.json
|
||||
# - ./databases.toml:/config/config.toml
|
||||
- cargo-registry:/usr/local/cargo/registry
|
||||
- cargo-git:/usr/local/cargo/git
|
||||
- ./databases.json:/config/config.json
|
||||
#- ./databases.toml:/config/config.toml
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
|
||||
# - cargo-target:/app/target
|
||||
# - sqlite-data:/sqlite-data/workspace/data
|
||||
# - ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data
|
||||
# - cargo-target:/app/target
|
||||
- databases_sqlite-data:/sqlite-data/workspace/data
|
||||
- ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data
|
||||
environment:
|
||||
APP_ENV: development
|
||||
LOG: debug
|
||||
TZ: "Europe/Paris"
|
||||
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiNGI1OTM2MGItNTNkMi00ZTZmLWE1ODctODcyMmQ1NDc1MTNmIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ=="
|
||||
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiMWM4NmQ5NGEtMGVjOC00NzkxLTk0ZTEtOWRlYmJmMWY0M2I5IiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ=="
|
||||
#CHUNK_SIZE_MB: "1"
|
||||
#POOLING: 1
|
||||
#DATABASES_CONFIG_FILE: "config.toml"
|
||||
extra_hosts:
|
||||
- "localhost:host-gateway"
|
||||
networks:
|
||||
- portabase
|
||||
cpus: "1.50"
|
||||
mem_limit: 4g
|
||||
memswap_limit: 4g
|
||||
pids_limit: 512
|
||||
|
||||
db-postgres:
|
||||
container_name: db-postgres
|
||||
image: postgres:17-alpine
|
||||
ports:
|
||||
- "5436:5432"
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
environment:
|
||||
- POSTGRES_DB=devdb
|
||||
- POSTGRES_USER=devuser
|
||||
- POSTGRES_PASSWORD=changeme
|
||||
networks:
|
||||
- portabase
|
||||
#
|
||||
# db-mariadb:
|
||||
# container_name: db-mariadb
|
||||
# image: mariadb:latest
|
||||
# ports:
|
||||
# - "3311:3306"
|
||||
# environment:
|
||||
# - MYSQL_DATABASE=mariadb
|
||||
# - MYSQL_USER=mariadb
|
||||
# - MYSQL_PASSWORD=changeme
|
||||
# - MYSQL_RANDOM_ROOT_PASSWORD=yes
|
||||
# volumes:
|
||||
# - mariadb-data:/var/lib/mysql
|
||||
# networks:
|
||||
# - portabase
|
||||
#
|
||||
#
|
||||
# db-mongodb-auth:
|
||||
# container_name: db-mongodb-auth
|
||||
# image: mongo:latest
|
||||
# ports:
|
||||
# - "27082:27017"
|
||||
# environment:
|
||||
# MONGO_INITDB_ROOT_USERNAME: root
|
||||
# MONGO_INITDB_ROOT_PASSWORD: rootpassword
|
||||
# MONGO_INITDB_DATABASE: testdbauth
|
||||
# command: mongod --auth
|
||||
# networks:
|
||||
# - portabase
|
||||
# volumes:
|
||||
# - mongodb-data-auth:/data/db
|
||||
# healthcheck:
|
||||
# test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ]
|
||||
# interval: 5s
|
||||
# timeout: 5s
|
||||
# retries: 10
|
||||
#
|
||||
# db-mongodb:
|
||||
# container_name: db-mongodb
|
||||
# image: mongo:latest
|
||||
# ports:
|
||||
# - "27083:27017"
|
||||
# volumes:
|
||||
# - mongodb-data:/data/db
|
||||
# healthcheck:
|
||||
# test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ]
|
||||
# interval: 5s
|
||||
# timeout: 5s
|
||||
# retries: 10
|
||||
# environment:
|
||||
# MONGO_INITDB_DATABASE: testdb
|
||||
# networks:
|
||||
# - portabase
|
||||
|
||||
# sqlite:
|
||||
# container_name: db-sqlite
|
||||
# image: keinos/sqlite3
|
||||
# volumes:
|
||||
# - sqlite-data:/workspace/data
|
||||
# working_dir: /workspace
|
||||
# command: tail -f /dev/null
|
||||
# stdin_open: true
|
||||
# tty: true
|
||||
|
||||
db-redis:
|
||||
image: redis:latest
|
||||
container_name: db-redis
|
||||
ports:
|
||||
- "6379:6379"
|
||||
volumes:
|
||||
- redis-data:/data
|
||||
command: [ "redis-server", "--appendonly", "yes" ]
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
db-redis-auth:
|
||||
image: redis:latest
|
||||
container_name: db-redis-auth
|
||||
ports:
|
||||
- "6380:6379"
|
||||
volumes:
|
||||
- redis-data-auth:/data
|
||||
environment:
|
||||
- REDIS_PASSWORD=supersecurepassword
|
||||
command: [ "redis-server", "--requirepass", "supersecurepassword", "--appendonly", "yes" ]
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
db-valkey:
|
||||
image: valkey/valkey
|
||||
container_name: db-valkey
|
||||
environment:
|
||||
- ALLOW_EMPTY_PASSWORD=yes
|
||||
ports:
|
||||
- '6381:6379'
|
||||
volumes:
|
||||
- valkey-data:/data
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
db-valkey-auth:
|
||||
image: valkey/valkey
|
||||
container_name: db-valkey-auth
|
||||
command: >
|
||||
--requirepass "supersecurepassword"
|
||||
ports:
|
||||
- '6382:6379'
|
||||
volumes:
|
||||
- valkey-data-auth:/data
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
volumes:
|
||||
cargo-registry:
|
||||
cargo-git:
|
||||
# cargo-target:
|
||||
postgres-data:
|
||||
# mariadb-data:
|
||||
# mongodb-data:
|
||||
# mongodb-data-auth:
|
||||
# sqlite-data:
|
||||
redis-data:
|
||||
redis-data-auth:
|
||||
valkey-data:
|
||||
valkey-data-auth:
|
||||
databases_sqlite-data:
|
||||
external: true
|
||||
|
||||
networks:
|
||||
portabase:
|
||||
|
||||
+43
-1
@@ -1,3 +1,11 @@
|
||||
# =========================
|
||||
# MySQL client tools
|
||||
# =========================
|
||||
FROM mysql:8.4 AS mysql-client-tools
|
||||
RUN mkdir -p /mysql-exports/bin /mysql-exports/lib \
|
||||
&& cp /usr/bin/mysqldump /mysql-exports/bin/ \
|
||||
&& find /usr/lib -name "libmysqlclient.so.21*" -exec cp {} /mysql-exports/lib/ \;
|
||||
|
||||
# =========================
|
||||
# Base image (shared)
|
||||
# =========================
|
||||
@@ -18,9 +26,24 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
sqlite3 \
|
||||
redis-tools \
|
||||
valkey \
|
||||
firebird3.0-utils \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV DOTNET_ROOT=/usr/local/dotnet
|
||||
RUN curl -sSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh \
|
||||
&& chmod +x /tmp/dotnet-install.sh \
|
||||
&& /tmp/dotnet-install.sh --channel 8.0 --install-dir /usr/local/dotnet \
|
||||
&& rm /tmp/dotnet-install.sh \
|
||||
&& /usr/local/dotnet/dotnet tool install --global microsoft.sqlpackage
|
||||
|
||||
ENV PATH="$PATH:/usr/local/dotnet:/root/.dotnet/tools"
|
||||
|
||||
RUN ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') \
|
||||
&& curl -sSL "https://github.com/microsoft/go-sqlcmd/releases/latest/download/sqlcmd-linux-${ARCH}.tar.bz2" \
|
||||
| tar -xjf - -C /usr/local/bin sqlcmd \
|
||||
&& chmod +x /usr/local/bin/sqlcmd
|
||||
|
||||
ARG TARGETARCH
|
||||
|
||||
# =========================
|
||||
@@ -46,12 +69,20 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \
|
||||
rm -rf /tmp/pg-x64 /tmp/pg-arm && \
|
||||
chmod +x /usr/lib/postgresql/*/bin/*
|
||||
|
||||
|
||||
# =========================
|
||||
# MongoDB client binaries
|
||||
# =========================
|
||||
COPY assets/tools/${TARGETARCH}/mongodb/ /usr/local/mongodb/
|
||||
RUN chmod +x /usr/local/mongodb/bin/*
|
||||
|
||||
# =========================
|
||||
# MySQL real mysqldump binary
|
||||
# =========================
|
||||
COPY --from=mysql-client-tools /mysql-exports/bin/mysqldump /usr/local/bin/mysqldump
|
||||
COPY --from=mysql-client-tools /mysql-exports/lib/ /usr/local/lib/
|
||||
RUN chmod +x /usr/local/bin/mysqldump && ldconfig
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# =========================
|
||||
@@ -95,12 +126,19 @@ RUN apt-get update && apt-get install -y \
|
||||
libreadline8 \
|
||||
libncurses6 \
|
||||
zlib1g \
|
||||
curl \
|
||||
mariadb-client \
|
||||
sqlite3 \
|
||||
redis-tools \
|
||||
valkey \
|
||||
firebird3.0-utils \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV DOTNET_ROOT=/usr/local/dotnet
|
||||
RUN curl -sSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh \
|
||||
&& chmod +x /tmp/dotnet-install.sh \
|
||||
&& /tmp/dotnet-install.sh --channel 8.0 --runtime dotnet --install-dir /usr/local/dotnet \
|
||||
&& rm /tmp/dotnet-install.sh
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
@@ -111,8 +149,12 @@ RUN chmod +x /entrypoint.sh
|
||||
|
||||
COPY --from=base /usr/lib/postgresql/ /usr/lib/postgresql/
|
||||
COPY --from=base /usr/local/mongodb/bin/ /usr/local/mongodb/bin/
|
||||
COPY --from=base /root/.dotnet/tools/ /root/.dotnet/tools/
|
||||
COPY --from=mysql-client-tools /mysql-exports/bin/mysqldump /usr/local/bin/mysqldump
|
||||
COPY --from=mysql-client-tools /mysql-exports/lib/ /usr/local/lib/
|
||||
RUN chmod +x /usr/local/bin/mysqldump && ldconfig
|
||||
|
||||
|
||||
ENV PATH="$PATH:/usr/local/dotnet:/root/.dotnet/tools"
|
||||
ENV APP_ENV=production
|
||||
|
||||
CMD ["/entrypoint.sh"]
|
||||
|
||||
@@ -34,6 +34,11 @@ check_network() {
|
||||
check_docker
|
||||
check_network
|
||||
|
||||
echo "Starting docker-compose..."
|
||||
docker compose -f ./docker-compose.yml up
|
||||
echo "Docker-compose started successfully."
|
||||
echo "Stopping old database containers..."
|
||||
docker compose -f ./docker-compose.databases.yml down
|
||||
|
||||
echo "Starting database containers..."
|
||||
docker compose -f ./docker-compose.databases.yml up -d
|
||||
|
||||
echo "Starting main services..."
|
||||
docker compose -f ./docker-compose.yml up
|
||||
@@ -0,0 +1,71 @@
|
||||
set dotenv-load := true
|
||||
set shell := ["bash", "-cu"]
|
||||
|
||||
CLUSTER_SCRIPT := "docker/entrypoints/app-dev-entrypoint.sh"
|
||||
|
||||
up:
|
||||
bash {{CLUSTER_SCRIPT}}
|
||||
|
||||
seed-mongo:
|
||||
echo "Seeding MongoDB..."
|
||||
bash ./scripts/mongo/seed-mongo.sh
|
||||
|
||||
seed-mongo-auth:
|
||||
echo "Seeding MongoDB with auth..."
|
||||
bash ./scripts/mongo/seed-mongo.sh auth
|
||||
|
||||
seed-mysql:
|
||||
echo "Seeding MySQL..."
|
||||
mysql -h 127.0.0.1 -P "$MYSQL_PORT" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DB" < ./scripts/mysql/seed-mysql.sql
|
||||
|
||||
seed-mysql-1gb:
|
||||
echo "Seeding MySQL (1GB)..."
|
||||
mysql -h 127.0.0.1 -P "$MYSQL_PORT" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DB" < ./scripts/mysql/seed-1gb.sql
|
||||
|
||||
seed-postgres:
|
||||
echo "Seeding Postgres..."
|
||||
docker exec -i -e PGPASSWORD="$PG_PASSWORD" "$PG_CONTAINER" \
|
||||
psql -U "$PG_USER" -d "$PG_DB" < ./scripts/postgres/seed.sql
|
||||
|
||||
seed-postgres-1gb:
|
||||
echo "Seeding Postgres (1GB)..."
|
||||
docker exec -i -e PGPASSWORD="$PG_PASSWORD" "$PG_CONTAINER" \
|
||||
psql -U "$PG_USER" -d "$PG_DB" < ./scripts/postgres/seed-1gb.sql
|
||||
|
||||
seed-sqlite:
|
||||
echo "Seeding SQLite..."
|
||||
bash ./scripts/sqlite/seed.sh
|
||||
echo "Done"
|
||||
|
||||
seed-firebird:
|
||||
echo "Seeding Firebird..."
|
||||
docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb < ./scripts/firebird/seed.sql
|
||||
|
||||
echo "Verifying Firebird tables..."
|
||||
echo "SELECT RDB\$RELATION_NAME FROM RDB\$RELATIONS WHERE RDB\$SYSTEM_FLAG = 0 AND RDB\$VIEW_BLR IS NULL;" \
|
||||
| docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb
|
||||
|
||||
seed-mssql:
|
||||
echo "Seeding MSSQL..."
|
||||
docker exec -i rust-dev sqlcmd -S "db-mssql,1433" -U sa -P "$MSSQL_SA_PASSWORD" -N disable -i /app/scripts/mssql/seed.sql
|
||||
echo "Done"
|
||||
|
||||
seed-all:
|
||||
just seed-mongo
|
||||
just seed-mysql
|
||||
just seed-postgres
|
||||
just seed-postgres-1gb
|
||||
just seed-sqlite
|
||||
just seed-mongo
|
||||
just seed-firebird
|
||||
just seed-mssql
|
||||
|
||||
test:
|
||||
echo "Build test image (with grcov included)"
|
||||
docker compose -f docker-compose.test.yml build agent-test
|
||||
echo "Start agent-test container"
|
||||
docker compose -f docker-compose.test.yml up -d agent-test
|
||||
echo "Run tests inside container"
|
||||
docker compose -f docker-compose.test.yml exec -e CARGO_INCREMENTAL -e RUSTFLAGS -e LLVM_PROFILE_FILE agent-test bash -c "cargo test --verbose && sync"
|
||||
echo "Down volumes tests databases"
|
||||
docker compose -f docker-compose.test.yml down --volumes
|
||||
@@ -0,0 +1,11 @@
|
||||
CREATE TABLE users (
|
||||
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
email VARCHAR(255) NOT NULL UNIQUE,
|
||||
name VARCHAR(255),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
INSERT INTO users (email, name) VALUES ('alice@example.com', 'Alice');
|
||||
INSERT INTO users (email, name) VALUES ('bob@example.com', 'Bob');
|
||||
|
||||
COMMIT;
|
||||
@@ -0,0 +1,20 @@
|
||||
IF NOT EXISTS (SELECT name FROM sys.databases WHERE name = N'myappdb')
|
||||
CREATE DATABASE [myappdb];
|
||||
GO
|
||||
|
||||
USE [myappdb];
|
||||
GO
|
||||
|
||||
IF OBJECT_ID('users', 'U') IS NULL
|
||||
CREATE TABLE users (
|
||||
id INT IDENTITY(1,1) PRIMARY KEY,
|
||||
email NVARCHAR(255) NOT NULL UNIQUE,
|
||||
name NVARCHAR(255),
|
||||
created_at DATETIME DEFAULT GETDATE()
|
||||
);
|
||||
GO
|
||||
|
||||
INSERT INTO users (email, name) VALUES ('alice@example.com', 'Alice');
|
||||
INSERT INTO users (email, name) VALUES ('bob@example.com', 'Bob');
|
||||
INSERT INTO users (email, name) VALUES ('charlie@example.com', 'Charlie');
|
||||
GO
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SEED_VALUE="${SEED:-}"
|
||||
|
||||
if [ "$SEED_VALUE" = "big" ]; then
|
||||
SQLITE_SEED_FILE="./scripts/sqlite/seed-big.sql"
|
||||
else
|
||||
SQLITE_SEED_FILE="./scripts/sqlite/seed.sql"
|
||||
fi
|
||||
|
||||
docker exec -u 0 db-sqlite sh -c "chmod -R 777 /workspace/data"
|
||||
docker exec -u 0 db-sqlite sh -c "touch /workspace/data/app.db"
|
||||
docker exec -i db-sqlite sh -c "sqlite3 /workspace/data/app.db" < "$SQLITE_SEED_FILE"
|
||||
@@ -0,0 +1,68 @@
|
||||
use crate::domain::docker_volume::docker::{
|
||||
client, create_helper, remove_helper, resolve_helper_image, start_container, stop_container,
|
||||
};
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use bollard::query_parameters::DownloadFromContainerOptions;
|
||||
use futures_util::StreamExt;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tokio::fs::File;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, backup_dir: PathBuf, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||
futures::executor::block_on(async move {
|
||||
logger.log("info", format!("Starting docker-volume backup for {}", cfg.name));
|
||||
|
||||
let docker = client()?;
|
||||
let image = resolve_helper_image(&docker).await?;
|
||||
logger.log("debug", format!("Helper image: {image}"));
|
||||
|
||||
if let Some(name) = &cfg.container_name {
|
||||
logger.log("info", format!("Stopping container {name} for consistent backup"));
|
||||
stop_container(&docker, name).await?;
|
||||
}
|
||||
|
||||
let result = async {
|
||||
let helper = create_helper(&docker, &image, &cfg.volume_name, &cfg.generated_id, true, None).await?;
|
||||
|
||||
let file_path = backup_dir.join(format!("{}.tar", cfg.generated_id));
|
||||
let start = Instant::now();
|
||||
|
||||
let dl_opts = DownloadFromContainerOptions { path: "/vol".to_string() };
|
||||
let mut stream = docker.download_from_container(&helper.id, Some(dl_opts));
|
||||
|
||||
let mut out = File::create(&file_path)
|
||||
.await
|
||||
.with_context(|| format!("Failed to create backup file {}", file_path.display()))?;
|
||||
let mut bytes_written: u64 = 0;
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.context("Error streaming volume archive from Docker")?;
|
||||
bytes_written += chunk.len() as u64;
|
||||
out.write_all(&chunk).await?;
|
||||
}
|
||||
out.flush().await?;
|
||||
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
logger.log_command("docker download_from_container", None, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("Volume backup wrote {bytes_written} bytes to {}", file_path.display()));
|
||||
|
||||
remove_helper(&docker, &helper.id).await;
|
||||
anyhow::Ok(file_path)
|
||||
}
|
||||
.await;
|
||||
|
||||
if let Some(name) = &cfg.container_name {
|
||||
if let Err(e) = start_container(&docker, name).await {
|
||||
logger.log("error", format!("Failed to restart container {name}: {e}"));
|
||||
}
|
||||
}
|
||||
|
||||
result
|
||||
})
|
||||
})
|
||||
.await?
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::{backup, ping, restore};
|
||||
use crate::domain::factory::Database;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::utils::locks::{DbOpLock, FileLock};
|
||||
|
||||
pub struct DockerVolumeDatabase {
|
||||
cfg: DatabaseConfig,
|
||||
}
|
||||
|
||||
impl DockerVolumeDatabase {
|
||||
pub fn new(cfg: DatabaseConfig) -> Self {
|
||||
Self { cfg }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Database for DockerVolumeDatabase {
|
||||
fn file_extension(&self) -> &'static str {
|
||||
".tar"
|
||||
}
|
||||
|
||||
async fn ping(&self) -> Result<bool> {
|
||||
ping::run(self.cfg.clone()).await
|
||||
}
|
||||
|
||||
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), logger).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
|
||||
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use bollard::Docker;
|
||||
use bollard::models::{ContainerCreateBody, HostConfig};
|
||||
use bollard::query_parameters::{
|
||||
CreateContainerOptions, InspectContainerOptions, ListContainersOptions,
|
||||
RemoveContainerOptions, StartContainerOptions, StopContainerOptions,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use tracing::{info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub const EPHEMERAL_LABEL: &str = "io.portabase.ephemeral";
|
||||
const HELPER_MOUNT: &str = "/vol";
|
||||
|
||||
pub fn client() -> Result<Docker> {
|
||||
Docker::connect_with_unix_defaults().context("Failed to connect to Docker daemon socket")
|
||||
}
|
||||
|
||||
pub fn parse_container_id(mountinfo: &str, cgroup: &str) -> Option<String> {
|
||||
for src in [mountinfo, cgroup] {
|
||||
for line in src.lines() {
|
||||
for marker in ["/containers/", "/docker/"] {
|
||||
if let Some(idx) = line.find(marker) {
|
||||
let rest = &line[idx + marker.len()..];
|
||||
let id: String = rest.chars().take_while(|c| c.is_ascii_hexdigit()).collect();
|
||||
if id.len() >= 64 {
|
||||
return Some(id[..64].to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub async fn resolve_helper_image(docker: &Docker) -> Result<String> {
|
||||
if let Ok(img) = std::env::var("PORTABASE_HELPER_IMAGE") {
|
||||
if !img.trim().is_empty() {
|
||||
return Ok(img);
|
||||
}
|
||||
}
|
||||
let mountinfo = std::fs::read_to_string("/proc/self/mountinfo").unwrap_or_default();
|
||||
let cgroup = std::fs::read_to_string("/proc/self/cgroup").unwrap_or_default();
|
||||
let id = parse_container_id(&mountinfo, &cgroup).context(
|
||||
"Could not determine own container id; set PORTABASE_HELPER_IMAGE to a locally-present image",
|
||||
)?;
|
||||
let info = docker
|
||||
.inspect_container(&id, None::<InspectContainerOptions>)
|
||||
.await
|
||||
.with_context(|| format!("Failed to inspect self container {id}"))?;
|
||||
info.image
|
||||
.context("Self container inspection returned no image reference")
|
||||
}
|
||||
|
||||
pub struct Helper {
|
||||
pub id: String,
|
||||
}
|
||||
|
||||
pub async fn create_helper(
|
||||
docker: &Docker,
|
||||
image: &str,
|
||||
volume_name: &str,
|
||||
generated_id: &str,
|
||||
read_only: bool,
|
||||
cmd: Option<Vec<String>>,
|
||||
) -> Result<Helper> {
|
||||
let bind = format!(
|
||||
"{volume_name}:{HELPER_MOUNT}{}",
|
||||
if read_only { ":ro" } else { "" }
|
||||
);
|
||||
let mut labels = HashMap::new();
|
||||
labels.insert(EPHEMERAL_LABEL.to_string(), "true".to_string());
|
||||
labels.insert("com.docker.compose.project".to_string(), String::new());
|
||||
labels.insert("com.docker.compose.service".to_string(), String::new());
|
||||
labels.insert("com.docker.compose.oneoff".to_string(), String::new());
|
||||
|
||||
let name = format!(
|
||||
"portabase-vol-{generated_id}-{}",
|
||||
&Uuid::new_v4().to_string()[..8]
|
||||
);
|
||||
|
||||
let body = ContainerCreateBody {
|
||||
image: Some(image.to_string()),
|
||||
cmd,
|
||||
labels: Some(labels),
|
||||
host_config: Some(HostConfig {
|
||||
binds: Some(vec![bind]),
|
||||
auto_remove: Some(false),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let opts = CreateContainerOptions {
|
||||
name: Some(name),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let res = docker
|
||||
.create_container(Some(opts), body)
|
||||
.await
|
||||
.with_context(|| format!("Failed to create helper container for volume {volume_name}"))?;
|
||||
|
||||
Ok(Helper { id: res.id })
|
||||
}
|
||||
|
||||
|
||||
pub async fn remove_helper(docker: &Docker, id: &str) {
|
||||
let stop_opts = StopContainerOptions {
|
||||
t: Some(2),
|
||||
..Default::default()
|
||||
};
|
||||
let _ = docker.stop_container(id, Some(stop_opts)).await;
|
||||
|
||||
if let Ok(info) = docker
|
||||
.inspect_container(id, None::<InspectContainerOptions>)
|
||||
.await
|
||||
{
|
||||
let name = info.name.unwrap_or_default();
|
||||
let name = name.trim_start_matches('/');
|
||||
let code = info.state.and_then(|s| s.exit_code).unwrap_or_default();
|
||||
info!("Helper container {name} exited with code {code}");
|
||||
}
|
||||
|
||||
let opts = RemoveContainerOptions {
|
||||
force: true,
|
||||
..Default::default()
|
||||
};
|
||||
if let Err(e) = docker.remove_container(id, Some(opts)).await {
|
||||
warn!("Failed to remove helper container {id}: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn stop_container(docker: &Docker, name: &str) -> Result<()> {
|
||||
docker
|
||||
.stop_container(name, None::<StopContainerOptions>)
|
||||
.await
|
||||
.with_context(|| format!("Failed to stop container {name}"))
|
||||
}
|
||||
|
||||
pub async fn start_container(docker: &Docker, name: &str) -> Result<()> {
|
||||
docker
|
||||
.start_container(name, None::<StartContainerOptions>)
|
||||
.await
|
||||
.with_context(|| format!("Failed to start container {name}"))
|
||||
}
|
||||
|
||||
pub async fn sweep_ephemeral(docker: &Docker) -> Result<usize> {
|
||||
let mut filters = HashMap::new();
|
||||
filters.insert("label".to_string(), vec![format!("{EPHEMERAL_LABEL}=true")]);
|
||||
|
||||
let opts = ListContainersOptions {
|
||||
all: true,
|
||||
filters: Some(filters),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let list = docker.list_containers(Some(opts)).await?;
|
||||
let mut removed = 0;
|
||||
for c in list {
|
||||
if let Some(id) = c.id {
|
||||
remove_helper(docker, &id).await;
|
||||
removed += 1;
|
||||
}
|
||||
}
|
||||
Ok(removed)
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
pub mod backup;
|
||||
pub mod database;
|
||||
pub mod docker;
|
||||
pub mod ping;
|
||||
pub mod restore;
|
||||
@@ -0,0 +1,12 @@
|
||||
use crate::domain::docker_volume::docker::client;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::Result;
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
|
||||
let docker = client()?;
|
||||
match docker.inspect_volume(&cfg.volume_name).await {
|
||||
Ok(_) => Ok(true),
|
||||
Err(bollard::errors::Error::DockerResponseServerError { status_code: 404, .. }) => Ok(false),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
use crate::domain::docker_volume::docker::{
|
||||
client, create_helper, remove_helper, resolve_helper_image, start_container, stop_container,
|
||||
};
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use bollard::exec::StartExecResults;
|
||||
use bollard::models::ExecConfig;
|
||||
use bollard::query_parameters::UploadToContainerOptions;
|
||||
use futures_util::StreamExt;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, archive: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
futures::executor::block_on(async move {
|
||||
logger.log("info", format!("Starting docker-volume restore for {}", cfg.name));
|
||||
|
||||
let docker = client()?;
|
||||
let image = resolve_helper_image(&docker).await?;
|
||||
|
||||
logger.log("debug", format!("Restore archive: {}", archive.display()));
|
||||
|
||||
if let Some(name) = &cfg.container_name {
|
||||
logger.log("info", format!("Stopping container {name} for restore"));
|
||||
stop_container(&docker, name).await?;
|
||||
}
|
||||
|
||||
let result = async {
|
||||
let helper = create_helper(
|
||||
&docker,
|
||||
&image,
|
||||
&cfg.volume_name,
|
||||
&cfg.generated_id,
|
||||
false,
|
||||
Some(vec![
|
||||
"sh".into(),
|
||||
"-c".into(),
|
||||
"trap 'exit 0' TERM; sleep 2147483647 & wait".into(),
|
||||
]),
|
||||
)
|
||||
.await?;
|
||||
start_container(&docker, &helper.id).await?;
|
||||
|
||||
|
||||
let exec = docker
|
||||
.create_exec(
|
||||
&helper.id,
|
||||
ExecConfig {
|
||||
cmd: Some(vec![
|
||||
"sh".to_string(),
|
||||
"-c".to_string(),
|
||||
"rm -rf /vol/* /vol/.[!.]* 2>/dev/null || true".to_string(),
|
||||
]),
|
||||
attach_stdout: Some(true),
|
||||
attach_stderr: Some(true),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.context("Failed to create wipe exec")?;
|
||||
|
||||
if let StartExecResults::Attached { mut output, .. } =
|
||||
docker.start_exec(&exec.id, None).await.context("Failed to run wipe exec")?
|
||||
{
|
||||
while output.next().await.is_some() {}
|
||||
}
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
let file = tokio::fs::File::open(&archive)
|
||||
.await
|
||||
.with_context(|| format!("Failed to open {}", archive.display()))?;
|
||||
let stream = tokio_util::io::ReaderStream::new(file);
|
||||
|
||||
let up_opts = UploadToContainerOptions { path: "/".to_string(), ..Default::default() };
|
||||
docker
|
||||
.upload_to_container(&helper.id, Some(up_opts), bollard::body_try_stream(stream))
|
||||
.await
|
||||
.context("Failed to upload volume archive")?;
|
||||
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
logger.log_command("docker upload_to_container", None, Some(0), Some(duration_ms));
|
||||
|
||||
remove_helper(&docker, &helper.id).await;
|
||||
logger.log("info", format!("Volume restore completed for {}", cfg.name));
|
||||
anyhow::Ok(())
|
||||
}
|
||||
.await;
|
||||
|
||||
if let Some(name) = &cfg.container_name {
|
||||
if let Err(e) = start_container(&docker, name).await {
|
||||
logger.log("error", format!("Failed to restart container {name}: {e}"));
|
||||
}
|
||||
}
|
||||
|
||||
result
|
||||
})
|
||||
})
|
||||
.await?
|
||||
}
|
||||
+18
-4
@@ -1,10 +1,16 @@
|
||||
use crate::domain::docker_volume::database::DockerVolumeDatabase;
|
||||
use crate::domain::mongodb::database::MongoDatabase;
|
||||
use crate::domain::mysql::database::MySQLDatabase;
|
||||
use crate::domain::postgres::cluster::database::PostgresClusterDatabase;
|
||||
use crate::domain::postgres::database::PostgresDatabase;
|
||||
use crate::domain::postgres::{detect_format_from_file, detect_format_from_size};
|
||||
use crate::domain::redis::database::RedisDatabase;
|
||||
use crate::domain::sqlite::database::SqliteDatabase;
|
||||
use crate::domain::valkey::database::ValkeyDatabase;
|
||||
use crate::domain::firebird::database::FirebirdDatabase;
|
||||
use crate::domain::mariadb::database::MariaDBDatabase;
|
||||
use crate::domain::mssql::database::MssqlDatabase;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::{DatabaseConfig, DbType};
|
||||
use anyhow::Result;
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -14,8 +20,8 @@ use std::sync::Arc;
|
||||
pub trait Database: Send + Sync {
|
||||
fn file_extension(&self) -> &'static str;
|
||||
async fn ping(&self) -> Result<bool>;
|
||||
async fn backup(&self, backup_dir: &Path) -> Result<PathBuf>;
|
||||
async fn restore(&self, restore_file: &Path) -> Result<()>;
|
||||
async fn backup(&self, backup_dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf>;
|
||||
async fn restore(&self, restore_file: &Path, logger: Arc<JobLogger>) -> Result<()>;
|
||||
}
|
||||
|
||||
pub struct DatabaseFactory;
|
||||
@@ -27,12 +33,16 @@ impl DatabaseFactory {
|
||||
let format = detect_format_from_size(&cfg).await;
|
||||
Arc::new(PostgresDatabase::new(cfg, format))
|
||||
}
|
||||
DbType::PostgresqlCluster => Arc::new(PostgresClusterDatabase::new(cfg)),
|
||||
DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)),
|
||||
DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)),
|
||||
DbType::Mariadb => Arc::new(MariaDBDatabase::new(cfg)),
|
||||
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
|
||||
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
|
||||
DbType::Redis => Arc::new(RedisDatabase::new(cfg)),
|
||||
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)),
|
||||
DbType::Firebird => Arc::new(FirebirdDatabase::new(cfg)),
|
||||
DbType::Mssql => Arc::new(MssqlDatabase::new(cfg)),
|
||||
DbType::DockerVolume => Arc::new(DockerVolumeDatabase::new(cfg)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -42,12 +52,16 @@ impl DatabaseFactory {
|
||||
let format = detect_format_from_file(restore_file);
|
||||
Arc::new(PostgresDatabase::new(cfg, format))
|
||||
}
|
||||
DbType::PostgresqlCluster => Arc::new(PostgresClusterDatabase::new(cfg)),
|
||||
DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)),
|
||||
DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)),
|
||||
DbType::Mariadb => Arc::new(MariaDBDatabase::new(cfg)),
|
||||
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
|
||||
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
|
||||
DbType::Redis => Arc::new(RedisDatabase::new(cfg)),
|
||||
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)),
|
||||
DbType::Firebird => Arc::new(FirebirdDatabase::new(cfg)),
|
||||
DbType::Mssql => Arc::new(MssqlDatabase::new(cfg)),
|
||||
DbType::DockerVolume => Arc::new(DockerVolumeDatabase::new(cfg)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
backup_dir: PathBuf,
|
||||
file_extension: &'static str,
|
||||
logger: Arc<JobLogger>,
|
||||
) -> Result<PathBuf> {
|
||||
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||
logger.log("info", format!("Starting Firebird backup for database: {}", cfg.name));
|
||||
|
||||
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
|
||||
let db_path = format!("{}/{}:{}", cfg.host, cfg.port, cfg.database);
|
||||
|
||||
logger.log("info", format!("Firebird target: {} → {}", db_path, file_path.display()));
|
||||
|
||||
let start = Instant::now();
|
||||
let output = Command::new("gbak")
|
||||
.arg("-b")
|
||||
.arg("-v")
|
||||
.arg("-user").arg(&cfg.username)
|
||||
.arg("-password").arg(&cfg.password)
|
||||
.arg(&db_path)
|
||||
.arg(&file_path)
|
||||
.output()
|
||||
.with_context(|| format!("Failed to run gbak for {}", cfg.name))?;
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
let exit_code = output.status.code().unwrap_or(-1);
|
||||
|
||||
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
|
||||
let combined_output = if stderr.is_empty() && stdout.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(format!("{}{}", stdout, stderr).trim().to_string())
|
||||
};
|
||||
|
||||
if !output.status.success() {
|
||||
logger.log_command("gbak", combined_output, Some(exit_code), Some(duration_ms));
|
||||
anyhow::bail!("Firebird backup failed for {}: {}", cfg.name, stderr);
|
||||
}
|
||||
|
||||
logger.log_command("gbak", combined_output, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("Firebird backup completed for {}", cfg.name));
|
||||
Ok(file_path)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
use super::{backup, ping, restore};
|
||||
use crate::domain::factory::Database;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::utils::locks::{DbOpLock, FileLock};
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub struct FirebirdDatabase {
|
||||
cfg: DatabaseConfig,
|
||||
}
|
||||
|
||||
impl FirebirdDatabase {
|
||||
pub fn new(cfg: DatabaseConfig) -> Self {
|
||||
Self { cfg }
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Database for FirebirdDatabase {
|
||||
fn file_extension(&self) -> &'static str {
|
||||
".fbk"
|
||||
}
|
||||
|
||||
async fn ping(&self) -> Result<bool> {
|
||||
ping::run(self.cfg.clone()).await
|
||||
}
|
||||
|
||||
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
let res = backup::run(
|
||||
self.cfg.clone(),
|
||||
dir.to_path_buf(),
|
||||
self.file_extension(),
|
||||
logger,
|
||||
)
|
||||
.await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
|
||||
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
pub mod database;
|
||||
mod ping;
|
||||
mod backup;
|
||||
mod restore;
|
||||
@@ -0,0 +1,57 @@
|
||||
use std::process::Stdio;
|
||||
use tracing::{error, info};
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use tokio::process::Command;
|
||||
use tokio::time::{Duration, timeout};
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig) -> anyhow::Result<bool> {
|
||||
let db_path = format!("{}/{}:{}", cfg.host, cfg.port, cfg.database);
|
||||
|
||||
info!("Running Ping database from {}", db_path);
|
||||
|
||||
let mut child = Command::new("isql-fb")
|
||||
.arg("-q")
|
||||
.arg("-user")
|
||||
.arg(&cfg.username)
|
||||
.arg("-password")
|
||||
.arg(&cfg.password)
|
||||
.arg(&db_path)
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()?;
|
||||
|
||||
let query = b"SELECT 1 FROM RDB$DATABASE;\nQUIT;\n";
|
||||
|
||||
if let Some(mut stdin) = child.stdin.take() {
|
||||
stdin.write_all(query).await?;
|
||||
}
|
||||
|
||||
let output = match timeout(Duration::from_secs(5), child.wait_with_output()).await {
|
||||
Ok(res) => res?,
|
||||
Err(_) => return Ok(false),
|
||||
};
|
||||
|
||||
if !output.status.success() {
|
||||
error!("Error output for firebird: {:?}", output);
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
|
||||
if stderr.to_lowercase().contains("error") {
|
||||
error!("Error output for firebird: {:?}", output);
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
if stdout.contains("1") {
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
info!("stdout {}", stdout);
|
||||
error!("stderr {}", stderr);
|
||||
|
||||
Ok(false)
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
logger.log("debug", format!("Starting Firebird restore for database {}", cfg.name));
|
||||
|
||||
let db_path = format!("{}/{}:{}", cfg.host, cfg.port, cfg.database);
|
||||
|
||||
logger.log("info", format!("Restore source: {}", restore_file.display()));
|
||||
logger.log("info", format!("Restore target: {}", db_path));
|
||||
|
||||
let start = Instant::now();
|
||||
let output = Command::new("gbak")
|
||||
.arg("-c")
|
||||
.arg("-v")
|
||||
.arg("-replace_database")
|
||||
.arg("-user")
|
||||
.arg(&cfg.username)
|
||||
.arg("-password")
|
||||
.arg(&cfg.password)
|
||||
.arg(&restore_file)
|
||||
.arg(&db_path)
|
||||
.output()
|
||||
.with_context(|| format!("Failed to run gbak restore for {}", cfg.name))?;
|
||||
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
let exit_code = output.status.code().unwrap_or(-1);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||
|
||||
if !output.status.success() {
|
||||
logger.log_command("gbak", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
|
||||
logger.log("error", format!("Firebird restore failed for {}: {}", cfg.name, stderr));
|
||||
anyhow::bail!("Firebird restore failed for {}: {}", cfg.name, stderr);
|
||||
}
|
||||
|
||||
logger.log_command("gbak", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("Firebird restore completed for {}", cfg.name));
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.await?
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
use crate::domain::mariadb::connection::{select_mariadb_path, server_version};
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
backup_dir: PathBuf,
|
||||
env: HashMap<String, String>,
|
||||
file_extension: &'static str,
|
||||
logger: Arc<JobLogger>,
|
||||
) -> Result<PathBuf> {
|
||||
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||
logger.log("info", format!("Starting backup for database {}", cfg.name));
|
||||
|
||||
let version = match futures::executor::block_on(server_version(&cfg)) {
|
||||
Ok(v) => {
|
||||
logger.log("info", format!("MariaDB version detected: {}", v));
|
||||
v
|
||||
}
|
||||
Err(e) => {
|
||||
logger.log("error", format!("Failed to get server version: {}", e));
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
|
||||
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
|
||||
let _mariadb_dump = select_mariadb_path(&version).join("mariadb-dump");
|
||||
|
||||
logger.log("debug", format!("Using mariadb-dump at {}", _mariadb_dump.display()));
|
||||
|
||||
if let Ok(out) = Command::new("mariadb-dump").arg("--version").output() {
|
||||
logger.log("debug", format!("mariadb-dump client: {}", String::from_utf8_lossy(&out.stdout).trim()));
|
||||
}
|
||||
|
||||
logger.log("info", format!("Running mariadb-dump for {}", cfg.name));
|
||||
|
||||
let start = Instant::now();
|
||||
let output = Command::new("mariadb-dump")
|
||||
.arg("--host").arg(&cfg.host)
|
||||
.arg("--port").arg(cfg.port.to_string())
|
||||
.arg("--user").arg(&cfg.username)
|
||||
.arg("--routines")
|
||||
.arg("--events")
|
||||
.arg("--triggers")
|
||||
.arg("--single-transaction")
|
||||
.arg("--quick")
|
||||
.arg("--skip-lock-tables")
|
||||
.arg("--no-create-db")
|
||||
.arg("--skip-add-drop-table")
|
||||
.arg("--compress")
|
||||
.arg("--verbose")
|
||||
.arg(format!("--max-allowed-packet={}", cfg.max_packet_size))
|
||||
.arg("--net-buffer-length=16K")
|
||||
.arg("--default-character-set=utf8mb4")
|
||||
.arg(&cfg.database)
|
||||
.arg("-r").arg(&file_path)
|
||||
.envs(env)
|
||||
.output()
|
||||
.with_context(|| format!("Failed to run mariadb-dump for {}", cfg.name))?;
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
let exit_code = output.status.code().unwrap_or(-1);
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||
logger.log_command("mariadb-dump", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
|
||||
anyhow::bail!("Mariadb backup failed for {}: {}", cfg.name, stderr);
|
||||
}
|
||||
|
||||
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||
logger.log_command("mariadb-dump", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("mariadb-dump completed for {}", cfg.name));
|
||||
Ok(file_path)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
use std::path::PathBuf;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::Result;
|
||||
use std::process::Command;
|
||||
|
||||
pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
|
||||
let output = Command::new("mariadb")
|
||||
.arg("--host")
|
||||
.arg(&cfg.host)
|
||||
.arg("--port")
|
||||
.arg(cfg.port.to_string())
|
||||
.arg("--user")
|
||||
.arg(&cfg.username)
|
||||
.arg("-e")
|
||||
.arg("SELECT VERSION();")
|
||||
.env("MYSQL_PWD", &cfg.password)
|
||||
.output()?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
anyhow::bail!("Version query failed: {}", stderr);
|
||||
}
|
||||
|
||||
let version = String::from_utf8_lossy(&output.stdout)
|
||||
.lines()
|
||||
.nth(1)
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.to_string();
|
||||
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
|
||||
pub fn select_mariadb_path(version: &str) -> PathBuf {
|
||||
let mut parts = version.split('.');
|
||||
let major = parts.next().and_then(|v| v.parse::<u32>().ok()).unwrap_or(10);
|
||||
let minor = parts.next().and_then(|v| v.parse::<u32>().ok()).unwrap_or(0);
|
||||
|
||||
if major < 10 || (major == 10 && minor <= 6) {
|
||||
"/usr/local/mariadb-10.6/bin".into()
|
||||
} else {
|
||||
"/usr/local/mariadb-12.1/bin".into()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
use super::{backup, ping, restore};
|
||||
use crate::domain::factory::Database;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::utils::locks::{DbOpLock, FileLock};
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub struct MariaDBDatabase {
|
||||
cfg: DatabaseConfig,
|
||||
}
|
||||
|
||||
impl MariaDBDatabase {
|
||||
pub fn new(cfg: DatabaseConfig) -> Self {
|
||||
Self { cfg }
|
||||
}
|
||||
|
||||
fn build_env(&self) -> HashMap<String, String> {
|
||||
let mut envs = std::env::vars().collect::<HashMap<_, _>>();
|
||||
envs.insert("MYSQL_PWD".to_string(), self.cfg.password.to_string());
|
||||
envs
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Database for MariaDBDatabase {
|
||||
fn file_extension(&self) -> &'static str {
|
||||
".sql"
|
||||
}
|
||||
|
||||
async fn ping(&self) -> Result<bool> {
|
||||
ping::run(self.cfg.clone(), self.build_env().clone()).await
|
||||
}
|
||||
|
||||
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
let res = backup::run(
|
||||
self.cfg.clone(),
|
||||
dir.to_path_buf(),
|
||||
self.build_env().clone(),
|
||||
self.file_extension(),
|
||||
logger,
|
||||
)
|
||||
.await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
|
||||
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
pub mod backup;
|
||||
mod connection;
|
||||
pub mod database;
|
||||
mod ping;
|
||||
mod restore;
|
||||
@@ -0,0 +1,28 @@
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use std::collections::HashMap;
|
||||
use tokio::process::Command;
|
||||
use tokio::time::{Duration, timeout};
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::Result<bool> {
|
||||
let mut cmd = Command::new("mysqladmin");
|
||||
cmd.arg("--host")
|
||||
.arg(cfg.host)
|
||||
.arg("--port")
|
||||
.arg(cfg.port.to_string())
|
||||
.arg("--user")
|
||||
.arg(cfg.username)
|
||||
.arg("ping")
|
||||
.envs(env);
|
||||
|
||||
let result = timeout(Duration::from_secs(10), cmd.output()).await;
|
||||
|
||||
match result {
|
||||
Ok(output) => {
|
||||
let output = output?;
|
||||
Ok(output.status.success())
|
||||
}
|
||||
Err(_) => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::fs::File;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
|
||||
let handle = tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
logger.log("info", format!("Starting restore for database {}", cfg.name));
|
||||
|
||||
let mut file = File::open(&restore_file)
|
||||
.with_context(|| format!("Failed to open restore file {}", restore_file.display()))?;
|
||||
|
||||
let drop_create_cmd = format!(
|
||||
"DROP DATABASE IF EXISTS `{0}`; CREATE DATABASE `{0}`;",
|
||||
cfg.database
|
||||
);
|
||||
|
||||
let drop_start = Instant::now();
|
||||
let drop_output = Command::new("mariadb")
|
||||
.arg("--host")
|
||||
.arg(&cfg.host)
|
||||
.arg("--port")
|
||||
.arg(cfg.port.to_string())
|
||||
.arg("--user")
|
||||
.arg(&cfg.username)
|
||||
.arg("-e")
|
||||
.arg(&drop_create_cmd)
|
||||
.env("MYSQL_PWD", &cfg.password)
|
||||
.output()
|
||||
.with_context(|| format!("Failed to drop/recreate database {}", cfg.name))?;
|
||||
|
||||
let drop_duration_ms = drop_start.elapsed().as_millis() as f64;
|
||||
let drop_exit_code = drop_output.status.code().unwrap_or(-1);
|
||||
let drop_stderr = String::from_utf8_lossy(&drop_output.stderr).to_string();
|
||||
|
||||
if !drop_output.status.success() {
|
||||
logger.log_command("mariadb", Some(drop_stderr.clone()), Some(drop_exit_code), Some(drop_duration_ms));
|
||||
logger.log("error", format!("Drop/create database failed for {}: {}", cfg.name, drop_stderr));
|
||||
anyhow::bail!("Failed to drop/recreate database {}", cfg.name);
|
||||
}
|
||||
logger.log_command("mariadb", if drop_stderr.is_empty() { None } else { Some(drop_stderr) }, Some(0), Some(drop_duration_ms));
|
||||
logger.log("info", format!("Database {} dropped and recreated", cfg.name));
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
let mut child = Command::new("mariadb")
|
||||
.arg("--host")
|
||||
.arg(&cfg.host)
|
||||
.arg("--port")
|
||||
.arg(cfg.port.to_string())
|
||||
.arg("--user")
|
||||
.arg(&cfg.username)
|
||||
.arg("--database")
|
||||
.arg(&cfg.database)
|
||||
.env("MYSQL_PWD", &cfg.password)
|
||||
.stdin(std::process::Stdio::piped())
|
||||
.spawn()
|
||||
.with_context(|| format!("Failed to start MariaDB restore for {}", cfg.name))?;
|
||||
|
||||
let mut stdin = child.stdin.take().context("Failed to open child stdin")?;
|
||||
std::io::copy(&mut file, &mut stdin)
|
||||
.context("Failed to stream SQL content to MariaDB stdin")?;
|
||||
drop(stdin);
|
||||
|
||||
let output = child
|
||||
.wait_with_output()
|
||||
.with_context(|| format!("Failed to complete MariaDB restore for {}", cfg.name))?;
|
||||
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
let exit_code = output.status.code().unwrap_or(-1);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||
|
||||
if !output.status.success() {
|
||||
logger.log_command("mariadb", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
|
||||
logger.log("error", format!("MariaDB restore failed for {}: {}", cfg.name, stderr));
|
||||
anyhow::bail!("MariaDB restore failed for {}", cfg.name);
|
||||
}
|
||||
|
||||
logger.log_command("mariadb", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("Restore finished successfully for database {}", cfg.name));
|
||||
Ok(())
|
||||
});
|
||||
|
||||
handle.await??;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
pub mod docker_volume;
|
||||
pub mod factory;
|
||||
mod mongodb;
|
||||
pub mod mysql;
|
||||
@@ -5,3 +6,6 @@ pub mod postgres;
|
||||
mod redis;
|
||||
mod sqlite;
|
||||
mod valkey;
|
||||
mod mariadb;
|
||||
mod firebird;
|
||||
pub mod mssql;
|
||||
|
||||
@@ -1,36 +1,48 @@
|
||||
use crate::domain::mongodb::connection::{get_mongo_uri, select_mongo_path};
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use tracing::{debug, error, info};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
backup_dir: PathBuf,
|
||||
file_extension: &'static str,
|
||||
logger: Arc<JobLogger>,
|
||||
) -> Result<PathBuf> {
|
||||
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||
debug!("Starting MongoDB backup for database {}", cfg.name);
|
||||
logger.log("info", format!("Starting MongoDB backup for database {}", cfg.name));
|
||||
|
||||
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
|
||||
let mongodump = select_mongo_path().join("mongodump");
|
||||
info!("{:?}", mongodump);
|
||||
let uri = get_mongo_uri(cfg.clone())?;
|
||||
|
||||
logger.log("info", format!("Running mongodump for {}", cfg.name));
|
||||
|
||||
let start = Instant::now();
|
||||
let output = Command::new(mongodump)
|
||||
.arg(format!("--uri={}", uri))
|
||||
.arg(format!("--archive={}", file_path.display()))
|
||||
.arg("--gzip")
|
||||
.arg("--verbose")
|
||||
.output()
|
||||
.context("MongoDB backup failed")?;
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
let exit_code = output.status.code().unwrap_or(-1);
|
||||
|
||||
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
error!("MongoDB backup failed for {}: {}", cfg.name, stderr);
|
||||
logger.log("error", format!("MongoDB backup failed for {}: {}", cfg.name, stderr));
|
||||
logger.log_command("mongodump", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
|
||||
anyhow::bail!("MongoDB backup failed for {}: {}", cfg.name, stderr);
|
||||
}
|
||||
info!("MongoDB backup completed for {}", cfg.name);
|
||||
|
||||
logger.log_command("mongodump", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("MongoDB backup completed for {}", cfg.name));
|
||||
Ok(file_path)
|
||||
})
|
||||
.await?
|
||||
|
||||
@@ -28,3 +28,18 @@ pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result<String> {
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
pub fn extract_db_name(dry_output: &str) -> Option<String> {
|
||||
let mut dbs = std::collections::HashSet::new();
|
||||
for line in dry_output.lines() {
|
||||
if let Some(pos) = line.find("archive prelude ") {
|
||||
let rest = &line[pos + "archive prelude ".len()..];
|
||||
if let Some(dot) = rest.find('.') {
|
||||
let db = &rest[..dot];
|
||||
dbs.insert(db.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
dbs.into_iter().next()
|
||||
}
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::{backup, ping, restore};
|
||||
use crate::domain::factory::Database;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::utils::locks::{DbOpLock, FileLock};
|
||||
|
||||
@@ -27,16 +29,16 @@ impl Database for MongoDatabase {
|
||||
ping::run(self.cfg.clone()).await
|
||||
}
|
||||
|
||||
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
|
||||
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
|
||||
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension(), logger).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
|
||||
async fn restore(&self, file: &Path) -> Result<()> {
|
||||
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
|
||||
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
|
||||
@@ -1,32 +1,74 @@
|
||||
use crate::domain::mongodb::connection::{get_mongo_uri, select_mongo_path};
|
||||
use crate::domain::mongodb::connection::{extract_db_name, get_mongo_uri, select_mongo_path};
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use tracing::{debug, error, info};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
|
||||
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
debug!("Starting MongoDB restore for database {}", cfg.name);
|
||||
logger.log("debug", format!("Starting MongoDB restore for database {}", cfg.name));
|
||||
|
||||
let mongorestore = select_mongo_path().join("mongorestore");
|
||||
let uri = get_mongo_uri(cfg.clone())?;
|
||||
|
||||
let output = Command::new(mongorestore)
|
||||
let dry_start = Instant::now();
|
||||
let dry_run = Command::new(&mongorestore)
|
||||
.arg(format!(
|
||||
"--uri={}",
|
||||
format!(
|
||||
"mongodb://{}:{}@{}:{}/?authSource=admin",
|
||||
cfg.username, cfg.password, cfg.host, cfg.port
|
||||
)
|
||||
))
|
||||
.arg(format!("--archive={}", restore_file.display()))
|
||||
.arg("--gzip")
|
||||
.arg("--dryRun")
|
||||
.arg("--verbose")
|
||||
.output()?;
|
||||
|
||||
let dry_duration_ms = dry_start.elapsed().as_millis() as f64;
|
||||
let dry_exit_code = dry_run.status.code().unwrap_or(-1);
|
||||
let dry_output = String::from_utf8_lossy(&dry_run.stderr);
|
||||
logger.log_command(
|
||||
"mongorestore --dryRun",
|
||||
if dry_output.is_empty() { None } else { Some(dry_output.to_string()) },
|
||||
Some(dry_exit_code),
|
||||
Some(dry_duration_ms),
|
||||
);
|
||||
let source_db = extract_db_name(&dry_output).unwrap_or_else(|| {
|
||||
logger.log("info", format!("Could not detect source database from archive, falling back to configured database: {}", cfg.database));
|
||||
cfg.database.clone()
|
||||
});
|
||||
|
||||
logger.log("info", format!("Using source database in archive: {}", source_db));
|
||||
|
||||
let start = Instant::now();
|
||||
let output = Command::new(&mongorestore)
|
||||
.arg(format!("--uri={}", uri))
|
||||
.arg(format!("--archive={}", restore_file.display()))
|
||||
.arg("--gzip")
|
||||
.arg("--drop")
|
||||
.arg(format!("--nsInclude={}.*", source_db))
|
||||
.arg(format!("--nsFrom={}.*", source_db))
|
||||
.arg(format!("--nsTo={}.*", cfg.database))
|
||||
.output()
|
||||
.with_context(|| format!("Failed to run mongorestore for {}", cfg.name))?;
|
||||
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
let exit_code = output.status.code().unwrap_or(-1);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
error!("MongoDB restore failed for {}: {}", cfg.name, stderr);
|
||||
anyhow::bail!("MongoDB restore failed for : {}", cfg.name);
|
||||
logger.log_command("mongorestore", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
|
||||
logger.log("error", format!("MongoDB restore failed for {}: {}", cfg.name, stderr));
|
||||
anyhow::bail!("MongoDB restore failed for: {}", cfg.name);
|
||||
}
|
||||
|
||||
info!("MongoDB restore completed for {}", cfg.name);
|
||||
logger.log_command("mongorestore", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("MongoDB restore completed for {}", cfg.name));
|
||||
Ok(())
|
||||
})
|
||||
.await?
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
backup_dir: PathBuf,
|
||||
file_extension: &'static str,
|
||||
logger: Arc<JobLogger>,
|
||||
) -> Result<PathBuf> {
|
||||
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||
logger.log("info", format!("Starting MSSQL backup for database {}", cfg.name));
|
||||
|
||||
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
|
||||
let connection_string = format!(
|
||||
"Server=tcp:{},{};Database={};User Id={};Password={};TrustServerCertificate=True;Encrypt=False",
|
||||
cfg.host, cfg.port, cfg.database, cfg.username, cfg.password
|
||||
);
|
||||
|
||||
logger.log("info", format!("MSSQL backup: {}:{}/{} → {}", cfg.host, cfg.port, cfg.database, file_path.display()));
|
||||
|
||||
let start = Instant::now();
|
||||
let output = Command::new("sqlpackage")
|
||||
.arg("/a:Export")
|
||||
.arg(format!("/scs:{}", connection_string))
|
||||
.arg(format!("/tf:{}", file_path.display()))
|
||||
.output()
|
||||
.with_context(|| format!("Failed to run sqlpackage for {}", cfg.name))?;
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
let exit_code = output.status.code().unwrap_or(-1);
|
||||
|
||||
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
|
||||
|
||||
if !output.status.success() {
|
||||
logger.log("error", format!("MSSQL backup failed — stderr: {} stdout: {}", stderr, stdout));
|
||||
let out = format!("stderr: {} stdout: {}", stderr, stdout);
|
||||
logger.log_command("sqlpackage", Some(out), Some(exit_code), Some(duration_ms));
|
||||
anyhow::bail!("MSSQL backup failed for {}: {}", cfg.name, stderr);
|
||||
}
|
||||
|
||||
let combined = if stdout.is_empty() && stderr.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(format!("{}{}", stdout, stderr).trim().to_string())
|
||||
};
|
||||
logger.log_command("sqlpackage", combined, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("MSSQL backup completed for {}", cfg.name));
|
||||
Ok(file_path)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::Result;
|
||||
use tiberius::{AuthMethod, Client, Config};
|
||||
use tokio::net::TcpStream;
|
||||
use tokio_util::compat::{Compat, TokioAsyncWriteCompatExt};
|
||||
|
||||
pub async fn build_client(cfg: &DatabaseConfig) -> Result<Client<Compat<TcpStream>>> {
|
||||
let mut config = Config::new();
|
||||
config.host(&cfg.host);
|
||||
config.port(cfg.port);
|
||||
config.authentication(AuthMethod::sql_server(&cfg.username, &cfg.password));
|
||||
config.trust_cert();
|
||||
|
||||
let tcp = TcpStream::connect(config.get_addr()).await?;
|
||||
tcp.set_nodelay(true)?;
|
||||
let client = Client::connect(config, tcp.compat_write()).await?;
|
||||
Ok(client)
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
use super::{backup, ping, restore};
|
||||
use crate::domain::factory::Database;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::utils::locks::{DbOpLock, FileLock};
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub struct MssqlDatabase {
|
||||
cfg: DatabaseConfig,
|
||||
}
|
||||
|
||||
impl MssqlDatabase {
|
||||
pub fn new(cfg: DatabaseConfig) -> Self {
|
||||
Self { cfg }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Database for MssqlDatabase {
|
||||
fn file_extension(&self) -> &'static str {
|
||||
".bacpac"
|
||||
}
|
||||
|
||||
async fn ping(&self) -> Result<bool> {
|
||||
ping::run(self.cfg.clone()).await
|
||||
}
|
||||
|
||||
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
let res = backup::run(
|
||||
self.cfg.clone(),
|
||||
dir.to_path_buf(),
|
||||
self.file_extension(),
|
||||
logger,
|
||||
)
|
||||
.await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
|
||||
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
pub mod database;
|
||||
mod connection;
|
||||
mod ping;
|
||||
mod backup;
|
||||
mod restore;
|
||||
@@ -0,0 +1,24 @@
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::Result;
|
||||
use tracing::{error, info};
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
|
||||
info!("Running ping for MSSQL database {}", cfg.name);
|
||||
|
||||
match super::connection::build_client(&cfg).await {
|
||||
Ok(mut client) => match client.simple_query("SELECT 1").await {
|
||||
Ok(_) => {
|
||||
info!("MSSQL ping succeeded for {}", cfg.name);
|
||||
Ok(true)
|
||||
}
|
||||
Err(e) => {
|
||||
error!("MSSQL ping query failed for {}: {:?}", cfg.name, e);
|
||||
Ok(false)
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
error!("MSSQL connection failed for {}: {:?}", cfg.name, e);
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
logger.log("debug", format!("Starting MSSQL restore for database {}", cfg.name));
|
||||
|
||||
let connection_string = format!(
|
||||
"Server=tcp:{},{};Database={};User Id={};Password={};TrustServerCertificate=True;Encrypt=False",
|
||||
cfg.host, cfg.port, cfg.database, cfg.username, cfg.password
|
||||
);
|
||||
|
||||
logger.log("info", format!(
|
||||
"MSSQL restore: {} → {}:{}/{}",
|
||||
restore_file.display(),
|
||||
cfg.host,
|
||||
cfg.port,
|
||||
cfg.database
|
||||
));
|
||||
|
||||
let start = Instant::now();
|
||||
let output = Command::new("sqlpackage")
|
||||
.arg("/a:Import")
|
||||
.arg(format!("/tcs:{}", connection_string))
|
||||
.arg(format!("/sf:{}", restore_file.display()))
|
||||
.output()
|
||||
.with_context(|| format!("Failed to run sqlpackage restore for {}", cfg.name))?;
|
||||
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
let exit_code = output.status.code().unwrap_or(-1);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
|
||||
let combined = format!("{}{}", stdout, stderr);
|
||||
|
||||
if !output.status.success() {
|
||||
logger.log_command("sqlpackage", if combined.is_empty() { None } else { Some(combined.clone()) }, Some(exit_code), Some(duration_ms));
|
||||
logger.log("error", format!(
|
||||
"MSSQL restore failed for {} — stderr: {} stdout: {}",
|
||||
cfg.name, stderr, stdout
|
||||
));
|
||||
anyhow::bail!("MSSQL restore failed for {}: {}", cfg.name, stderr);
|
||||
}
|
||||
|
||||
logger.log_command("sqlpackage", if combined.is_empty() { None } else { Some(combined) }, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("MSSQL restore completed for {}", cfg.name));
|
||||
Ok(())
|
||||
})
|
||||
.await?
|
||||
}
|
||||
+37
-19
@@ -1,62 +1,80 @@
|
||||
use crate::domain::mysql::connection::server_version;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use tracing::{debug, error, info};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
backup_dir: PathBuf,
|
||||
env: HashMap<String, String>,
|
||||
file_extension: &'static str,
|
||||
logger: Arc<JobLogger>,
|
||||
) -> Result<PathBuf> {
|
||||
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||
debug!("Starting backup for database {}", cfg.name);
|
||||
logger.log("info", format!("Starting backup for database {}", cfg.name));
|
||||
|
||||
let version = match futures::executor::block_on(server_version(&cfg)) {
|
||||
let _version = match futures::executor::block_on(server_version(&cfg)) {
|
||||
Ok(v) => {
|
||||
debug!("Mysql version detected: {}", v);
|
||||
logger.log("debug", format!("MySQL version detected: {}", v));
|
||||
v
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to get server version for {}: {:?}", cfg.name, e);
|
||||
logger.log("error", format!("Failed to get server version: {}", e));
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
|
||||
info!("Mysql version found: {}", version);
|
||||
|
||||
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
|
||||
|
||||
if let Ok(out) = Command::new("mysqldump").arg("--version").output() {
|
||||
logger.log("debug", format!("mysqldump client: {}", String::from_utf8_lossy(&out.stdout).trim()));
|
||||
}
|
||||
|
||||
logger.log("info", format!("Running mysqldump for {}", cfg.name));
|
||||
|
||||
let start = Instant::now();
|
||||
let output = Command::new("mysqldump")
|
||||
.arg("--host")
|
||||
.arg(cfg.host)
|
||||
.arg("--port")
|
||||
.arg(cfg.port.to_string())
|
||||
.arg("--user")
|
||||
.arg(cfg.username)
|
||||
.arg("--host").arg(&cfg.host)
|
||||
.arg("--port").arg(cfg.port.to_string())
|
||||
.arg("--user").arg(&cfg.username)
|
||||
.arg("--routines")
|
||||
.arg("--events")
|
||||
.arg("--triggers")
|
||||
.arg("--verbose")
|
||||
.arg("--single-transaction")
|
||||
.arg("--set-gtid-purged=OFF")
|
||||
.arg("--no-tablespaces")
|
||||
.arg("--quick")
|
||||
.arg("--add-drop-database")
|
||||
.arg("--databases")
|
||||
.arg(cfg.database)
|
||||
.arg("-r")
|
||||
.arg(&file_path)
|
||||
.arg("--skip-lock-tables")
|
||||
.arg("--skip-add-drop-table")
|
||||
.arg("--no-create-db")
|
||||
.arg("--default-character-set=utf8mb4")
|
||||
.arg("--network-timeout")
|
||||
.arg(format!("--max-allowed-packet={}", cfg.max_packet_size))
|
||||
.arg(&cfg.database)
|
||||
.arg("-r").arg(&file_path)
|
||||
.envs(env)
|
||||
.output()
|
||||
.with_context(|| format!("Failed to run mysqldump for {}", cfg.name))?;
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
let exit_code = output.status.code().unwrap_or(-1);
|
||||
|
||||
let _stdout = String::from_utf8_lossy(&output.stdout).to_string();
|
||||
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
logger.log_command("mysqldump", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
|
||||
anyhow::bail!("MySQL backup failed for {}: {}", cfg.name, stderr);
|
||||
}
|
||||
|
||||
logger.log_command("mysqldump", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("mysqldump completed for {}", cfg.name));
|
||||
|
||||
Ok(file_path)
|
||||
})
|
||||
.await?
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
use super::{backup, ping, restore};
|
||||
use crate::domain::factory::Database;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::utils::locks::{DbOpLock, FileLock};
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub struct MySQLDatabase {
|
||||
cfg: DatabaseConfig,
|
||||
@@ -33,22 +35,23 @@ impl Database for MySQLDatabase {
|
||||
ping::run(self.cfg.clone(), self.build_env().clone()).await
|
||||
}
|
||||
|
||||
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
|
||||
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
let res = backup::run(
|
||||
self.cfg.clone(),
|
||||
dir.to_path_buf(),
|
||||
self.build_env().clone(),
|
||||
self.file_extension(),
|
||||
logger,
|
||||
)
|
||||
.await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
|
||||
async fn restore(&self, file: &Path) -> Result<()> {
|
||||
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
|
||||
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ use tokio::process::Command;
|
||||
use tokio::time::{Duration, timeout};
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::Result<bool> {
|
||||
let mut cmd = Command::new("mysqladmin");
|
||||
let mut cmd = Command::new("mariadb-admin");
|
||||
cmd.arg("--host")
|
||||
.arg(cfg.host)
|
||||
.arg("--port")
|
||||
|
||||
+31
-20
@@ -1,27 +1,26 @@
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::fs::File;
|
||||
use std::io::{Read, Write};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use tracing::{debug, error, info};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
|
||||
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
|
||||
let handle = tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
debug!("Starting restore for database {}", cfg.name);
|
||||
logger.log("info", format!("Starting restore for database {}", cfg.name));
|
||||
|
||||
let mut sql_content = String::new();
|
||||
let mut file = File::open(&restore_file)
|
||||
.with_context(|| format!("Failed to open restore file {}", restore_file.display()))?;
|
||||
file.read_to_string(&mut sql_content)
|
||||
.with_context(|| format!("Failed to read restore file {}", restore_file.display()))?;
|
||||
|
||||
let drop_create_cmd = format!(
|
||||
"DROP DATABASE IF EXISTS {0}; CREATE DATABASE {0};",
|
||||
"DROP DATABASE IF EXISTS `{0}`; CREATE DATABASE `{0}`;",
|
||||
cfg.database
|
||||
);
|
||||
|
||||
let drop_status = Command::new("mysql")
|
||||
let drop_start = Instant::now();
|
||||
let drop_output = Command::new("mysql")
|
||||
.arg("--host")
|
||||
.arg(&cfg.host)
|
||||
.arg("--port")
|
||||
@@ -31,14 +30,22 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
|
||||
.arg("-e")
|
||||
.arg(&drop_create_cmd)
|
||||
.env("MYSQL_PWD", &cfg.password)
|
||||
.status()
|
||||
.output()
|
||||
.with_context(|| format!("Failed to drop/recreate database {}", cfg.name))?;
|
||||
|
||||
if !drop_status.success() {
|
||||
error!("Drop/create database failed for {}", cfg.name);
|
||||
let drop_duration_ms = drop_start.elapsed().as_millis() as f64;
|
||||
let drop_exit_code = drop_output.status.code().unwrap_or(-1);
|
||||
let drop_stderr = String::from_utf8_lossy(&drop_output.stderr).to_string();
|
||||
|
||||
if !drop_output.status.success() {
|
||||
logger.log_command("mysql", Some(drop_stderr.clone()), Some(drop_exit_code), Some(drop_duration_ms));
|
||||
logger.log("error", format!("Drop/create database failed for {}: {}", cfg.name, drop_stderr));
|
||||
anyhow::bail!("Failed to drop/recreate database {}", cfg.name);
|
||||
}
|
||||
info!("Database {} dropped and recreated", cfg.name);
|
||||
logger.log_command("mysql", if drop_stderr.is_empty() { None } else { Some(drop_stderr) }, Some(0), Some(drop_duration_ms));
|
||||
logger.log("info", format!("Database {} dropped and recreated", cfg.name));
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
let mut child = Command::new("mysql")
|
||||
.arg("--host")
|
||||
@@ -47,6 +54,7 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
|
||||
.arg(cfg.port.to_string())
|
||||
.arg("--user")
|
||||
.arg(&cfg.username)
|
||||
.arg("--database")
|
||||
.arg(&cfg.database)
|
||||
.env("MYSQL_PWD", &cfg.password)
|
||||
.stdin(std::process::Stdio::piped())
|
||||
@@ -54,23 +62,26 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
|
||||
.with_context(|| format!("Failed to start mysql restore for {}", cfg.name))?;
|
||||
|
||||
let mut stdin = child.stdin.take().context("Failed to open child stdin")?;
|
||||
stdin
|
||||
.write_all(sql_content.as_bytes())
|
||||
.context("Failed to write SQL content to mysql stdin")?;
|
||||
stdin.flush()?;
|
||||
std::io::copy(&mut file, &mut stdin)
|
||||
.context("Failed to stream SQL content to mysql stdin")?;
|
||||
drop(stdin);
|
||||
|
||||
let output = child
|
||||
.wait_with_output()
|
||||
.with_context(|| format!("Failed to complete mysql restore for {}", cfg.name))?;
|
||||
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
let exit_code = output.status.code().unwrap_or(-1);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
error!("MySQL restore failed for {}: {}", cfg.name, stderr);
|
||||
logger.log_command("mysql", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
|
||||
logger.log("error", format!("MySQL restore failed for {}: {}", cfg.name, stderr));
|
||||
anyhow::bail!("MySQL restore failed for {}", cfg.name);
|
||||
}
|
||||
|
||||
info!("Restore finished successfully for database {}", cfg.name);
|
||||
logger.log_command("mysql", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("Restore finished successfully for database {}", cfg.name));
|
||||
Ok(())
|
||||
});
|
||||
|
||||
|
||||
@@ -1,142 +1,150 @@
|
||||
use anyhow::Result;
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use tracing::{debug, error, info};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use super::connection::{select_pg_path, server_version};
|
||||
use super::format::PostgresDumpFormat;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
format: PostgresDumpFormat,
|
||||
backup_dir: PathBuf,
|
||||
env: HashMap<String, String>,
|
||||
logger: Arc<JobLogger>,
|
||||
) -> Result<PathBuf> {
|
||||
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||
debug!("Starting backup for database {}", cfg.name);
|
||||
logger.log("info", format!("Starting backup for database {}", cfg.name));
|
||||
|
||||
let version = match futures::executor::block_on(server_version(&cfg)) {
|
||||
Ok(v) => {
|
||||
debug!("Postgres version detected: {}", v);
|
||||
logger.log("debug", format!("Postgres version detected: {}", v));
|
||||
v
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to get server version for {}: {:?}", cfg.name, e);
|
||||
logger.log("error", format!("Failed to get server version: {}", e));
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
|
||||
let pg_dump = select_pg_path(&version).join("pg_dump");
|
||||
|
||||
debug!("Using pg_dump at {:?}", pg_dump);
|
||||
logger.log("debug", format!("Using pg_dump at {:?}", pg_dump));
|
||||
|
||||
match format {
|
||||
PostgresDumpFormat::Fc => {
|
||||
info!("Running FC backup for {}", cfg.name);
|
||||
let file_path = backup_dir.join(format!("{}.dump", cfg.generated_id));
|
||||
let url = format!(
|
||||
"postgresql://{}:{}@{}:{}/{}",
|
||||
cfg.username, cfg.password, cfg.host, cfg.port, cfg.database
|
||||
);
|
||||
logger.log("info", format!("Running FC backup for {}", cfg.name));
|
||||
|
||||
let status = Command::new(&pg_dump)
|
||||
.arg("--dbname")
|
||||
.arg(&url)
|
||||
let file_path = backup_dir.join(format!("{}.dump", cfg.generated_id));
|
||||
|
||||
let start = Instant::now();
|
||||
let output = Command::new(&pg_dump)
|
||||
.arg("--host").arg(&cfg.host)
|
||||
.arg("--port").arg(cfg.port.to_string())
|
||||
.arg("--username").arg(&cfg.username)
|
||||
.arg("--dbname").arg(&cfg.database)
|
||||
.arg("-Fc")
|
||||
.arg("-f")
|
||||
.arg(&file_path)
|
||||
.arg("-f").arg(&file_path)
|
||||
.arg("-v")
|
||||
.arg("--compress=3")
|
||||
.status();
|
||||
.envs(env)
|
||||
.output();
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
|
||||
match status {
|
||||
Ok(s) if s.success() => info!(
|
||||
"FC backup completed successfully for {} at {:?}",
|
||||
cfg.name, file_path
|
||||
),
|
||||
Ok(s) => {
|
||||
error!("FC backup failed with status {:?} for {}", s, cfg.name);
|
||||
anyhow::bail!("Postgres backup failed for {}", cfg.name);
|
||||
match output {
|
||||
Ok(o) => {
|
||||
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
|
||||
let exit_code = o.status.code().unwrap_or(-1);
|
||||
|
||||
if o.status.success() {
|
||||
logger.log("info", format!("FC backup completed successfully for {} at {:?}", cfg.name, file_path));
|
||||
logger.log_command("pg_dump", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
|
||||
} else {
|
||||
logger.log("error", format!("FC backup failed with status {:?} for {}", o.status, cfg.name));
|
||||
logger.log_command("pg_dump", Some(stderr), Some(exit_code), Some(duration_ms));
|
||||
anyhow::bail!("Postgres backup failed for {}", cfg.name);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Error executing pg_dump for {}: {:?}", cfg.name, e);
|
||||
logger.log("error", format!("Error executing pg_dump for {}: {:?}", cfg.name, e));
|
||||
logger.log_command("pg_dump", Some(e.to_string()), Some(-1), Some(duration_ms));
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
info!("Backup finished for database {}", cfg.name);
|
||||
logger.log("info", format!("Backup finished for database {}", cfg.name));
|
||||
Ok(file_path)
|
||||
}
|
||||
|
||||
PostgresDumpFormat::Fd => {
|
||||
info!("Running FD backup for {}", cfg.name);
|
||||
logger.log("info", format!("Running FD backup for {}", cfg.name));
|
||||
|
||||
let dump_dir = backup_dir.join(format!("{}_dir", cfg.generated_id));
|
||||
let tar_file = backup_dir.join(format!("{}.tar.gz", cfg.generated_id));
|
||||
|
||||
if let Err(e) = std::fs::create_dir_all(&dump_dir) {
|
||||
error!(
|
||||
"Failed to create dump directory {:?} for {}: {:?}",
|
||||
dump_dir, cfg.name, e
|
||||
);
|
||||
logger.log("error", format!("Failed to create dump directory {:?} for {}: {:?}", dump_dir, cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
|
||||
let url = format!(
|
||||
"postgresql://{}:{}@{}:{}/{}",
|
||||
cfg.username, cfg.password, cfg.host, cfg.port, cfg.database
|
||||
);
|
||||
let cmd_label = format!("pg_dump -Fd {}@{}:{}/{}", cfg.username, cfg.host, cfg.port, cfg.database);
|
||||
|
||||
let status = Command::new(&pg_dump)
|
||||
.arg("--dbname")
|
||||
.arg(&url)
|
||||
let start = Instant::now();
|
||||
let output = Command::new(&pg_dump)
|
||||
.arg("--host").arg(&cfg.host)
|
||||
.arg("--port").arg(cfg.port.to_string())
|
||||
.arg("--username").arg(&cfg.username)
|
||||
.arg("--dbname").arg(&cfg.database)
|
||||
.arg("-Fd")
|
||||
.arg("-j")
|
||||
.arg("4")
|
||||
.arg("-f")
|
||||
.arg(&dump_dir)
|
||||
.arg("-j").arg("4")
|
||||
.arg("-f").arg(&dump_dir)
|
||||
.arg("-v")
|
||||
.status();
|
||||
.envs(env)
|
||||
.output();
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
|
||||
match status {
|
||||
Ok(s) if s.success() => {
|
||||
info!("FD backup pg_dump completed successfully for {}", cfg.name)
|
||||
}
|
||||
Ok(s) => {
|
||||
error!(
|
||||
"FD backup pg_dump failed with status {:?} for {}",
|
||||
s, cfg.name
|
||||
);
|
||||
anyhow::bail!("Postgres FD backup failed for {}", cfg.name);
|
||||
match output {
|
||||
Ok(o) => {
|
||||
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
|
||||
let exit_code = o.status.code().unwrap_or(-1);
|
||||
if o.status.success() {
|
||||
logger.log("info", format!("FD backup pg_dump completed successfully for {}", cfg.name));
|
||||
logger.log_command(cmd_label, if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
|
||||
} else {
|
||||
logger.log("error", format!("FD backup pg_dump failed with status {:?} for {}", o.status, cfg.name));
|
||||
logger.log_command(cmd_label, Some(stderr), Some(exit_code), Some(duration_ms));
|
||||
anyhow::bail!("Postgres FD backup failed for {}", cfg.name);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Error executing pg_dump for {}: {:?}", cfg.name, e);
|
||||
logger.log("error", format!("Error executing pg_dump for {}: {:?}", cfg.name, e));
|
||||
logger.log_command(cmd_label, Some(e.to_string()), Some(-1), Some(duration_ms));
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
|
||||
match std::fs::File::create(&tar_file) {
|
||||
Ok(tar_gz) => {
|
||||
let enc =
|
||||
flate2::write::GzEncoder::new(tar_gz, flate2::Compression::default());
|
||||
let enc = flate2::write::GzEncoder::new(tar_gz, flate2::Compression::default());
|
||||
let mut tar = tar::Builder::new(enc);
|
||||
if let Err(e) = tar.append_dir_all(".", &dump_dir) {
|
||||
error!("Failed to append dump_dir to tar for {}: {:?}", cfg.name, e);
|
||||
logger.log("error", format!("Failed to append dump_dir to tar for {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
if let Err(e) = tar.finish() {
|
||||
error!("Failed to finish tar archive for {}: {:?}", cfg.name, e);
|
||||
logger.log("error", format!("Failed to finish tar archive for {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
info!("FD backup archive created at {:?}", tar_file);
|
||||
logger.log("info", format!("FD backup archive created at {:?}", tar_file));
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
"Failed to create tar.gz file {:?} for {}: {:?}",
|
||||
tar_file, cfg.name, e
|
||||
);
|
||||
logger.log("error", format!("Failed to create tar.gz file {:?} for {}: {:?}", tar_file, cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
info!("Backup finished for database {}", cfg.name);
|
||||
logger.log("info", format!("Backup finished for database {}", cfg.name));
|
||||
Ok(tar_file)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
use anyhow::Result;
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use super::super::connection::{
|
||||
is_superuser, pg_dumpall_binary_name, select_pg_path, server_version,
|
||||
};
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
backup_dir: PathBuf,
|
||||
env: HashMap<String, String>,
|
||||
logger: Arc<JobLogger>,
|
||||
) -> Result<PathBuf> {
|
||||
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||
logger.log("info", format!("Starting cluster backup for {}", cfg.name));
|
||||
|
||||
let version = match futures::executor::block_on(server_version(&cfg)) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
logger.log("error", format!("Failed to get server version for {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
|
||||
match futures::executor::block_on(is_superuser(&cfg)) {
|
||||
Ok(true) => {}
|
||||
Ok(false) => {
|
||||
logger.log("error", format!("postgresql-cluster backup requires a superuser role for {}", cfg.name));
|
||||
anyhow::bail!("postgresql-cluster backup requires a superuser role for {}", cfg.name);
|
||||
}
|
||||
Err(e) => {
|
||||
logger.log("error", format!("Failed to check superuser status for {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
|
||||
let pg_dumpall = select_pg_path(&version).join(pg_dumpall_binary_name());
|
||||
let file_path = backup_dir.join(format!("{}.sql", cfg.generated_id));
|
||||
|
||||
logger.log("info", format!("Running pg_dumpall for cluster {} via {:?}", cfg.name, pg_dumpall));
|
||||
|
||||
let start = Instant::now();
|
||||
let output = Command::new(&pg_dumpall)
|
||||
.arg("--host").arg(&cfg.host)
|
||||
.arg("--port").arg(cfg.port.to_string())
|
||||
.arg("--username").arg(&cfg.username)
|
||||
.arg("--clean")
|
||||
.arg("--if-exists")
|
||||
.arg("-v")
|
||||
.arg("-f").arg(&file_path)
|
||||
.envs(env)
|
||||
.output();
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
|
||||
match output {
|
||||
Ok(o) => {
|
||||
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
|
||||
let exit_code = o.status.code().unwrap_or(-1);
|
||||
if o.status.success() {
|
||||
logger.log_command("pg_dumpall", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("Cluster backup completed for {} at {:?}", cfg.name, file_path));
|
||||
Ok(file_path)
|
||||
} else {
|
||||
logger.log_command("pg_dumpall", Some(stderr), Some(exit_code), Some(duration_ms));
|
||||
anyhow::bail!("Cluster backup (pg_dumpall) failed for {}", cfg.name);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
logger.log_command("pg_dumpall", Some(e.to_string()), Some(-1), Some(duration_ms));
|
||||
Err(e.into())
|
||||
}
|
||||
}
|
||||
})
|
||||
.await?
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::super::ping;
|
||||
use super::{backup, restore};
|
||||
use crate::domain::factory::Database;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::utils::locks::{DbOpLock, FileLock};
|
||||
|
||||
pub struct PostgresClusterDatabase {
|
||||
pub cfg: DatabaseConfig,
|
||||
}
|
||||
|
||||
impl PostgresClusterDatabase {
|
||||
pub fn new(cfg: DatabaseConfig) -> Self {
|
||||
Self { cfg }
|
||||
}
|
||||
|
||||
fn build_env(&self) -> HashMap<String, String> {
|
||||
let mut envs = std::env::vars().collect::<HashMap<_, _>>();
|
||||
envs.insert("PGPASSWORD".to_string(), self.cfg.password.to_string());
|
||||
envs
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Database for PostgresClusterDatabase {
|
||||
fn file_extension(&self) -> &'static str {
|
||||
".sql"
|
||||
}
|
||||
|
||||
async fn ping(&self) -> Result<bool> {
|
||||
ping::run(self.cfg.clone()).await
|
||||
}
|
||||
|
||||
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.build_env(), logger).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
|
||||
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||
let res = restore::run(self.cfg.clone(), file.to_path_buf(), self.build_env(), logger).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
pub mod backup;
|
||||
pub mod database;
|
||||
pub mod restore;
|
||||
@@ -0,0 +1,84 @@
|
||||
use anyhow::Result;
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use super::super::connection::{is_superuser, psql_binary_name, select_pg_path, server_version, terminate_all_connections};
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
restore_file: PathBuf,
|
||||
env: HashMap<String, String>,
|
||||
logger: Arc<JobLogger>,
|
||||
) -> Result<()> {
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
logger.log("info", format!("Starting cluster restore for {}", cfg.name));
|
||||
|
||||
let version = match futures::executor::block_on(server_version(&cfg)) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
logger.log("error", format!("Failed to get server version for {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
|
||||
match futures::executor::block_on(is_superuser(&cfg)) {
|
||||
Ok(true) => {}
|
||||
Ok(false) => {
|
||||
logger.log("error", format!("postgresql-cluster restore requires a superuser role for {}", cfg.name));
|
||||
anyhow::bail!("postgresql-cluster restore requires a superuser role for {}", cfg.name);
|
||||
}
|
||||
Err(e) => {
|
||||
logger.log("error", format!("Failed to check superuser status for {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
|
||||
let psql = select_pg_path(&version).join(psql_binary_name());
|
||||
|
||||
if let Err(e) = futures::executor::block_on(terminate_all_connections(&cfg)) {
|
||||
logger.log("error", format!("Failed to terminate connections for cluster {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
logger.log("info", format!("All user database connections terminated for cluster {}", cfg.name));
|
||||
|
||||
logger.log("info", format!("Replaying cluster dump for {} via {:?}", cfg.name, psql));
|
||||
|
||||
let start = Instant::now();
|
||||
let output = Command::new(&psql)
|
||||
.arg("--host").arg(&cfg.host)
|
||||
.arg("--port").arg(cfg.port.to_string())
|
||||
.arg("--username").arg(&cfg.username)
|
||||
.arg("--dbname").arg("postgres")
|
||||
.arg("-f").arg(&restore_file)
|
||||
.envs(env)
|
||||
.output();
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
|
||||
match output {
|
||||
Ok(o) => {
|
||||
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
|
||||
let stdout = String::from_utf8_lossy(&o.stdout).to_string();
|
||||
let combined = format!("{}{}", stdout, stderr);
|
||||
let exit_code = o.status.code().unwrap_or(-1);
|
||||
if o.status.success() {
|
||||
logger.log_command("psql", if combined.is_empty() { None } else { Some(combined) }, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("Cluster restore completed for {}", cfg.name));
|
||||
Ok(())
|
||||
} else {
|
||||
logger.log_command("psql", if combined.is_empty() { None } else { Some(combined) }, Some(exit_code), Some(duration_ms));
|
||||
anyhow::bail!("Cluster restore (psql) failed for {}", cfg.name);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
logger.log_command("psql", Some(e.to_string()), Some(-1), Some(duration_ms));
|
||||
Err(e.into())
|
||||
}
|
||||
}
|
||||
})
|
||||
.await?
|
||||
}
|
||||
@@ -1,18 +1,23 @@
|
||||
use crate::domain::postgres::format::PostgresDumpFormat;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::settings::CONFIG;
|
||||
use anyhow::Result;
|
||||
use std::path::Path;
|
||||
use tokio_postgres::{Client, NoTls};
|
||||
use tokio_postgres::{Client, Config, NoTls};
|
||||
use tracing::{error, info};
|
||||
|
||||
pub async fn connect(cfg: &DatabaseConfig) -> Result<Client> {
|
||||
info!("Connecting to postgres database {}:{}", cfg.host, cfg.port);
|
||||
let dsn = format!(
|
||||
"host={} port={} user={} password={} dbname={}",
|
||||
cfg.host, cfg.port, cfg.username, cfg.password, cfg.database
|
||||
);
|
||||
|
||||
let (client, connection) = tokio_postgres::connect(&dsn, NoTls).await?;
|
||||
let mut config = Config::new();
|
||||
config
|
||||
.host(&cfg.host)
|
||||
.port(cfg.port)
|
||||
.user(&cfg.username)
|
||||
.password(&cfg.password)
|
||||
.dbname(&cfg.database);
|
||||
|
||||
let (client, connection) = config.connect(NoTls).await?;
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = connection.await {
|
||||
error!("Postgres connection error: {}", e);
|
||||
@@ -28,11 +33,96 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
pub async fn is_superuser(cfg: &DatabaseConfig) -> Result<bool> {
|
||||
let client = connect(cfg).await?;
|
||||
let is_super: bool = client
|
||||
.query_one("SELECT current_setting('is_superuser') = 'on';", &[])
|
||||
.await?
|
||||
.get(0);
|
||||
|
||||
Ok(is_super)
|
||||
}
|
||||
|
||||
|
||||
pub fn select_pg_path(version: &str) -> std::path::PathBuf {
|
||||
select_pg_path_with(version, &CONFIG.pg_bin_dir)
|
||||
}
|
||||
|
||||
pub(crate) fn select_pg_path_with(version: &str, pg_bin_dir: &str) -> std::path::PathBuf {
|
||||
let major = version.split('.').next().unwrap_or("17");
|
||||
|
||||
if !pg_bin_dir.is_empty() {
|
||||
return pg_bin_dir.into();
|
||||
}
|
||||
|
||||
let candidates: Vec<std::path::PathBuf> = if cfg!(target_os = "windows") {
|
||||
vec![
|
||||
// Default install path used by the official EDB Windows installer
|
||||
format!(r"C:\Program Files\PostgreSQL\{major}\bin").into(),
|
||||
format!(r"C:\Program Files (x86)\PostgreSQL\{major}\bin").into(),
|
||||
]
|
||||
} else if cfg!(target_os = "macos") {
|
||||
vec![
|
||||
// Homebrew on Apple Silicon
|
||||
format!("/opt/homebrew/opt/postgresql@{major}/bin").into(),
|
||||
// Homebrew on Intel
|
||||
format!("/usr/local/opt/postgresql@{major}/bin").into(),
|
||||
// Postgres.app
|
||||
format!("/Applications/Postgres.app/Contents/Versions/{major}/bin").into(),
|
||||
]
|
||||
} else {
|
||||
vec![
|
||||
// Debian/Ubuntu packages
|
||||
format!("/usr/lib/postgresql/{major}/bin").into(),
|
||||
// Common RPM-based distro layout
|
||||
format!("/usr/pgsql-{major}/bin").into(),
|
||||
]
|
||||
};
|
||||
|
||||
if let Some(found) = candidates.into_iter().find(|p| pg_dump_exists_in(p)) {
|
||||
return found;
|
||||
}
|
||||
|
||||
if let Some(dir) = find_pg_dump_in_path() {
|
||||
return dir;
|
||||
}
|
||||
|
||||
format!("/usr/lib/postgresql/{}/bin", major).into()
|
||||
}
|
||||
|
||||
pub(crate) fn pg_dump_binary_name() -> &'static str {
|
||||
if cfg!(target_os = "windows") {
|
||||
"pg_dump.exe"
|
||||
} else {
|
||||
"pg_dump"
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn pg_dumpall_binary_name() -> &'static str {
|
||||
if cfg!(target_os = "windows") {
|
||||
"pg_dumpall.exe"
|
||||
} else {
|
||||
"pg_dumpall"
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn psql_binary_name() -> &'static str {
|
||||
if cfg!(target_os = "windows") {
|
||||
"psql.exe"
|
||||
} else {
|
||||
"psql"
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn pg_dump_exists_in(dir: &std::path::Path) -> bool {
|
||||
dir.join(pg_dump_binary_name()).is_file()
|
||||
}
|
||||
|
||||
fn find_pg_dump_in_path() -> Option<std::path::PathBuf> {
|
||||
let path_var = std::env::var_os("PATH")?;
|
||||
std::env::split_paths(&path_var).find(|dir| pg_dump_exists_in(dir))
|
||||
}
|
||||
|
||||
pub async fn terminate_connections(cfg: &DatabaseConfig) -> Result<()> {
|
||||
let mut admin = cfg.clone();
|
||||
admin.database = "postgres".to_string().into();
|
||||
@@ -54,6 +144,27 @@ pub async fn terminate_connections(cfg: &DatabaseConfig) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn terminate_all_connections(cfg: &DatabaseConfig) -> Result<()> {
|
||||
let mut admin = cfg.clone();
|
||||
admin.database = "postgres".to_string().into();
|
||||
|
||||
let client = connect(&admin).await?;
|
||||
|
||||
client
|
||||
.execute(
|
||||
r#"
|
||||
SELECT pg_terminate_backend(pid)
|
||||
FROM pg_stat_activity
|
||||
WHERE datname NOT IN ('postgres', 'template0', 'template1')
|
||||
AND pid <> pg_backend_pid();
|
||||
"#,
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn detect_format_from_file(restore_file: &Path) -> PostgresDumpFormat {
|
||||
match restore_file.extension().and_then(|e| e.to_str()) {
|
||||
Some("dump") => PostgresDumpFormat::Fc,
|
||||
@@ -68,6 +179,7 @@ pub async fn detect_format_from_size(cfg: &DatabaseConfig) -> PostgresDumpFormat
|
||||
"Detecting database format {:?} - {:?}",
|
||||
cfg.name, cfg.generated_id
|
||||
);
|
||||
|
||||
let client = match connect(cfg).await {
|
||||
Ok(c) => c,
|
||||
Err(_) => return PostgresDumpFormat::Fc,
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use std::sync::Arc;
|
||||
use super::{backup, format::PostgresDumpFormat, ping, restore};
|
||||
use crate::domain::factory::Database;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::utils::locks::{DbOpLock, FileLock};
|
||||
|
||||
@@ -16,6 +18,12 @@ impl PostgresDatabase {
|
||||
pub fn new(cfg: DatabaseConfig, format: PostgresDumpFormat) -> Self {
|
||||
Self { cfg, format }
|
||||
}
|
||||
|
||||
fn build_env(&self) -> HashMap<String, String> {
|
||||
let mut envs = std::env::vars().collect::<HashMap<_, _>>();
|
||||
envs.insert("PGPASSWORD".to_string(), self.cfg.password.to_string());
|
||||
envs
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -31,16 +39,30 @@ impl Database for PostgresDatabase {
|
||||
ping::run(self.cfg.clone()).await
|
||||
}
|
||||
|
||||
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
|
||||
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
let res = backup::run(self.cfg.clone(), self.format, dir.to_path_buf()).await;
|
||||
let res = backup::run(
|
||||
self.cfg.clone(),
|
||||
self.format,
|
||||
dir.to_path_buf(),
|
||||
self.build_env(),
|
||||
logger,
|
||||
)
|
||||
.await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
|
||||
async fn restore(&self, file: &Path) -> Result<()> {
|
||||
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||
let res = restore::run(self.cfg.clone(), self.format, file.to_path_buf()).await;
|
||||
let res = restore::run(
|
||||
self.cfg.clone(),
|
||||
self.format,
|
||||
file.to_path_buf(),
|
||||
self.build_env(),
|
||||
logger,
|
||||
)
|
||||
.await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
pub mod backup;
|
||||
mod connection;
|
||||
pub(crate) mod cluster;
|
||||
pub(crate) mod connection;
|
||||
pub mod database;
|
||||
mod format;
|
||||
mod ping;
|
||||
|
||||
@@ -1,94 +1,119 @@
|
||||
use anyhow::Result;
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use tracing::{debug, error, info};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use super::connection::{select_pg_path, server_version, terminate_connections};
|
||||
use super::format::PostgresDumpFormat;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
format: PostgresDumpFormat,
|
||||
restore_file: PathBuf,
|
||||
env: HashMap<String, String>,
|
||||
logger: Arc<JobLogger>,
|
||||
) -> Result<()> {
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
debug!("Starting restore for database {}", cfg.name);
|
||||
logger.log("info", format!("Starting restore for database {}", cfg.name));
|
||||
|
||||
let version = match futures::executor::block_on(server_version(&cfg)) {
|
||||
Ok(v) => {
|
||||
debug!("Postgres version detected: {}", v);
|
||||
logger.log("debug", format!("Postgres version detected: {}", v));
|
||||
v
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to get server version for {}: {:?}", cfg.name, e);
|
||||
logger.log("error", format!("Failed to get server version for {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
|
||||
let pg_restore = select_pg_path(&version).join("pg_restore");
|
||||
|
||||
debug!("Using pg_restore at {:?}", pg_restore);
|
||||
logger.log("debug", format!("Using pg_restore at {:?}", pg_restore));
|
||||
|
||||
if let Err(e) = futures::executor::block_on(terminate_connections(&cfg)) {
|
||||
error!("Failed to terminate connections for {}: {:?}", cfg.name, e);
|
||||
logger.log("error", format!("Failed to terminate connections for {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
info!("Connections terminated for database {}", cfg.name);
|
||||
logger.log("info", format!("Connections terminated for database {}", cfg.name));
|
||||
|
||||
let url = format!(
|
||||
"postgresql://{}:{}@{}:{}/postgres",
|
||||
cfg.username, cfg.password, cfg.host, cfg.port
|
||||
);
|
||||
let keep_ownership = cfg.options
|
||||
.get("keep_ownership")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
|
||||
debug!("Restore URL: {}", url);
|
||||
if keep_ownership {
|
||||
logger.log("info", format!("Restoring ownership and privileges for {}", cfg.name));
|
||||
} else {
|
||||
logger.log("info", format!("Stripping ownership and privileges for {} (--no-owner --no-privileges)", cfg.name));
|
||||
}
|
||||
|
||||
match format {
|
||||
PostgresDumpFormat::Fc => {
|
||||
info!("Running FC restore for {}", cfg.name);
|
||||
let status = Command::new(&pg_restore)
|
||||
.arg("--no-owner")
|
||||
.arg("--no-privileges")
|
||||
logger.log("info", format!("Running FC restore for {}", cfg.name));
|
||||
let start = Instant::now();
|
||||
let mut cmd = Command::new(&pg_restore);
|
||||
if !keep_ownership {
|
||||
cmd.arg("--no-owner").arg("--no-privileges");
|
||||
}
|
||||
let output = cmd
|
||||
.arg("--clean")
|
||||
.arg("--if-exists")
|
||||
.arg("--create")
|
||||
.arg("--dbname")
|
||||
.arg(&url)
|
||||
// .arg("--create")
|
||||
.arg("--host").arg(&cfg.host)
|
||||
.arg("--port").arg(cfg.port.to_string())
|
||||
.arg("--username").arg(&cfg.username)
|
||||
.arg("--dbname").arg(&cfg.database)
|
||||
.arg("-v")
|
||||
.arg(&restore_file)
|
||||
.env("PGPASSWORD", &cfg.password)
|
||||
.status();
|
||||
.envs(env)
|
||||
.output();
|
||||
|
||||
match status {
|
||||
Ok(s) if s.success() => {
|
||||
info!("FC restore completed successfully for {}", cfg.name)
|
||||
}
|
||||
Ok(s) => {
|
||||
error!("FC restore failed with status {:?} for {}", s, cfg.name);
|
||||
anyhow::bail!("Postgres restore failed for {}", cfg.name);
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
|
||||
match output {
|
||||
Ok(o) => {
|
||||
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
|
||||
let stdout = String::from_utf8_lossy(&o.stdout).to_string();
|
||||
let combined = format!("{}{}", stdout, stderr);
|
||||
let exit_code = o.status.code().unwrap_or(-1);
|
||||
|
||||
if o.status.success() {
|
||||
logger.log_command("pg_restore", if combined.is_empty() { None } else { Some(combined) }, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("FC restore completed successfully for {}", cfg.name))
|
||||
} else {
|
||||
logger.log_command("pg_restore", if combined.is_empty() { None } else { Some(combined) }, Some(exit_code), Some(duration_ms));
|
||||
logger.log("error", format!("FC restore failed with status {:?} for {}", o.status, cfg.name));
|
||||
anyhow::bail!("Postgres restore failed for {}", cfg.name);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Error executing pg_restore for {}: {:?}", cfg.name, e);
|
||||
logger.log_command("pg_restore", Some(e.to_string()), Some(-1), Some(duration_ms));
|
||||
logger.log("error", format!("Error executing pg_restore for {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
PostgresDumpFormat::Fd => {
|
||||
info!("Running FD restore for {}", cfg.name);
|
||||
logger.log("info", format!("Running FD restore for {}", cfg.name));
|
||||
|
||||
let tar_gz = match std::fs::File::open(&restore_file) {
|
||||
Ok(f) => f,
|
||||
Err(e) => {
|
||||
error!(
|
||||
logger.log("error", format!(
|
||||
"Failed to open restore file {:?} for {}: {:?}",
|
||||
restore_file, cfg.name, e
|
||||
);
|
||||
));
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
|
||||
info!("tar_gz {:?}", tar_gz);
|
||||
logger.log("info", format!("tar_gz {:?}", tar_gz));
|
||||
|
||||
let dec = flate2::read::GzDecoder::new(tar_gz);
|
||||
let mut archive = tar::Archive::new(dec);
|
||||
@@ -96,31 +121,31 @@ pub async fn run(
|
||||
let tmp_dir = match tempfile::TempDir::new() {
|
||||
Ok(d) => d,
|
||||
Err(e) => {
|
||||
error!(
|
||||
logger.log("error", format!(
|
||||
"Failed to create temporary directory for FD restore of {}: {:?}",
|
||||
cfg.name, e
|
||||
);
|
||||
));
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(e) = archive.unpack(tmp_dir.path()) {
|
||||
error!("Failed to unpack FD archive for {}: {:?}", cfg.name, e);
|
||||
logger.log("error", format!("Failed to unpack FD archive for {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
|
||||
debug!("Listing contents of temp dir: {}", tmp_dir.path().display());
|
||||
logger.log("debug", format!("Listing contents of temp dir: {}", tmp_dir.path().display()));
|
||||
|
||||
for entry in std::fs::read_dir(tmp_dir.path())? {
|
||||
if let Ok(entry) = entry {
|
||||
let path = entry.path();
|
||||
let file_type = entry.file_type()?;
|
||||
debug!(
|
||||
logger.log("debug", format!(
|
||||
" - {} | is_dir: {} | is_file: {}",
|
||||
path.display(),
|
||||
file_type.is_dir(),
|
||||
file_type.is_file()
|
||||
);
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -134,38 +159,54 @@ pub async fn run(
|
||||
.ok_or_else(|| anyhow::anyhow!("Invalid FD archive: toc.dat not found"))?
|
||||
};
|
||||
|
||||
let status = Command::new(&pg_restore)
|
||||
.arg("--no-owner")
|
||||
.arg("--no-privileges")
|
||||
let start = Instant::now();
|
||||
let mut cmd = Command::new(&pg_restore);
|
||||
if !keep_ownership {
|
||||
cmd.arg("--no-owner").arg("--no-privileges");
|
||||
}
|
||||
let output = cmd
|
||||
.arg("--clean")
|
||||
.arg("--if-exists")
|
||||
.arg("--create")
|
||||
.arg("--dbname")
|
||||
.arg(&url)
|
||||
// .arg("--create")
|
||||
.arg("--host").arg(&cfg.host)
|
||||
.arg("--port").arg(cfg.port.to_string())
|
||||
.arg("--username").arg(&cfg.username)
|
||||
.arg("--dbname").arg(&cfg.database)
|
||||
.arg("-v")
|
||||
.arg("-j")
|
||||
.arg("4")
|
||||
.arg(dump_dir)
|
||||
.env("PGPASSWORD", &cfg.password)
|
||||
.status();
|
||||
.envs(env)
|
||||
.output();
|
||||
|
||||
match status {
|
||||
Ok(s) if s.success() => {
|
||||
info!("FD restore completed successfully for {}", cfg.name)
|
||||
}
|
||||
Ok(s) => {
|
||||
error!("FD restore failed with status {:?} for {}", s, cfg.name);
|
||||
anyhow::bail!("Postgres FD restore failed for {}", cfg.name);
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
|
||||
match output {
|
||||
Ok(o) => {
|
||||
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
|
||||
let stdout = String::from_utf8_lossy(&o.stdout).to_string();
|
||||
let combined = format!("{}{}", stdout, stderr);
|
||||
let exit_code = o.status.code().unwrap_or(-1);
|
||||
|
||||
if o.status.success() {
|
||||
logger.log_command("pg_restore", if combined.is_empty() { None } else { Some(combined) }, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("FD restore completed successfully for {}", cfg.name))
|
||||
} else {
|
||||
logger.log_command("pg_restore", if combined.is_empty() { None } else { Some(combined) }, Some(exit_code), Some(duration_ms));
|
||||
logger.log("error", format!("FD restore failed with status {:?} for {}", o.status, cfg.name));
|
||||
anyhow::bail!("Postgres FD restore failed for {}", cfg.name);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Error executing pg_restore for {}: {:?}", cfg.name, e);
|
||||
logger.log_command("pg_restore", Some(e.to_string()), Some(-1), Some(duration_ms));
|
||||
logger.log("error", format!("Error executing pg_restore for {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
info!("Restore finished for database {}", cfg.name);
|
||||
logger.log("info", format!("Restore finished for database {}", cfg.name));
|
||||
|
||||
Ok(())
|
||||
})
|
||||
|
||||
+44
-13
@@ -1,21 +1,26 @@
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use tracing::{debug, error, info};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
backup_dir: PathBuf,
|
||||
file_extension: &'static str,
|
||||
logger: Arc<JobLogger>,
|
||||
) -> Result<PathBuf> {
|
||||
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||
debug!("Starting Redis backup for database {}", cfg.name);
|
||||
logger.log(
|
||||
"info",
|
||||
format!("Starting Redis backup for database {}", cfg.name),
|
||||
);
|
||||
|
||||
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
|
||||
|
||||
let mut cmd = Command::new("redis-cli");
|
||||
|
||||
cmd.arg("-h")
|
||||
.arg(&cfg.host)
|
||||
.arg("-p")
|
||||
@@ -24,40 +29,66 @@ pub async fn run(
|
||||
if !cfg.username.is_empty() {
|
||||
cmd.arg("--user").arg(&cfg.username);
|
||||
}
|
||||
|
||||
if !cfg.password.is_empty() {
|
||||
cmd.arg("-a").arg(&cfg.password);
|
||||
}
|
||||
|
||||
cmd.arg("--rdb").arg(&file_path);
|
||||
|
||||
debug!("Command Backup: {:?}", cmd);
|
||||
logger.log("info", format!("Running redis-cli --rdb for {}", cfg.name));
|
||||
|
||||
let start = Instant::now();
|
||||
let output = cmd.output().context("Redis backup command failed")?;
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
let exit_code = output.status.code().unwrap_or(-1);
|
||||
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
|
||||
if !output.status.success() {
|
||||
if stderr.contains("NOAUTH") {
|
||||
error!(
|
||||
"Redis backup failed for {}: Authentication required (NOAUTH)",
|
||||
cfg.name
|
||||
logger.log(
|
||||
"error",
|
||||
format!(
|
||||
"Redis backup failed for {}: Authentication required (NOAUTH)",
|
||||
cfg.name
|
||||
),
|
||||
);
|
||||
logger.log_command(
|
||||
"redis-cli",
|
||||
Some("Authentication required (NOAUTH)".into()),
|
||||
Some(exit_code),
|
||||
Some(duration_ms),
|
||||
);
|
||||
anyhow::bail!(
|
||||
"Redis backup failed for {}: Authentication required",
|
||||
cfg.name
|
||||
);
|
||||
} else {
|
||||
error!("Redis backup failed for {}: {}", cfg.name, stderr);
|
||||
logger.log(
|
||||
"error",
|
||||
format!("Redis backup failed for {}: {}", cfg.name, stderr),
|
||||
);
|
||||
logger.log_command(
|
||||
"redis-cli",
|
||||
Some(stderr.to_string()),
|
||||
Some(exit_code),
|
||||
Some(duration_ms),
|
||||
);
|
||||
anyhow::bail!("Redis backup failed for {}: {}", cfg.name, stderr);
|
||||
}
|
||||
}
|
||||
|
||||
info!(
|
||||
"Redis backup completed for {}. Output: {}",
|
||||
cfg.name, stdout
|
||||
logger.log_command(
|
||||
"redis-cli",
|
||||
if stdout.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(stdout.to_string())
|
||||
},
|
||||
Some(0),
|
||||
Some(duration_ms),
|
||||
);
|
||||
logger.log("info", format!("Redis backup completed for {}", cfg.name));
|
||||
|
||||
Ok(file_path)
|
||||
})
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
use anyhow::{Result, bail};
|
||||
use async_trait::async_trait;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::domain::factory::Database;
|
||||
use crate::domain::redis::{backup, ping};
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::utils::locks::{DbOpLock, FileLock};
|
||||
|
||||
@@ -27,14 +29,14 @@ impl Database for RedisDatabase {
|
||||
ping::run(self.cfg.clone()).await
|
||||
}
|
||||
|
||||
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
|
||||
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
|
||||
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension(), logger).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
|
||||
async fn restore(&self, _file: &Path) -> Result<()> {
|
||||
async fn restore(&self, _file: &Path, _logger: Arc<JobLogger>) -> Result<()> {
|
||||
bail!("Restore not supported for Redis databases")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,10 +30,7 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
|
||||
let output = output.context("Failed to execute redis-cli")?;
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
|
||||
if !stdout.is_empty() {
|
||||
error!("Redis stderr: {}", stderr);
|
||||
}
|
||||
|
||||
|
||||
if stderr.contains("NOAUTH") {
|
||||
error!("Redis authentication failed (NOAUTH required)");
|
||||
|
||||
@@ -1,16 +1,19 @@
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use tracing::{debug, error, info};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
backup_dir: PathBuf,
|
||||
file_extension: &'static str,
|
||||
logger: Arc<JobLogger>,
|
||||
) -> Result<PathBuf> {
|
||||
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||
debug!("Starting SQLite backup for database {}", cfg.name);
|
||||
logger.log("info", format!("Starting SQLite backup for database {}", cfg.name));
|
||||
|
||||
let db_path_str = if cfg.path.is_empty() {
|
||||
anyhow::bail!("Database path not configured");
|
||||
@@ -19,26 +22,35 @@ pub async fn run(
|
||||
};
|
||||
|
||||
let db_path = PathBuf::from(db_path_str);
|
||||
logger.log("info", format!("Database path: {}", db_path.display()));
|
||||
|
||||
if !db_path.exists() {
|
||||
logger.log("error", format!("SQLite database file not found: {}", db_path.display()));
|
||||
anyhow::bail!("SQLite database file not found: {}", db_path.display());
|
||||
}
|
||||
|
||||
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
|
||||
|
||||
logger.log("info", format!("Running sqlite3 backup for {}", cfg.name));
|
||||
|
||||
let start = Instant::now();
|
||||
let output = Command::new("sqlite3")
|
||||
.arg(db_path.as_os_str())
|
||||
.arg(format!(".backup '{}'", file_path.display()))
|
||||
.output()
|
||||
.context("SQLite backup command failed to start")?;
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
let exit_code = output.status.code().unwrap_or(-1);
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
error!("SQLite backup failed for {}: {}", cfg.name, stderr);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||
logger.log("error", format!("SQLite backup failed for {}: {}", cfg.name, stderr));
|
||||
logger.log_command("sqlite3", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
|
||||
anyhow::bail!("SQLite backup failed for {}: {}", cfg.name, stderr);
|
||||
}
|
||||
|
||||
info!("SQLite backup completed for {}", cfg.name);
|
||||
logger.log_command("sqlite3", None, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("SQLite backup completed for {}", cfg.name));
|
||||
Ok(file_path)
|
||||
})
|
||||
.await?
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::{backup, ping, restore};
|
||||
use crate::domain::factory::Database;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::utils::locks::{DbOpLock, FileLock};
|
||||
|
||||
@@ -27,15 +29,15 @@ impl Database for SqliteDatabase {
|
||||
ping::run(self.cfg.clone()).await
|
||||
}
|
||||
|
||||
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
|
||||
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
|
||||
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension(), logger).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
async fn restore(&self, file: &Path) -> Result<()> {
|
||||
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
|
||||
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use tracing::{debug, error, info};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
|
||||
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
debug!("Starting SQLite restore for database {}", cfg.name);
|
||||
logger.log("debug", format!("Starting SQLite restore for database {}", cfg.name));
|
||||
|
||||
let db_path_str = if cfg.path.is_empty() {
|
||||
anyhow::bail!("Database path not configured");
|
||||
@@ -25,19 +27,25 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
|
||||
.with_context(|| format!("Failed to remove existing DB {}", db_path.display()))?;
|
||||
}
|
||||
|
||||
let start = Instant::now();
|
||||
let output = Command::new("sqlite3")
|
||||
.arg(db_path.as_os_str())
|
||||
.arg(format!(".restore '{}'", restore_file.display()))
|
||||
.output()
|
||||
.with_context(|| format!("Failed to run sqlite3 restore for {}", cfg.name))?;
|
||||
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
let exit_code = output.status.code().unwrap_or(-1);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
error!("SQLite restore failed for {}: {}", cfg.name, stderr);
|
||||
logger.log_command("sqlite3", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
|
||||
logger.log("error", format!("SQLite restore failed for {}: {}", cfg.name, stderr));
|
||||
anyhow::bail!("SQLite restore failed for {}", cfg.name);
|
||||
}
|
||||
|
||||
info!("SQLite restore completed for {}", cfg.name);
|
||||
logger.log_command("sqlite3", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("SQLite restore completed for {}", cfg.name));
|
||||
Ok(())
|
||||
})
|
||||
.await?
|
||||
|
||||
+44
-13
@@ -1,21 +1,26 @@
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use tracing::{debug, error, info};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
backup_dir: PathBuf,
|
||||
file_extension: &'static str,
|
||||
logger: Arc<JobLogger>,
|
||||
) -> Result<PathBuf> {
|
||||
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||
debug!("Starting Valkey backup for database {}", cfg.name);
|
||||
logger.log(
|
||||
"info",
|
||||
format!("Starting Valkey backup for database {}", cfg.name),
|
||||
);
|
||||
|
||||
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
|
||||
|
||||
let mut cmd = Command::new("valkey-cli");
|
||||
|
||||
cmd.arg("-h")
|
||||
.arg(&cfg.host)
|
||||
.arg("-p")
|
||||
@@ -24,40 +29,66 @@ pub async fn run(
|
||||
if !cfg.username.is_empty() {
|
||||
cmd.arg("--user").arg(&cfg.username);
|
||||
}
|
||||
|
||||
if !cfg.password.is_empty() {
|
||||
cmd.arg("-a").arg(&cfg.password);
|
||||
}
|
||||
|
||||
cmd.arg("--rdb").arg(&file_path);
|
||||
|
||||
debug!("Command Backup: {:?}", cmd);
|
||||
logger.log("info", format!("Running valkey-cli --rdb for {}", cfg.name));
|
||||
|
||||
let start = Instant::now();
|
||||
let output = cmd.output().context("Valkey backup command failed")?;
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
let exit_code = output.status.code().unwrap_or(-1);
|
||||
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
|
||||
if !output.status.success() {
|
||||
if stderr.contains("NOAUTH") {
|
||||
error!(
|
||||
"Valkey backup failed for {}: Authentication required (NOAUTH)",
|
||||
cfg.name
|
||||
logger.log(
|
||||
"error",
|
||||
format!(
|
||||
"Valkey backup failed for {}: Authentication required (NOAUTH)",
|
||||
cfg.name
|
||||
),
|
||||
);
|
||||
logger.log_command(
|
||||
"valkey-cli",
|
||||
Some("Authentication required (NOAUTH)".into()),
|
||||
Some(exit_code),
|
||||
Some(duration_ms),
|
||||
);
|
||||
anyhow::bail!(
|
||||
"Valkey backup failed for {}: Authentication required",
|
||||
cfg.name
|
||||
);
|
||||
} else {
|
||||
error!("Valkey backup failed for {}: {}", cfg.name, stderr);
|
||||
logger.log(
|
||||
"error",
|
||||
format!("Valkey backup failed for {}: {}", cfg.name, stderr),
|
||||
);
|
||||
logger.log_command(
|
||||
"valkey-cli",
|
||||
Some(stderr.to_string()),
|
||||
Some(exit_code),
|
||||
Some(duration_ms),
|
||||
);
|
||||
anyhow::bail!("Valkey backup failed for {}: {}", cfg.name, stderr);
|
||||
}
|
||||
}
|
||||
|
||||
info!(
|
||||
"Valkey backup completed for {}. Output: {}",
|
||||
cfg.name, stdout
|
||||
logger.log_command(
|
||||
"valkey-cli",
|
||||
if stdout.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(stdout.to_string())
|
||||
},
|
||||
Some(0),
|
||||
Some(duration_ms),
|
||||
);
|
||||
logger.log("info", format!("Valkey backup completed for {}", cfg.name));
|
||||
|
||||
Ok(file_path)
|
||||
})
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
use anyhow::{Result, bail};
|
||||
use async_trait::async_trait;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::domain::factory::Database;
|
||||
use crate::domain::valkey::{backup, ping};
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::utils::locks::{DbOpLock, FileLock};
|
||||
|
||||
@@ -27,14 +29,14 @@ impl Database for ValkeyDatabase {
|
||||
ping::run(self.cfg.clone()).await
|
||||
}
|
||||
|
||||
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
|
||||
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
|
||||
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension(), logger).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
|
||||
async fn restore(&self, _file: &Path) -> Result<()> {
|
||||
async fn restore(&self, _file: &Path, _logger: Arc<JobLogger>) -> Result<()> {
|
||||
bail!("Restore not supported for Valkey databases")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,11 +30,7 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
|
||||
let output = output.context("Failed to execute valkey-cli")?;
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
|
||||
if !stdout.is_empty() {
|
||||
error!("Valkey stderr: {}", stderr);
|
||||
}
|
||||
|
||||
|
||||
if stderr.contains("NOAUTH") {
|
||||
error!("Valkey authentication failed (NOAUTH required)");
|
||||
return Ok(false);
|
||||
|
||||
+10
@@ -22,6 +22,16 @@ async fn main() {
|
||||
eprintln!("Failed to clean locks on startup: {:?}", e);
|
||||
}
|
||||
|
||||
// Best-effort cleanup of ephemeral helper containers orphaned by a crash.
|
||||
match crate::domain::docker_volume::docker::client() {
|
||||
Ok(docker) => match crate::domain::docker_volume::docker::sweep_ephemeral(&docker).await {
|
||||
Ok(n) if n > 0 => tracing::info!("Removed {n} orphaned ephemeral helper container(s)"),
|
||||
Ok(_) => {}
|
||||
Err(e) => tracing::warn!("Ephemeral helper sweep failed: {e}"),
|
||||
},
|
||||
Err(e) => tracing::debug!("Docker socket unavailable, skipping helper sweep: {e}"),
|
||||
}
|
||||
|
||||
tokio::join!(ping_server(), async {
|
||||
let conn = redis_client::redis_connection().await;
|
||||
scheduler::scheduler_loop(conn).await;
|
||||
|
||||
@@ -2,6 +2,7 @@ pub mod upload;
|
||||
|
||||
use crate::services::api::models::agent::backup::BackupResponse;
|
||||
use crate::services::api::{ApiClient, ApiError};
|
||||
use crate::services::backup::logger::JobLogEntry;
|
||||
use anyhow::Result;
|
||||
use reqwest::Method;
|
||||
use serde::Serialize;
|
||||
@@ -21,6 +22,9 @@ pub struct BackupUpdateRequest {
|
||||
pub size: Option<u64>,
|
||||
#[serde(rename = "generatedId")]
|
||||
pub generated_id: String,
|
||||
pub logs: Vec<JobLogEntry>,
|
||||
#[serde(rename = "durationMs")]
|
||||
pub duration_ms: f64,
|
||||
}
|
||||
|
||||
impl ApiClient {
|
||||
@@ -49,12 +53,16 @@ impl ApiClient {
|
||||
status: impl Into<String>,
|
||||
file_size: impl Into<Option<u64>>,
|
||||
generated_id: impl Into<String>,
|
||||
job_logs: Vec<JobLogEntry>,
|
||||
duration_ms: f64,
|
||||
) -> Result<Option<BackupResponse>, ApiError> {
|
||||
let body = BackupUpdateRequest {
|
||||
backup_id: backup_id.into(),
|
||||
status: status.into(),
|
||||
size: file_size.into(),
|
||||
generated_id: generated_id.into(),
|
||||
logs: job_logs,
|
||||
duration_ms: duration_ms,
|
||||
};
|
||||
|
||||
let agent_id = agent_id.into();
|
||||
|
||||
@@ -3,12 +3,16 @@ use crate::services::api::{ApiClient, ApiError};
|
||||
use anyhow::Result;
|
||||
use reqwest::Method;
|
||||
use serde::Serialize;
|
||||
use crate::services::backup::logger::JobLogEntry;
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct ResultRestoreRequest {
|
||||
#[serde(rename = "generatedId")]
|
||||
pub generated_id: String,
|
||||
pub status: String,
|
||||
pub logs: Vec<JobLogEntry>,
|
||||
#[serde(rename = "durationMs")]
|
||||
pub duration_ms: f64,
|
||||
}
|
||||
|
||||
impl ApiClient {
|
||||
@@ -17,10 +21,14 @@ impl ApiClient {
|
||||
agent_id: impl Into<String>,
|
||||
generated_id: impl Into<String>,
|
||||
status: impl Into<String>,
|
||||
job_logs: Vec<JobLogEntry>,
|
||||
duration_ms: f64,
|
||||
) -> Result<Option<ResultRestoreResponse>, ApiError> {
|
||||
let body = ResultRestoreRequest {
|
||||
generated_id: generated_id.into(),
|
||||
status: status.into(),
|
||||
logs: job_logs,
|
||||
duration_ms,
|
||||
};
|
||||
|
||||
let agent_id = agent_id.into();
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use crate::utils::deserializer::deserialize_snake_case;
|
||||
use crate::utils::deserializer::{deserialize_snake_case, string_or_number_to_string};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use toml::Value;
|
||||
|
||||
@@ -54,4 +54,6 @@ pub struct RestoreInfo {
|
||||
pub file: Option<String>,
|
||||
#[serde(rename = "metaFile")]
|
||||
pub meta_file: Option<String>,
|
||||
#[serde(default, deserialize_with = "string_or_number_to_string")]
|
||||
pub size: Option<String>,
|
||||
}
|
||||
|
||||
@@ -1,14 +1,17 @@
|
||||
use super::logger::JobLogger;
|
||||
use super::service::BackupService;
|
||||
use crate::utils::compress::compress_to_tar_gz_large;
|
||||
use anyhow::Result;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
impl BackupService {
|
||||
pub async fn compress_backup(&self, backup_file: Option<PathBuf>) -> Result<PathBuf> {
|
||||
pub async fn compress_backup(&self, backup_file: Option<PathBuf>, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||
let file = backup_file.ok_or_else(|| anyhow::anyhow!("No backup file generated"))?;
|
||||
|
||||
let compression = compress_to_tar_gz_large(&file).await?;
|
||||
|
||||
logger.log("info", "Start compressing archive".to_string());
|
||||
let compression = compress_to_tar_gz_large(&file, logger).await?;
|
||||
|
||||
Ok(compression.compressed_path)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use super::logger::JobLogger;
|
||||
use super::service::BackupService;
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
@@ -5,6 +6,8 @@ use crate::utils::common::BackupMethod;
|
||||
use crate::utils::locks::FileLock;
|
||||
|
||||
use anyhow::Result;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tempfile::TempDir;
|
||||
|
||||
impl BackupService {
|
||||
@@ -16,9 +19,13 @@ impl BackupService {
|
||||
storages: Vec<DatabaseStorage>,
|
||||
encrypt: bool,
|
||||
) -> Result<()> {
|
||||
let logger = Arc::new(JobLogger::new());
|
||||
|
||||
if FileLock::is_locked(&generated_id).await? {
|
||||
anyhow::bail!("backup already running");
|
||||
}
|
||||
let start = Instant::now();
|
||||
logger.log("info", "Database backup job started".to_string());
|
||||
|
||||
let backup = self.create_backup_record(&generated_id, &method).await?;
|
||||
let backup_id = backup.backup.id;
|
||||
@@ -26,21 +33,27 @@ impl BackupService {
|
||||
let temp_dir = TempDir::new()?;
|
||||
let tmp_path = temp_dir.path();
|
||||
|
||||
let mut result = Self::run(db_cfg, tmp_path).await?;
|
||||
let mut result = Self::run(db_cfg, tmp_path, Arc::clone(&logger)).await?;
|
||||
|
||||
if result.status == "failed" {
|
||||
self.send_result(result, vec![], &backup_id).await?;
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
let logs = Arc::try_unwrap(logger).unwrap_or_else(|_| JobLogger::new()).into_entries();
|
||||
self.send_result(result, vec![], &backup_id, logs, duration_ms).await?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let compressed = self.compress_backup(result.backup_file.take()).await?;
|
||||
let compressed = self.compress_backup(result.backup_file.take(), Arc::clone(&logger)).await?;
|
||||
result.backup_file = Some(compressed);
|
||||
|
||||
let uploads = self
|
||||
.upload(result.clone(), method, storages, encrypt, &backup_id)
|
||||
.upload(result.clone(), method, storages, encrypt, &backup_id, Arc::clone(&logger))
|
||||
.await?;
|
||||
|
||||
self.send_result(result, uploads, &backup_id).await?;
|
||||
logger.log("info", "Database backup job finished".to_string());
|
||||
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
let logs = Arc::try_unwrap(logger).unwrap_or_else(|_| JobLogger::new()).into_entries();
|
||||
self.send_result(result, uploads, &backup_id, logs, duration_ms).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
use chrono::Utc;
|
||||
use serde::Serialize;
|
||||
use std::sync::Mutex;
|
||||
use tracing::{event, Level};
|
||||
|
||||
#[derive(Serialize, Clone, Debug)]
|
||||
pub struct JobLogEntry {
|
||||
pub timestamp: String,
|
||||
#[serde(rename = "type")]
|
||||
pub entry_type: &'static str,
|
||||
pub level: &'static str,
|
||||
pub message: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub command: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub output: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub exit_code: Option<i32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub duration_ms: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Default, Debug)]
|
||||
pub struct JobLogger {
|
||||
entries: Mutex<Vec<JobLogEntry>>,
|
||||
}
|
||||
|
||||
impl JobLogger {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
fn trace_log(level: &'static str, message: &str) {
|
||||
match level {
|
||||
"trace" => event!(Level::TRACE, "{message}"),
|
||||
"debug" => event!(Level::DEBUG, "{message}"),
|
||||
"info" => event!(Level::INFO, "{message}"),
|
||||
"warn" | "warning" => event!(Level::WARN, "{message}"),
|
||||
"error" => event!(Level::ERROR, "{message}"),
|
||||
_ => event!(Level::INFO, "{message}"),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn log(&self, level: &'static str, message: impl Into<String>) {
|
||||
let message = message.into();
|
||||
|
||||
Self::trace_log(level, &message);
|
||||
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
entries.push(JobLogEntry {
|
||||
timestamp: Utc::now().to_rfc3339(),
|
||||
entry_type: "log",
|
||||
level,
|
||||
message,
|
||||
command: None,
|
||||
output: None,
|
||||
exit_code: None,
|
||||
duration_ms: None,
|
||||
});
|
||||
}
|
||||
|
||||
pub fn log_command(
|
||||
&self,
|
||||
command: impl Into<String>,
|
||||
output: Option<String>,
|
||||
exit_code: Option<i32>,
|
||||
duration_ms: Option<f64>,
|
||||
) {
|
||||
let level = match exit_code {
|
||||
Some(0) | None => "debug",
|
||||
_ => "error",
|
||||
};
|
||||
|
||||
|
||||
let cmd = command.into();
|
||||
let message = format!("Executed: {}", cmd);
|
||||
|
||||
match level {
|
||||
"debug" => {
|
||||
tracing::debug!(
|
||||
command = %cmd,
|
||||
exit_code = ?exit_code,
|
||||
duration_ms = ?duration_ms,
|
||||
"Command executed"
|
||||
);
|
||||
}
|
||||
"error" => {
|
||||
tracing::error!(
|
||||
command = %cmd,
|
||||
exit_code = ?exit_code,
|
||||
duration_ms = ?duration_ms,
|
||||
"Command failed"
|
||||
);
|
||||
}
|
||||
_ => {
|
||||
tracing::info!(
|
||||
command = %cmd,
|
||||
exit_code = ?exit_code,
|
||||
duration_ms = ?duration_ms,
|
||||
"Command executed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(output_text) = output.as_deref() {
|
||||
for line in output_text.lines() {
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
match level {
|
||||
"debug" => tracing::debug!("{line}"),
|
||||
"error" => tracing::error!("{line}"),
|
||||
_ => tracing::info!("{line}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
entries.push(JobLogEntry {
|
||||
timestamp: Utc::now().to_rfc3339(),
|
||||
entry_type: "command",
|
||||
level,
|
||||
message,
|
||||
command: Some(cmd),
|
||||
output,
|
||||
exit_code,
|
||||
duration_ms,
|
||||
});
|
||||
}
|
||||
|
||||
pub fn into_entries(self) -> Vec<JobLogEntry> {
|
||||
self.entries.into_inner().unwrap()
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
pub mod compressor;
|
||||
pub mod logger;
|
||||
pub mod dispatcher;
|
||||
pub mod executor;
|
||||
pub mod helpers;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use super::logger::JobLogEntry;
|
||||
use super::models::{BackupResult, UploadResult};
|
||||
use super::service::BackupService;
|
||||
use crate::services::api::ApiError;
|
||||
@@ -11,6 +12,8 @@ impl BackupService {
|
||||
result: BackupResult,
|
||||
upload_results: Vec<UploadResult>,
|
||||
backup_id: &String,
|
||||
logs: Vec<JobLogEntry>,
|
||||
duration_ms: f64,
|
||||
) -> Result<Option<BackupResponse>, ApiError> {
|
||||
let status = if upload_results.iter().any(|r| r.success) {
|
||||
"success"
|
||||
@@ -32,6 +35,8 @@ impl BackupService {
|
||||
status,
|
||||
file_size,
|
||||
&result.generated_id,
|
||||
logs,
|
||||
duration_ms
|
||||
)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use super::logger::JobLogger;
|
||||
use super::models::BackupResult;
|
||||
use super::service::BackupService;
|
||||
|
||||
@@ -6,10 +7,11 @@ use crate::services::config::DatabaseConfig;
|
||||
|
||||
use anyhow::Result;
|
||||
use std::path::Path;
|
||||
use tracing::{error, info};
|
||||
use std::sync::Arc;
|
||||
use tracing::error;
|
||||
|
||||
impl BackupService {
|
||||
pub async fn run(cfg: DatabaseConfig, tmp_path: &Path) -> Result<BackupResult> {
|
||||
pub async fn run(cfg: DatabaseConfig, tmp_path: &Path, logger: Arc<JobLogger>) -> Result<BackupResult> {
|
||||
let db = DatabaseFactory::create_for_backup(cfg.clone()).await;
|
||||
|
||||
let generated_id = cfg.generated_id.clone();
|
||||
@@ -19,13 +21,15 @@ impl BackupService {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
error!("Ping failed: {}", e);
|
||||
logger.log("error", format!("Ping failed: {}", e));
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
|
||||
info!("Reachable: {}", reachable);
|
||||
logger.log("info", format!("Database reachable: {}", reachable));
|
||||
|
||||
if !reachable {
|
||||
logger.log("error", "Database unreachable, backup aborted");
|
||||
return Ok(BackupResult {
|
||||
generated_id,
|
||||
db_type,
|
||||
@@ -35,7 +39,7 @@ impl BackupService {
|
||||
});
|
||||
}
|
||||
|
||||
match db.backup(tmp_path).await {
|
||||
match db.backup(tmp_path, Arc::clone(&logger)).await {
|
||||
Ok(file) => Ok(BackupResult {
|
||||
generated_id,
|
||||
db_type,
|
||||
@@ -44,21 +48,27 @@ impl BackupService {
|
||||
code: None,
|
||||
}),
|
||||
|
||||
Err(e) if e.to_string() == "backup_already_in_progress" => Ok(BackupResult {
|
||||
generated_id,
|
||||
db_type,
|
||||
status: "failed".into(),
|
||||
backup_file: None,
|
||||
code: Some("backup_already_in_progress".into()),
|
||||
}),
|
||||
Err(e) if e.to_string() == "backup_already_in_progress" => {
|
||||
logger.log("warn", "Backup already in progress");
|
||||
Ok(BackupResult {
|
||||
generated_id,
|
||||
db_type,
|
||||
status: "failed".into(),
|
||||
backup_file: None,
|
||||
code: Some("backup_already_in_progress".into()),
|
||||
})
|
||||
}
|
||||
|
||||
Err(_) => Ok(BackupResult {
|
||||
generated_id,
|
||||
db_type,
|
||||
status: "failed".into(),
|
||||
backup_file: None,
|
||||
code: None,
|
||||
}),
|
||||
Err(e) => {
|
||||
logger.log("error", format!("Backup failed: {}", e));
|
||||
Ok(BackupResult {
|
||||
generated_id,
|
||||
db_type,
|
||||
status: "failed".into(),
|
||||
backup_file: None,
|
||||
code: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
use super::logger::JobLogger;
|
||||
use super::models::{BackupResult, UploadResult};
|
||||
use super::service::BackupService;
|
||||
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::services::storage;
|
||||
use crate::utils::common::BackupMethod;
|
||||
|
||||
use anyhow::{Result, bail};
|
||||
use futures::future::join_all;
|
||||
use tracing::{error, info};
|
||||
use std::sync::Arc;
|
||||
use tracing::info;
|
||||
|
||||
impl BackupService {
|
||||
pub async fn upload(
|
||||
@@ -17,6 +17,7 @@ impl BackupService {
|
||||
storages: Vec<DatabaseStorage>,
|
||||
encrypt: bool,
|
||||
backup_id: &String,
|
||||
logger: Arc<JobLogger>,
|
||||
) -> Result<Vec<UploadResult>> {
|
||||
if result.code.as_deref() == Some("backup_already_in_progress") {
|
||||
info!("Skipping send: backup already in progress");
|
||||
@@ -29,15 +30,13 @@ impl BackupService {
|
||||
let ctx_clone = ctx.clone();
|
||||
let result_clone = result.clone();
|
||||
let provider = storage::get_provider(&storage);
|
||||
let logger_clone = Arc::clone(&logger);
|
||||
|
||||
let storage_id = storage.id.clone();
|
||||
let generated_id = result_clone.generated_id.clone();
|
||||
|
||||
async move {
|
||||
info!(
|
||||
"Uploading storage -> {:?} for {:?}",
|
||||
storage.provider, storage_id
|
||||
);
|
||||
logger_clone.log("info", format!("Uploading storage {:?} (id: {})", storage.provider, storage_id));
|
||||
|
||||
/*
|
||||
INIT STEP
|
||||
@@ -54,7 +53,7 @@ impl BackupService {
|
||||
{
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
error!("backup_upload_init failed: {}", e);
|
||||
logger_clone.log("error", format!("Upload init failed: {}", e));
|
||||
|
||||
return UploadResult {
|
||||
storage_id,
|
||||
@@ -69,6 +68,7 @@ impl BackupService {
|
||||
let backup_storage_id = match init {
|
||||
Some(v) => v.backup_storage.id,
|
||||
None => {
|
||||
logger_clone.log("error", "Upload init returned empty response");
|
||||
return UploadResult {
|
||||
storage_id,
|
||||
success: false,
|
||||
@@ -83,7 +83,7 @@ impl BackupService {
|
||||
PROVIDER CHECK
|
||||
*/
|
||||
let Some(provider) = provider else {
|
||||
error!("Skipping storage due to missing provider");
|
||||
logger_clone.log("error", format!("Missing provider for storage {}", storage_id));
|
||||
|
||||
return UploadResult {
|
||||
storage_id,
|
||||
@@ -104,23 +104,49 @@ impl BackupService {
|
||||
method,
|
||||
&storage,
|
||||
Some(encrypt),
|
||||
&backup_storage_id,
|
||||
)
|
||||
.await;
|
||||
|
||||
let status = if upload_result.success {
|
||||
"success"
|
||||
} else {
|
||||
"failed"
|
||||
};
|
||||
let status = if upload_result.success { "success" } else { "failed" };
|
||||
|
||||
if status != "success" {
|
||||
logger_clone.log("error", format!(
|
||||
"Upload failed for storage {}: {}",
|
||||
storage_id,
|
||||
upload_result.error.as_deref().unwrap_or("unknown error")
|
||||
));
|
||||
|
||||
// `backup_upload_init` opened a per-storage record; close it as "failed"
|
||||
// so the server is notified of the failure (no path/size on this path).
|
||||
if let Err(err) = ctx_clone
|
||||
.api
|
||||
.backup_upload_status(
|
||||
ctx_clone.edge_key.agent_id.clone(),
|
||||
generated_id.clone(),
|
||||
backup_storage_id.clone(),
|
||||
status,
|
||||
String::new(),
|
||||
0u64,
|
||||
backup_id,
|
||||
)
|
||||
.await
|
||||
{
|
||||
logger_clone.log("error", format!(
|
||||
"Failed-status update failed for {}: {}",
|
||||
storage_id, err
|
||||
));
|
||||
}
|
||||
|
||||
return upload_result;
|
||||
}
|
||||
|
||||
info!(
|
||||
"Storage {} uploaded to remote path {:?}",
|
||||
storage_id, upload_result.remote_file_path
|
||||
);
|
||||
logger_clone.log("info", format!(
|
||||
"Storage {} uploaded to {:?} ({} bytes)",
|
||||
storage_id,
|
||||
upload_result.remote_file_path.clone().unwrap().to_string(),
|
||||
upload_result.total_size.unwrap_or(0)
|
||||
));
|
||||
|
||||
/*
|
||||
METADATA VALIDATION
|
||||
@@ -129,6 +155,7 @@ impl BackupService {
|
||||
match (&upload_result.remote_file_path, upload_result.total_size) {
|
||||
(Some(path), Some(size)) => (path.clone(), size),
|
||||
_ => {
|
||||
logger_clone.log("error", format!("Missing remote_file_path or total_size for storage {}", storage_id));
|
||||
return UploadResult {
|
||||
storage_id,
|
||||
success: false,
|
||||
@@ -158,10 +185,7 @@ impl BackupService {
|
||||
Ok(_) => upload_result,
|
||||
|
||||
Err(err) => {
|
||||
error!(
|
||||
"backup_upload_status failed (storage_id={}): {}",
|
||||
storage_id, err
|
||||
);
|
||||
logger_clone.log("error", format!("Upload status update failed for {}: {}", storage_id, err));
|
||||
|
||||
UploadResult {
|
||||
storage_id,
|
||||
|
||||
+64
-12
@@ -3,6 +3,7 @@
|
||||
use crate::core::context::Context;
|
||||
use serde::Deserialize;
|
||||
use serde_json;
|
||||
use std::collections::HashMap;
|
||||
use std::fs::File;
|
||||
use std::io::Read;
|
||||
use std::path::Path;
|
||||
@@ -17,22 +18,32 @@ pub enum DbType {
|
||||
Mysql,
|
||||
Mariadb,
|
||||
Postgresql,
|
||||
#[serde(rename = "postgresql-cluster")]
|
||||
PostgresqlCluster,
|
||||
MongoDB,
|
||||
Sqlite,
|
||||
Redis,
|
||||
Valkey,
|
||||
Firebird,
|
||||
Mssql,
|
||||
#[serde(rename = "docker-volume")]
|
||||
DockerVolume,
|
||||
}
|
||||
|
||||
impl DbType {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
DbType::Mysql => "mysql",
|
||||
DbType::Mariadb => "mysql",
|
||||
DbType::Mariadb => "mariadb",
|
||||
DbType::Postgresql => "postgresql",
|
||||
DbType::PostgresqlCluster => "postgresql-cluster",
|
||||
DbType::MongoDB => "mongodb",
|
||||
DbType::Sqlite => "sqlite",
|
||||
DbType::Redis => "redis",
|
||||
DbType::Valkey => "valkey",
|
||||
DbType::Firebird => "firebird",
|
||||
DbType::Mssql => "mssql",
|
||||
DbType::DockerVolume => "docker-volume",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -50,6 +61,10 @@ pub struct DatabaseConfig {
|
||||
pub host: String,
|
||||
pub generated_id: String,
|
||||
pub path: String,
|
||||
pub max_packet_size: String,
|
||||
pub volume_name: String,
|
||||
pub container_name: Option<String>,
|
||||
pub options: HashMap<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
@@ -71,6 +86,10 @@ pub struct InputDatabaseConfig {
|
||||
pub host: Option<String>,
|
||||
pub generated_id: String,
|
||||
pub path: Option<String>,
|
||||
pub max_packet_size: Option<String>,
|
||||
pub volume_name: Option<String>,
|
||||
pub container_name: Option<String>,
|
||||
pub options: Option<HashMap<String, serde_json::Value>>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
@@ -163,41 +182,57 @@ impl ConfigService {
|
||||
}
|
||||
|
||||
let username = match db.db_type {
|
||||
DbType::Postgresql | DbType::Mysql | DbType::Mariadb => {
|
||||
required(&db.username, &db.name, "username")?
|
||||
}
|
||||
DbType::Postgresql
|
||||
| DbType::PostgresqlCluster
|
||||
| DbType::Mysql
|
||||
| DbType::Mariadb
|
||||
| DbType::Mssql => required(&db.username, &db.name, "username")?,
|
||||
_ => optional(&db.username),
|
||||
};
|
||||
|
||||
let password = match db.db_type {
|
||||
DbType::Postgresql | DbType::Mysql | DbType::Mariadb => {
|
||||
required(&db.password, &db.name, "password")?
|
||||
}
|
||||
DbType::Postgresql
|
||||
| DbType::PostgresqlCluster
|
||||
| DbType::Mysql
|
||||
| DbType::Mariadb
|
||||
| DbType::Mssql => required(&db.password, &db.name, "password")?,
|
||||
_ => optional(&db.password),
|
||||
};
|
||||
|
||||
let host = match db.db_type {
|
||||
DbType::Postgresql
|
||||
| DbType::PostgresqlCluster
|
||||
| DbType::Mysql
|
||||
| DbType::Mariadb
|
||||
| DbType::MongoDB
|
||||
| DbType::Redis
|
||||
| DbType::Valkey => required(&db.host, &db.name, "host")?,
|
||||
DbType::Sqlite => optional(&db.host),
|
||||
| DbType::Firebird
|
||||
| DbType::Valkey
|
||||
| DbType::Mssql => required(&db.host, &db.name, "host")?,
|
||||
DbType::Sqlite | DbType::DockerVolume => optional(&db.host),
|
||||
};
|
||||
|
||||
let port = match db.db_type {
|
||||
DbType::Postgresql
|
||||
| DbType::PostgresqlCluster
|
||||
| DbType::Mysql
|
||||
| DbType::Mariadb
|
||||
| DbType::MongoDB
|
||||
| DbType::Redis
|
||||
| DbType::Valkey => required(&db.port, &db.name, "port")?,
|
||||
DbType::Sqlite => db.port.unwrap_or(0),
|
||||
| DbType::Firebird
|
||||
| DbType::Valkey
|
||||
| DbType::Mssql => required(&db.port, &db.name, "port")?,
|
||||
DbType::Sqlite | DbType::DockerVolume => db.port.unwrap_or(0),
|
||||
};
|
||||
|
||||
let database_name = match db.db_type {
|
||||
DbType::Sqlite | DbType::Redis | DbType::Valkey => optional(&db.database),
|
||||
DbType::Sqlite | DbType::Redis | DbType::Valkey | DbType::DockerVolume => {
|
||||
optional(&db.database)
|
||||
}
|
||||
DbType::PostgresqlCluster => db
|
||||
.database
|
||||
.clone()
|
||||
.unwrap_or_else(|| "postgres".to_string()),
|
||||
_ => required(&db.database, &db.name, "database")?,
|
||||
};
|
||||
|
||||
@@ -206,6 +241,19 @@ impl ConfigService {
|
||||
_ => optional(&db.path),
|
||||
};
|
||||
|
||||
let max_packet_size = match db.db_type {
|
||||
DbType::Mysql | DbType::Mariadb => {
|
||||
db.max_packet_size.unwrap_or_else(|| "512M".to_string())
|
||||
}
|
||||
_ => String::new(),
|
||||
};
|
||||
|
||||
let volume_name = match db.db_type {
|
||||
DbType::DockerVolume => required(&db.volume_name, &db.name, "volume_name")?,
|
||||
_ => optional(&db.volume_name),
|
||||
};
|
||||
let container_name = db.container_name.clone();
|
||||
|
||||
databases.push(DatabaseConfig {
|
||||
name: db.name,
|
||||
database: database_name,
|
||||
@@ -216,6 +264,10 @@ impl ConfigService {
|
||||
port,
|
||||
generated_id: db.generated_id,
|
||||
path: path_val,
|
||||
max_packet_size,
|
||||
volume_name,
|
||||
container_name,
|
||||
options: db.options.unwrap_or_default(),
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -4,4 +4,4 @@ pub mod config;
|
||||
pub mod cron;
|
||||
pub mod restore;
|
||||
pub mod status;
|
||||
mod storage;
|
||||
pub mod storage;
|
||||
|
||||
@@ -1,26 +1,35 @@
|
||||
use super::service::RestoreService;
|
||||
|
||||
use crate::utils::compress::decompress_large_tar_gz;
|
||||
use crate::utils::file::decrypt_file_stream_gcm;
|
||||
|
||||
use anyhow::Result;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DbType;
|
||||
use crate::utils::common::choose_restore_path;
|
||||
|
||||
impl RestoreService {
|
||||
pub async fn prepare_archive(
|
||||
&self,
|
||||
downloaded_file: PathBuf,
|
||||
tmp_path: &Path,
|
||||
db_type: &DbType,
|
||||
logger: Arc<JobLogger>
|
||||
) -> Result<PathBuf> {
|
||||
logger.log("info", "Start preparing backup archive".to_string());
|
||||
|
||||
let filename = downloaded_file
|
||||
.file_name()
|
||||
.unwrap()
|
||||
.to_string_lossy()
|
||||
.to_string();
|
||||
|
||||
logger.log("debug", format!("Archive filename: {}", filename));
|
||||
|
||||
let is_legacy = filename.ends_with(".sql") || filename.ends_with(".dump");
|
||||
|
||||
if is_legacy {
|
||||
logger.log("info", "Legacy archive detected, skipping extraction".to_string());
|
||||
return Ok(downloaded_file);
|
||||
}
|
||||
|
||||
@@ -29,30 +38,54 @@ impl RestoreService {
|
||||
let mut archive = downloaded_file.clone();
|
||||
|
||||
if encrypted {
|
||||
logger.log("info", "Archive is encrypted, decrypting".to_string());
|
||||
|
||||
let new_name = filename.strip_suffix(".enc").unwrap();
|
||||
|
||||
let decrypted = tmp_path.join(new_name);
|
||||
|
||||
decrypt_file_stream_gcm(
|
||||
if let Err(e) = decrypt_file_stream_gcm(
|
||||
downloaded_file,
|
||||
decrypted.clone(),
|
||||
self.ctx.edge_key.master_key_b64.clone(),
|
||||
)
|
||||
.await?;
|
||||
.await
|
||||
{
|
||||
logger.log("error", format!("Failed to decrypt archive: {}", e));
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
logger.log("info", format!("Archive decrypted to {}", decrypted.display()));
|
||||
|
||||
archive = decrypted;
|
||||
}
|
||||
|
||||
let files = decompress_large_tar_gz(archive.as_path(), tmp_path).await?;
|
||||
if matches!(db_type, DbType::DockerVolume) {
|
||||
let raw_tar = tmp_path.join("volume.tar");
|
||||
crate::utils::compress::gunzip_to_file(archive.as_path(), &raw_tar).await?;
|
||||
logger.log("info", format!("Docker volume archive gunzipped to {}", raw_tar.display()));
|
||||
return Ok(raw_tar);
|
||||
}
|
||||
|
||||
logger.log("info", format!("Decompressing archive {}", archive.display()));
|
||||
|
||||
let files = match decompress_large_tar_gz(archive.as_path(), tmp_path).await {
|
||||
Ok(f) => f,
|
||||
Err(e) => {
|
||||
logger.log("error", format!("Failed to decompress archive: {}", e));
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
|
||||
if files.is_empty() {
|
||||
logger.log("error", "Archive is empty after decompression".to_string());
|
||||
anyhow::bail!("archive empty");
|
||||
}
|
||||
|
||||
if files.len() == 1 {
|
||||
Ok(files[0].clone())
|
||||
} else {
|
||||
Ok(archive)
|
||||
}
|
||||
logger.log("info", format!("Archive prepared, {} file(s) extracted", files.len()));
|
||||
|
||||
let chosen = choose_restore_path(&files, tmp_path, &archive);
|
||||
logger.log("debug", format!("Restore source resolved to: {}", chosen.display()));
|
||||
Ok(chosen)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,6 +20,8 @@ impl RestoreService {
|
||||
return;
|
||||
};
|
||||
|
||||
let expected_size = db.data.restore.size.clone();
|
||||
|
||||
let service = Self {
|
||||
ctx: self.ctx.clone(),
|
||||
};
|
||||
@@ -27,7 +29,10 @@ impl RestoreService {
|
||||
let db_cfg = cfg.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = service.execute_restore(db_cfg, file_to_restore).await {
|
||||
if let Err(e) = service
|
||||
.execute_restore(db_cfg, file_to_restore, expected_size)
|
||||
.await
|
||||
{
|
||||
error!("Restore failed: {}", e);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1,17 +1,41 @@
|
||||
use super::service::RestoreService;
|
||||
|
||||
use anyhow::Result;
|
||||
use futures::StreamExt;
|
||||
use reqwest::{Client, Url};
|
||||
use std::path::{Path, PathBuf};
|
||||
use tracing::info;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
|
||||
fn human_size(bytes: u64) -> String {
|
||||
if bytes >= 1024 * 1024 {
|
||||
format!("{} MB", bytes / 1024 / 1024)
|
||||
} else if bytes >= 1024 {
|
||||
format!("{} KB", bytes / 1024)
|
||||
} else {
|
||||
format!("{bytes} B")
|
||||
}
|
||||
}
|
||||
|
||||
impl RestoreService {
|
||||
pub async fn download_backup(&self, file_url: &str, tmp_path: &Path) -> Result<PathBuf> {
|
||||
pub async fn download_backup(
|
||||
&self,
|
||||
file_url: &str,
|
||||
tmp_path: &Path,
|
||||
logger: Arc<JobLogger>,
|
||||
expected_size: Option<String>,
|
||||
) -> Result<PathBuf> {
|
||||
logger.log("info", "Start downloading backup archive".to_string());
|
||||
|
||||
let client = Client::new();
|
||||
|
||||
let response = client.get(file_url).send().await?;
|
||||
let status = response.status();
|
||||
|
||||
if !response.status().is_success() {
|
||||
if !status.is_success() {
|
||||
logger.log("error", "Failed to download".to_string());
|
||||
anyhow::bail!("download failed");
|
||||
}
|
||||
|
||||
@@ -35,11 +59,65 @@ impl RestoreService {
|
||||
|
||||
let path = tmp_path.join(&filename);
|
||||
|
||||
let bytes = response.bytes().await?;
|
||||
let total = expected_size
|
||||
.as_deref()
|
||||
.and_then(|s| s.trim().parse::<u64>().ok())
|
||||
.filter(|&n| n > 0);
|
||||
|
||||
tokio::fs::write(&path, &bytes).await?;
|
||||
logger.log(
|
||||
"info",
|
||||
format!(
|
||||
"Downloading backup '{}' ({})",
|
||||
filename,
|
||||
total.map(human_size).unwrap_or_else(|| "unknown size".to_string())
|
||||
),
|
||||
);
|
||||
|
||||
info!("Backup downloaded to {}", path.display());
|
||||
let start = Instant::now();
|
||||
let mut file = tokio::fs::File::create(&path).await?;
|
||||
let mut stream = response.bytes_stream();
|
||||
let mut downloaded: u64 = 0;
|
||||
let mut next_pct: u64 = 10;
|
||||
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk?;
|
||||
file.write_all(&chunk).await?;
|
||||
downloaded += chunk.len() as u64;
|
||||
|
||||
if let Some(total) = total {
|
||||
let pct = (downloaded.saturating_mul(100) / total).min(100);
|
||||
let milestone = pct / 10 * 10;
|
||||
if milestone >= next_pct {
|
||||
logger.log(
|
||||
"info",
|
||||
format!(
|
||||
"Download progress: {}% ({} / {} bytes)",
|
||||
milestone, downloaded, total
|
||||
),
|
||||
);
|
||||
next_pct = milestone + 10;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
file.flush().await?;
|
||||
|
||||
if downloaded == 0 {
|
||||
logger.log(
|
||||
"warn",
|
||||
format!("Downloaded 0 bytes (status {status}); backup body was empty"),
|
||||
);
|
||||
}
|
||||
|
||||
logger.log(
|
||||
"info",
|
||||
format!(
|
||||
"Backup downloaded to {} ( {} bytes in {:.1}s)",
|
||||
path.display(),
|
||||
downloaded,
|
||||
start.elapsed().as_secs_f64()
|
||||
),
|
||||
);
|
||||
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
@@ -1,23 +1,45 @@
|
||||
use super::service::RestoreService;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::Result;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tempfile::TempDir;
|
||||
use tracing::info;
|
||||
|
||||
impl RestoreService {
|
||||
pub async fn execute_restore(&self, cfg: DatabaseConfig, file_url: String) -> Result<()> {
|
||||
pub async fn execute_restore(
|
||||
&self,
|
||||
cfg: DatabaseConfig,
|
||||
file_url: String,
|
||||
expected_size: Option<String>,
|
||||
) -> Result<()> {
|
||||
let logger = Arc::new(JobLogger::new());
|
||||
let start = Instant::now();
|
||||
|
||||
logger.log("info", "Database restoration job started".to_string());
|
||||
|
||||
let temp_dir = TempDir::new()?;
|
||||
let tmp_path = temp_dir.path();
|
||||
|
||||
info!("Created temp directory {}", tmp_path.display());
|
||||
logger.log("info", format!("Created temp directory {}", tmp_path.display()));
|
||||
|
||||
let downloaded = self.download_backup(&file_url, tmp_path).await?;
|
||||
let downloaded = self
|
||||
.download_backup(&file_url, tmp_path, Arc::clone(&logger), expected_size)
|
||||
.await?;
|
||||
|
||||
let backup_file = self.prepare_archive(downloaded, tmp_path).await?;
|
||||
let backup_file = self
|
||||
.prepare_archive(downloaded, tmp_path, &cfg.db_type, Arc::clone(&logger))
|
||||
.await?;
|
||||
|
||||
let result = self.run_restore(cfg, backup_file).await?;
|
||||
let result = self.run_restore(cfg, backup_file, Arc::clone(&logger)).await?;
|
||||
|
||||
self.send_result(result).await;
|
||||
logger.log("info", "Database restore job finished".to_string());
|
||||
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
let logs = Arc::try_unwrap(logger)
|
||||
.unwrap_or_else(|_| JobLogger::new())
|
||||
.into_entries();
|
||||
self.send_result(result, logs, duration_ms).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1,31 +1,38 @@
|
||||
use super::models::RestoreResult;
|
||||
use super::service::RestoreService;
|
||||
use crate::services::api::ApiError;
|
||||
use crate::services::api::models::agent::restore::ResultRestoreResponse;
|
||||
|
||||
use tracing::{error, info};
|
||||
use crate::services::backup::logger::JobLogEntry;
|
||||
|
||||
impl RestoreService {
|
||||
pub async fn send_result(&self, result: RestoreResult) {
|
||||
pub async fn send_result(
|
||||
&self,
|
||||
result: RestoreResult,
|
||||
logs: Vec<JobLogEntry>,
|
||||
duration_ms: f64,
|
||||
) -> Result<Option<ResultRestoreResponse>, ApiError> {
|
||||
info!(
|
||||
"[RestoreService] DB: {} | Status: {}",
|
||||
result.generated_id, result.status
|
||||
"[RestoreService] DB: {} | Status: {} | Duration: {}ms",
|
||||
result.generated_id,
|
||||
result.status,
|
||||
duration_ms
|
||||
);
|
||||
|
||||
match self
|
||||
.ctx
|
||||
self.ctx
|
||||
.api
|
||||
.restore_result(
|
||||
self.ctx.edge_key.agent_id.clone(),
|
||||
&result.generated_id,
|
||||
&result.status,
|
||||
logs,
|
||||
duration_ms
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
info!("Restoration result sent successfully");
|
||||
}
|
||||
Err(e) => {
|
||||
.map_err(|e| {
|
||||
error!("Failed to send restoration result: {}", e);
|
||||
}
|
||||
}
|
||||
e.into()
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,39 +4,65 @@ use super::service::RestoreService;
|
||||
use crate::domain::factory::DatabaseFactory;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use anyhow::Result;
|
||||
use std::path::PathBuf;
|
||||
use tracing::{error, info};
|
||||
use std::sync::Arc;
|
||||
|
||||
impl RestoreService {
|
||||
pub async fn run_restore(
|
||||
&self,
|
||||
cfg: DatabaseConfig,
|
||||
backup_file: PathBuf,
|
||||
logger: Arc<JobLogger>,
|
||||
) -> Result<RestoreResult> {
|
||||
let generated_id = cfg.generated_id.clone();
|
||||
|
||||
logger.log(
|
||||
"info",
|
||||
format!("Preparing restore for database {}", cfg.name),
|
||||
);
|
||||
|
||||
let db = DatabaseFactory::create_for_restore(cfg.clone(), &backup_file).await;
|
||||
|
||||
logger.log(
|
||||
"debug",
|
||||
format!("Checking reachability for database {}", cfg.name),
|
||||
);
|
||||
|
||||
let reachable = db.ping().await.unwrap_or(false);
|
||||
|
||||
info!("Reachable: {}", reachable);
|
||||
logger.log("info", format!("Reachable: {}", reachable));
|
||||
|
||||
if !reachable {
|
||||
logger.log(
|
||||
"error",
|
||||
format!("Database {} unreachable, aborting restore", cfg.name),
|
||||
);
|
||||
|
||||
return Ok(RestoreResult {
|
||||
generated_id,
|
||||
status: "failed".into(),
|
||||
});
|
||||
}
|
||||
|
||||
match db.restore(&backup_file).await {
|
||||
Ok(_) => Ok(RestoreResult {
|
||||
generated_id,
|
||||
status: "success".into(),
|
||||
}),
|
||||
match db.restore(&backup_file, Arc::clone(&logger)).await {
|
||||
Ok(_) => {
|
||||
logger.log(
|
||||
"info",
|
||||
format!("Restore completed successfully for database {}", cfg.name),
|
||||
);
|
||||
Ok(RestoreResult {
|
||||
generated_id,
|
||||
status: "success".into(),
|
||||
})
|
||||
}
|
||||
|
||||
Err(e) => {
|
||||
error!("Restore failed: {:?}", e);
|
||||
logger.log(
|
||||
"error",
|
||||
format!("Restore failed for database {}: {:?}", cfg.name, e),
|
||||
);
|
||||
|
||||
Ok(RestoreResult {
|
||||
generated_id,
|
||||
|
||||
@@ -9,6 +9,7 @@ use reqwest::Client;
|
||||
use std::error::Error;
|
||||
use std::sync::Arc;
|
||||
use futures_util::future::try_join_all;
|
||||
use tracing::info;
|
||||
use crate::domain::factory::DatabaseFactory;
|
||||
|
||||
pub struct StatusService {
|
||||
@@ -32,6 +33,7 @@ impl StatusService {
|
||||
let db_engine = DatabaseFactory::create_for_backup(db.clone()).await;
|
||||
|
||||
let reachable = db_engine.ping().await?;
|
||||
info!("Ping {} => {:?}",db.name, reachable);
|
||||
|
||||
Ok::<DatabasePayload, anyhow::Error>(DatabasePayload {
|
||||
name: &db.name,
|
||||
|
||||
@@ -5,6 +5,8 @@ use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::services::backup::models::{BackupResult, UploadResult};
|
||||
use crate::utils::common::BackupMethod;
|
||||
use async_trait::async_trait;
|
||||
use providers::azure_blob;
|
||||
use providers::google_cloud_storage;
|
||||
use providers::google_drive;
|
||||
use providers::local;
|
||||
use providers::s3;
|
||||
@@ -20,6 +22,7 @@ pub trait StorageProvider: Send + Sync {
|
||||
method: BackupMethod,
|
||||
config: &DatabaseStorage,
|
||||
encrypt: Option<bool>,
|
||||
backup_storage_id: &str,
|
||||
) -> UploadResult;
|
||||
}
|
||||
|
||||
@@ -31,7 +34,11 @@ pub fn get_provider(storage: &DatabaseStorage) -> Option<Box<dyn StorageProvider
|
||||
match storage.provider.as_str() {
|
||||
"local" => Some(Box::new(local::LocalProvider {})),
|
||||
"s3" => Some(Box::new(s3::S3Provider {})),
|
||||
"blob" => Some(Box::new(azure_blob::AzureBlobProvider {})),
|
||||
"google-drive" => Some(Box::new(google_drive::GoogleDriveProvider {})),
|
||||
"google-cloud-storage" => Some(Box::new(
|
||||
google_cloud_storage::GoogleCloudStorageProvider {},
|
||||
)),
|
||||
_ => {
|
||||
error!("Unknown storage provider: {}", storage.provider);
|
||||
None
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
use anyhow::{Context as _, Result, anyhow};
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose::STANDARD;
|
||||
use chrono::{Duration, Utc};
|
||||
use openssl::hash::MessageDigest;
|
||||
use openssl::pkey::PKey;
|
||||
use openssl::sign::Signer;
|
||||
use url::Url;
|
||||
use azure_core::http::RequestContent;
|
||||
use azure_storage_blob::clients::{BlobClient, BlockBlobClient};
|
||||
use azure_storage_blob::models::BlockLookupList;
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use futures::{Stream, StreamExt};
|
||||
use std::pin::Pin;
|
||||
use tracing::info;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ResolvedAzure {
|
||||
pub account_name: String,
|
||||
pub account_key: String,
|
||||
pub blob_endpoint: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub enum SasResource {
|
||||
Blob,
|
||||
#[allow(dead_code)]
|
||||
Container,
|
||||
}
|
||||
|
||||
impl SasResource {
|
||||
fn code(self) -> &'static str {
|
||||
match self { SasResource::Blob => "b", SasResource::Container => "c" }
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) const SAS_VERSION: &str = "2022-11-02";
|
||||
|
||||
pub(crate) fn hmac_sha256_b64(key: &[u8], data: &str) -> Result<String> {
|
||||
let pkey = PKey::hmac(key).context("hmac key")?;
|
||||
let mut signer = Signer::new(MessageDigest::sha256(), &pkey).context("signer")?;
|
||||
signer.update(data.as_bytes()).context("signer update")?;
|
||||
let sig = signer.sign_to_vec().context("sign")?;
|
||||
Ok(STANDARD.encode(sig))
|
||||
}
|
||||
|
||||
pub fn build_service_sas(
|
||||
resolved: &ResolvedAzure,
|
||||
canonical_resource: &str,
|
||||
resource: SasResource,
|
||||
permissions: &str,
|
||||
) -> Result<Vec<(String, String)>> {
|
||||
let key = STANDARD
|
||||
.decode(&resolved.account_key)
|
||||
.map_err(|_| anyhow!("account key is not valid base64"))?;
|
||||
|
||||
let signed_start = String::new();
|
||||
let signed_expiry = (Utc::now() + Duration::hours(1))
|
||||
.format("%Y-%m-%dT%H:%M:%SZ").to_string();
|
||||
let signed_protocol = "https,http"; // Azurite is http
|
||||
let signed_resource = resource.code();
|
||||
|
||||
let string_to_sign = format!(
|
||||
"{sp}\n{st}\n{se}\n{canon}\n{si}\n{sip}\n{spr}\n{sv}\n{sr}\n{snap}\n{enc}\n{rscc}\n{rscd}\n{rsce}\n{rscl}\n{rsct}",
|
||||
sp = permissions, st = signed_start, se = signed_expiry, canon = canonical_resource,
|
||||
si = "", sip = "", spr = signed_protocol, sv = SAS_VERSION, sr = signed_resource,
|
||||
snap = "", enc = "", rscc = "", rscd = "", rsce = "", rscl = "", rsct = "",
|
||||
);
|
||||
|
||||
let sig = hmac_sha256_b64(&key, &string_to_sign)?;
|
||||
|
||||
Ok(vec![
|
||||
("sv".into(), SAS_VERSION.into()),
|
||||
("sr".into(), signed_resource.into()),
|
||||
("sp".into(), permissions.into()),
|
||||
("se".into(), signed_expiry),
|
||||
("spr".into(), signed_protocol.into()),
|
||||
("sig".into(), sig),
|
||||
])
|
||||
}
|
||||
|
||||
pub fn build_sas_url(
|
||||
resolved: &ResolvedAzure,
|
||||
container: &str,
|
||||
blob: &str,
|
||||
resource: SasResource,
|
||||
permissions: &str,
|
||||
) -> Result<Url> {
|
||||
let canonical = if blob.is_empty() {
|
||||
format!("/blob/{}/{}", resolved.account_name, container)
|
||||
} else {
|
||||
format!("/blob/{}/{}/{}", resolved.account_name, container, blob)
|
||||
};
|
||||
let pairs = build_service_sas(resolved, &canonical, resource, permissions)?;
|
||||
|
||||
let base = if blob.is_empty() {
|
||||
format!("{}/{}", resolved.blob_endpoint.trim_end_matches('/'), container)
|
||||
} else {
|
||||
format!("{}/{}/{}", resolved.blob_endpoint.trim_end_matches('/'), container, blob)
|
||||
};
|
||||
|
||||
let mut url = Url::parse(&base).context("invalid blob endpoint/url")?;
|
||||
{
|
||||
let mut qp = url.query_pairs_mut();
|
||||
for (k, v) in pairs { qp.append_pair(&k, &v); }
|
||||
}
|
||||
Ok(url)
|
||||
}
|
||||
|
||||
pub const BLOCK_SIZE: usize = 100 * 1024 * 1024;
|
||||
|
||||
type ByteStream = Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>;
|
||||
|
||||
async fn stage_block(
|
||||
bbc: &BlockBlobClient,
|
||||
index: u32,
|
||||
block: Bytes,
|
||||
block_ids: &mut Vec<Vec<u8>>,
|
||||
) -> Result<()> {
|
||||
let raw_id = format!("{index:032}").into_bytes();
|
||||
let len = block.len() as u64;
|
||||
bbc.stage_block(&raw_id, len, RequestContent::from(block.to_vec()), None)
|
||||
.await
|
||||
.map_err(|e| anyhow!("stage_block {index} failed: {e}"))?;
|
||||
block_ids.push(raw_id);
|
||||
info!("staged azure block {index} ({len} bytes)");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn upload_stream_to_azure(
|
||||
resolved: &ResolvedAzure,
|
||||
container: &str,
|
||||
blob: &str,
|
||||
mut body: ByteStream,
|
||||
block_size: usize,
|
||||
) -> Result<()> {
|
||||
let url = build_sas_url(resolved, container, blob, SasResource::Blob, "cw")?;
|
||||
let blob_client = BlobClient::new(url, None, None).context("blob client")?;
|
||||
let bbc = blob_client.block_blob_client();
|
||||
|
||||
let mut buffer = BytesMut::with_capacity(block_size);
|
||||
let mut block_ids: Vec<Vec<u8>> = Vec::new();
|
||||
let mut index: u32 = 0;
|
||||
|
||||
while let Some(item) = body.next().await {
|
||||
let bytes = item.context("stream error during upload")?;
|
||||
buffer.extend_from_slice(&bytes);
|
||||
|
||||
while buffer.len() >= block_size {
|
||||
let block = buffer.split_to(block_size).freeze();
|
||||
stage_block(&bbc, index, block, &mut block_ids).await?;
|
||||
index += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if !buffer.is_empty() {
|
||||
let block = buffer.split().freeze();
|
||||
stage_block(&bbc, index, block, &mut block_ids).await?;
|
||||
}
|
||||
|
||||
if block_ids.is_empty() {
|
||||
stage_block(&bbc, 0, Bytes::new(), &mut block_ids).await?;
|
||||
}
|
||||
|
||||
let block_list = BlockLookupList {
|
||||
latest: Some(block_ids),
|
||||
..Default::default()
|
||||
};
|
||||
bbc.commit_block_list(block_list.try_into()?, None)
|
||||
.await
|
||||
.map_err(|e| anyhow!("commit_block_list failed: {e}"))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
pub mod helpers;
|
||||
pub(crate) mod models;
|
||||
|
||||
use crate::core::context::Context;
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::services::backup::models::{BackupResult, UploadResult};
|
||||
use crate::services::storage::StorageProvider;
|
||||
use crate::services::storage::providers::azure_blob::helpers::{BLOCK_SIZE, upload_stream_to_azure};
|
||||
use crate::services::storage::providers::azure_blob::models::AzureBlobProviderConfig;
|
||||
use crate::utils::common::BackupMethod;
|
||||
use crate::utils::file::{full_file_name, full_file_path};
|
||||
use crate::utils::stream::build_stream;
|
||||
use async_trait::async_trait;
|
||||
use std::sync::Arc;
|
||||
use tokio::fs;
|
||||
use tracing::{error, info};
|
||||
|
||||
pub struct AzureBlobProvider {}
|
||||
|
||||
#[async_trait]
|
||||
impl StorageProvider for AzureBlobProvider {
|
||||
async fn upload(
|
||||
&self,
|
||||
ctx: Arc<Context>,
|
||||
result: BackupResult,
|
||||
_method: BackupMethod,
|
||||
storage: &DatabaseStorage,
|
||||
encrypt: Option<bool>,
|
||||
_backup_storage_id: &str,
|
||||
) -> UploadResult {
|
||||
let Some(file_path) = result.backup_file else {
|
||||
return UploadResult {
|
||||
storage_id: storage.id.clone(),
|
||||
success: false,
|
||||
error: Some("Missing backup file path".to_string()),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
};
|
||||
};
|
||||
|
||||
let total_size = match fs::metadata(&file_path).await {
|
||||
Ok(meta) => meta.len(),
|
||||
Err(e) => {
|
||||
error!("Failed to get file size: {}", e);
|
||||
return UploadResult {
|
||||
storage_id: storage.id.clone(),
|
||||
success: false,
|
||||
error: Some(e.to_string()),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let encrypt = encrypt.unwrap_or(false);
|
||||
|
||||
let upload = match build_stream(&file_path, encrypt, &ctx.edge_key.master_key_b64).await {
|
||||
Ok(u) => u,
|
||||
Err(e) => {
|
||||
error!("Stream build failed: {}", e);
|
||||
return UploadResult {
|
||||
storage_id: storage.id.clone(),
|
||||
success: false,
|
||||
error: Some(e.to_string()),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let config: AzureBlobProviderConfig = match storage.clone().config.try_into() {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
return UploadResult {
|
||||
storage_id: storage.id.clone(),
|
||||
success: false,
|
||||
error: Some(e.to_string()),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let resolved = match config.resolve() {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
return UploadResult {
|
||||
storage_id: storage.id.clone(),
|
||||
success: false,
|
||||
error: Some(e.to_string()),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let file_name = full_file_name(encrypt);
|
||||
let remote_file_path = full_file_path(&file_name);
|
||||
info!(
|
||||
"Starting block upload to azure blob {}/{}",
|
||||
config.container_name, remote_file_path
|
||||
);
|
||||
|
||||
match upload_stream_to_azure(
|
||||
&resolved,
|
||||
&config.container_name,
|
||||
&remote_file_path,
|
||||
upload.stream,
|
||||
BLOCK_SIZE,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
info!("Azure blob upload successful: {}", remote_file_path);
|
||||
UploadResult {
|
||||
storage_id: storage.id.clone(),
|
||||
success: true,
|
||||
error: None,
|
||||
remote_file_path: Some(remote_file_path),
|
||||
total_size: Some(total_size),
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Azure blob upload failed: {:?}", e);
|
||||
UploadResult {
|
||||
storage_id: storage.id.clone(),
|
||||
success: false,
|
||||
error: Some(e.to_string()),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
use crate::services::storage::providers::azure_blob::helpers::ResolvedAzure;
|
||||
use anyhow::{Result, anyhow};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use url::Url;
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
pub struct AzureBlobProviderConfig {
|
||||
#[serde(default)]
|
||||
pub account_name: String,
|
||||
#[serde(default)]
|
||||
pub account_key: String,
|
||||
pub container_name: String,
|
||||
#[serde(default)]
|
||||
pub auth_mode: Option<String>,
|
||||
#[serde(default)]
|
||||
pub connection_string: String,
|
||||
#[serde(default)]
|
||||
pub endpoint_url: Option<String>,
|
||||
}
|
||||
|
||||
fn parse_connection_string(cs: &str) -> std::collections::HashMap<String, String> {
|
||||
cs.split(';')
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.filter_map(|pair| {
|
||||
let mut it = pair.splitn(2, '=');
|
||||
let k = it.next()?.trim().to_string();
|
||||
let v = it.next()?.trim().to_string();
|
||||
Some((k, v))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
|
||||
pub(crate) fn ensure_account_in_endpoint(endpoint: &str, account: &str) -> String {
|
||||
let trimmed = endpoint.trim_end_matches('/');
|
||||
if account.is_empty() {
|
||||
return trimmed.to_string();
|
||||
}
|
||||
if let Ok(url) = Url::parse(trimmed) {
|
||||
let host = url.host_str().unwrap_or("");
|
||||
if host.contains(account) {
|
||||
return trimmed.to_string();
|
||||
}
|
||||
let path = url.path().trim_matches('/');
|
||||
if path == account || path.starts_with(&format!("{account}/")) {
|
||||
return trimmed.to_string();
|
||||
}
|
||||
}
|
||||
format!("{trimmed}/{account}")
|
||||
}
|
||||
|
||||
impl AzureBlobProviderConfig {
|
||||
pub fn resolve(&self) -> Result<ResolvedAzure> {
|
||||
let mode = self.auth_mode.as_deref().unwrap_or("").trim();
|
||||
let has_connection_string = !self.connection_string.trim().is_empty();
|
||||
|
||||
if mode == "connectionString" || (mode.is_empty() && has_connection_string) {
|
||||
if !has_connection_string {
|
||||
return Err(anyhow!(
|
||||
"authMode is connectionString but connectionString is empty"
|
||||
));
|
||||
}
|
||||
let map = parse_connection_string(&self.connection_string);
|
||||
let account_name = map
|
||||
.get("AccountName")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| self.account_name.clone());
|
||||
let account_key = map
|
||||
.get("AccountKey")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| self.account_key.clone());
|
||||
let blob_endpoint = map
|
||||
.get("BlobEndpoint")
|
||||
.cloned()
|
||||
.ok_or_else(|| anyhow!("connection string missing BlobEndpoint"))?;
|
||||
return Ok(ResolvedAzure {
|
||||
account_name,
|
||||
account_key,
|
||||
blob_endpoint,
|
||||
});
|
||||
}
|
||||
|
||||
if self.account_name.trim().is_empty() {
|
||||
return Err(anyhow!("accountName required for accountKey auth"));
|
||||
}
|
||||
if self.account_key.trim().is_empty() {
|
||||
return Err(anyhow!("accountKey required for accountKey auth"));
|
||||
}
|
||||
|
||||
let blob_endpoint = match self
|
||||
.endpoint_url
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
{
|
||||
Some(endpoint) => ensure_account_in_endpoint(endpoint, &self.account_name),
|
||||
None => format!("https://{}.blob.core.windows.net", self.account_name),
|
||||
};
|
||||
|
||||
Ok(ResolvedAzure {
|
||||
account_name: self.account_name.clone(),
|
||||
account_key: self.account_key.clone(),
|
||||
blob_endpoint,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
use crate::services::storage::providers::google_cloud_storage::models::GoogleCloudStorageProviderConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use bytes::Bytes;
|
||||
use futures::Stream;
|
||||
use futures::StreamExt;
|
||||
use google_cloud_auth::credentials::Credentials;
|
||||
use google_cloud_storage::client::Storage;
|
||||
use google_cloud_storage::streaming_source::{SizeHint, StreamingSource};
|
||||
use std::pin::Pin;
|
||||
|
||||
pub fn build_credentials(cfg: &GoogleCloudStorageProviderConfig) -> Result<Credentials> {
|
||||
// Service-account JSON stores the PEM with `\n` escape sequences. When the key is
|
||||
// carried through config as a JSON string those can arrive as literal two-char `\n`
|
||||
// sequences rather than real newlines, so the PEM parser finds no `-----BEGIN-----`
|
||||
// line ("no items found"). Normalize them back to real newlines. A PEM that already
|
||||
// has real newlines contains no literal `\n` pairs, so this is a no-op for it.
|
||||
let private_key = cfg.private_key.replace("\\n", "\n");
|
||||
|
||||
let key = serde_json::json!({
|
||||
"type": "service_account",
|
||||
"project_id": cfg.project_id,
|
||||
"client_email": cfg.client_email,
|
||||
"private_key": private_key,
|
||||
"private_key_id": "",
|
||||
"token_uri": "https://oauth2.googleapis.com/token",
|
||||
"universe_domain": "googleapis.com",
|
||||
});
|
||||
|
||||
google_cloud_auth::credentials::service_account::Builder::new(key)
|
||||
.build()
|
||||
.context("failed to build GCS service account credentials")
|
||||
}
|
||||
|
||||
pub async fn build_client(cfg: &GoogleCloudStorageProviderConfig) -> Result<Storage> {
|
||||
let endpoint = cfg.api_endpoint.as_deref().filter(|s| !s.trim().is_empty());
|
||||
|
||||
// A custom endpoint means a local emulator (fake-gcs-server), which does not verify
|
||||
// credentials. Use anonymous creds so a dummy/empty `private_key` in the emulator
|
||||
// config doesn't trip the service-account PEM parser. Real GCS still uses the
|
||||
// service-account key built from config.
|
||||
let builder = if let Some(ep) = endpoint {
|
||||
let creds = google_cloud_auth::credentials::anonymous::Builder::new().build();
|
||||
Storage::builder()
|
||||
.with_credentials(creds)
|
||||
.with_endpoint(ep.to_string())
|
||||
} else {
|
||||
Storage::builder().with_credentials(build_credentials(cfg)?)
|
||||
};
|
||||
|
||||
builder.build().await.context("failed to build GCS client")
|
||||
}
|
||||
|
||||
/// Bridges `build_stream`'s `Send`-only byte stream into the SDK's `StreamingSource`
|
||||
/// (which `send_buffered` requires to be `Send + Sync + 'static`) via a bounded mpsc
|
||||
/// channel. Also reports an exact `size_hint`: the SDK picks single-shot vs resumable
|
||||
/// upload from `size_hint().upper()` — an unknown bound forces resumable unconditionally
|
||||
/// (see `upload_with_client`).
|
||||
pub struct StreamSource {
|
||||
rx: tokio::sync::mpsc::Receiver<Result<Bytes, std::io::Error>>,
|
||||
total_size: u64,
|
||||
}
|
||||
|
||||
impl StreamSource {
|
||||
pub fn from_stream(
|
||||
mut stream: Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>,
|
||||
total_size: u64,
|
||||
) -> Self {
|
||||
let (tx, rx) = tokio::sync::mpsc::channel(8);
|
||||
tokio::spawn(async move {
|
||||
while let Some(item) = stream.next().await {
|
||||
if tx.send(item).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
StreamSource { rx, total_size }
|
||||
}
|
||||
}
|
||||
|
||||
impl StreamingSource for StreamSource {
|
||||
type Error = std::io::Error;
|
||||
async fn next(&mut self) -> Option<Result<Bytes, Self::Error>> {
|
||||
self.rx.recv().await
|
||||
}
|
||||
|
||||
// Report the exact size so the SDK can choose single-shot uploads. The default
|
||||
// impl returns an unknown bound, which forces the resumable path unconditionally.
|
||||
async fn size_hint(&self) -> Result<SizeHint, Self::Error> {
|
||||
Ok(SizeHint::with_exact(self.total_size))
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn upload_with_client(
|
||||
client: &Storage,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
source: StreamSource,
|
||||
force_single_shot: bool,
|
||||
) -> Result<()> {
|
||||
// `write_object` uses gRPC-style resource names: the bucket must be passed as
|
||||
// `projects/_/buckets/<name>`, not the bare bucket id.
|
||||
let bucket_resource = format!("projects/_/buckets/{bucket}");
|
||||
|
||||
let mut builder = client.write_object(bucket_resource, object, source);
|
||||
|
||||
// Resumable uploads follow a server-generated `Location` URL. When pointed at a
|
||||
// custom `apiEndpoint` on a non-443 port, the SDK's transport drops the port from
|
||||
// the `Host` header (google-cloud-gax-internal `host.rs`), so emulators that build
|
||||
// the `Location` from `Host` hand back a portless URL the SDK then hangs on. A
|
||||
// single-shot upload issues one request to the configured endpoint (no `Location`
|
||||
// to follow), sidestepping the bug. We force it only for custom endpoints; against
|
||||
// real GCS we keep resumable (bounded memory + resume on large backups).
|
||||
if force_single_shot {
|
||||
builder = builder.with_resumable_upload_threshold(usize::MAX);
|
||||
}
|
||||
|
||||
builder
|
||||
.send_buffered()
|
||||
.await
|
||||
.context("GCS write_object failed")?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
pub mod helpers;
|
||||
mod models;
|
||||
|
||||
use crate::core::context::Context;
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::services::backup::models::{BackupResult, UploadResult};
|
||||
use crate::services::storage::StorageProvider;
|
||||
use crate::services::storage::providers::google_cloud_storage::helpers::{
|
||||
StreamSource, build_client, upload_with_client,
|
||||
};
|
||||
use crate::services::storage::providers::google_cloud_storage::models::GoogleCloudStorageProviderConfig;
|
||||
use crate::utils::common::BackupMethod;
|
||||
use crate::utils::file::{full_file_name, full_file_path};
|
||||
use crate::utils::stream::build_stream;
|
||||
use async_trait::async_trait;
|
||||
use std::sync::Arc;
|
||||
use tokio::fs;
|
||||
use tracing::{error, info};
|
||||
|
||||
pub struct GoogleCloudStorageProvider {}
|
||||
|
||||
#[async_trait]
|
||||
impl StorageProvider for GoogleCloudStorageProvider {
|
||||
async fn upload(
|
||||
&self,
|
||||
ctx: Arc<Context>,
|
||||
result: BackupResult,
|
||||
_method: BackupMethod,
|
||||
storage: &DatabaseStorage,
|
||||
encrypt: Option<bool>,
|
||||
_backup_storage_id: &str,
|
||||
) -> UploadResult {
|
||||
let Some(file_path) = result.backup_file else {
|
||||
return UploadResult {
|
||||
storage_id: storage.id.clone(),
|
||||
success: false,
|
||||
error: Some("Missing backup file path".to_string()),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
};
|
||||
};
|
||||
|
||||
let total_size = match fs::metadata(&file_path).await {
|
||||
Ok(meta) => meta.len(),
|
||||
Err(e) => {
|
||||
error!("Failed to get file size: {}", e);
|
||||
return UploadResult {
|
||||
storage_id: storage.id.clone(),
|
||||
success: false,
|
||||
error: Some(e.to_string()),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let encrypt = encrypt.unwrap_or(false);
|
||||
|
||||
let upload = match build_stream(&file_path, encrypt, &ctx.edge_key.master_key_b64).await {
|
||||
Ok(u) => u,
|
||||
Err(e) => {
|
||||
error!("Stream build failed: {}", e);
|
||||
return UploadResult {
|
||||
storage_id: storage.id.clone(),
|
||||
success: false,
|
||||
error: Some(e.to_string()),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let config: GoogleCloudStorageProviderConfig = match storage.clone().config.try_into() {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
return UploadResult {
|
||||
storage_id: storage.id.clone(),
|
||||
success: false,
|
||||
error: Some(e.to_string()),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let file_name = full_file_name(encrypt);
|
||||
info!("Uploading file {}", file_name);
|
||||
let remote_file_path = full_file_path(&file_name);
|
||||
|
||||
let client = match build_client(&config).await {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
error!("GCS client build failed: {:?}", e);
|
||||
return UploadResult {
|
||||
storage_id: storage.id.clone(),
|
||||
success: false,
|
||||
error: Some(e.to_string()),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let source = StreamSource::from_stream(upload.stream, total_size);
|
||||
|
||||
// A custom apiEndpoint (self-hosted / emulator) on a non-443 port trips an
|
||||
// upstream SDK bug in the resumable-upload path; force single-shot for it.
|
||||
let force_single_shot = config
|
||||
.api_endpoint
|
||||
.as_deref()
|
||||
.is_some_and(|s| !s.trim().is_empty());
|
||||
|
||||
info!(
|
||||
"Starting GCS upload to {}/{} (single_shot={})",
|
||||
config.bucket_name, remote_file_path, force_single_shot
|
||||
);
|
||||
|
||||
match upload_with_client(
|
||||
&client,
|
||||
&config.bucket_name,
|
||||
&remote_file_path,
|
||||
source,
|
||||
force_single_shot,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
info!("GCS upload successful: {}", remote_file_path);
|
||||
UploadResult {
|
||||
storage_id: storage.id.clone(),
|
||||
success: true,
|
||||
error: None,
|
||||
remote_file_path: Some(remote_file_path),
|
||||
total_size: Some(total_size),
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("GCS upload failed: {:?}", e);
|
||||
UploadResult {
|
||||
storage_id: storage.id.clone(),
|
||||
success: false,
|
||||
error: Some(e.to_string()),
|
||||
remote_file_path: None,
|
||||
total_size: Some(total_size),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
pub struct GoogleCloudStorageProviderConfig {
|
||||
pub project_id: String,
|
||||
pub bucket_name: String,
|
||||
pub client_email: String,
|
||||
pub private_key: String,
|
||||
#[serde(default)]
|
||||
pub api_endpoint: Option<String>,
|
||||
}
|
||||
@@ -26,6 +26,7 @@ impl StorageProvider for GoogleDriveProvider {
|
||||
_method: BackupMethod,
|
||||
storage: &DatabaseStorage,
|
||||
encrypt: Option<bool>,
|
||||
_backup_storage_id: &str,
|
||||
) -> UploadResult {
|
||||
let Some(file_path) = result.backup_file else {
|
||||
return UploadResult {
|
||||
|
||||
@@ -23,6 +23,7 @@ impl StorageProvider for LocalProvider {
|
||||
method: BackupMethod,
|
||||
storage: &DatabaseStorage,
|
||||
encrypt: Option<bool>,
|
||||
backup_storage_id: &str,
|
||||
) -> UploadResult {
|
||||
let Some(file_path) = result.backup_file else {
|
||||
return UploadResult {
|
||||
@@ -82,6 +83,10 @@ impl StorageProvider for LocalProvider {
|
||||
"X-Generated-Id",
|
||||
HeaderValue::from_str(&result.generated_id).unwrap(),
|
||||
);
|
||||
extra_headers.insert(
|
||||
"X-Backup-Storage-Id",
|
||||
HeaderValue::from_str(backup_storage_id).unwrap(),
|
||||
);
|
||||
extra_headers.insert("X-Status", HeaderValue::from_str(&result.status).unwrap());
|
||||
extra_headers.insert(
|
||||
"X-Method",
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user