Compare commits

..

79 Commits

Author SHA1 Message Date
github-actions[bot] e504d09cb3 chore: release 1.12.3 2026-06-20 13:00:11 +00:00
Charles GTE 0363b300a4 Merge pull request #65 from Portabase/dev
Dev
2026-06-20 14:58:03 +02:00
Charles GTE bf6e7d41ab fix: Dockerfile 2026-06-20 14:40:37 +02:00
Charles GTE 2c7065ae95 Merge pull request #64 from pcrosthwaite/feature-backup-timeout-on-large-db
Support large MySQL and MariaDb databases
2026-06-20 14:38:26 +02:00
Charles GTE 1fece577cc fix: tests and config.rs and backup.rs args 2026-06-20 14:36:09 +02:00
Charles GTE 273475fa3b Merge branch 'dev' into feature-backup-timeout-on-large-db 2026-06-20 13:02:14 +02:00
Charles GTE 628c017584 Merge pull request #63 from tobfel/tf-main
docs: add Windows build and run guide / and fix(postgres): resolve pg_dump/pg_restore path cross-platform
2026-06-20 12:53:31 +02:00
Charles GTE f02218708a fix: windows-release.yml 2026-06-20 12:52:53 +02:00
Charles GTE 16152328b0 fix: extract the env variable to the settings.rs and some refactoring, and the trigger in windows-release.yml 2026-06-20 12:31:01 +02:00
Peter Crosthwaite 7328827435 Fix mysqldump timeout on large database backups
- Add mysql:8.4 client tools stage to Dockerfile so the real mysqldump
  binary (with --network-timeout) takes PATH precedence over MariaDB's
  mysqldump symlink, which lacks this flag
- Use --network-timeout in MySQL backup command; sets net_read_timeout and
  net_write_timeout to 86400s for the duration of the dump
- Simplify config: replace net_read_timeout/net_write_timeout/max_allowed_packet
  with a single max_packet_size field (optional, defaults to 512M)
- Log mysqldump/mariadb-dump client version as a debug message before each backup
2026-06-20 18:06:21 +10:00
Peter Crosthwaite 348eaac81b Added config options for setting timeouts on mysql/mariadb 2026-06-20 07:51:18 +10:00
tf 0f6c93ecd0 fix encoding 2026-06-19 16:31:21 +02:00
tf befec0deae Windows Build workflow 2026-06-19 16:01:45 +02:00
tf 2052ab0ff5 fix(postgres): resolve pg_dump/pg_restore path cross-platform 2026-06-19 15:27:45 +02:00
tf 461e92d67e docs: add Windows build and run guide 2026-06-19 15:22:24 +02:00
github-actions[bot] 23678bf2d6 chore: release 1.12.2 2026-06-19 10:00:23 +00:00
Charles GTE 02105a8171 Merge pull request #62 from Portabase/fix/password-slash
fix: password-slash
2026-06-19 11:58:26 +02:00
charles-gauthereau f94656a39b fix: postgres password slash 2026-06-19 11:29:51 +02:00
Charles GTE b94ff4e987 Merge branch 'main' into dev 2026-06-09 19:18:38 +02:00
github-actions[bot] dc32c442f3 chore: release 1.12.1 2026-06-07 18:06:37 +00:00
Charles GTE 1453851555 Merge pull request #61 from Portabase/fix/cpu-ram-crash
fix: cpu-ram-crash
2026-06-07 20:04:25 +02:00
github-actions[bot] 185a7ee612 chore: release 1.12.0 2026-06-07 15:46:22 +00:00
Charles GTE d8328f60ca Merge pull request #60 from Portabase/feat/backup-restore-job-logs
feat: backup-restore-job-logs
2026-06-07 17:44:26 +02:00
charles-gauthereau 82c7122790 fix: refactoring 2026-06-07 17:32:16 +02:00
Charles GTE fba72f5c26 feat: add job logs for restoration process 2026-06-07 11:16:02 +02:00
Charles GTE b537980c71 fix: test ci 2026-06-06 21:14:38 +02:00
Charles GTE 0261a07fc2 fix: test ci 2026-06-06 20:59:30 +02:00
Charles GTE c840f91680 fix: test ci 2026-06-06 20:20:52 +02:00
Charles GTE 2d52eccbb2 fix: test ci 2026-06-06 19:43:54 +02:00
Charles GTE f9bf481c79 fix: test ci 2026-06-06 19:26:44 +02:00
Charles GTE 3c853ee7ca fix: tests and refactoring 2026-06-06 19:15:22 +02:00
Charles GTE 45d791358c fix: tracing logs 2026-06-06 11:01:55 +02:00
Charles GTE 0b915d87c9 feat: job logs 2026-06-06 10:22:27 +02:00
Charles GTE 531a25f292 Merge branch 'main' into dev 2026-05-24 16:52:40 +02:00
Charles GTE 41e2b197cb Merge remote-tracking branch 'origin/main' 2026-05-24 16:52:35 +02:00
github-actions[bot] f149363b12 chore: release 1.11.1 2026-05-17 16:51:27 +00:00
Charles GTE aa3ff39bb3 fix: Dockerfile (#58)
* fix: s3 port type mismatch, add string_or_number_to_string deserializer

* chore: docker-compose.databases.yml

* fix: tests and refactoring, add justfile instead of makefile

* fix: databases.json

* fix: add gbak in production Dockerfile

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-05-17 18:49:18 +02:00
Charles GTE 740ee43038 fix: add gbak in production Dockerfile 2026-05-17 18:26:29 +02:00
Charles GTE 1fccc0bc19 fix: databases.json 2026-05-15 22:17:02 +02:00
Charles GTE 2c15ea64ae Merge branch 'main' into dev 2026-05-15 22:09:41 +02:00
Charles GTE 18bfdf3b7e Merge remote-tracking branch 'origin/main' 2026-05-11 22:09:18 +02:00
github-actions[bot] db26910679 chore: release 1.11.0 2026-05-11 20:07:01 +00:00
Charles GTE eac2853b83 feat: add-mssql-support (#56)
* fix: tests and refactoring, add justfile instead of makefile

* chore: add tiberius dependency for MSSQL support

* feat: add Mssql variant to DbType enum and config validation

* feat: add MSSQL domain adapter with tiberius ping and sqlpackage backup/restore

* feat: install sqlpackage in Docker base and prod stages

* feat: add MSSQL service to docker-compose and config examples

* fix: log backup errors instead of silently discarding them

* fix: use dotnet-install.sh instead of Microsoft apt feed (SHA1 key rejected by Debian Trixie)

* fix: add /TrustServerCertificate:true to sqlpackage commands for self-signed certs

* fix: use connection string with TrustServerCertificate for sqlpackage

* fix: use user database in MSSQL config example (system databases cannot be exported)

* feat: add seed-mssql task and seed script

* fix: docker-compose.yml
2026-05-11 22:04:36 +02:00
Charles GTE 8690a15888 Merge remote-tracking branch 'origin/main' 2026-05-11 19:45:48 +02:00
github-actions[bot] a4bbb37a4e chore: release 1.10.1 2026-04-29 20:42:31 +00:00
Charles GTE ac8f7fd8d8 Merge branch 'main' into dev 2026-04-29 22:41:16 +02:00
Charles GTE c6baa8fdc3 Merge remote-tracking branch 'origin/main'
# Conflicts:
#	docker-compose.yml
2026-04-29 22:40:31 +02:00
Charles GTE 14068846b9 fix: migration-process (#55)
* fix: s3 port type mismatch, add string_or_number_to_string deserializer

* chore: docker-compose.databases.yml

* fix: tests and refactoring, add justfile instead of makefile

* fix: restoration process for postgres/mysql and mariadb

* fix: restoration process for postgres/mysql and mariadb

* fix: postgres restore

* fix: mariadb/mysql restore, added backticks to drop command

* fix: mongodb and migration test for firebird

* fix: docker-compose.yml

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-04-29 22:40:02 +02:00
Charles GTE 6edf2890f1 Merge remote-tracking branch 'origin/dev' into dev 2026-04-26 17:42:52 +02:00
Charles GTE 5f579690ff Merge branch 'main' into dev 2026-04-26 17:42:46 +02:00
Charles GTE bfe63d7fd3 fix: tests and refactoring, add justfile instead of makefile 2026-04-26 17:42:30 +02:00
github-actions[bot] 0a422448b4 chore: release 1.10.0 2026-04-16 09:22:34 +00:00
Charles GTE 14a6c9ab0a feat: add chunk size in .env variables (#54)
Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-04-16 11:20:14 +02:00
charles-gauthereau 5c35375df7 Merge branch 'main' into dev 2026-04-08 19:57:08 +02:00
Charles GTE 86027fd6bd fix: docker compose files [skip-release] (#53)
* fix: s3 port type mismatch, add string_or_number_to_string deserializer

* chore: docker-compose.databases.yml

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-04-02 19:30:07 +02:00
charles-gauthereau 61c7224104 chore: docker-compose.databases.yml 2026-04-02 19:28:20 +02:00
charles-gauthereau d7958e03b9 Merge branch 'main' into dev 2026-04-01 17:25:32 +02:00
Charles GTE 8eab97a3d6 chore: README.md (#52) [skip-release]
Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-04-01 17:24:48 +02:00
github-actions[bot] 25c95108f0 chore: release 1.9.0 2026-03-31 13:12:45 +00:00
Charles GTE f37e4d2de9 feat: add firebird (#51)
* feat: add firebird

* feat: add firebird tests

* fix: tests and refactoring, add justfile instead of makefile
2026-03-31 15:10:05 +02:00
Charles GTE b269d98d3c Merge branch 'main' into dev 2026-03-29 09:32:25 +02:00
github-actions[bot] a1aa66ff3f chore: release 1.8.0 2026-03-29 07:30:57 +00:00
Charles GTE d4abcbcebc feat: separate mysql and mariadb (#49)
* feat: separate mysql and mariadb

* feat: separate mysql and mariadb
2026-03-29 09:28:30 +02:00
Charles GTE 2d44b62844 Merge branch 'main' into dev 2026-03-28 17:23:31 +01:00
github-actions[bot] 8bf6503f1f chore: release 1.7.0 2026-03-28 15:47:43 +00:00
Charles GTE fdf35228d4 feat: add-heathcheck-database (#48)
feat: add-heathcheck-database
2026-03-28 16:45:13 +01:00
Charles GTE 1855c5120a fix: docker-compose.yml 2026-03-28 16:44:07 +01:00
Charles GTE 3d5a7eeded feat: add health check ping for databases 2026-03-28 16:07:37 +01:00
Charles GTE 517f3779cd feat: sending database ping result for each databases to portabase server 2026-03-27 23:07:06 +01:00
github-actions[bot] a1e18754b1 chore: release 1.6.3 2026-03-27 21:27:23 +00:00
Charles GTE 092f760431 Merge branch 'main' into dev 2026-03-27 22:25:48 +01:00
Charles GTE 725312e1d5 fix: tests (#47)
* fix: s3 port type mismatch, add string_or_number_to_string deserializer

* fix: adding some tests

* fix: adding some tests

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-27 22:24:59 +01:00
Charles GTE 2c5c805308 Merge branch 'main' into dev 2026-03-21 10:20:08 +01:00
github-actions[bot] 8e8ebb5920 chore: release 1.6.2 2026-03-21 07:53:35 +00:00
Charles GTE 8489a6f6e3 fix: adding retry config for s3 provider (#46) 2026-03-21 08:51:15 +01:00
Charles GTE 38a3274c43 Merge branch 'main' into dev 2026-03-21 08:34:41 +01:00
Charles GTE bb45c1961f Merge branch 'main' into dev 2026-03-18 19:11:42 +01:00
charles-gauthereau ee7016d1fa Merge branch 'main' into dev
# Conflicts:
#	docker-compose.yml
2026-03-16 14:36:05 +01:00
charlesgauthereau c46c301371 fix: s3 port type mismatch, add string_or_number_to_string deserializer 2026-03-16 08:55:16 +01:00
174 changed files with 3936 additions and 1168 deletions
+1 -1
View File
@@ -60,7 +60,7 @@ representative at an online or offline event.
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported to the community leaders responsible for enforcement at
contact@soluce-technologies.com.
contact@portabase.io.
All complaints will be reviewed and investigated promptly and fairly.
All community leaders are obligated to respect the privacy and security of the
+1 -1
View File
@@ -18,7 +18,7 @@ We take security seriously and aim to support the following versions of the proj
If you discover a security vulnerability in this project, we appreciate your help in disclosing it responsibly.
1. **Contact Us**
Please report the vulnerability by emailing **[contact@soluce-technologies.com](mailto:contact@soluce-technologies.com)**. Include the following details:
Please report the vulnerability by emailing **[contact@portabase.io](mailto:contact@portabase.io)**. Include the following details:
- A detailed description of the issue.
- Steps to reproduce the vulnerability (if applicable).
- Any potential impacts or risks.
+66 -34
View File
@@ -8,51 +8,83 @@ on:
env:
CARGO_TERM_COLOR: always
CARGO_TARGET_DIR: /app/target
CARGO_INCREMENTAL: 0
RUSTFLAGS: "-C instrument-coverage -C link-dead-code"
LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw"
jobs:
build:
coverage:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: actions-rs/toolchain@v1
with:
toolchain: stable
override: true
components: llvm-tools-preview
- name: Build test image
run: docker compose -f docker-compose.test.yml build agent-test
- name: Install grcov
run: cargo install grcov
- name: Start agent-test container
run: docker compose -f docker-compose.test.yml up -d agent-test
- name: Install test requirements
run: bash scripts/tests/requirements.sh
- name: Build
run: cargo build --verbose
- name: Run tests
env:
CARGO_INCREMENTAL: 0
RUSTFLAGS: "-C instrument-coverage"
LLVM_PROFILE_FILE: "cargo-test-%p-%m.profraw"
run: cargo test --verbose
- name: Generate coverage
- name: Run tests inside container
run: |
grcov . \
--binary-path ./target/debug/ \
-s . \
-t lcov \
--branch \
--ignore-not-existing \
-o lcov.info
docker compose -f docker-compose.test.yml exec -T \
-e CARGO_TARGET_DIR \
-e CARGO_INCREMENTAL \
-e RUSTFLAGS \
-e LLVM_PROFILE_FILE \
agent-test bash -c "
mkdir -p /app/coverage &&
rm -rf /app/target/* /app/coverage/* &&
cargo test --verbose &&
sync
"
- name: Upload to Codecov
- name: Verify profraw files exist
run: |
docker compose -f docker-compose.test.yml exec -T \
agent-test bash -c '
count=$(find /app/coverage -type f -name "*.profraw" | wc -l)
echo "profraw files: $count"
test "$count" -gt 0
'
- name: Generate coverage report inside container
run: |
docker compose -f docker-compose.test.yml exec -T \
agent-test bash -c "
rustup component add llvm-tools-preview &&
grcov /app/coverage \
--binary-path /app/target/debug \
-s /app \
--llvm \
-t lcov \
--branch \
--ignore-not-existing \
--ignore '/app/target/*' \
-o /app/lcov.info
"
- name: Copy lcov.info from container to host
run: docker compose -f docker-compose.test.yml cp agent-test:/app/lcov.info ./lcov.info
- name: Show basic coverage report info
run: |
echo "lcov.info size:" $(wc -c ./lcov.info | awk '{print $1}')
head -n 30 ./lcov.info || true
test -s ./lcov.info
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v5
with:
files: lcov.info
files: ./lcov.info
flags: unittests
name: rust-unit-coverage
verbose: true
fail_ci_if_error: true
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
token: ${{ secrets.CODECOV_TOKEN }}
use_pypi: true
- name: Stop and remove container
if: always()
run: docker compose -f docker-compose.test.yml down --volumes
+96
View File
@@ -0,0 +1,96 @@
name: Build Windows release
on:
workflow_dispatch:
push:
tags:
- '[0-9]+.[0-9]+.[0-9]+'
branches:
- main
- master
jobs:
build-windows:
runs-on: windows-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up Rust toolchain (MSVC)
uses: actions-rs/toolchain@v1
with:
toolchain: stable-x86_64-pc-windows-msvc
profile: minimal
override: true
- name: Install vcpkg and OpenSSL (x64)
shell: pwsh
run: |
# Install vcpkg and the prebuilt OpenSSL package
git clone https://github.com/microsoft/vcpkg C:\vcpkg
C:\vcpkg\bootstrap-vcpkg.bat
C:\vcpkg\vcpkg install openssl:x64-windows
# Export variables for subsequent steps
'VCPKG_ROOT=C:\vcpkg' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
'OPENSSL_DIR=C:\vcpkg\installed\x64-windows' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
- name: Build (cargo release)
shell: pwsh
env:
# Cargo / openssl-sys will pick up OPENSSL_DIR from the environment
OPENSSL_DIR: ${{ env.OPENSSL_DIR }}
run: |
# Ensure the environment variable is present for this step
if (-Not $env:OPENSSL_DIR) { Write-Host "OPENSSL_DIR not set, printing env for debugging"; Get-ChildItem Env: | ForEach-Object { Write-Host $_ } }
# Build the declared bin target explicitly (Cargo.toml [[bin]] name = "app")
cargo build --release --bin app
- name: Prepare artifact zip
id: prepare_artifact
shell: pwsh
env:
RELEASE_TAG: ${{ github.ref_name }}
run: |
$tag = $env:RELEASE_TAG
if (-not $tag) { $tag = $env:GITHUB_SHA }
# Package the declared bin target deterministically (Cargo.toml [[bin]] name = "app")
$exe = "target\release\app.exe"
if (-not (Test-Path $exe)) { Write-Error "Built binary $exe not found in target/release"; exit 1 }
$outDir = "artifact"
New-Item -ItemType Directory -Path $outDir -Force | Out-Null
# Ship under the package name, not the internal bin name "app"
Copy-Item -Path $exe -Destination "$outDir\portabase-agent.exe"
$zipName = "windows-release-$tag.zip"
if (Test-Path $zipName) { Remove-Item $zipName }
Compress-Archive -Path "$outDir\*" -DestinationPath $zipName -Force
Write-Host "ZIP=$zipName"
Write-Output "zip=$zipName" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
- name: Upload build artifact
uses: actions/upload-artifact@v4
with:
name: windows-release
path: windows-release-*.zip
- name: Create GitHub Release
if: startsWith(github.ref, 'refs/tags/')
id: create_release
uses: softprops/action-gh-release@v1
with:
tag_name: ${{ github.ref_name }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Upload release asset
if: startsWith(github.ref, 'refs/tags/')
uses: actions/upload-release-asset@v1
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: windows-release-${{ github.ref_name }}.zip
asset_name: windows-release-${{ github.ref_name }}.zip
asset_content_type: application/zip
+5 -1
View File
@@ -3,4 +3,8 @@
/src/data/
.DS_Store
.env
.env
.claude
/docs
+1 -1
View File
@@ -27,5 +27,5 @@ keywords:
- self-hosted
- portabase
license: Apache-2.0
version: 1.6.1
version: 1.12.3
date-released: '2026-02-24'
Generated
+123 -8
View File
@@ -169,6 +169,19 @@ dependencies = [
"syn",
]
[[package]]
name = "asynchronous-codec"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4057f2c32adbb2fc158e22fb38433c8e9bbf76b75a4732c7c0cbaf695fb65568"
dependencies = [
"bytes",
"futures-sink",
"futures-util",
"memchr",
"pin-project-lite",
]
[[package]]
name = "atomic-waker"
version = "1.1.2"
@@ -496,7 +509,7 @@ dependencies = [
"pin-project-lite",
"rustls 0.21.12",
"rustls 0.23.37",
"rustls-native-certs",
"rustls-native-certs 0.8.3",
"rustls-pki-types",
"tokio",
"tokio-rustls 0.26.4",
@@ -770,7 +783,7 @@ dependencies = [
"pin-project-lite",
"rand 0.9.2",
"rustls 0.23.37",
"rustls-native-certs",
"rustls-native-certs 0.8.3",
"rustls-pki-types",
"serde",
"serde_derive",
@@ -968,6 +981,12 @@ version = "0.4.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d"
[[package]]
name = "connection-string"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "510ca239cf13b7f8d16a2b48f263de7b4f8c566f0af58d901031473c76afb1e3"
[[package]]
name = "const-oid"
version = "0.9.6"
@@ -1436,6 +1455,26 @@ dependencies = [
"syn",
]
[[package]]
name = "enumflags2"
version = "0.7.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef"
dependencies = [
"enumflags2_derive",
]
[[package]]
name = "enumflags2_derive"
version = "0.7.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "equivalent"
version = "1.0.2"
@@ -2050,7 +2089,7 @@ dependencies = [
"hyper 1.8.1",
"hyper-util",
"rustls 0.23.37",
"rustls-native-certs",
"rustls-native-certs 0.8.3",
"rustls-pki-types",
"tokio",
"tokio-rustls 0.26.4",
@@ -2854,6 +2893,12 @@ dependencies = [
"syn",
]
[[package]]
name = "openssl-probe"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e"
[[package]]
name = "openssl-probe"
version = "0.2.1"
@@ -3038,7 +3083,7 @@ dependencies = [
[[package]]
name = "portabase-agent"
version = "1.6.1"
version = "1.12.3"
dependencies = [
"aes",
"aes-gcm",
@@ -3076,6 +3121,7 @@ dependencies = [
"testcontainers",
"testcontainers-modules",
"thiserror 2.0.18",
"tiberius",
"time",
"tokio",
"tokio-postgres",
@@ -3191,6 +3237,12 @@ dependencies = [
"termtree",
]
[[package]]
name = "pretty-hex"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6fa0831dd7cc608c38a5e323422a0077678fa5744aa2be4ad91c4ece8eec8d5"
[[package]]
name = "prettyplease"
version = "0.2.37"
@@ -3677,16 +3729,37 @@ dependencies = [
"zeroize",
]
[[package]]
name = "rustls-native-certs"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00"
dependencies = [
"openssl-probe 0.1.6",
"rustls-pemfile",
"schannel",
"security-framework 2.11.1",
]
[[package]]
name = "rustls-native-certs"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63"
dependencies = [
"openssl-probe",
"openssl-probe 0.2.1",
"rustls-pki-types",
"schannel",
"security-framework",
"security-framework 3.7.0",
]
[[package]]
name = "rustls-pemfile"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c"
dependencies = [
"base64 0.21.7",
]
[[package]]
@@ -3711,10 +3784,10 @@ dependencies = [
"log",
"once_cell",
"rustls 0.23.37",
"rustls-native-certs",
"rustls-native-certs 0.8.3",
"rustls-platform-verifier-android",
"rustls-webpki 0.103.9",
"security-framework",
"security-framework 3.7.0",
"security-framework-sys",
"webpki-root-certs",
"windows-sys 0.61.2",
@@ -3832,6 +3905,19 @@ dependencies = [
"zeroize",
]
[[package]]
name = "security-framework"
version = "2.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"
dependencies = [
"bitflags 2.11.0",
"core-foundation 0.9.4",
"core-foundation-sys",
"libc",
"security-framework-sys",
]
[[package]]
name = "security-framework"
version = "3.7.0"
@@ -4353,6 +4439,34 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "tiberius"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1446cb4198848d1562301a3340424b4f425ef79f35ef9ee034769a9dd92c10d"
dependencies = [
"async-trait",
"asynchronous-codec",
"byteorder",
"bytes",
"chrono",
"connection-string",
"encoding_rs",
"enumflags2",
"futures-util",
"num-traits",
"once_cell",
"pin-project-lite",
"pretty-hex",
"rustls-native-certs 0.6.3",
"rustls-pemfile",
"thiserror 1.0.69",
"tokio-rustls 0.24.1",
"tokio-util",
"tracing",
"uuid",
]
[[package]]
name = "time"
version = "0.3.47"
@@ -4668,6 +4782,7 @@ version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
dependencies = [
"log",
"pin-project-lite",
"tracing-attributes",
"tracing-core",
+5 -4
View File
@@ -1,6 +1,6 @@
[package]
name = "portabase-agent"
version = "1.6.1"
version = "1.12.3"
edition = "2024"
[dependencies]
@@ -35,7 +35,8 @@ rand = "0.9.2"
bytes = "1.11.0"
async-stream = "0.3.6"
uuid = { version = "1.20.0", features = ["v4"] }
tokio-util = "0.7.18"
tokio-util = { version = "0.7.18", features = ["compat"] }
tiberius = { version = "0.12", default-features = false, features = ["rustls", "chrono"] }
aws-config = "1.8.13"
aws-sdk-s3 = { version = "1.122.0", features = ["behavior-version-latest"] }
async-compression = { version = "0.4.37", features = ["tokio", "gzip"] }
@@ -49,7 +50,7 @@ tokio-stream = "0.1.18"
aes = "0.9.0-rc.4"
typenum = "1.19.0"
testcontainers = "0.27.1"
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "valkey"] }
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "valkey", "mysql", "mariadb", "mongo"] }
postgres = "0.19.12"
url = "2.5.8"
@@ -57,7 +58,7 @@ url = "2.5.8"
tokio = { version = "1", features = ["full"] }
mockall = "0.13"
testcontainers = "0.27.1"
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis"] }
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "mysql", "mariadb", "mongo"] }
wiremock = "0.6"
[profile.release]
-54
View File
@@ -1,54 +0,0 @@
include .env
export $(shell sed 's/=.*//' .env)
CLUSTER_SCRIPT=docker/entrypoints/app-dev-entrypoint.sh
.PHONY: seed-mongo seed-mysql seed-postgres
up:
@bash $(CLUSTER_SCRIPT)
seed-mongo:
@echo "Seeding MongoDB..."
bash ./scripts/mongo/seed-mongo.sh
seed-mongo-auth:
@echo "Seeding MongoDB with auth..."
bash ./scripts/mongo/seed-mongo.sh auth
seed-mysql:
@echo "Seeding MySQL..."
mysql -h 127.0.0.1 -P "$$MYSQL_PORT" -u "$$MYSQL_USER" -p"$$MYSQL_PASSWORD" "$$MYSQL_DB" < ./scripts/mysql/seed-mysql.sql
seed-mysql-1gb:
@echo "Seeding MySQL..."
mysql -h 127.0.0.1 -P "$$MYSQL_PORT" -u "$$MYSQL_USER" -p"$$MYSQL_PASSWORD" "$$MYSQL_DB" < ./scripts/mysql/seed-1gb.sql
seed-postgres:
@echo "Seeding Postgres..."
docker exec -i -e PGPASSWORD=$$PG_PASSWORD $$PG_CONTAINER \
psql -U $$PG_USER -d $$PG_DB < ./scripts/postgres/seed.sql
seed-postgres-1gb:
@echo "Seeding Postgres..."
docker exec -i -e PGPASSWORD=$$PG_PASSWORD $$PG_CONTAINER \
psql -U $$PG_USER -d $$PG_DB < ./scripts/postgres/seed-1gb.sql
SQLITE_SEED_FILE := $(if $(filter big,$(SEED)),./scripts/sqlite/seed-big.sql,./scripts/sqlite/seed.sql)
seed-sqlite:
@echo "Seeding Sqlite..."
@echo "Run as root to fix permissions inside the volume"
docker exec -u 0 -it db-sqlite sh -c "chmod -R 777 /workspace/data"
@echo "Create the database file (if it doesnt exist)"
docker exec -u 0 -it db-sqlite sh -c "touch /workspace/data/app.db"
@echo "Seed the database"
docker exec -i db-sqlite sh -c "sqlite3 /workspace/data/app.db" < $(SQLITE_SEED_FILE)
@echo "Verify"
docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT name FROM sqlite_master WHERE type='table';"
@echo "Done"
seed-all: seed-mongo seed-mysql seed-postgres seed-postgres-1gb
+6 -2
View File
@@ -13,6 +13,7 @@
[![License: Apache](https://img.shields.io/badge/License-apache-yellow.svg)](LICENSE)
[![Docker Pulls](https://img.shields.io/docker/pulls/portabase/agent?color=brightgreen)](https://hub.docker.com/r/portabase/agent)
[![Helm Chart](https://img.shields.io/badge/Helm-Kubernetes-326ce5?logo=helm&logoColor=white)](https://github.com/Portabase/agent/pkgs/container/charts%2Fportabase-agent)
[![Platform](https://img.shields.io/badge/platform-linux%20%7C%20macos%20%7C%20windows-lightgrey)](https://github.com/Portabase/portabase)
[![Support Portabase](https://img.shields.io/badge/Support-Portabase-orange)](https://www.buymeacoffee.com/portabase)
@@ -20,9 +21,12 @@
[![MySQL](https://img.shields.io/badge/MySQL-4479A1?logo=mysql&logoColor=white)](https://www.mysql.com/)
[![SQLite](https://img.shields.io/badge/-SQLite-blue?logo=sqlite&logoColor=white)](https://sqlite.org/)
[![Redis](https://img.shields.io/badge/Redis-DC382D?style=flat&logo=Redis&logoColor=white)](https://redis.io/)
[![Valkey](https://img.shields.io/badge/Valkey-6284fc?style=flat&logo=Valkey&logoColor=white)](https://valkey.io/)
[![MariaDB](https://img.shields.io/badge/MariaDB-003545?logo=mariadb&logoColor=white)](https://mariadb.org/)
[![MongoDB](https://img.shields.io/badge/-MongoDB-13aa52?logo=mongodb&logoColor=white)](https://www.mongodb.com/)
[![Valkey](https://img.shields.io/badge/Valkey-6284fc?style=flat&logo=Valkey&logoColor=white)](https://valkey.io/)
[![Firebird](https://img.shields.io/badge/Firebird-f55b14?style=flat&logo=Firebird&logoColor=white)](https://firebirdsql.org/)
[![Self Hosted](https://img.shields.io/badge/self--hosted-yes-brightgreen)](https://github.com/Portabase/portabase)
[![Open Source](https://img.shields.io/badge/open%20source-❤️-red)](https://github.com/Portabase/portabase)
@@ -50,7 +54,7 @@ You have 4 ways to install Portabase Agent:
- Automated CLI (recommended) - [details](https://portabase.io/docs/agent/setup#cli)
- Docker Compose setup - [details](https://portabase.io/docs/agent/setup#docker)
- Kubernetes with Helm (soon)
- Kubernetes with Helm [details](https://portabase.io/docs/agent/setup#helm)
- Development setup - [details](https://portabase.io/docs/agent/setup#development)
**Ensure Docker is installed on your machine before getting started.**
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
+39 -9
View File
@@ -13,7 +13,7 @@
{
"name": "Test database 2 - MariaDB",
"database": "mariadb",
"type": "mysql",
"type": "mariadb",
"username": "mariadb",
"password": "changeme",
"port": 3306,
@@ -21,7 +21,7 @@
"generated_id": "16678124-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 4 - MongoDB Auth",
"name": "Test database 3 - MongoDB Auth",
"database": "testdbauth",
"type": "mongodb",
"username": "root",
@@ -31,7 +31,7 @@
"generated_id": "16678145-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 5 - MongoDB",
"name": "Test database 4 - MongoDB",
"database": "testdb",
"type": "mongodb",
"port": 27017,
@@ -39,26 +39,26 @@
"generated_id": "16678147-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 6 - SQLite DB",
"name": "Test database 5 - SQLite DB",
"type": "sqlite",
"path": "/sqlite-data/workspace/data/app.db",
"generated_id": "16678178-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 7 - SQLite DB",
"name": "Test database 6 - SQLite DB",
"type": "sqlite",
"path": "/sqlite-data-2/workspace/data/app.db",
"generated_id": "16678179-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 8 - Redis",
"name": "Test database 7 - Redis",
"type": "redis",
"port": 6379,
"host": "db-redis",
"generated_id": "16678166-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 9 - Redis Auth",
"name": "Test database 8 - Redis Auth",
"type": "redis",
"password": "supersecurepassword",
"port": 6379,
@@ -67,20 +67,50 @@
"generated_id": "16678160-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 10 - Valkey",
"name": "Test database 9 - Valkey",
"type": "valkey",
"port": 6379,
"host": "db-valkey",
"generated_id": "16678560-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 11 - Valkey Auth",
"name": "Test database 10 - Valkey Auth",
"type": "valkey",
"password": "supersecurepassword",
"port": 6379,
"username": "default",
"host": "db-valkey-auth",
"generated_id": "16678561-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 11 - Mysql",
"database": "mysqldb",
"type": "mysql",
"username": "mysqldb",
"password": "changeme",
"port": 3306,
"host": "db-mysql",
"generated_id": "16656124-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 12 - Firebird",
"database": "/var/lib/firebird/data/mirror.fdb",
"type": "firebird",
"username": "alice",
"password": "fake_password",
"port": 3050,
"host": "db-firebird",
"generated_id": "16706124-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 13 - MsSQL",
"database": "myappdb",
"type": "mssql",
"username": "sa",
"password": "Portabase!Strong1",
"port": 1433,
"host": "db-mssql",
"generated_id": "16706125-ff7e-4c97-8c83-0adeff214682"
}
]
}
+77 -3
View File
@@ -29,11 +29,85 @@ host = "db-mongodb-auth"
generated_id = "16678145-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 3 - MongoDB"
name = "Test database 4 - MongoDB"
database = "testdb"
type = "mongodb"
username = ""
password = ""
port = 27017
host = "db-mongodb"
generated_id = "16678147-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 5 - SQLite DB"
type = "sqlite"
path = "/sqlite-data/workspace/data/app.db"
generated_id = "16678178-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 6 - SQLite DB 2"
type = "sqlite"
path = "/sqlite-data-2/workspace/data/app.db"
generated_id = "16678179-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 7 - Redis"
type = "redis"
port = 6379
host = "db-redis"
generated_id = "16678166-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 8 - Redis Auth"
type = "redis"
password = "supersecurepassword"
username = "default"
port = 6379
host = "db-redis-auth"
generated_id = "16678160-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 9 - Valkey"
type = "valkey"
port = 6379
host = "db-valkey"
generated_id = "16678560-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 10 - Valkey Auth"
type = "valkey"
password = "supersecurepassword"
username = "default"
port = 6379
host = "db-valkey-auth"
generated_id = "16678561-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 11 - Mysql"
database = "mysqldb"
type = "mysql"
password = "changeme"
username = "mysqldb"
port = 3306
host = "db-mysql"
generated_id = "16656124-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 12 - Firebird"
database = "/var/lib/firebird/data/mirror.fdb"
type = "firebird"
password = "fake_password"
username = "alice"
port = 3050
host = "db-firebird"
generated_id = "16706124-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 13 - MSSQL"
database = "myappdb"
type = "mssql"
username = "sa"
password = "Portabase!Strong1"
port = 1433
host = "db-mssql"
generated_id = "16706125-ff7e-4c97-8c83-0adeff214682"
+198
View File
@@ -0,0 +1,198 @@
name: databases
services:
db-postgres:
container_name: db-postgres
image: postgres:17-alpine
ports:
- "5436:5432"
volumes:
- postgres-data:/var/lib/postgresql/data
environment:
- POSTGRES_DB=devdb
- POSTGRES_USER=devuser
- POSTGRES_PASSWORD=changeme
networks:
- portabase
db-mariadb:
container_name: db-mariadb
image: mariadb:latest
ports:
- "3311:3306"
environment:
- MYSQL_DATABASE=mariadb
- MYSQL_USER=mariadb
- MYSQL_PASSWORD=changeme
- MYSQL_RANDOM_ROOT_PASSWORD=yes
volumes:
- mariadb-data:/var/lib/mysql
networks:
- portabase
db-mysql:
container_name: db-mysql
image: mysql:9.5
ports:
- "3312:3306"
environment:
- MYSQL_DATABASE=mysqldb
- MYSQL_USER=mysqldb
- MYSQL_PASSWORD=changeme
- MYSQL_RANDOM_ROOT_PASSWORD=yes
volumes:
- mysql-data:/var/lib/mysql
networks:
- portabase
db-mongodb-auth:
container_name: db-mongodb-auth
image: mongo:latest
ports:
- "27082:27017"
environment:
MONGO_INITDB_ROOT_USERNAME: root
MONGO_INITDB_ROOT_PASSWORD: rootpassword
MONGO_INITDB_DATABASE: testdbauth
command: mongod --auth
networks:
- portabase
volumes:
- mongodb-data-auth:/data/db
healthcheck:
test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ]
interval: 5s
timeout: 5s
retries: 10
db-mongodb:
container_name: db-mongodb
image: mongo:latest
ports:
- "27083:27017"
volumes:
- mongodb-data:/data/db
healthcheck:
test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ]
interval: 5s
timeout: 5s
retries: 10
environment:
MONGO_INITDB_DATABASE: testdb
networks:
- portabase
sqlite:
container_name: db-sqlite
image: keinos/sqlite3
volumes:
- sqlite-data:/workspace/data
working_dir: /workspace
command: tail -f /dev/null
stdin_open: true
tty: true
db-redis:
image: redis:latest
container_name: db-redis
ports:
- "6378:6379"
volumes:
- redis-data:/data
command: [ "redis-server", "--appendonly", "yes" ]
networks:
- portabase
db-redis-auth:
image: redis:latest
container_name: db-redis-auth
ports:
- "6380:6379"
volumes:
- redis-data-auth:/data
environment:
- REDIS_PASSWORD=supersecurepassword
command: [ "redis-server", "--requirepass", "supersecurepassword", "--appendonly", "yes" ]
networks:
- portabase
db-valkey:
image: valkey/valkey
container_name: db-valkey
environment:
- ALLOW_EMPTY_PASSWORD=yes
ports:
- '6381:6379'
volumes:
- valkey-data:/data
networks:
- portabase
db-valkey-auth:
image: valkey/valkey
container_name: db-valkey-auth
command: >
--requirepass "supersecurepassword"
ports:
- '6382:6379'
volumes:
- valkey-data-auth:/data
networks:
- portabase
db-firebird:
image: firebirdsql/firebird
container_name: db-firebird
restart: always
environment:
- FIREBIRD_ROOT_PASSWORD=fake_root_password
- FIREBIRD_USER=alice
- FIREBIRD_PASSWORD=fake_password
- FIREBIRD_DATABASE=mirror.fdb
- FIREBIRD_DATABASE_DEFAULT_CHARSET=UTF8
volumes:
- firebird-data:/var/lib/firebird/data
ports:
- "3060:3050"
networks:
- portabase
db-mssql:
container_name: db-mssql
image: mcr.microsoft.com/azure-sql-edge:latest
ports:
- "1433:1433"
environment:
ACCEPT_EULA: "Y"
MSSQL_SA_PASSWORD: "Portabase!Strong1"
volumes:
- mssql-data:/var/opt/mssql
networks:
- portabase
healthcheck:
test: ["CMD-SHELL", "cat /proc/net/tcp6 | grep -q '059901' || exit 1"]
interval: 10s
timeout: 5s
retries: 20
volumes:
postgres-data:
mariadb-data:
mysql-data:
mongodb-data:
mongodb-data-auth:
sqlite-data:
redis-data:
redis-data-auth:
valkey-data:
valkey-data-auth:
firebird-data:
mssql-data:
networks:
portabase:
name: portabase_network
external: true
+2 -5
View File
@@ -5,23 +5,20 @@ services:
# dockerfile: docker/Dockerfile
# target: prod
image: portabase/agent:latest
# platform: linux/arm64
container_name: rust-prod
volumes:
- ./databases.json:/config/config.json
environment:
# APP_ENV: production
LOG: info
TZ: "Europe/Paris"
# DATABASES_CONFIG_FILE: "config.toml"
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiNWU1OGU2MGEtODhiMy00YTBjLWI0NDktNTQ3OWZhOTQzZDBkIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ=="
#DATABASES_CONFIG_FILE: "config.toml"
EDGE_KEY: "<to complete>"
extra_hosts:
- "localhost:host-gateway"
networks:
- portabase
db-mongodb-auth:
container_name: db-mongodb-auth
image: mongo:latest
+22
View File
@@ -0,0 +1,22 @@
services:
agent-test:
build:
context: .
dockerfile: docker/Dockerfile
target: dev
working_dir: /app
container_name: agent-test
volumes:
- .:/app
- cargo-registry:/usr/local/cargo/registry
- cargo-git:/usr/local/cargo/git
- /var/run/docker.sock:/var/run/docker.sock
environment:
APP_ENV: test
LOG: debug
TZ: Europe/Paris
EDGE_KEY: ""
volumes:
cargo-registry:
cargo-git:
+13 -135
View File
@@ -7,18 +7,20 @@ services:
container_name: rust-dev
volumes:
- .:/app
- ./databases.json:/config/config.json
# - ./databases.toml:/config/config.toml
- cargo-registry:/usr/local/cargo/registry
- cargo-git:/usr/local/cargo/git
- ./databases.json:/config/config.json
#- ./databases.toml:/config/config.toml
#- /var/run/docker.sock:/var/run/docker.sock
# - cargo-target:/app/target
# - sqlite-data:/sqlite-data/workspace/data
# - ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data
- databases_sqlite-data:/sqlite-data/workspace/data
- ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data
environment:
APP_ENV: development
LOG: debug
TZ: "Europe/Paris"
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiMmVkZWFlYjctNWYwNi00MjI1LWIzNWItOGYyM2MwOTMyMTA2IiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ=="
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiNWRkZTE1NTctZWQ1ZC00MjUxLThiZDMtMDE0MjkxOTg2OGZjIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ=="
#CHUNK_SIZE_MB: "1"
#POOLING: 1
#DATABASES_CONFIG_FILE: "config.toml"
extra_hosts:
@@ -26,144 +28,20 @@ services:
networks:
- portabase
db-postgres:
container_name: db-postgres
image: postgres:17-alpine
ports:
- "5436:5432"
volumes:
- postgres-data:/var/lib/postgresql/data
environment:
- POSTGRES_DB=devdb
- POSTGRES_USER=devuser
- POSTGRES_PASSWORD=changeme
networks:
- portabase
#
# db-mariadb:
# container_name: db-mariadb
# image: mariadb:latest
# ports:
# - "3311:3306"
# environment:
# - MYSQL_DATABASE=mariadb
# - MYSQL_USER=mariadb
# - MYSQL_PASSWORD=changeme
# - MYSQL_RANDOM_ROOT_PASSWORD=yes
# volumes:
# - mariadb-data:/var/lib/mysql
# networks:
# - portabase
#
#
# db-mongodb-auth:
# container_name: db-mongodb-auth
# image: mongo:latest
# ports:
# - "27082:27017"
# environment:
# MONGO_INITDB_ROOT_USERNAME: root
# MONGO_INITDB_ROOT_PASSWORD: rootpassword
# MONGO_INITDB_DATABASE: testdbauth
# command: mongod --auth
# networks:
# - portabase
# volumes:
# - mongodb-data-auth:/data/db
# healthcheck:
# test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ]
# interval: 5s
# timeout: 5s
# retries: 10
#
# db-mongodb:
# container_name: db-mongodb
# image: mongo:latest
# ports:
# - "27083:27017"
# volumes:
# - mongodb-data:/data/db
# healthcheck:
# test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ]
# interval: 5s
# timeout: 5s
# retries: 10
# environment:
# MONGO_INITDB_DATABASE: testdb
# networks:
# - portabase
cpus: "1.50"
# sqlite:
# container_name: db-sqlite
# image: keinos/sqlite3
# volumes:
# - sqlite-data:/workspace/data
# working_dir: /workspace
# command: tail -f /dev/null
# stdin_open: true
# tty: true
mem_limit: 4g
memswap_limit: 4g
db-redis:
image: redis:latest
container_name: db-redis
ports:
- "6379:6379"
volumes:
- redis-data:/data
command: [ "redis-server", "--appendonly", "yes" ]
networks:
- portabase
pids_limit: 512
db-redis-auth:
image: redis:latest
container_name: db-redis-auth
ports:
- "6380:6379"
volumes:
- redis-data-auth:/data
environment:
- REDIS_PASSWORD=supersecurepassword
command: [ "redis-server", "--requirepass", "supersecurepassword", "--appendonly", "yes" ]
networks:
- portabase
db-valkey:
image: valkey/valkey
container_name: db-valkey
environment:
- ALLOW_EMPTY_PASSWORD=yes
ports:
- '6381:6379'
volumes:
- valkey-data:/data
networks:
- portabase
db-valkey-auth:
image: valkey/valkey
container_name: db-valkey-auth
command: >
--requirepass "supersecurepassword"
ports:
- '6382:6379'
volumes:
- valkey-data-auth:/data
networks:
- portabase
volumes:
cargo-registry:
cargo-git:
# cargo-target:
postgres-data:
# mariadb-data:
# mongodb-data:
# mongodb-data-auth:
# sqlite-data:
redis-data:
redis-data-auth:
valkey-data:
valkey-data-auth:
databases_sqlite-data:
external: true
networks:
portabase:
+44 -1
View File
@@ -1,3 +1,11 @@
# =========================
# MySQL client tools
# =========================
FROM mysql:8.4 AS mysql-client-tools
RUN mkdir -p /mysql-exports/bin /mysql-exports/lib \
&& cp /usr/bin/mysqldump /mysql-exports/bin/ \
&& find /usr/lib -name "libmysqlclient.so.21*" -exec cp {} /mysql-exports/lib/ \;
# =========================
# Base image (shared)
# =========================
@@ -18,9 +26,24 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
sqlite3 \
redis-tools \
valkey \
firebird3.0-utils \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
ENV DOTNET_ROOT=/usr/local/dotnet
RUN curl -sSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh \
&& chmod +x /tmp/dotnet-install.sh \
&& /tmp/dotnet-install.sh --channel 8.0 --install-dir /usr/local/dotnet \
&& rm /tmp/dotnet-install.sh \
&& /usr/local/dotnet/dotnet tool install --global microsoft.sqlpackage
ENV PATH="$PATH:/usr/local/dotnet:/root/.dotnet/tools"
RUN ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') \
&& curl -sSL "https://github.com/microsoft/go-sqlcmd/releases/latest/download/sqlcmd-linux-${ARCH}.tar.bz2" \
| tar -xjf - -C /usr/local/bin sqlcmd \
&& chmod +x /usr/local/bin/sqlcmd
ARG TARGETARCH
# =========================
@@ -46,12 +69,20 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \
rm -rf /tmp/pg-x64 /tmp/pg-arm && \
chmod +x /usr/lib/postgresql/*/bin/*
# =========================
# MongoDB client binaries
# =========================
COPY assets/tools/${TARGETARCH}/mongodb/ /usr/local/mongodb/
RUN chmod +x /usr/local/mongodb/bin/*
# =========================
# MySQL real mysqldump binary
# =========================
COPY --from=mysql-client-tools /mysql-exports/bin/mysqldump /usr/local/bin/mysqldump
COPY --from=mysql-client-tools /mysql-exports/lib/ /usr/local/lib/
RUN chmod +x /usr/local/bin/mysqldump && ldconfig
WORKDIR /app
# =========================
@@ -60,6 +91,7 @@ WORKDIR /app
FROM base AS dev
RUN cargo install cargo-watch
RUN cargo install grcov --locked
COPY Cargo.toml Cargo.lock ./
RUN mkdir src && echo "fn main() {}" > src/main.rs
@@ -94,12 +126,19 @@ RUN apt-get update && apt-get install -y \
libreadline8 \
libncurses6 \
zlib1g \
curl \
mariadb-client \
sqlite3 \
redis-tools \
valkey \
firebird3.0-utils \
&& rm -rf /var/lib/apt/lists/*
ENV DOTNET_ROOT=/usr/local/dotnet
RUN curl -sSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh \
&& chmod +x /tmp/dotnet-install.sh \
&& /tmp/dotnet-install.sh --channel 8.0 --runtime dotnet --install-dir /usr/local/dotnet \
&& rm /tmp/dotnet-install.sh
WORKDIR /app
@@ -110,8 +149,12 @@ RUN chmod +x /entrypoint.sh
COPY --from=base /usr/lib/postgresql/ /usr/lib/postgresql/
COPY --from=base /usr/local/mongodb/bin/ /usr/local/mongodb/bin/
COPY --from=base /root/.dotnet/tools/ /root/.dotnet/tools/
COPY --from=mysql-client-tools /mysql-exports/bin/mysqldump /usr/local/bin/mysqldump
COPY --from=mysql-client-tools /mysql-exports/lib/ /usr/local/lib/
RUN chmod +x /usr/local/bin/mysqldump && ldconfig
ENV PATH="$PATH:/usr/local/dotnet:/root/.dotnet/tools"
ENV APP_ENV=production
CMD ["/entrypoint.sh"]
+8 -3
View File
@@ -34,6 +34,11 @@ check_network() {
check_docker
check_network
echo "Starting docker-compose..."
docker compose -f ./docker-compose.yml up
echo "Docker-compose started successfully."
echo "Stopping old database containers..."
docker compose -f ./docker-compose.databases.yml down
echo "Starting database containers..."
docker compose -f ./docker-compose.databases.yml up -d
echo "Starting main services..."
docker compose -f ./docker-compose.yml up
+71
View File
@@ -0,0 +1,71 @@
set dotenv-load := true
set shell := ["bash", "-cu"]
CLUSTER_SCRIPT := "docker/entrypoints/app-dev-entrypoint.sh"
up:
bash {{CLUSTER_SCRIPT}}
seed-mongo:
echo "Seeding MongoDB..."
bash ./scripts/mongo/seed-mongo.sh
seed-mongo-auth:
echo "Seeding MongoDB with auth..."
bash ./scripts/mongo/seed-mongo.sh auth
seed-mysql:
echo "Seeding MySQL..."
mysql -h 127.0.0.1 -P "$MYSQL_PORT" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DB" < ./scripts/mysql/seed-mysql.sql
seed-mysql-1gb:
echo "Seeding MySQL (1GB)..."
mysql -h 127.0.0.1 -P "$MYSQL_PORT" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DB" < ./scripts/mysql/seed-1gb.sql
seed-postgres:
echo "Seeding Postgres..."
docker exec -i -e PGPASSWORD="$PG_PASSWORD" "$PG_CONTAINER" \
psql -U "$PG_USER" -d "$PG_DB" < ./scripts/postgres/seed.sql
seed-postgres-1gb:
echo "Seeding Postgres (1GB)..."
docker exec -i -e PGPASSWORD="$PG_PASSWORD" "$PG_CONTAINER" \
psql -U "$PG_USER" -d "$PG_DB" < ./scripts/postgres/seed-1gb.sql
seed-sqlite:
echo "Seeding SQLite..."
bash ./scripts/sqlite/seed.sh
echo "Done"
seed-firebird:
echo "Seeding Firebird..."
docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb < ./scripts/firebird/seed.sql
echo "Verifying Firebird tables..."
echo "SELECT RDB\$RELATION_NAME FROM RDB\$RELATIONS WHERE RDB\$SYSTEM_FLAG = 0 AND RDB\$VIEW_BLR IS NULL;" \
| docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb
seed-mssql:
echo "Seeding MSSQL..."
docker exec -i rust-dev sqlcmd -S "db-mssql,1433" -U sa -P "$MSSQL_SA_PASSWORD" -N disable -i /app/scripts/mssql/seed.sql
echo "Done"
seed-all:
just seed-mongo
just seed-mysql
just seed-postgres
just seed-postgres-1gb
just seed-sqlite
just seed-mongo
just seed-firebird
just seed-mssql
test:
echo "Build test image (with grcov included)"
docker compose -f docker-compose.test.yml build agent-test
echo "Start agent-test container"
docker compose -f docker-compose.test.yml up -d agent-test
echo "Run tests inside container"
docker compose -f docker-compose.test.yml exec -e CARGO_INCREMENTAL -e RUSTFLAGS -e LLVM_PROFILE_FILE agent-test bash -c "cargo test --verbose && sync"
echo "Down volumes tests databases"
docker compose -f docker-compose.test.yml down --volumes
+4
View File
@@ -24,3 +24,7 @@ docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT * FROM users LI
```bash
docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT name FROM sqlite_master WHERE type='table';"
```
```bash
docker compose -f docker-compose.test.yml run agent-test bash -c "cargo clean && cargo test"
```
+11
View File
@@ -0,0 +1,11 @@
CREATE TABLE users (
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
email VARCHAR(255) NOT NULL UNIQUE,
name VARCHAR(255),
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
INSERT INTO users (email, name) VALUES ('alice@example.com', 'Alice');
INSERT INTO users (email, name) VALUES ('bob@example.com', 'Bob');
COMMIT;
+20
View File
@@ -0,0 +1,20 @@
IF NOT EXISTS (SELECT name FROM sys.databases WHERE name = N'myappdb')
CREATE DATABASE [myappdb];
GO
USE [myappdb];
GO
IF OBJECT_ID('users', 'U') IS NULL
CREATE TABLE users (
id INT IDENTITY(1,1) PRIMARY KEY,
email NVARCHAR(255) NOT NULL UNIQUE,
name NVARCHAR(255),
created_at DATETIME DEFAULT GETDATE()
);
GO
INSERT INTO users (email, name) VALUES ('alice@example.com', 'Alice');
INSERT INTO users (email, name) VALUES ('bob@example.com', 'Bob');
INSERT INTO users (email, name) VALUES ('charlie@example.com', 'Charlie');
GO
+14
View File
@@ -0,0 +1,14 @@
#!/usr/bin/env bash
set -euo pipefail
SEED_VALUE="${SEED:-}"
if [ "$SEED_VALUE" = "big" ]; then
SQLITE_SEED_FILE="./scripts/sqlite/seed-big.sql"
else
SQLITE_SEED_FILE="./scripts/sqlite/seed.sql"
fi
docker exec -u 0 db-sqlite sh -c "chmod -R 777 /workspace/data"
docker exec -u 0 db-sqlite sh -c "touch /workspace/data/app.db"
docker exec -i db-sqlite sh -c "sqlite3 /workspace/data/app.db" < "$SQLITE_SEED_FILE"
-88
View File
@@ -1,88 +0,0 @@
#!/usr/bin/env bash
set -e
POSTGRES_BASE="/usr/local/postgresql"
echo "Detecting OS and architecture..."
OS_TYPE="$(uname -s)"
ARCH="$(uname -m)"
install_pg_binaries() {
echo "Installing PostgreSQL binaries for versions 12-18..."
for v in 12 13 14 15 16 17 18; do
TARGET_DIR="$POSTGRES_BASE/$v/bin"
sudo mkdir -p "$TARGET_DIR"
if [[ "$OS_TYPE" == "Linux" ]]; then
if [[ "$ARCH" == "x86_64" ]]; then
SRC_DIR="./assets/tools/amd64/postgresql/postgresql-$v/bin"
elif [[ "$ARCH" == "aarch64" ]]; then
SRC_DIR="./assets/tools/arm64/postgresql/postgresql-$v/bin"
else
echo "Unsupported architecture: $ARCH"
continue
fi
if [[ -d "$SRC_DIR" ]]; then
echo "Copying PostgreSQL $v binaries from $SRC_DIR to $TARGET_DIR"
sudo cp -r "$SRC_DIR"/* "$TARGET_DIR/"
else
echo "Binaries for PostgreSQL $v not found for Linux, skipping..."
continue
fi
elif [[ "$OS_TYPE" == "Darwin" ]]; then
PG_SRC="$(brew --prefix postgresql@$v)/bin" 2>/dev/null || true
if [[ ! -d "$PG_SRC" ]]; then
echo "PostgreSQL $v not installed via Homebrew. Trying to install..."
if ! brew install postgresql@$v; then
echo "PostgreSQL $v not available, skipping..."
continue
fi
PG_SRC="$(brew --prefix postgresql@$v)/bin"
fi
echo "Copying PostgreSQL $v binaries from $PG_SRC to $TARGET_DIR"
sudo cp -r "$PG_SRC"/* "$TARGET_DIR/"
fi
sudo chown -R "$(whoami)" "$TARGET_DIR"
chmod +x "$TARGET_DIR"/*
done
echo "PostgreSQL binaries installed under $POSTGRES_BASE"
}
if [[ "$OS_TYPE" == "Linux" ]]; then
if command -v apt >/dev/null 2>&1; then
echo "Linux detected with apt. Installing prerequisites..."
sudo apt update
sudo apt install -y wget gnupg lsb-release redis-tools valkey
install_pg_binaries
else
echo "Unsupported Linux distribution. Only apt-based distros are supported."
exit 1
fi
elif [[ "$OS_TYPE" == "Darwin" ]]; then
if command -v brew >/dev/null 2>&1; then
echo "macOS detected. Installing prerequisites..."
brew install redis
brew install valkey
sudo mkdir -p "$POSTGRES_BASE"
sudo chown -R "$(whoami)" "$POSTGRES_BASE"
install_pg_binaries
else
echo "Homebrew not found. Please install Homebrew first: https://brew.sh/"
exit 1
fi
else
echo "Unsupported OS: $OS_TYPE"
exit 1
fi
echo "Tools installation completed successfully."
+15 -8
View File
@@ -4,11 +4,11 @@ use crate::core::context::Context;
use crate::services::backup::BackupService;
use crate::services::config::ConfigService;
use crate::services::cron::CronService;
use crate::services::restore::RestoreService;
use crate::services::status::StatusService;
use crate::utils::common::BackupMethod;
use std::sync::Arc;
use tracing::info;
use crate::services::restore::RestoreService;
pub struct Agent {
ctx: Arc<Context>,
@@ -43,23 +43,30 @@ impl Agent {
let ping_result = self.status_service.ping(&config.databases).await?;
for db in ping_result.databases.iter() {
let database = config.databases.iter().find(|cfg_db|cfg_db.generated_id == db.generated_id).unwrap();
let database = config
.databases
.iter()
.find(|cfg_db| cfg_db.generated_id == db.generated_id)
.unwrap();
info!(
"Generated Id: {} | backup action: {} | restore action: {} | Database Name: {}",
db.generated_id, db.data.backup.action, db.data.restore.action, database.name,
db.generated_id, db.data.backup.action, db.data.restore.action, database.name,
);
let _ = self.cron_service.sync(db).await;
if db.data.backup.action {
let _ = self
.backup_service
.dispatch(&db.generated_id, &config, method.clone(), &db.storages, db.encrypt)
.dispatch(
&db.generated_id,
&config,
method.clone(),
&db.storages,
db.encrypt,
)
.await;
} else if db.data.restore.action {
let _ = self
.restore_service
.dispatch(db, &config)
.await;
let _ = self.restore_service.dispatch(db, &config).await;
}
}
+1 -1
View File
@@ -35,7 +35,7 @@ impl Context {
panic!("Cannot initialize AgentContext due to invalid EDGE_KEY");
}
};
let server_url = format!("{}/api", edge_key.server_url);
let api_client = ApiClient::new(server_url);
+1 -1
View File
@@ -1,2 +1,2 @@
pub mod context;
pub mod agent;
pub mod context;
+15 -8
View File
@@ -4,18 +4,22 @@ use crate::domain::postgres::database::PostgresDatabase;
use crate::domain::postgres::{detect_format_from_file, detect_format_from_size};
use crate::domain::redis::database::RedisDatabase;
use crate::domain::sqlite::database::SqliteDatabase;
use crate::domain::valkey::database::ValkeyDatabase;
use crate::domain::firebird::database::FirebirdDatabase;
use crate::domain::mariadb::database::MariaDBDatabase;
use crate::domain::mssql::database::MssqlDatabase;
use crate::services::backup::logger::JobLogger;
use crate::services::config::{DatabaseConfig, DbType};
use anyhow::Result;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use crate::domain::valkey::database::ValkeyDatabase;
#[async_trait::async_trait]
pub trait Database: Send + Sync {
fn file_extension(&self) -> &'static str;
async fn ping(&self) -> Result<bool>;
async fn backup(&self, backup_dir: &Path, is_test: Option<bool>) -> Result<PathBuf>;
async fn restore(&self, restore_file: &Path, is_test: Option<bool>) -> Result<()>;
async fn backup(&self, backup_dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf>;
async fn restore(&self, restore_file: &Path, logger: Arc<JobLogger>) -> Result<()>;
}
pub struct DatabaseFactory;
@@ -28,11 +32,13 @@ impl DatabaseFactory {
Arc::new(PostgresDatabase::new(cfg, format))
}
DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)),
DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)),
DbType::Mariadb => Arc::new(MariaDBDatabase::new(cfg)),
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
DbType::Redis => Arc::new(RedisDatabase::new(cfg)),
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg))
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)),
DbType::Firebird => Arc::new(FirebirdDatabase::new(cfg)),
DbType::Mssql => Arc::new(MssqlDatabase::new(cfg)),
}
}
@@ -43,12 +49,13 @@ impl DatabaseFactory {
Arc::new(PostgresDatabase::new(cfg, format))
}
DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)),
DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)),
DbType::Mariadb => Arc::new(MariaDBDatabase::new(cfg)),
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
DbType::Redis => Arc::new(RedisDatabase::new(cfg)),
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg))
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)),
DbType::Firebird => Arc::new(FirebirdDatabase::new(cfg)),
DbType::Mssql => Arc::new(MssqlDatabase::new(cfg)),
}
}
}
+54
View File
@@ -0,0 +1,54 @@
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use std::time::Instant;
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
file_extension: &'static str,
logger: Arc<JobLogger>,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
logger.log("info", format!("Starting Firebird backup for database: {}", cfg.name));
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
let db_path = format!("{}/{}:{}", cfg.host, cfg.port, cfg.database);
logger.log("info", format!("Firebird target: {}{}", db_path, file_path.display()));
let start = Instant::now();
let output = Command::new("gbak")
.arg("-b")
.arg("-v")
.arg("-user").arg(&cfg.username)
.arg("-password").arg(&cfg.password)
.arg(&db_path)
.arg(&file_path)
.output()
.with_context(|| format!("Failed to run gbak for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
let combined_output = if stderr.is_empty() && stdout.is_empty() {
None
} else {
Some(format!("{}{}", stdout, stderr).trim().to_string())
};
if !output.status.success() {
logger.log_command("gbak", combined_output, Some(exit_code), Some(duration_ms));
anyhow::bail!("Firebird backup failed for {}: {}", cfg.name, stderr);
}
logger.log_command("gbak", combined_output, Some(0), Some(duration_ms));
logger.log("info", format!("Firebird backup completed for {}", cfg.name));
Ok(file_path)
})
.await?
}
+51
View File
@@ -0,0 +1,51 @@
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
use anyhow::Result;
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use std::sync::Arc;
pub struct FirebirdDatabase {
cfg: DatabaseConfig,
}
impl FirebirdDatabase {
pub fn new(cfg: DatabaseConfig) -> Self {
Self { cfg }
}
}
#[async_trait]
impl Database for FirebirdDatabase {
fn file_extension(&self) -> &'static str {
".fbk"
}
async fn ping(&self) -> Result<bool> {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(
self.cfg.clone(),
dir.to_path_buf(),
self.file_extension(),
logger,
)
.await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+4
View File
@@ -0,0 +1,4 @@
pub mod database;
mod ping;
mod backup;
mod restore;
+57
View File
@@ -0,0 +1,57 @@
use std::process::Stdio;
use tracing::{error, info};
use tokio::io::AsyncWriteExt;
use crate::services::config::DatabaseConfig;
use tokio::process::Command;
use tokio::time::{Duration, timeout};
pub async fn run(cfg: DatabaseConfig) -> anyhow::Result<bool> {
let db_path = format!("{}/{}:{}", cfg.host, cfg.port, cfg.database);
info!("Running Ping database from {}", db_path);
let mut child = Command::new("isql-fb")
.arg("-q")
.arg("-user")
.arg(&cfg.username)
.arg("-password")
.arg(&cfg.password)
.arg(&db_path)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?;
let query = b"SELECT 1 FROM RDB$DATABASE;\nQUIT;\n";
if let Some(mut stdin) = child.stdin.take() {
stdin.write_all(query).await?;
}
let output = match timeout(Duration::from_secs(5), child.wait_with_output()).await {
Ok(res) => res?,
Err(_) => return Ok(false),
};
if !output.status.success() {
error!("Error output for firebird: {:?}", output);
return Ok(false);
}
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
if stderr.to_lowercase().contains("error") {
error!("Error output for firebird: {:?}", output);
return Ok(false);
}
if stdout.contains("1") {
return Ok(true);
}
info!("stdout {}", stdout);
error!("stderr {}", stderr);
Ok(false)
}
+48
View File
@@ -0,0 +1,48 @@
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use std::time::Instant;
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
tokio::task::spawn_blocking(move || -> Result<()> {
logger.log("debug", format!("Starting Firebird restore for database {}", cfg.name));
let db_path = format!("{}/{}:{}", cfg.host, cfg.port, cfg.database);
logger.log("info", format!("Restore source: {}", restore_file.display()));
logger.log("info", format!("Restore target: {}", db_path));
let start = Instant::now();
let output = Command::new("gbak")
.arg("-c")
.arg("-v")
.arg("-replace_database")
.arg("-user")
.arg(&cfg.username)
.arg("-password")
.arg(&cfg.password)
.arg(&restore_file)
.arg(&db_path)
.output()
.with_context(|| format!("Failed to run gbak restore for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
if !output.status.success() {
logger.log_command("gbak", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
logger.log("error", format!("Firebird restore failed for {}: {}", cfg.name, stderr));
anyhow::bail!("Firebird restore failed for {}: {}", cfg.name, stderr);
}
logger.log_command("gbak", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
logger.log("info", format!("Firebird restore completed for {}", cfg.name));
Ok(())
})
.await?
}
+81
View File
@@ -0,0 +1,81 @@
use crate::domain::mariadb::connection::{select_mariadb_path, server_version};
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::collections::HashMap;
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use std::time::Instant;
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
env: HashMap<String, String>,
file_extension: &'static str,
logger: Arc<JobLogger>,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
logger.log("info", format!("Starting backup for database {}", cfg.name));
let version = match futures::executor::block_on(server_version(&cfg)) {
Ok(v) => {
logger.log("info", format!("MariaDB version detected: {}", v));
v
}
Err(e) => {
logger.log("error", format!("Failed to get server version: {}", e));
return Err(e.into());
}
};
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
let _mariadb_dump = select_mariadb_path(&version).join("mariadb-dump");
logger.log("debug", format!("Using mariadb-dump at {}", _mariadb_dump.display()));
if let Ok(out) = Command::new("mariadb-dump").arg("--version").output() {
logger.log("debug", format!("mariadb-dump client: {}", String::from_utf8_lossy(&out.stdout).trim()));
}
logger.log("info", format!("Running mariadb-dump for {}", cfg.name));
let start = Instant::now();
let output = Command::new("mariadb-dump")
.arg("--host").arg(&cfg.host)
.arg("--port").arg(cfg.port.to_string())
.arg("--user").arg(&cfg.username)
.arg("--routines")
.arg("--events")
.arg("--triggers")
.arg("--single-transaction")
.arg("--quick")
.arg("--skip-lock-tables")
.arg("--no-create-db")
.arg("--skip-add-drop-table")
.arg("--compress")
.arg("--verbose")
.arg(format!("--max-allowed-packet={}", cfg.max_packet_size))
.arg("--net-buffer-length=16K")
.arg("--default-character-set=utf8mb4")
.arg(&cfg.database)
.arg("-r").arg(&file_path)
.envs(env)
.output()
.with_context(|| format!("Failed to run mariadb-dump for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
logger.log_command("mariadb-dump", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
anyhow::bail!("Mariadb backup failed for {}: {}", cfg.name, stderr);
}
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
logger.log_command("mariadb-dump", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
logger.log("info", format!("mariadb-dump completed for {}", cfg.name));
Ok(file_path)
})
.await?
}
+45
View File
@@ -0,0 +1,45 @@
use std::path::PathBuf;
use crate::services::config::DatabaseConfig;
use anyhow::Result;
use std::process::Command;
pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
let output = Command::new("mariadb")
.arg("--host")
.arg(&cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(&cfg.username)
.arg("-e")
.arg("SELECT VERSION();")
.env("MYSQL_PWD", &cfg.password)
.output()?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
anyhow::bail!("Version query failed: {}", stderr);
}
let version = String::from_utf8_lossy(&output.stdout)
.lines()
.nth(1)
.unwrap_or_default()
.trim()
.to_string();
Ok(version)
}
pub fn select_mariadb_path(version: &str) -> PathBuf {
let mut parts = version.split('.');
let major = parts.next().and_then(|v| v.parse::<u32>().ok()).unwrap_or(10);
let minor = parts.next().and_then(|v| v.parse::<u32>().ok()).unwrap_or(0);
if major < 10 || (major == 10 && minor <= 6) {
"/usr/local/mariadb-10.6/bin".into()
} else {
"/usr/local/mariadb-12.1/bin".into()
}
}
+58
View File
@@ -0,0 +1,58 @@
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
use anyhow::Result;
use async_trait::async_trait;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
pub struct MariaDBDatabase {
cfg: DatabaseConfig,
}
impl MariaDBDatabase {
pub fn new(cfg: DatabaseConfig) -> Self {
Self { cfg }
}
fn build_env(&self) -> HashMap<String, String> {
let mut envs = std::env::vars().collect::<HashMap<_, _>>();
envs.insert("MYSQL_PWD".to_string(), self.cfg.password.to_string());
envs
}
}
#[async_trait]
impl Database for MariaDBDatabase {
fn file_extension(&self) -> &'static str {
".sql"
}
async fn ping(&self) -> Result<bool> {
ping::run(self.cfg.clone(), self.build_env().clone()).await
}
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(
self.cfg.clone(),
dir.to_path_buf(),
self.build_env().clone(),
self.file_extension(),
logger,
)
.await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+5
View File
@@ -0,0 +1,5 @@
pub mod backup;
mod connection;
pub mod database;
mod ping;
mod restore;
+28
View File
@@ -0,0 +1,28 @@
use crate::services::config::DatabaseConfig;
use std::collections::HashMap;
use tokio::process::Command;
use tokio::time::{Duration, timeout};
pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::Result<bool> {
let mut cmd = Command::new("mysqladmin");
cmd.arg("--host")
.arg(cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(cfg.username)
.arg("ping")
.envs(env);
let result = timeout(Duration::from_secs(10), cmd.output()).await;
match result {
Ok(output) => {
let output = output?;
Ok(output.status.success())
}
Err(_) => Ok(false),
}
}
+91
View File
@@ -0,0 +1,91 @@
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::fs::File;
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use std::time::Instant;
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
let handle = tokio::task::spawn_blocking(move || -> Result<()> {
logger.log("info", format!("Starting restore for database {}", cfg.name));
let mut file = File::open(&restore_file)
.with_context(|| format!("Failed to open restore file {}", restore_file.display()))?;
let drop_create_cmd = format!(
"DROP DATABASE IF EXISTS `{0}`; CREATE DATABASE `{0}`;",
cfg.database
);
let drop_start = Instant::now();
let drop_output = Command::new("mariadb")
.arg("--host")
.arg(&cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(&cfg.username)
.arg("-e")
.arg(&drop_create_cmd)
.env("MYSQL_PWD", &cfg.password)
.output()
.with_context(|| format!("Failed to drop/recreate database {}", cfg.name))?;
let drop_duration_ms = drop_start.elapsed().as_millis() as f64;
let drop_exit_code = drop_output.status.code().unwrap_or(-1);
let drop_stderr = String::from_utf8_lossy(&drop_output.stderr).to_string();
if !drop_output.status.success() {
logger.log_command("mariadb", Some(drop_stderr.clone()), Some(drop_exit_code), Some(drop_duration_ms));
logger.log("error", format!("Drop/create database failed for {}: {}", cfg.name, drop_stderr));
anyhow::bail!("Failed to drop/recreate database {}", cfg.name);
}
logger.log_command("mariadb", if drop_stderr.is_empty() { None } else { Some(drop_stderr) }, Some(0), Some(drop_duration_ms));
logger.log("info", format!("Database {} dropped and recreated", cfg.name));
let start = Instant::now();
let mut child = Command::new("mariadb")
.arg("--host")
.arg(&cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(&cfg.username)
.arg("--database")
.arg(&cfg.database)
.env("MYSQL_PWD", &cfg.password)
.stdin(std::process::Stdio::piped())
.spawn()
.with_context(|| format!("Failed to start MariaDB restore for {}", cfg.name))?;
let mut stdin = child.stdin.take().context("Failed to open child stdin")?;
std::io::copy(&mut file, &mut stdin)
.context("Failed to stream SQL content to MariaDB stdin")?;
drop(stdin);
let output = child
.wait_with_output()
.with_context(|| format!("Failed to complete MariaDB restore for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
if !output.status.success() {
logger.log_command("mariadb", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
logger.log("error", format!("MariaDB restore failed for {}: {}", cfg.name, stderr));
anyhow::bail!("MariaDB restore failed for {}", cfg.name);
}
logger.log_command("mariadb", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
logger.log("info", format!("Restore finished successfully for database {}", cfg.name));
Ok(())
});
handle.await??;
Ok(())
}
+6 -4
View File
@@ -1,8 +1,10 @@
pub mod factory;
pub mod postgres;
pub mod mysql;
mod mongodb;
mod sqlite;
pub mod mysql;
pub mod postgres;
mod redis;
mod sqlite;
mod valkey;
mod mariadb;
mod firebird;
pub mod mssql;
+18 -5
View File
@@ -1,35 +1,48 @@
use crate::domain::mongodb::connection::{get_mongo_uri, select_mongo_path};
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
use std::sync::Arc;
use std::time::Instant;
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
file_extension: &'static str,
logger: Arc<JobLogger>,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
debug!("Starting MongoDB backup for database {}", cfg.name);
logger.log("info", format!("Starting MongoDB backup for database {}", cfg.name));
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
let mongodump = select_mongo_path().join("mongodump");
let uri = get_mongo_uri(cfg.clone())?;
logger.log("info", format!("Running mongodump for {}", cfg.name));
let start = Instant::now();
let output = Command::new(mongodump)
.arg(format!("--uri={}", uri))
.arg(format!("--archive={}", file_path.display()))
.arg("--gzip")
.arg("--verbose")
.output()
.context("MongoDB backup failed")?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
error!("MongoDB backup failed for {}: {}", cfg.name, stderr);
logger.log("error", format!("MongoDB backup failed for {}: {}", cfg.name, stderr));
logger.log_command("mongodump", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
anyhow::bail!("MongoDB backup failed for {}: {}", cfg.name, stderr);
}
info!("MongoDB backup completed for {}", cfg.name);
logger.log_command("mongodump", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
logger.log("info", format!("MongoDB backup completed for {}", cfg.name));
Ok(file_path)
})
.await?
+17 -2
View File
@@ -16,9 +16,11 @@ pub fn select_mongo_path() -> std::path::PathBuf {
}
pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result<String> {
if cfg.username.is_empty() || cfg.password.is_empty() {
Ok(format!("mongodb://{}:{}/{}", cfg.host, cfg.port, cfg.database))
Ok(format!(
"mongodb://{}:{}/{}",
cfg.host, cfg.port, cfg.database
))
} else {
Ok(format!(
"mongodb://{}:{}@{}:{}/{}?authSource=admin",
@@ -28,3 +30,16 @@ pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result<String> {
}
pub fn extract_db_name(dry_output: &str) -> Option<String> {
let mut dbs = std::collections::HashSet::new();
for line in dry_output.lines() {
if let Some(pos) = line.find("archive prelude ") {
let rest = &line[pos + "archive prelude ".len()..];
if let Some(dot) = rest.find('.') {
let db = &rest[..dot];
dbs.insert(db.to_string());
}
}
}
dbs.into_iter().next()
}
+10 -18
View File
@@ -1,9 +1,11 @@
use anyhow::Result;
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
@@ -27,27 +29,17 @@ impl Database for MongoDatabase {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
}
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, is_test: Option<bool>) -> Result<()> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
}
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+2 -2
View File
@@ -1,5 +1,5 @@
mod backup;
mod restore;
mod connection;
pub mod database;
mod ping;
mod connection;
mod restore;
+1 -1
View File
@@ -24,7 +24,7 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
error!("Full Error: {}", e);
error!("Check you database network connectivity");
error!("----------------------------------------");
Err(anyhow::anyhow!("Ping failed for {}: {}", cfg.name, e))
Ok(false)
}
}
}
+51 -9
View File
@@ -1,32 +1,74 @@
use crate::domain::mongodb::connection::{get_mongo_uri, select_mongo_path};
use crate::domain::mongodb::connection::{extract_db_name, get_mongo_uri, select_mongo_path};
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
use std::sync::Arc;
use std::time::Instant;
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
tokio::task::spawn_blocking(move || -> Result<()> {
debug!("Starting MongoDB restore for database {}", cfg.name);
logger.log("debug", format!("Starting MongoDB restore for database {}", cfg.name));
let mongorestore = select_mongo_path().join("mongorestore");
let uri = get_mongo_uri(cfg.clone())?;
let output = Command::new(mongorestore)
let dry_start = Instant::now();
let dry_run = Command::new(&mongorestore)
.arg(format!(
"--uri={}",
format!(
"mongodb://{}:{}@{}:{}/?authSource=admin",
cfg.username, cfg.password, cfg.host, cfg.port
)
))
.arg(format!("--archive={}", restore_file.display()))
.arg("--gzip")
.arg("--dryRun")
.arg("--verbose")
.output()?;
let dry_duration_ms = dry_start.elapsed().as_millis() as f64;
let dry_exit_code = dry_run.status.code().unwrap_or(-1);
let dry_output = String::from_utf8_lossy(&dry_run.stderr);
logger.log_command(
"mongorestore --dryRun",
if dry_output.is_empty() { None } else { Some(dry_output.to_string()) },
Some(dry_exit_code),
Some(dry_duration_ms),
);
let source_db = extract_db_name(&dry_output).unwrap_or_else(|| {
logger.log("info", format!("Could not detect source database from archive, falling back to configured database: {}", cfg.database));
cfg.database.clone()
});
logger.log("info", format!("Using source database in archive: {}", source_db));
let start = Instant::now();
let output = Command::new(&mongorestore)
.arg(format!("--uri={}", uri))
.arg(format!("--archive={}", restore_file.display()))
.arg("--gzip")
.arg("--drop")
.arg(format!("--nsInclude={}.*", source_db))
.arg(format!("--nsFrom={}.*", source_db))
.arg(format!("--nsTo={}.*", cfg.database))
.output()
.with_context(|| format!("Failed to run mongorestore for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
error!("MongoDB restore failed for {}: {}", cfg.name, stderr);
anyhow::bail!("MongoDB restore failed for : {}", cfg.name);
logger.log_command("mongorestore", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
logger.log("error", format!("MongoDB restore failed for {}: {}", cfg.name, stderr));
anyhow::bail!("MongoDB restore failed for: {}", cfg.name);
}
info!("MongoDB restore completed for {}", cfg.name);
logger.log_command("mongorestore", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
logger.log("info", format!("MongoDB restore completed for {}", cfg.name));
Ok(())
})
.await?
+56
View File
@@ -0,0 +1,56 @@
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use std::time::Instant;
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
file_extension: &'static str,
logger: Arc<JobLogger>,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
logger.log("info", format!("Starting MSSQL backup for database {}", cfg.name));
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
let connection_string = format!(
"Server=tcp:{},{};Database={};User Id={};Password={};TrustServerCertificate=True;Encrypt=False",
cfg.host, cfg.port, cfg.database, cfg.username, cfg.password
);
logger.log("info", format!("MSSQL backup: {}:{}/{}{}", cfg.host, cfg.port, cfg.database, file_path.display()));
let start = Instant::now();
let output = Command::new("sqlpackage")
.arg("/a:Export")
.arg(format!("/scs:{}", connection_string))
.arg(format!("/tf:{}", file_path.display()))
.output()
.with_context(|| format!("Failed to run sqlpackage for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
if !output.status.success() {
logger.log("error", format!("MSSQL backup failed — stderr: {} stdout: {}", stderr, stdout));
let out = format!("stderr: {} stdout: {}", stderr, stdout);
logger.log_command("sqlpackage", Some(out), Some(exit_code), Some(duration_ms));
anyhow::bail!("MSSQL backup failed for {}: {}", cfg.name, stderr);
}
let combined = if stdout.is_empty() && stderr.is_empty() {
None
} else {
Some(format!("{}{}", stdout, stderr).trim().to_string())
};
logger.log_command("sqlpackage", combined, Some(0), Some(duration_ms));
logger.log("info", format!("MSSQL backup completed for {}", cfg.name));
Ok(file_path)
})
.await?
}
+18
View File
@@ -0,0 +1,18 @@
use crate::services::config::DatabaseConfig;
use anyhow::Result;
use tiberius::{AuthMethod, Client, Config};
use tokio::net::TcpStream;
use tokio_util::compat::{Compat, TokioAsyncWriteCompatExt};
pub async fn build_client(cfg: &DatabaseConfig) -> Result<Client<Compat<TcpStream>>> {
let mut config = Config::new();
config.host(&cfg.host);
config.port(cfg.port);
config.authentication(AuthMethod::sql_server(&cfg.username, &cfg.password));
config.trust_cert();
let tcp = TcpStream::connect(config.get_addr()).await?;
tcp.set_nodelay(true)?;
let client = Client::connect(config, tcp.compat_write()).await?;
Ok(client)
}
+50
View File
@@ -0,0 +1,50 @@
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
use anyhow::Result;
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use std::sync::Arc;
pub struct MssqlDatabase {
cfg: DatabaseConfig,
}
impl MssqlDatabase {
pub fn new(cfg: DatabaseConfig) -> Self {
Self { cfg }
}
}
#[async_trait]
impl Database for MssqlDatabase {
fn file_extension(&self) -> &'static str {
".bacpac"
}
async fn ping(&self) -> Result<bool> {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(
self.cfg.clone(),
dir.to_path_buf(),
self.file_extension(),
logger,
)
.await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+5
View File
@@ -0,0 +1,5 @@
pub mod database;
mod connection;
mod ping;
mod backup;
mod restore;
+24
View File
@@ -0,0 +1,24 @@
use crate::services::config::DatabaseConfig;
use anyhow::Result;
use tracing::{error, info};
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
info!("Running ping for MSSQL database {}", cfg.name);
match super::connection::build_client(&cfg).await {
Ok(mut client) => match client.simple_query("SELECT 1").await {
Ok(_) => {
info!("MSSQL ping succeeded for {}", cfg.name);
Ok(true)
}
Err(e) => {
error!("MSSQL ping query failed for {}: {:?}", cfg.name, e);
Ok(false)
}
},
Err(e) => {
error!("MSSQL connection failed for {}: {:?}", cfg.name, e);
Ok(false)
}
}
}
+54
View File
@@ -0,0 +1,54 @@
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use std::time::Instant;
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
tokio::task::spawn_blocking(move || -> Result<()> {
logger.log("debug", format!("Starting MSSQL restore for database {}", cfg.name));
let connection_string = format!(
"Server=tcp:{},{};Database={};User Id={};Password={};TrustServerCertificate=True;Encrypt=False",
cfg.host, cfg.port, cfg.database, cfg.username, cfg.password
);
logger.log("info", format!(
"MSSQL restore: {} → {}:{}/{}",
restore_file.display(),
cfg.host,
cfg.port,
cfg.database
));
let start = Instant::now();
let output = Command::new("sqlpackage")
.arg("/a:Import")
.arg(format!("/tcs:{}", connection_string))
.arg(format!("/sf:{}", restore_file.display()))
.output()
.with_context(|| format!("Failed to run sqlpackage restore for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
let combined = format!("{}{}", stdout, stderr);
if !output.status.success() {
logger.log_command("sqlpackage", if combined.is_empty() { None } else { Some(combined.clone()) }, Some(exit_code), Some(duration_ms));
logger.log("error", format!(
"MSSQL restore failed for {} — stderr: {} stdout: {}",
cfg.name, stderr, stdout
));
anyhow::bail!("MSSQL restore failed for {}: {}", cfg.name, stderr);
}
logger.log_command("sqlpackage", if combined.is_empty() { None } else { Some(combined) }, Some(0), Some(duration_ms));
logger.log("info", format!("MSSQL restore completed for {}", cfg.name));
Ok(())
})
.await?
}
+38 -20
View File
@@ -1,62 +1,80 @@
use crate::domain::mysql::connection::{server_version};
use crate::domain::mysql::connection::server_version;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::collections::HashMap;
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
use std::sync::Arc;
use std::time::Instant;
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
env: HashMap<String, String>,
file_extension: &'static str,
logger: Arc<JobLogger>,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
debug!("Starting backup for database {}", cfg.name);
logger.log("info", format!("Starting backup for database {}", cfg.name));
let version = match futures::executor::block_on(server_version(&cfg)) {
let _version = match futures::executor::block_on(server_version(&cfg)) {
Ok(v) => {
debug!("Mysql version detected: {}", v);
logger.log("debug", format!("MySQL version detected: {}", v));
v
}
Err(e) => {
error!("Failed to get server version for {}: {:?}", cfg.name, e);
logger.log("error", format!("Failed to get server version: {}", e));
return Err(e.into());
}
};
info!("Mysql version found: {}", version);
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
if let Ok(out) = Command::new("mysqldump").arg("--version").output() {
logger.log("debug", format!("mysqldump client: {}", String::from_utf8_lossy(&out.stdout).trim()));
}
logger.log("info", format!("Running mysqldump for {}", cfg.name));
let start = Instant::now();
let output = Command::new("mysqldump")
.arg("--host")
.arg(cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(cfg.username)
.arg("--host").arg(&cfg.host)
.arg("--port").arg(cfg.port.to_string())
.arg("--user").arg(&cfg.username)
.arg("--routines")
.arg("--events")
.arg("--triggers")
.arg("--verbose")
.arg("--single-transaction")
.arg("--set-gtid-purged=OFF")
.arg("--no-tablespaces")
.arg("--quick")
.arg("--add-drop-database")
.arg("--databases")
.arg(cfg.database)
.arg("-r")
.arg(&file_path)
.arg("--skip-lock-tables")
.arg("--skip-add-drop-table")
.arg("--no-create-db")
.arg("--default-character-set=utf8mb4")
.arg("--network-timeout")
.arg(format!("--max-allowed-packet={}", cfg.max_packet_size))
.arg(&cfg.database)
.arg("-r").arg(&file_path)
.envs(env)
.output()
.with_context(|| format!("Failed to run mysqldump for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let _stdout = String::from_utf8_lossy(&output.stdout).to_string();
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
logger.log_command("mysqldump", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
anyhow::bail!("MySQL backup failed for {}: {}", cfg.name, stderr);
}
logger.log_command("mysqldump", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
logger.log("info", format!("mysqldump completed for {}", cfg.name));
Ok(file_path)
})
.await?
+10 -8
View File
@@ -1,14 +1,17 @@
use crate::services::config::DatabaseConfig;
use std::process::Command;
use anyhow::Result;
use std::process::Command;
pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
let output = Command::new("mysql")
.arg("--host").arg(&cfg.host)
.arg("--port").arg(cfg.port.to_string())
.arg("--user").arg(&cfg.username)
.arg("-e").arg("SELECT VERSION();")
.arg("--host")
.arg(&cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(&cfg.username)
.arg("-e")
.arg("SELECT VERSION();")
.env("MYSQL_PWD", &cfg.password)
.output()?;
@@ -19,11 +22,10 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
let version = String::from_utf8_lossy(&output.stdout)
.lines()
.nth(1) // skip column header
.nth(1)
.unwrap_or_default()
.trim()
.to_string();
Ok(version)
}
+22 -27
View File
@@ -1,14 +1,13 @@
use std::collections::HashMap;
use anyhow::Result;
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use super::{
backup,
ping, restore,
};
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
use anyhow::Result;
use async_trait::async_trait;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
pub struct MySQLDatabase {
cfg: DatabaseConfig,
@@ -36,28 +35,24 @@ impl Database for MySQLDatabase {
ping::run(self.cfg.clone(), self.build_env().clone()).await
}
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
}
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.build_env().clone(), self.file_extension()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(
self.cfg.clone(),
dir.to_path_buf(),
self.build_env().clone(),
self.file_extension(),
logger,
)
.await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, is_test: Option<bool>) -> Result<()> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
}
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+2 -2
View File
@@ -1,5 +1,5 @@
pub mod backup;
mod connection;
pub mod database;
mod restore;
mod ping;
mod connection;
mod restore;
+1 -2
View File
@@ -4,8 +4,7 @@ use tokio::process::Command;
use tokio::time::{Duration, timeout};
pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::Result<bool> {
let mut cmd = Command::new("mysqladmin");
let mut cmd = Command::new("mariadb-admin");
cmd.arg("--host")
.arg(cfg.host)
.arg("--port")
+31 -20
View File
@@ -1,27 +1,26 @@
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::fs::File;
use std::io::{Read, Write};
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
use std::sync::Arc;
use std::time::Instant;
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
let handle = tokio::task::spawn_blocking(move || -> Result<()> {
debug!("Starting restore for database {}", cfg.name);
logger.log("info", format!("Starting restore for database {}", cfg.name));
let mut sql_content = String::new();
let mut file = File::open(&restore_file)
.with_context(|| format!("Failed to open restore file {}", restore_file.display()))?;
file.read_to_string(&mut sql_content)
.with_context(|| format!("Failed to read restore file {}", restore_file.display()))?;
let drop_create_cmd = format!(
"DROP DATABASE IF EXISTS {0}; CREATE DATABASE {0};",
"DROP DATABASE IF EXISTS `{0}`; CREATE DATABASE `{0}`;",
cfg.database
);
let drop_status = Command::new("mysql")
let drop_start = Instant::now();
let drop_output = Command::new("mysql")
.arg("--host")
.arg(&cfg.host)
.arg("--port")
@@ -31,14 +30,22 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
.arg("-e")
.arg(&drop_create_cmd)
.env("MYSQL_PWD", &cfg.password)
.status()
.output()
.with_context(|| format!("Failed to drop/recreate database {}", cfg.name))?;
if !drop_status.success() {
error!("Drop/create database failed for {}", cfg.name);
let drop_duration_ms = drop_start.elapsed().as_millis() as f64;
let drop_exit_code = drop_output.status.code().unwrap_or(-1);
let drop_stderr = String::from_utf8_lossy(&drop_output.stderr).to_string();
if !drop_output.status.success() {
logger.log_command("mysql", Some(drop_stderr.clone()), Some(drop_exit_code), Some(drop_duration_ms));
logger.log("error", format!("Drop/create database failed for {}: {}", cfg.name, drop_stderr));
anyhow::bail!("Failed to drop/recreate database {}", cfg.name);
}
info!("Database {} dropped and recreated", cfg.name);
logger.log_command("mysql", if drop_stderr.is_empty() { None } else { Some(drop_stderr) }, Some(0), Some(drop_duration_ms));
logger.log("info", format!("Database {} dropped and recreated", cfg.name));
let start = Instant::now();
let mut child = Command::new("mysql")
.arg("--host")
@@ -47,6 +54,7 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
.arg(cfg.port.to_string())
.arg("--user")
.arg(&cfg.username)
.arg("--database")
.arg(&cfg.database)
.env("MYSQL_PWD", &cfg.password)
.stdin(std::process::Stdio::piped())
@@ -54,23 +62,26 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
.with_context(|| format!("Failed to start mysql restore for {}", cfg.name))?;
let mut stdin = child.stdin.take().context("Failed to open child stdin")?;
stdin
.write_all(sql_content.as_bytes())
.context("Failed to write SQL content to mysql stdin")?;
stdin.flush()?;
std::io::copy(&mut file, &mut stdin)
.context("Failed to stream SQL content to mysql stdin")?;
drop(stdin);
let output = child
.wait_with_output()
.with_context(|| format!("Failed to complete mysql restore for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
error!("MySQL restore failed for {}: {}", cfg.name, stderr);
logger.log_command("mysql", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
logger.log("error", format!("MySQL restore failed for {}: {}", cfg.name, stderr));
anyhow::bail!("MySQL restore failed for {}", cfg.name);
}
info!("Restore finished successfully for database {}", cfg.name);
logger.log_command("mysql", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
logger.log("info", format!("Restore finished successfully for database {}", cfg.name));
Ok(())
});
+75 -68
View File
@@ -1,143 +1,150 @@
use anyhow::Result;
use std::collections::HashMap;
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
use std::sync::Arc;
use std::time::Instant;
use super::connection::{select_pg_path, server_version};
use super::format::PostgresDumpFormat;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
pub async fn run(
cfg: DatabaseConfig,
format: PostgresDumpFormat,
backup_dir: PathBuf,
is_test: Option<bool>
env: HashMap<String, String>,
logger: Arc<JobLogger>,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
debug!("Starting backup for database {}", cfg.name);
logger.log("info", format!("Starting backup for database {}", cfg.name));
let version = match futures::executor::block_on(server_version(&cfg)) {
Ok(v) => {
debug!("Postgres version detected: {}", v);
logger.log("debug", format!("Postgres version detected: {}", v));
v
}
Err(e) => {
error!("Failed to get server version for {}: {:?}", cfg.name, e);
logger.log("error", format!("Failed to get server version: {}", e));
return Err(e.into());
}
};
let pg_dump = select_pg_path(&version, is_test).join("pg_dump");
debug!("Using pg_dump at {:?}", pg_dump);
let pg_dump = select_pg_path(&version).join("pg_dump");
logger.log("debug", format!("Using pg_dump at {:?}", pg_dump));
match format {
PostgresDumpFormat::Fc => {
info!("Running FC backup for {}", cfg.name);
let file_path = backup_dir.join(format!("{}.dump", cfg.generated_id));
let url = format!(
"postgresql://{}:{}@{}:{}/{}",
cfg.username, cfg.password, cfg.host, cfg.port, cfg.database
);
logger.log("info", format!("Running FC backup for {}", cfg.name));
let status = Command::new(&pg_dump)
.arg("--dbname")
.arg(&url)
let file_path = backup_dir.join(format!("{}.dump", cfg.generated_id));
let start = Instant::now();
let output = Command::new(&pg_dump)
.arg("--host").arg(&cfg.host)
.arg("--port").arg(cfg.port.to_string())
.arg("--username").arg(&cfg.username)
.arg("--dbname").arg(&cfg.database)
.arg("-Fc")
.arg("-f")
.arg(&file_path)
.arg("-f").arg(&file_path)
.arg("-v")
.arg("--compress=3")
.status();
.envs(env)
.output();
let duration_ms = start.elapsed().as_millis() as f64;
match status {
Ok(s) if s.success() => info!(
"FC backup completed successfully for {} at {:?}",
cfg.name, file_path
),
Ok(s) => {
error!("FC backup failed with status {:?} for {}", s, cfg.name);
anyhow::bail!("Postgres backup failed for {}", cfg.name);
match output {
Ok(o) => {
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
let exit_code = o.status.code().unwrap_or(-1);
if o.status.success() {
logger.log("info", format!("FC backup completed successfully for {} at {:?}", cfg.name, file_path));
logger.log_command("pg_dump", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
} else {
logger.log("error", format!("FC backup failed with status {:?} for {}", o.status, cfg.name));
logger.log_command("pg_dump", Some(stderr), Some(exit_code), Some(duration_ms));
anyhow::bail!("Postgres backup failed for {}", cfg.name);
}
}
Err(e) => {
error!("Error executing pg_dump for {}: {:?}", cfg.name, e);
logger.log("error", format!("Error executing pg_dump for {}: {:?}", cfg.name, e));
logger.log_command("pg_dump", Some(e.to_string()), Some(-1), Some(duration_ms));
return Err(e.into());
}
}
info!("Backup finished for database {}", cfg.name);
logger.log("info", format!("Backup finished for database {}", cfg.name));
Ok(file_path)
}
PostgresDumpFormat::Fd => {
info!("Running FD backup for {}", cfg.name);
logger.log("info", format!("Running FD backup for {}", cfg.name));
let dump_dir = backup_dir.join(format!("{}_dir", cfg.generated_id));
let tar_file = backup_dir.join(format!("{}.tar.gz", cfg.generated_id));
if let Err(e) = std::fs::create_dir_all(&dump_dir) {
error!(
"Failed to create dump directory {:?} for {}: {:?}",
dump_dir, cfg.name, e
);
logger.log("error", format!("Failed to create dump directory {:?} for {}: {:?}", dump_dir, cfg.name, e));
return Err(e.into());
}
let url = format!(
"postgresql://{}:{}@{}:{}/{}",
cfg.username, cfg.password, cfg.host, cfg.port, cfg.database
);
let cmd_label = format!("pg_dump -Fd {}@{}:{}/{}", cfg.username, cfg.host, cfg.port, cfg.database);
let status = Command::new(&pg_dump)
.arg("--dbname")
.arg(&url)
let start = Instant::now();
let output = Command::new(&pg_dump)
.arg("--host").arg(&cfg.host)
.arg("--port").arg(cfg.port.to_string())
.arg("--username").arg(&cfg.username)
.arg("--dbname").arg(&cfg.database)
.arg("-Fd")
.arg("-j")
.arg("4")
.arg("-f")
.arg(&dump_dir)
.arg("-j").arg("4")
.arg("-f").arg(&dump_dir)
.arg("-v")
.status();
.envs(env)
.output();
let duration_ms = start.elapsed().as_millis() as f64;
match status {
Ok(s) if s.success() => {
info!("FD backup pg_dump completed successfully for {}", cfg.name)
}
Ok(s) => {
error!(
"FD backup pg_dump failed with status {:?} for {}",
s, cfg.name
);
anyhow::bail!("Postgres FD backup failed for {}", cfg.name);
match output {
Ok(o) => {
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
let exit_code = o.status.code().unwrap_or(-1);
if o.status.success() {
logger.log("info", format!("FD backup pg_dump completed successfully for {}", cfg.name));
logger.log_command(cmd_label, if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
} else {
logger.log("error", format!("FD backup pg_dump failed with status {:?} for {}", o.status, cfg.name));
logger.log_command(cmd_label, Some(stderr), Some(exit_code), Some(duration_ms));
anyhow::bail!("Postgres FD backup failed for {}", cfg.name);
}
}
Err(e) => {
error!("Error executing pg_dump for {}: {:?}", cfg.name, e);
logger.log("error", format!("Error executing pg_dump for {}: {:?}", cfg.name, e));
logger.log_command(cmd_label, Some(e.to_string()), Some(-1), Some(duration_ms));
return Err(e.into());
}
}
match std::fs::File::create(&tar_file) {
Ok(tar_gz) => {
let enc =
flate2::write::GzEncoder::new(tar_gz, flate2::Compression::default());
let enc = flate2::write::GzEncoder::new(tar_gz, flate2::Compression::default());
let mut tar = tar::Builder::new(enc);
if let Err(e) = tar.append_dir_all(".", &dump_dir) {
error!("Failed to append dump_dir to tar for {}: {:?}", cfg.name, e);
logger.log("error", format!("Failed to append dump_dir to tar for {}: {:?}", cfg.name, e));
return Err(e.into());
}
if let Err(e) = tar.finish() {
error!("Failed to finish tar archive for {}: {:?}", cfg.name, e);
logger.log("error", format!("Failed to finish tar archive for {}: {:?}", cfg.name, e));
return Err(e.into());
}
info!("FD backup archive created at {:?}", tar_file);
logger.log("info", format!("FD backup archive created at {:?}", tar_file));
}
Err(e) => {
error!(
"Failed to create tar.gz file {:?} for {}: {:?}",
tar_file, cfg.name, e
);
logger.log("error", format!("Failed to create tar.gz file {:?} for {}: {:?}", tar_file, cfg.name, e));
return Err(e.into());
}
}
info!("Backup finished for database {}", cfg.name);
logger.log("info", format!("Backup finished for database {}", cfg.name));
Ok(tar_file)
}
}
+92 -11
View File
@@ -1,18 +1,23 @@
use crate::domain::postgres::format::PostgresDumpFormat;
use crate::services::config::DatabaseConfig;
use crate::settings::CONFIG;
use anyhow::Result;
use std::path::Path;
use tokio_postgres::{Client, NoTls};
use tokio_postgres::{Client, Config, NoTls};
use tracing::{error, info};
pub async fn connect(cfg: &DatabaseConfig) -> Result<Client> {
info!("Connecting to postgres database {}:{}", cfg.host, cfg.port);
let dsn = format!(
"host={} port={} user={} password={} dbname={}",
cfg.host, cfg.port, cfg.username, cfg.password, cfg.database
);
let (client, connection) = tokio_postgres::connect(&dsn, NoTls).await?;
let mut config = Config::new();
config
.host(&cfg.host)
.port(cfg.port)
.user(&cfg.username)
.password(&cfg.password)
.dbname(&cfg.database);
let (client, connection) = config.connect(NoTls).await?;
tokio::spawn(async move {
if let Err(e) = connection.await {
error!("Postgres connection error: {}", e);
@@ -28,14 +33,89 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
Ok(version)
}
pub fn select_pg_path(version: &str, is_test: Option<bool>) -> std::path::PathBuf {
/// Resolves the `bin` directory of a PostgreSQL installation for the given
/// major version, in a cross-platform way.
///
/// Resolution order:
/// 1. The `PG_BIN_DIR` environment variable, if set, is used as-is. This
/// allows users/CI to override detection for non-standard installs
/// (e.g. portable PostgreSQL distributions, custom install locations).
/// 2. Platform-specific default install locations (Debian/Ubuntu packages,
/// the official Windows installer, Homebrew/Postgres.app on macOS, and
/// common RPM-based layouts on other Linux distros).
/// 3. A `PATH` lookup for `pg_dump` (`pg_dump.exe` on Windows), returning
/// its parent directory.
/// 4. The historical Debian/Ubuntu path as a last-resort fallback, so the
/// function keeps returning a `PathBuf` (never panics) even when nothing
/// was found, preserving the previous behavior for callers.
///
/// The override is sourced from `CONFIG.pg_bin_dir` (the `PG_BIN_DIR`
/// environment variable). An empty value means "unset" and falls through to
/// detection.
pub fn select_pg_path(version: &str) -> std::path::PathBuf {
select_pg_path_with(version, &CONFIG.pg_bin_dir)
}
/// Inner resolver behind [`select_pg_path`], parameterized over the
/// `PG_BIN_DIR` override. Kept pure (no env / no `CONFIG` access) so it is
/// unit-testable without mutating process-global state.
pub(crate) fn select_pg_path_with(version: &str, pg_bin_dir: &str) -> std::path::PathBuf {
let major = version.split('.').next().unwrap_or("17");
if is_test.unwrap_or(false) {
format!("/usr/local/postgresql/{}/bin", major).into()
} else {
format!("/usr/lib/postgresql/{}/bin", major).into()
if !pg_bin_dir.is_empty() {
return pg_bin_dir.into();
}
let candidates: Vec<std::path::PathBuf> = if cfg!(target_os = "windows") {
vec![
// Default install path used by the official EDB Windows installer
format!(r"C:\Program Files\PostgreSQL\{major}\bin").into(),
format!(r"C:\Program Files (x86)\PostgreSQL\{major}\bin").into(),
]
} else if cfg!(target_os = "macos") {
vec![
// Homebrew on Apple Silicon
format!("/opt/homebrew/opt/postgresql@{major}/bin").into(),
// Homebrew on Intel
format!("/usr/local/opt/postgresql@{major}/bin").into(),
// Postgres.app
format!("/Applications/Postgres.app/Contents/Versions/{major}/bin").into(),
]
} else {
vec![
// Debian/Ubuntu packages
format!("/usr/lib/postgresql/{major}/bin").into(),
// Common RPM-based distro layout
format!("/usr/pgsql-{major}/bin").into(),
]
};
if let Some(found) = candidates.into_iter().find(|p| pg_dump_exists_in(p)) {
return found;
}
if let Some(dir) = find_pg_dump_in_path() {
return dir;
}
format!("/usr/lib/postgresql/{}/bin", major).into()
}
pub(crate) fn pg_dump_binary_name() -> &'static str {
if cfg!(target_os = "windows") {
"pg_dump.exe"
} else {
"pg_dump"
}
}
pub(crate) fn pg_dump_exists_in(dir: &std::path::Path) -> bool {
dir.join(pg_dump_binary_name()).is_file()
}
fn find_pg_dump_in_path() -> Option<std::path::PathBuf> {
let path_var = std::env::var_os("PATH")?;
std::env::split_paths(&path_var).find(|dir| pg_dump_exists_in(dir))
}
pub async fn terminate_connections(cfg: &DatabaseConfig) -> Result<()> {
@@ -73,6 +153,7 @@ pub async fn detect_format_from_size(cfg: &DatabaseConfig) -> PostgresDumpFormat
"Detecting database format {:?} - {:?}",
cfg.name, cfg.generated_id
);
let client = match connect(cfg).await {
Ok(c) => c,
Err(_) => return PostgresDumpFormat::Fc,
+31 -19
View File
@@ -1,9 +1,11 @@
use anyhow::Result;
use async_trait::async_trait;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use super::{backup, format::PostgresDumpFormat, ping, restore};
use crate::domain::factory::Database;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
@@ -16,6 +18,12 @@ impl PostgresDatabase {
pub fn new(cfg: DatabaseConfig, format: PostgresDumpFormat) -> Self {
Self { cfg, format }
}
fn build_env(&self) -> HashMap<String, String> {
let mut envs = std::env::vars().collect::<HashMap<_, _>>();
envs.insert("PGPASSWORD".to_string(), self.cfg.password.to_string());
envs
}
}
#[async_trait]
@@ -31,27 +39,31 @@ impl Database for PostgresDatabase {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
}
let res = backup::run(self.cfg.clone(), self.format, dir.to_path_buf(), is_test).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(
self.cfg.clone(),
self.format,
dir.to_path_buf(),
self.build_env(),
logger,
)
.await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, is_test: Option<bool>) -> Result<()> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
}
let res = restore::run(self.cfg.clone(), self.format, file.to_path_buf(), is_test).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(
self.cfg.clone(),
self.format,
file.to_path_buf(),
self.build_env(),
logger,
)
.await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+1 -1
View File
@@ -2,4 +2,4 @@
pub enum PostgresDumpFormat {
Fc,
Fd,
}
}
+3 -3
View File
@@ -1,8 +1,8 @@
pub mod backup;
pub(crate) mod connection;
pub mod database;
mod restore;
mod connection;
mod format;
mod ping;
mod restore;
pub use connection::{detect_format_from_size, detect_format_from_file};
pub use connection::{detect_format_from_file, detect_format_from_size};
+2 -4
View File
@@ -1,8 +1,6 @@
use super::connection::connect;
use crate::services::config::DatabaseConfig;
pub async fn run(
cfg: DatabaseConfig,
) -> anyhow::Result<bool> {
pub async fn run(cfg: DatabaseConfig) -> anyhow::Result<bool> {
Ok(connect(&cfg).await.is_ok())
}
}
+81 -56
View File
@@ -1,95 +1,106 @@
use anyhow::Result;
use std::collections::HashMap;
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
use std::sync::Arc;
use std::time::Instant;
use super::connection::{select_pg_path, server_version, terminate_connections};
use super::format::PostgresDumpFormat;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
pub async fn run(
cfg: DatabaseConfig,
format: PostgresDumpFormat,
restore_file: PathBuf,
is_test: Option<bool>,
env: HashMap<String, String>,
logger: Arc<JobLogger>,
) -> Result<()> {
tokio::task::spawn_blocking(move || -> Result<()> {
debug!("Starting restore for database {}", cfg.name);
logger.log("info", format!("Starting restore for database {}", cfg.name));
let version = match futures::executor::block_on(server_version(&cfg)) {
Ok(v) => {
debug!("Postgres version detected: {}", v);
logger.log("debug", format!("Postgres version detected: {}", v));
v
}
Err(e) => {
error!("Failed to get server version for {}: {:?}", cfg.name, e);
logger.log("error", format!("Failed to get server version for {}: {:?}", cfg.name, e));
return Err(e.into());
}
};
let pg_restore = select_pg_path(&version, is_test).join("pg_restore");
let pg_restore = select_pg_path(&version).join("pg_restore");
debug!("Using pg_restore at {:?}", pg_restore);
logger.log("debug", format!("Using pg_restore at {:?}", pg_restore));
if let Err(e) = futures::executor::block_on(terminate_connections(&cfg)) {
error!("Failed to terminate connections for {}: {:?}", cfg.name, e);
logger.log("error", format!("Failed to terminate connections for {}: {:?}", cfg.name, e));
return Err(e.into());
}
info!("Connections terminated for database {}", cfg.name);
let url = format!(
"postgresql://{}:{}@{}:{}/postgres",
cfg.username, cfg.password, cfg.host, cfg.port
);
debug!("Restore URL: {}", url);
logger.log("info", format!("Connections terminated for database {}", cfg.name));
match format {
PostgresDumpFormat::Fc => {
info!("Running FC restore for {}", cfg.name);
let status = Command::new(&pg_restore)
logger.log("info", format!("Running FC restore for {}", cfg.name));
let start = Instant::now();
let output = Command::new(&pg_restore)
.arg("--no-owner")
.arg("--no-privileges")
.arg("--clean")
.arg("--if-exists")
.arg("--create")
.arg("--dbname")
.arg(&url)
// .arg("--create")
.arg("--host").arg(&cfg.host)
.arg("--port").arg(cfg.port.to_string())
.arg("--username").arg(&cfg.username)
.arg("--dbname").arg(&cfg.database)
.arg("-v")
.arg(&restore_file)
.env("PGPASSWORD", &cfg.password)
.status();
.envs(env)
.output();
match status {
Ok(s) if s.success() => {
info!("FC restore completed successfully for {}", cfg.name)
}
Ok(s) => {
error!("FC restore failed with status {:?} for {}", s, cfg.name);
anyhow::bail!("Postgres restore failed for {}", cfg.name);
let duration_ms = start.elapsed().as_millis() as f64;
match output {
Ok(o) => {
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
let stdout = String::from_utf8_lossy(&o.stdout).to_string();
let combined = format!("{}{}", stdout, stderr);
let exit_code = o.status.code().unwrap_or(-1);
if o.status.success() {
logger.log_command("pg_restore", if combined.is_empty() { None } else { Some(combined) }, Some(0), Some(duration_ms));
logger.log("info", format!("FC restore completed successfully for {}", cfg.name))
} else {
logger.log_command("pg_restore", if combined.is_empty() { None } else { Some(combined) }, Some(exit_code), Some(duration_ms));
logger.log("error", format!("FC restore failed with status {:?} for {}", o.status, cfg.name));
anyhow::bail!("Postgres restore failed for {}", cfg.name);
}
}
Err(e) => {
error!("Error executing pg_restore for {}: {:?}", cfg.name, e);
logger.log_command("pg_restore", Some(e.to_string()), Some(-1), Some(duration_ms));
logger.log("error", format!("Error executing pg_restore for {}: {:?}", cfg.name, e));
return Err(e.into());
}
}
}
PostgresDumpFormat::Fd => {
info!("Running FD restore for {}", cfg.name);
logger.log("info", format!("Running FD restore for {}", cfg.name));
let tar_gz = match std::fs::File::open(&restore_file) {
Ok(f) => f,
Err(e) => {
error!(
logger.log("error", format!(
"Failed to open restore file {:?} for {}: {:?}",
restore_file, cfg.name, e
);
));
return Err(e.into());
}
};
info!("tar_gz {:?}", tar_gz);
logger.log("info", format!("tar_gz {:?}", tar_gz));
let dec = flate2::read::GzDecoder::new(tar_gz);
let mut archive = tar::Archive::new(dec);
@@ -97,31 +108,31 @@ pub async fn run(
let tmp_dir = match tempfile::TempDir::new() {
Ok(d) => d,
Err(e) => {
error!(
logger.log("error", format!(
"Failed to create temporary directory for FD restore of {}: {:?}",
cfg.name, e
);
));
return Err(e.into());
}
};
if let Err(e) = archive.unpack(tmp_dir.path()) {
error!("Failed to unpack FD archive for {}: {:?}", cfg.name, e);
logger.log("error", format!("Failed to unpack FD archive for {}: {:?}", cfg.name, e));
return Err(e.into());
}
debug!("Listing contents of temp dir: {}", tmp_dir.path().display());
logger.log("debug", format!("Listing contents of temp dir: {}", tmp_dir.path().display()));
for entry in std::fs::read_dir(tmp_dir.path())? {
if let Ok(entry) = entry {
let path = entry.path();
let file_type = entry.file_type()?;
debug!(
logger.log("debug", format!(
" - {} | is_dir: {} | is_file: {}",
path.display(),
file_type.is_dir(),
file_type.is_file()
);
));
}
}
@@ -135,38 +146,52 @@ pub async fn run(
.ok_or_else(|| anyhow::anyhow!("Invalid FD archive: toc.dat not found"))?
};
let status = Command::new(&pg_restore)
let start = Instant::now();
let output = Command::new(&pg_restore)
.arg("--no-owner")
.arg("--no-privileges")
.arg("--clean")
.arg("--if-exists")
.arg("--create")
.arg("--dbname")
.arg(&url)
// .arg("--create")
.arg("--host").arg(&cfg.host)
.arg("--port").arg(cfg.port.to_string())
.arg("--username").arg(&cfg.username)
.arg("--dbname").arg(&cfg.database)
.arg("-v")
.arg("-j")
.arg("4")
.arg(dump_dir)
.env("PGPASSWORD", &cfg.password)
.status();
.envs(env)
.output();
match status {
Ok(s) if s.success() => {
info!("FD restore completed successfully for {}", cfg.name)
}
Ok(s) => {
error!("FD restore failed with status {:?} for {}", s, cfg.name);
anyhow::bail!("Postgres FD restore failed for {}", cfg.name);
let duration_ms = start.elapsed().as_millis() as f64;
match output {
Ok(o) => {
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
let stdout = String::from_utf8_lossy(&o.stdout).to_string();
let combined = format!("{}{}", stdout, stderr);
let exit_code = o.status.code().unwrap_or(-1);
if o.status.success() {
logger.log_command("pg_restore", if combined.is_empty() { None } else { Some(combined) }, Some(0), Some(duration_ms));
logger.log("info", format!("FD restore completed successfully for {}", cfg.name))
} else {
logger.log_command("pg_restore", if combined.is_empty() { None } else { Some(combined) }, Some(exit_code), Some(duration_ms));
logger.log("error", format!("FD restore failed with status {:?} for {}", o.status, cfg.name));
anyhow::bail!("Postgres FD restore failed for {}", cfg.name);
}
}
Err(e) => {
error!("Error executing pg_restore for {}: {:?}", cfg.name, e);
logger.log_command("pg_restore", Some(e.to_string()), Some(-1), Some(duration_ms));
logger.log("error", format!("Error executing pg_restore for {}: {:?}", cfg.name, e));
return Err(e.into());
}
}
}
}
info!("Restore finished for database {}", cfg.name);
logger.log("info", format!("Restore finished for database {}", cfg.name));
Ok(())
})
+44 -13
View File
@@ -1,21 +1,26 @@
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
use std::sync::Arc;
use std::time::Instant;
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
file_extension: &'static str,
logger: Arc<JobLogger>,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
debug!("Starting Redis backup for database {}", cfg.name);
logger.log(
"info",
format!("Starting Redis backup for database {}", cfg.name),
);
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
let mut cmd = Command::new("redis-cli");
cmd.arg("-h")
.arg(&cfg.host)
.arg("-p")
@@ -24,40 +29,66 @@ pub async fn run(
if !cfg.username.is_empty() {
cmd.arg("--user").arg(&cfg.username);
}
if !cfg.password.is_empty() {
cmd.arg("-a").arg(&cfg.password);
}
cmd.arg("--rdb").arg(&file_path);
debug!("Command Backup: {:?}", cmd);
logger.log("info", format!("Running redis-cli --rdb for {}", cfg.name));
let start = Instant::now();
let output = cmd.output().context("Redis backup command failed")?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr);
let stdout = String::from_utf8_lossy(&output.stdout);
if !output.status.success() {
if stderr.contains("NOAUTH") {
error!(
"Redis backup failed for {}: Authentication required (NOAUTH)",
cfg.name
logger.log(
"error",
format!(
"Redis backup failed for {}: Authentication required (NOAUTH)",
cfg.name
),
);
logger.log_command(
"redis-cli",
Some("Authentication required (NOAUTH)".into()),
Some(exit_code),
Some(duration_ms),
);
anyhow::bail!(
"Redis backup failed for {}: Authentication required",
cfg.name
);
} else {
error!("Redis backup failed for {}: {}", cfg.name, stderr);
logger.log(
"error",
format!("Redis backup failed for {}: {}", cfg.name, stderr),
);
logger.log_command(
"redis-cli",
Some(stderr.to_string()),
Some(exit_code),
Some(duration_ms),
);
anyhow::bail!("Redis backup failed for {}: {}", cfg.name, stderr);
}
}
info!(
"Redis backup completed for {}. Output: {}",
cfg.name, stdout
logger.log_command(
"redis-cli",
if stdout.is_empty() {
None
} else {
Some(stdout.to_string())
},
Some(0),
Some(duration_ms),
);
logger.log("info", format!("Redis backup completed for {}", cfg.name));
Ok(file_path)
})
+7 -10
View File
@@ -1,9 +1,11 @@
use anyhow::{Result, bail};
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use crate::domain::factory::Database;
use crate::domain::redis::{backup, ping};
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
@@ -27,19 +29,14 @@ impl Database for RedisDatabase {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
}
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, _file: &Path, _is_test: Option<bool>) -> Result<()> {
async fn restore(&self, _file: &Path, _logger: Arc<JobLogger>) -> Result<()> {
bail!("Restore not supported for Redis databases")
}
}
+8 -11
View File
@@ -1,8 +1,8 @@
use tracing::{debug, info};
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use tokio::process::Command;
use tokio::time::{timeout, Duration};
use anyhow::{Result, Context};
use tokio::time::{Duration, timeout};
use tracing::{debug, error, info};
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
let mut cmd = Command::new("redis-cli");
@@ -21,8 +21,7 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
cmd.arg("PING");
debug!("Command Ping: {:?}", cmd);
debug!("Command Ping Redis: {:?}", cmd);
let result = timeout(Duration::from_secs(10), cmd.output()).await;
@@ -31,17 +30,15 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
let output = output.context("Failed to execute redis-cli")?;
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
info!("Redis stdout: {}", stdout);
info!("Redis stderr: {}", stderr);
if stderr.contains("NOAUTH") {
info!("Redis authentication failed (NOAUTH required)");
error!("Redis authentication failed (NOAUTH required)");
return Ok(false);
}
if !output.status.success() {
info!("Redis command failed with status: {:?}", output.status);
error!("Redis command failed with status: {:?}", output.status);
return Ok(false);
}
@@ -52,4 +49,4 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
Ok(false)
}
}
}
}
+19 -7
View File
@@ -1,16 +1,19 @@
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
use std::sync::Arc;
use std::time::Instant;
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
file_extension: &'static str,
logger: Arc<JobLogger>,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
debug!("Starting SQLite backup for database {}", cfg.name);
logger.log("info", format!("Starting SQLite backup for database {}", cfg.name));
let db_path_str = if cfg.path.is_empty() {
anyhow::bail!("Database path not configured");
@@ -19,27 +22,36 @@ pub async fn run(
};
let db_path = PathBuf::from(db_path_str);
logger.log("info", format!("Database path: {}", db_path.display()));
if !db_path.exists() {
logger.log("error", format!("SQLite database file not found: {}", db_path.display()));
anyhow::bail!("SQLite database file not found: {}", db_path.display());
}
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
logger.log("info", format!("Running sqlite3 backup for {}", cfg.name));
let start = Instant::now();
let output = Command::new("sqlite3")
.arg(db_path.as_os_str())
.arg(format!(".backup '{}'", file_path.display()))
.output()
.context("SQLite backup command failed to start")?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
error!("SQLite backup failed for {}: {}", cfg.name, stderr);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
logger.log("error", format!("SQLite backup failed for {}: {}", cfg.name, stderr));
logger.log_command("sqlite3", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
anyhow::bail!("SQLite backup failed for {}: {}", cfg.name, stderr);
}
info!("SQLite backup completed for {}", cfg.name);
logger.log_command("sqlite3", None, Some(0), Some(duration_ms));
logger.log("info", format!("SQLite backup completed for {}", cfg.name));
Ok(file_path)
})
.await?
}
.await?
}
+10 -18
View File
@@ -1,9 +1,11 @@
use anyhow::Result;
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
@@ -27,26 +29,16 @@ impl Database for SqliteDatabase {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
}
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, is_test: Option<bool>) -> Result<()> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
}
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+2 -2
View File
@@ -1,4 +1,4 @@
mod backup;
mod restore;
pub mod database;
mod ping;
pub mod database;
mod restore;
+14 -6
View File
@@ -1,12 +1,14 @@
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
use std::sync::Arc;
use std::time::Instant;
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
tokio::task::spawn_blocking(move || -> Result<()> {
debug!("Starting SQLite restore for database {}", cfg.name);
logger.log("debug", format!("Starting SQLite restore for database {}", cfg.name));
let db_path_str = if cfg.path.is_empty() {
anyhow::bail!("Database path not configured");
@@ -25,19 +27,25 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
.with_context(|| format!("Failed to remove existing DB {}", db_path.display()))?;
}
let start = Instant::now();
let output = Command::new("sqlite3")
.arg(db_path.as_os_str())
.arg(format!(".restore '{}'", restore_file.display()))
.output()
.with_context(|| format!("Failed to run sqlite3 restore for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
error!("SQLite restore failed for {}: {}", cfg.name, stderr);
logger.log_command("sqlite3", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
logger.log("error", format!("SQLite restore failed for {}: {}", cfg.name, stderr));
anyhow::bail!("SQLite restore failed for {}", cfg.name);
}
info!("SQLite restore completed for {}", cfg.name);
logger.log_command("sqlite3", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
logger.log("info", format!("SQLite restore completed for {}", cfg.name));
Ok(())
})
.await?
+44 -13
View File
@@ -1,21 +1,26 @@
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
use std::sync::Arc;
use std::time::Instant;
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
file_extension: &'static str,
logger: Arc<JobLogger>,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
debug!("Starting Valkey backup for database {}", cfg.name);
logger.log(
"info",
format!("Starting Valkey backup for database {}", cfg.name),
);
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
let mut cmd = Command::new("valkey-cli");
cmd.arg("-h")
.arg(&cfg.host)
.arg("-p")
@@ -24,40 +29,66 @@ pub async fn run(
if !cfg.username.is_empty() {
cmd.arg("--user").arg(&cfg.username);
}
if !cfg.password.is_empty() {
cmd.arg("-a").arg(&cfg.password);
}
cmd.arg("--rdb").arg(&file_path);
debug!("Command Backup: {:?}", cmd);
logger.log("info", format!("Running valkey-cli --rdb for {}", cfg.name));
let start = Instant::now();
let output = cmd.output().context("Valkey backup command failed")?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr);
let stdout = String::from_utf8_lossy(&output.stdout);
if !output.status.success() {
if stderr.contains("NOAUTH") {
error!(
"Valkey backup failed for {}: Authentication required (NOAUTH)",
cfg.name
logger.log(
"error",
format!(
"Valkey backup failed for {}: Authentication required (NOAUTH)",
cfg.name
),
);
logger.log_command(
"valkey-cli",
Some("Authentication required (NOAUTH)".into()),
Some(exit_code),
Some(duration_ms),
);
anyhow::bail!(
"Valkey backup failed for {}: Authentication required",
cfg.name
);
} else {
error!("Valkey backup failed for {}: {}", cfg.name, stderr);
logger.log(
"error",
format!("Valkey backup failed for {}: {}", cfg.name, stderr),
);
logger.log_command(
"valkey-cli",
Some(stderr.to_string()),
Some(exit_code),
Some(duration_ms),
);
anyhow::bail!("Valkey backup failed for {}: {}", cfg.name, stderr);
}
}
info!(
"Valkey backup completed for {}. Output: {}",
cfg.name, stdout
logger.log_command(
"valkey-cli",
if stdout.is_empty() {
None
} else {
Some(stdout.to_string())
},
Some(0),
Some(duration_ms),
);
logger.log("info", format!("Valkey backup completed for {}", cfg.name));
Ok(file_path)
})
+7 -10
View File
@@ -1,9 +1,11 @@
use anyhow::{Result, bail};
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use crate::domain::factory::Database;
use crate::domain::valkey::{backup, ping};
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
@@ -27,19 +29,14 @@ impl Database for ValkeyDatabase {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
}
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, _file: &Path, _is_test: Option<bool>) -> Result<()> {
async fn restore(&self, _file: &Path, _logger: Arc<JobLogger>) -> Result<()> {
bail!("Restore not supported for Valkey databases")
}
}
+10 -14
View File
@@ -1,8 +1,8 @@
use tracing::{debug, info};
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use tokio::process::Command;
use tokio::time::{timeout, Duration};
use anyhow::{Result, Context};
use tokio::time::{Duration, timeout};
use tracing::{debug, info, error};
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
let mut cmd = Command::new("valkey-cli");
@@ -21,35 +21,31 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
cmd.arg("PING");
debug!("Command Ping: {:?}", cmd);
debug!("Command Ping Valkey: {:?}", cmd);
let result = timeout(Duration::from_secs(10), cmd.output()).await;
match result {
Ok(output) => {
let output = output.context("Failed to execute redis-cli")?;
let output = output.context("Failed to execute valkey-cli")?;
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
info!("Redis stdout: {}", stdout);
info!("Redis stderr: {}", stderr);
if stderr.contains("NOAUTH") {
info!("Redis authentication failed (NOAUTH required)");
error!("Valkey authentication failed (NOAUTH required)");
return Ok(false);
}
if !output.status.success() {
info!("Redis command failed with status: {:?}", output.status);
error!("Valkey command failed with status: {:?}", output.status);
return Ok(false);
}
Ok(stdout.contains("PONG"))
}
Err(_) => {
info!("Timeout connecting to Redis at {}:{}", cfg.host, cfg.port);
info!("Timeout connecting to Valkey at {}:{}", cfg.host, cfg.port);
Ok(false)
}
}
}
}
+2 -3
View File
@@ -3,19 +3,18 @@ mod domain;
mod services;
mod settings;
mod tasks;
mod utils;
#[cfg(test)]
mod tests;
mod utils;
use crate::tasks::ping::ping_server;
use crate::utils::locks::FileLock;
use crate::utils::logging;
use utils::redis_client;
use utils::task_manager::scheduler;
use crate::utils::logging;
#[tokio::main]
async fn main() {
logging::init_logger();
// Remove all locks on startup
+2 -7
View File
@@ -1,9 +1,9 @@
#![allow(dead_code)]
use crate::services::api::ApiError;
use reqwest::{Client, Method};
use serde::de::DeserializeOwned;
use std::time::Duration;
use crate::services::api::ApiError;
#[derive(Clone, Debug)]
pub struct ApiClient {
@@ -57,12 +57,7 @@ impl ApiClient {
{
let url = format!("{}{}", self.base_url, path);
let res = self
.http
.request(method, &url)
.json(body)
.send()
.await?;
let res = self.http.request(method, &url).json(body).send().await?;
let status = res.status();
let body_text = res.text().await.unwrap_or_default();
@@ -2,6 +2,7 @@ pub mod upload;
use crate::services::api::models::agent::backup::BackupResponse;
use crate::services::api::{ApiClient, ApiError};
use crate::services::backup::logger::JobLogEntry;
use anyhow::Result;
use reqwest::Method;
use serde::Serialize;
@@ -21,9 +22,11 @@ pub struct BackupUpdateRequest {
pub size: Option<u64>,
#[serde(rename = "generatedId")]
pub generated_id: String,
pub logs: Vec<JobLogEntry>,
#[serde(rename = "durationMs")]
pub duration_ms: f64,
}
impl ApiClient {
pub async fn backup_create(
&self,
@@ -50,12 +53,16 @@ impl ApiClient {
status: impl Into<String>,
file_size: impl Into<Option<u64>>,
generated_id: impl Into<String>,
job_logs: Vec<JobLogEntry>,
duration_ms: f64,
) -> Result<Option<BackupResponse>, ApiError> {
let body = BackupUpdateRequest {
backup_id: backup_id.into(),
status: status.into(),
size: file_size.into(),
generated_id: generated_id.into(),
logs: job_logs,
duration_ms: duration_ms,
};
let agent_id = agent_id.into();
@@ -1,8 +1,8 @@
use crate::services::api::models::agent::backup::BackupUploadResponse;
use crate::services::api::{ApiClient, ApiError};
use anyhow::Result;
use reqwest::Method;
use serde::Serialize;
use crate::services::api::models::agent::backup::BackupUploadResponse;
#[derive(Serialize)]
pub struct InitUploadRequest {
@@ -1,2 +1,2 @@
pub mod init;
pub mod status;
pub mod status;
+2 -2
View File
@@ -1,3 +1,3 @@
pub mod status;
pub mod backup;
pub mod restore;
pub mod restore;
pub mod status;
@@ -3,12 +3,16 @@ use crate::services::api::{ApiClient, ApiError};
use anyhow::Result;
use reqwest::Method;
use serde::Serialize;
use crate::services::backup::logger::JobLogEntry;
#[derive(Serialize)]
pub struct ResultRestoreRequest {
#[serde(rename = "generatedId")]
pub generated_id: String,
pub status: String,
pub logs: Vec<JobLogEntry>,
#[serde(rename = "durationMs")]
pub duration_ms: f64,
}
impl ApiClient {
@@ -17,10 +21,14 @@ impl ApiClient {
agent_id: impl Into<String>,
generated_id: impl Into<String>,
status: impl Into<String>,
job_logs: Vec<JobLogEntry>,
duration_ms: f64,
) -> Result<Option<ResultRestoreResponse>, ApiError> {
let body = ResultRestoreRequest {
generated_id: generated_id.into(),
status: status.into(),
logs: job_logs,
duration_ms,
};
let agent_id = agent_id.into();
+5 -2
View File
@@ -11,6 +11,8 @@ pub struct DatabasePayload<'a> {
pub dbms: &'a str,
#[serde(rename = "generatedId")]
pub generated_id: &'a str,
#[serde(rename = "pingStatus")]
pub ping_status: bool
}
#[derive(Serialize)]
@@ -31,6 +33,7 @@ impl ApiClient {
let agent_id = agent_id.into();
let path = format!("/agent/{}/status", agent_id);
self.request_with_body(Method::POST, path.as_str(), &body).await
self.request_with_body(Method::POST, path.as_str(), &body)
.await
}
}
}
+1 -1
View File
@@ -1,3 +1,3 @@
pub mod agent;
pub use agent::status;
pub use agent::status;

Some files were not shown because too many files have changed in this diff Show More