Compare commits

..

19 Commits

Author SHA1 Message Date
github-actions[bot] 0a422448b4 chore: release 1.10.0 2026-04-16 09:22:34 +00:00
Charles GTE 14a6c9ab0a feat: add chunk size in .env variables (#54)
Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-04-16 11:20:14 +02:00
Charles GTE 86027fd6bd fix: docker compose files [skip-release] (#53)
* fix: s3 port type mismatch, add string_or_number_to_string deserializer

* chore: docker-compose.databases.yml

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-04-02 19:30:07 +02:00
Charles GTE 8eab97a3d6 chore: README.md (#52) [skip-release]
Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-04-01 17:24:48 +02:00
github-actions[bot] 25c95108f0 chore: release 1.9.0 2026-03-31 13:12:45 +00:00
Charles GTE f37e4d2de9 feat: add firebird (#51)
* feat: add firebird

* feat: add firebird tests

* fix: tests and refactoring, add justfile instead of makefile
2026-03-31 15:10:05 +02:00
github-actions[bot] a1aa66ff3f chore: release 1.8.0 2026-03-29 07:30:57 +00:00
Charles GTE d4abcbcebc feat: separate mysql and mariadb (#49)
* feat: separate mysql and mariadb

* feat: separate mysql and mariadb
2026-03-29 09:28:30 +02:00
github-actions[bot] 8bf6503f1f chore: release 1.7.0 2026-03-28 15:47:43 +00:00
Charles GTE fdf35228d4 feat: add-heathcheck-database (#48)
feat: add-heathcheck-database
2026-03-28 16:45:13 +01:00
Charles GTE 1855c5120a fix: docker-compose.yml 2026-03-28 16:44:07 +01:00
Charles GTE 3d5a7eeded feat: add health check ping for databases 2026-03-28 16:07:37 +01:00
Charles GTE 517f3779cd feat: sending database ping result for each databases to portabase server 2026-03-27 23:07:06 +01:00
github-actions[bot] a1e18754b1 chore: release 1.6.3 2026-03-27 21:27:23 +00:00
Charles GTE 725312e1d5 fix: tests (#47)
* fix: s3 port type mismatch, add string_or_number_to_string deserializer

* fix: adding some tests

* fix: adding some tests

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-27 22:24:59 +01:00
github-actions[bot] 8e8ebb5920 chore: release 1.6.2 2026-03-21 07:53:35 +00:00
Charles GTE 8489a6f6e3 fix: adding retry config for s3 provider (#46) 2026-03-21 08:51:15 +01:00
github-actions[bot] 3f5b9c91cc chore: release 1.6.1 2026-03-18 18:42:23 +00:00
Charles GTE 1c52f298d2 fix: Enable Link-Time Optimization (LTO) and codegen-units = 1 for Release builds (#45) 2026-03-18 19:39:49 +01:00
156 changed files with 2250 additions and 878 deletions
+1 -1
View File
@@ -60,7 +60,7 @@ representative at an online or offline event.
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported to the community leaders responsible for enforcement at
contact@soluce-technologies.com.
contact@portabase.io.
All complaints will be reviewed and investigated promptly and fairly.
All community leaders are obligated to respect the privacy and security of the
+1 -1
View File
@@ -18,7 +18,7 @@ We take security seriously and aim to support the following versions of the proj
If you discover a security vulnerability in this project, we appreciate your help in disclosing it responsibly.
1. **Contact Us**
Please report the vulnerability by emailing **[contact@soluce-technologies.com](mailto:contact@soluce-technologies.com)**. Include the following details:
Please report the vulnerability by emailing **[contact@portabase.io](mailto:contact@portabase.io)**. Include the following details:
- A detailed description of the issue.
- Steps to reproduce the vulnerability (if applicable).
- Any potential impacts or risks.
+145 -31
View File
@@ -1,3 +1,86 @@
#name: Codecov Rust
#
#on:
# push:
# branches: ["main"]
# pull_request:
# branches: ["main"]
#
#env:
# CARGO_TERM_COLOR: always
#
#jobs:
# coverage:
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v4
#
# - name: Install Rust toolchain
# uses: dtolnay/rust-toolchain@stable
# with:
# components: llvm-tools-preview
#
# - name: Build test image (with grcov included)
# run: docker compose -f docker-compose.test.yml build agent-test
#
# - name: Run tests in container
# env:
# CARGO_TARGET_DIR: /app/target
# CARGO_INCREMENTAL: 0
# RUSTFLAGS: "-C instrument-coverage -C link-dead-code"
# LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw"
# run: |
# docker compose -f docker-compose.test.yml run \
# -e CARGO_TARGET_DIR \
# -e CARGO_INCREMENTAL \
# -e RUSTFLAGS \
# -e LLVM_PROFILE_FILE \
# agent-test bash -c "cargo clean && cargo test --verbose && sync"
#
# - name: Verify profraw files exist
# run: |
# docker compose -f docker-compose.test.yml run agent-test \
# find /app/coverage -type f -name "*.profraw" | wc -l || true
#
# - name: Generate coverage report inside container
# run: |
# docker compose -f docker-compose.test.yml run agent-test bash -c "
# rustup component add llvm-tools &&
# grcov /app/coverage \
# --binary-path /app/target/debug \
# -s /app \
# --llvm \
# -t lcov \
# --branch \
# --ignore-not-existing \
# --ignore '/app/target/*' \
# --ignore '/*' \
# -o /app/lcov.info
# "
#
# - name: Copy lcov.info from container to host
# run: |
# docker compose -f docker-compose.test.yml cp agent-test:/app/lcov.info ./lcov.info
#
# - name: Remove container
# run: |
# docker rm agent-test-run
#
# - name: Show basic coverage report info (debug)
# run: |
# echo "lcov.info size:" $(wc -c ./lcov.info | awk '{print $1}')
# head -n 30 ./lcov.info || true
#
# - name: Upload coverage to Codecov
# uses: codecov/codecov-action@v5
# with:
# files: ./lcov.info
# flags: unittests
# name: rust-unit-coverage
# verbose: true
# fail_ci_if_error: true
# env:
# CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
name: Codecov Rust
on:
@@ -8,51 +91,82 @@ on:
env:
CARGO_TERM_COLOR: always
CARGO_TARGET_DIR: /app/target
CARGO_INCREMENTAL: 0
RUSTFLAGS: "-C instrument-coverage -C link-dead-code"
LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw"
jobs:
build:
coverage:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: actions-rs/toolchain@v1
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: stable
override: true
components: llvm-tools-preview
- name: Install grcov
run: cargo install grcov
- name: Build test image (with grcov included)
run: docker compose -f docker-compose.test.yml build agent-test
- name: Install test requirements
run: bash scripts/tests/requirements.sh
- name: Start agent-test container
run: docker compose -f docker-compose.test.yml up -d agent-test
- name: Build
run: cargo build --verbose
- name: Run tests
env:
CARGO_INCREMENTAL: 0
RUSTFLAGS: "-C instrument-coverage"
LLVM_PROFILE_FILE: "cargo-test-%p-%m.profraw"
run: cargo test --verbose
- name: Generate coverage
- name: Run tests inside container
run: |
grcov . \
--binary-path ./target/debug/ \
-s . \
-t lcov \
--branch \
--ignore-not-existing \
-o lcov.info
docker compose -f docker-compose.test.yml exec \
-e CARGO_TARGET_DIR \
-e CARGO_INCREMENTAL \
-e RUSTFLAGS \
-e LLVM_PROFILE_FILE \
agent-test bash -c "
cargo clean &&
cargo test --verbose &&
sync
"
- name: Upload to Codecov
- name: Verify profraw files exist
run: |
docker compose -f docker-compose.test.yml exec \
-e LLVM_PROFILE_FILE \
agent-test find /app/coverage -type f -name "*.profraw" | wc -l || true
- name: Generate coverage report inside container
run: |
docker compose -f docker-compose.test.yml exec \
agent-test bash -c "
rustup component add llvm-tools &&
grcov /app/coverage \
--binary-path /app/target/debug \
-s /app \
--llvm \
-t lcov \
--branch \
--ignore-not-existing \
--ignore '/app/target/*' \
--ignore '/*' \
-o /app/lcov.info
"
- name: Copy lcov.info from container to host
run: docker compose -f docker-compose.test.yml cp agent-test:/app/lcov.info ./lcov.info
- name: Show basic coverage report info (debug)
run: |
echo "lcov.info size:" $(wc -c ./lcov.info | awk '{print $1}')
head -n 30 ./lcov.info || true
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v5
with:
files: lcov.info
files: ./lcov.info
flags: unittests
name: rust-unit-coverage
verbose: true
fail_ci_if_error: true
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
- name: Stop and remove container
run: docker compose -f docker-compose.test.yml down
+1 -1
View File
@@ -27,5 +27,5 @@ keywords:
- self-hosted
- portabase
license: Apache-2.0
version: 1.6.0
version: 1.10.0
date-released: '2026-02-24'
Generated
+1 -1
View File
@@ -3038,7 +3038,7 @@ dependencies = [
[[package]]
name = "portabase-agent"
version = "1.6.0"
version = "1.10.0"
dependencies = [
"aes",
"aes-gcm",
+7 -3
View File
@@ -1,6 +1,6 @@
[package]
name = "portabase-agent"
version = "1.6.0"
version = "1.10.0"
edition = "2024"
[dependencies]
@@ -49,7 +49,7 @@ tokio-stream = "0.1.18"
aes = "0.9.0-rc.4"
typenum = "1.19.0"
testcontainers = "0.27.1"
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "valkey"] }
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "valkey", "mysql", "mariadb", "mongo"] }
postgres = "0.19.12"
url = "2.5.8"
@@ -57,9 +57,13 @@ url = "2.5.8"
tokio = { version = "1", features = ["full"] }
mockall = "0.13"
testcontainers = "0.27.1"
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis"] }
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "mysql", "mariadb", "mongo"] }
wiremock = "0.6"
[profile.release]
lto = true
codegen-units = 1
strip = "debuginfo"
[[bin]]
name = "app"
-54
View File
@@ -1,54 +0,0 @@
include .env
export $(shell sed 's/=.*//' .env)
CLUSTER_SCRIPT=docker/entrypoints/app-dev-entrypoint.sh
.PHONY: seed-mongo seed-mysql seed-postgres
up:
@bash $(CLUSTER_SCRIPT)
seed-mongo:
@echo "Seeding MongoDB..."
bash ./scripts/mongo/seed-mongo.sh
seed-mongo-auth:
@echo "Seeding MongoDB with auth..."
bash ./scripts/mongo/seed-mongo.sh auth
seed-mysql:
@echo "Seeding MySQL..."
mysql -h 127.0.0.1 -P "$$MYSQL_PORT" -u "$$MYSQL_USER" -p"$$MYSQL_PASSWORD" "$$MYSQL_DB" < ./scripts/mysql/seed-mysql.sql
seed-mysql-1gb:
@echo "Seeding MySQL..."
mysql -h 127.0.0.1 -P "$$MYSQL_PORT" -u "$$MYSQL_USER" -p"$$MYSQL_PASSWORD" "$$MYSQL_DB" < ./scripts/mysql/seed-1gb.sql
seed-postgres:
@echo "Seeding Postgres..."
docker exec -i -e PGPASSWORD=$$PG_PASSWORD $$PG_CONTAINER \
psql -U $$PG_USER -d $$PG_DB < ./scripts/postgres/seed.sql
seed-postgres-1gb:
@echo "Seeding Postgres..."
docker exec -i -e PGPASSWORD=$$PG_PASSWORD $$PG_CONTAINER \
psql -U $$PG_USER -d $$PG_DB < ./scripts/postgres/seed-1gb.sql
SQLITE_SEED_FILE := $(if $(filter big,$(SEED)),./scripts/sqlite/seed-big.sql,./scripts/sqlite/seed.sql)
seed-sqlite:
@echo "Seeding Sqlite..."
@echo "Run as root to fix permissions inside the volume"
docker exec -u 0 -it db-sqlite sh -c "chmod -R 777 /workspace/data"
@echo "Create the database file (if it doesnt exist)"
docker exec -u 0 -it db-sqlite sh -c "touch /workspace/data/app.db"
@echo "Seed the database"
docker exec -i db-sqlite sh -c "sqlite3 /workspace/data/app.db" < $(SQLITE_SEED_FILE)
@echo "Verify"
docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT name FROM sqlite_master WHERE type='table';"
@echo "Done"
seed-all: seed-mongo seed-mysql seed-postgres seed-postgres-1gb
+6 -2
View File
@@ -13,6 +13,7 @@
[![License: Apache](https://img.shields.io/badge/License-apache-yellow.svg)](LICENSE)
[![Docker Pulls](https://img.shields.io/docker/pulls/portabase/agent?color=brightgreen)](https://hub.docker.com/r/portabase/agent)
[![Helm Chart](https://img.shields.io/badge/Helm-Kubernetes-326ce5?logo=helm&logoColor=white)](https://github.com/Portabase/agent/pkgs/container/charts%2Fportabase-agent)
[![Platform](https://img.shields.io/badge/platform-linux%20%7C%20macos%20%7C%20windows-lightgrey)](https://github.com/Portabase/portabase)
[![Support Portabase](https://img.shields.io/badge/Support-Portabase-orange)](https://www.buymeacoffee.com/portabase)
@@ -20,9 +21,12 @@
[![MySQL](https://img.shields.io/badge/MySQL-4479A1?logo=mysql&logoColor=white)](https://www.mysql.com/)
[![SQLite](https://img.shields.io/badge/-SQLite-blue?logo=sqlite&logoColor=white)](https://sqlite.org/)
[![Redis](https://img.shields.io/badge/Redis-DC382D?style=flat&logo=Redis&logoColor=white)](https://redis.io/)
[![Valkey](https://img.shields.io/badge/Valkey-6284fc?style=flat&logo=Valkey&logoColor=white)](https://valkey.io/)
[![MariaDB](https://img.shields.io/badge/MariaDB-003545?logo=mariadb&logoColor=white)](https://mariadb.org/)
[![MongoDB](https://img.shields.io/badge/-MongoDB-13aa52?logo=mongodb&logoColor=white)](https://www.mongodb.com/)
[![Valkey](https://img.shields.io/badge/Valkey-6284fc?style=flat&logo=Valkey&logoColor=white)](https://valkey.io/)
[![Firebird](https://img.shields.io/badge/Firebird-f55b14?style=flat&logo=Firebird&logoColor=white)](https://firebirdsql.org/)
[![Self Hosted](https://img.shields.io/badge/self--hosted-yes-brightgreen)](https://github.com/Portabase/portabase)
[![Open Source](https://img.shields.io/badge/open%20source-❤️-red)](https://github.com/Portabase/portabase)
@@ -50,7 +54,7 @@ You have 4 ways to install Portabase Agent:
- Automated CLI (recommended) - [details](https://portabase.io/docs/agent/setup#cli)
- Docker Compose setup - [details](https://portabase.io/docs/agent/setup#docker)
- Kubernetes with Helm (soon)
- Kubernetes with Helm [details](https://portabase.io/docs/agent/setup#helm)
- Development setup - [details](https://portabase.io/docs/agent/setup#development)
**Ensure Docker is installed on your machine before getting started.**
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
+29 -9
View File
@@ -13,7 +13,7 @@
{
"name": "Test database 2 - MariaDB",
"database": "mariadb",
"type": "mysql",
"type": "mariadb",
"username": "mariadb",
"password": "changeme",
"port": 3306,
@@ -21,7 +21,7 @@
"generated_id": "16678124-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 4 - MongoDB Auth",
"name": "Test database 3 - MongoDB Auth",
"database": "testdbauth",
"type": "mongodb",
"username": "root",
@@ -31,7 +31,7 @@
"generated_id": "16678145-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 5 - MongoDB",
"name": "Test database 4 - MongoDB",
"database": "testdb",
"type": "mongodb",
"port": 27017,
@@ -39,26 +39,26 @@
"generated_id": "16678147-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 6 - SQLite DB",
"name": "Test database 5 - SQLite DB",
"type": "sqlite",
"path": "/sqlite-data/workspace/data/app.db",
"generated_id": "16678178-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 7 - SQLite DB",
"name": "Test database 6 - SQLite DB",
"type": "sqlite",
"path": "/sqlite-data-2/workspace/data/app.db",
"generated_id": "16678179-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 8 - Redis",
"name": "Test database 7 - Redis",
"type": "redis",
"port": 6379,
"host": "db-redis",
"generated_id": "16678166-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 9 - Redis Auth",
"name": "Test database 8 - Redis Auth",
"type": "redis",
"password": "supersecurepassword",
"port": 6379,
@@ -67,20 +67,40 @@
"generated_id": "16678160-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 10 - Valkey",
"name": "Test database 9 - Valkey",
"type": "valkey",
"port": 6379,
"host": "db-valkey",
"generated_id": "16678560-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 11 - Valkey Auth",
"name": "Test database 10 - Valkey Auth",
"type": "valkey",
"password": "supersecurepassword",
"port": 6379,
"username": "default",
"host": "db-valkey-auth",
"generated_id": "16678561-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 11 - Mysql",
"database": "mysqldb",
"type": "mysql",
"username": "mysqldb",
"password": "changeme",
"port": 3306,
"host": "db-mysql",
"generated_id": "16656124-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 12 - Firebird",
"database": "/var/lib/firebird/data/mirror.fdb",
"type": "firebird",
"username": "alice",
"password": "fake_password",
"port": 3050,
"host": "db-firebird",
"generated_id": "16706124-ff7e-4c97-8c83-0adeff214681"
}
]
}
+67 -3
View File
@@ -29,11 +29,75 @@ host = "db-mongodb-auth"
generated_id = "16678145-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 3 - MongoDB"
name = "Test database 4 - MongoDB"
database = "testdb"
type = "mongodb"
username = ""
password = ""
port = 27017
host = "db-mongodb"
generated_id = "16678147-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 5 - SQLite DB"
type = "sqlite"
path = "/sqlite-data/workspace/data/app.db"
generated_id = "16678178-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 6 - SQLite DB 2"
type = "sqlite"
path = "/sqlite-data-2/workspace/data/app.db"
generated_id = "16678179-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 7 - Redis"
type = "redis"
port = 6379
host = "db-redis"
generated_id = "16678166-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 8 - Redis Auth"
type = "redis"
password = "supersecurepassword"
username = "default"
port = 6379
host = "db-redis-auth"
generated_id = "16678160-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 9 - Valkey"
type = "valkey"
port = 6379
host = "db-valkey"
generated_id = "16678560-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 10 - Valkey Auth"
type = "valkey"
password = "supersecurepassword"
username = "default"
port = 6379
host = "db-valkey-auth"
generated_id = "16678561-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 11 - Mysql"
database = "mysqldb"
type = "mysql"
password = "changeme"
username = "mysqldb"
port = 3306
host = "db-mysql"
generated_id = "16656124-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 12 - Firebird"
database = "/var/lib/firebird/data/mirror.fdb"
type = "firebird"
password = "fake_password"
username = "alice"
port = 3050
host = "db-firebird"
generated_id = "16706124-ff7e-4c97-8c83-0adeff214681"
+179
View File
@@ -0,0 +1,179 @@
name: databases
services:
db-postgres:
container_name: db-postgres
image: postgres:17-alpine
ports:
- "5436:5432"
volumes:
- postgres-data:/var/lib/postgresql/data
environment:
- POSTGRES_DB=devdb
- POSTGRES_USER=devuser
- POSTGRES_PASSWORD=changeme
networks:
- portabase
db-mariadb:
container_name: db-mariadb
image: mariadb:latest
ports:
- "3311:3306"
environment:
- MYSQL_DATABASE=mariadb
- MYSQL_USER=mariadb
- MYSQL_PASSWORD=changeme
- MYSQL_RANDOM_ROOT_PASSWORD=yes
volumes:
- mariadb-data:/var/lib/mysql
networks:
- portabase
db-mysql:
container_name: db-mysql
image: mysql:9.5
ports:
- "3312:3306"
environment:
- MYSQL_DATABASE=mysqldb
- MYSQL_USER=mysqldb
- MYSQL_PASSWORD=changeme
- MYSQL_RANDOM_ROOT_PASSWORD=yes
volumes:
- mysql-data:/var/lib/mysql
networks:
- portabase
db-mongodb-auth:
container_name: db-mongodb-auth
image: mongo:latest
ports:
- "27082:27017"
environment:
MONGO_INITDB_ROOT_USERNAME: root
MONGO_INITDB_ROOT_PASSWORD: rootpassword
MONGO_INITDB_DATABASE: testdbauth
command: mongod --auth
networks:
- portabase
volumes:
- mongodb-data-auth:/data/db
healthcheck:
test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ]
interval: 5s
timeout: 5s
retries: 10
db-mongodb:
container_name: db-mongodb
image: mongo:latest
ports:
- "27083:27017"
volumes:
- mongodb-data:/data/db
healthcheck:
test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ]
interval: 5s
timeout: 5s
retries: 10
environment:
MONGO_INITDB_DATABASE: testdb
networks:
- portabase
sqlite:
container_name: db-sqlite
image: keinos/sqlite3
volumes:
- sqlite-data:/workspace/data
working_dir: /workspace
command: tail -f /dev/null
stdin_open: true
tty: true
db-redis:
image: redis:latest
container_name: db-redis
ports:
- "6378:6379"
volumes:
- redis-data:/data
command: [ "redis-server", "--appendonly", "yes" ]
networks:
- portabase
db-redis-auth:
image: redis:latest
container_name: db-redis-auth
ports:
- "6380:6379"
volumes:
- redis-data-auth:/data
environment:
- REDIS_PASSWORD=supersecurepassword
command: [ "redis-server", "--requirepass", "supersecurepassword", "--appendonly", "yes" ]
networks:
- portabase
db-valkey:
image: valkey/valkey
container_name: db-valkey
environment:
- ALLOW_EMPTY_PASSWORD=yes
ports:
- '6381:6379'
volumes:
- valkey-data:/data
networks:
- portabase
db-valkey-auth:
image: valkey/valkey
container_name: db-valkey-auth
command: >
--requirepass "supersecurepassword"
ports:
- '6382:6379'
volumes:
- valkey-data-auth:/data
networks:
- portabase
db-firebird:
image: firebirdsql/firebird
container_name: db-firebird
restart: always
environment:
- FIREBIRD_ROOT_PASSWORD=fake_root_password
- FIREBIRD_USER=alice
- FIREBIRD_PASSWORD=fake_password
- FIREBIRD_DATABASE=mirror.fdb
- FIREBIRD_DATABASE_DEFAULT_CHARSET=UTF8
volumes:
- firebird-data:/var/lib/firebird/data
ports:
- "3060:3050"
networks:
- portabase
volumes:
postgres-data:
mariadb-data:
mysql-data:
mongodb-data:
mongodb-data-auth:
sqlite-data:
redis-data:
redis-data-auth:
valkey-data:
valkey-data-auth:
firebird-data:
networks:
portabase:
name: portabase_network
external: true
+2 -6
View File
@@ -5,23 +5,20 @@ services:
# dockerfile: docker/Dockerfile
# target: prod
image: portabase/agent:latest
# platform: linux/arm64
container_name: rust-prod
volumes:
- ./databases.json:/config/config.json
environment:
# APP_ENV: production
LOG: info
TZ: "Europe/Paris"
# DATABASES_CONFIG_FILE: "config.toml"
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiOGZmMDE4NTQtYjJhMS00ZTE0LTkwMjctZTJiOWIxZjQ1YzdlIiwicHVibGljS2V5IjoiLS0tLS1CRUdJTiBSU0EgUFVCTElDIEtFWS0tLS0tXG5NSUlCQ2dLQ0FRRUE5TWV4M2pmdnVLdFB5YU1ERnh2Ulp2dmd3YkRJQ2JzQi81Wll5NDNSVVRBaXZRYjJiSDdYXG5qRHBQd1lJeCs4UFBrbHlRbDVMQzV1UWZEaCs4SVd4OG1LZ3FvMXpWMkdiZXdGbEdEWFYxVEdyU1ZEU25aSWR4XG52bWdYc29EeXhVMlJvWUFUMS9YMWxuc2YxenZKdkFMTkhXdEhRdk42SjVDZTFSMmFsendVRGFEVXlJNzRmSldQXG5tNTh0SDMrYklXL0VVTXdjaWNxM0oySWw3Vm9KNkZNUHJQL1ZSOWEvdFF1SU1qa200MXpFY2NscExPa2luRkxuXG54NmVUWkFSZUpya2UrbnRvZ2t4TGEyRWV5a1lUNzB4V3hKNWp5ZExBVnRvNkkyQlVLVVJoTkowTUFaU29NYUtvXG5iMGJRcnY1UzExZWllMnMrT2I3aTYzSFpkVUx0UmV1MVJ3SURBUUFCXG4tLS0tLUVORCBSU0EgUFVCTElDIEtFWS0tLS0tXG4ifQ"
#DATABASES_CONFIG_FILE: "config.toml"
EDGE_KEY: "<to complete>"
extra_hosts:
- "localhost:host-gateway"
networks:
- portabase
db-mongodb-auth:
container_name: db-mongodb-auth
image: mongo:latest
@@ -63,7 +60,6 @@ volumes:
mongodb-data:
mongodb-data-auth:
networks:
portabase:
name: portabase_network
+22
View File
@@ -0,0 +1,22 @@
services:
agent-test:
build:
context: .
dockerfile: docker/Dockerfile
target: dev
working_dir: /app
container_name: agent-test
volumes:
- .:/app
- cargo-registry:/usr/local/cargo/registry
- cargo-git:/usr/local/cargo/git
- /var/run/docker.sock:/var/run/docker.sock
environment:
APP_ENV: test
LOG: debug
TZ: Europe/Paris
EDGE_KEY: ""
volumes:
cargo-registry:
cargo-git:
+13 -141
View File
@@ -7,18 +7,20 @@ services:
container_name: rust-dev
volumes:
- .:/app
- ./databases.json:/config/config.json
# - ./databases.toml:/config/config.toml
- cargo-registry:/usr/local/cargo/registry
- cargo-git:/usr/local/cargo/git
# - cargo-target:/app/target
# - sqlite-data:/sqlite-data/workspace/data
# - ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data
- ./databases.json:/config/config.json
#- ./databases.toml:/config/config.toml
#- /var/run/docker.sock:/var/run/docker.sock
#- cargo-target:/app/target
- databases_sqlite-data:/sqlite-data/workspace/data
- ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data
environment:
APP_ENV: development
LOG: debug
TZ: "Europe/Paris"
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiMmVkZWFlYjctNWYwNi00MjI1LWIzNWItOGYyM2MwOTMyMTA2IiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ=="
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiNzM0NjU3Y2YtMGQzYy00Y2UwLTkyODQtZDJmOGYyMjI2MzgzIiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ=="
#CHUNK_SIZE_MB: "1"
#POOLING: 1
#DATABASES_CONFIG_FILE: "config.toml"
extra_hosts:
@@ -26,144 +28,14 @@ services:
networks:
- portabase
db-postgres:
container_name: db-postgres
image: postgres:17-alpine
ports:
- "5436:5432"
volumes:
- postgres-data:/var/lib/postgresql/data
environment:
- POSTGRES_DB=devdb
- POSTGRES_USER=devuser
- POSTGRES_PASSWORD=changeme
networks:
- portabase
#
# db-mariadb:
# container_name: db-mariadb
# image: mariadb:latest
# ports:
# - "3311:3306"
# environment:
# - MYSQL_DATABASE=mariadb
# - MYSQL_USER=mariadb
# - MYSQL_PASSWORD=changeme
# - MYSQL_RANDOM_ROOT_PASSWORD=yes
# volumes:
# - mariadb-data:/var/lib/mysql
# networks:
# - portabase
#
#
# db-mongodb-auth:
# container_name: db-mongodb-auth
# image: mongo:latest
# ports:
# - "27082:27017"
# environment:
# MONGO_INITDB_ROOT_USERNAME: root
# MONGO_INITDB_ROOT_PASSWORD: rootpassword
# MONGO_INITDB_DATABASE: testdbauth
# command: mongod --auth
# networks:
# - portabase
# volumes:
# - mongodb-data-auth:/data/db
# healthcheck:
# test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ]
# interval: 5s
# timeout: 5s
# retries: 10
#
# db-mongodb:
# container_name: db-mongodb
# image: mongo:latest
# ports:
# - "27083:27017"
# volumes:
# - mongodb-data:/data/db
# healthcheck:
# test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ]
# interval: 5s
# timeout: 5s
# retries: 10
# environment:
# MONGO_INITDB_DATABASE: testdb
# networks:
# - portabase
# sqlite:
# container_name: db-sqlite
# image: keinos/sqlite3
# volumes:
# - sqlite-data:/workspace/data
# working_dir: /workspace
# command: tail -f /dev/null
# stdin_open: true
# tty: true
db-redis:
image: redis:latest
container_name: db-redis
ports:
- "6379:6379"
volumes:
- redis-data:/data
command: [ "redis-server", "--appendonly", "yes" ]
networks:
- portabase
db-redis-auth:
image: redis:latest
container_name: db-redis-auth
ports:
- "6380:6379"
volumes:
- redis-data-auth:/data
environment:
- REDIS_PASSWORD=supersecurepassword
command: [ "redis-server", "--requirepass", "supersecurepassword", "--appendonly", "yes" ]
networks:
- portabase
db-valkey:
image: valkey/valkey
container_name: db-valkey
environment:
- ALLOW_EMPTY_PASSWORD=yes
ports:
- '6381:6379'
volumes:
- valkey-data:/data
networks:
- portabase
db-valkey-auth:
image: valkey/valkey
container_name: db-valkey-auth
command: >
--requirepass "supersecurepassword"
ports:
- '6382:6379'
volumes:
- valkey-data-auth:/data
networks:
- portabase
volumes:
cargo-registry:
cargo-git:
# cargo-target:
postgres-data:
# mariadb-data:
# mongodb-data:
# mongodb-data-auth:
# sqlite-data:
redis-data:
redis-data-auth:
valkey-data:
valkey-data-auth:
#cargo-target:
databases_sqlite-data:
external: true
networks:
portabase:
+3
View File
@@ -18,6 +18,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
sqlite3 \
redis-tools \
valkey \
firebird3.0-utils \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
@@ -46,6 +47,7 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \
rm -rf /tmp/pg-x64 /tmp/pg-arm && \
chmod +x /usr/lib/postgresql/*/bin/*
# =========================
# MongoDB client binaries
# =========================
@@ -60,6 +62,7 @@ WORKDIR /app
FROM base AS dev
RUN cargo install cargo-watch
RUN cargo install grcov --locked
COPY Cargo.toml Cargo.lock ./
RUN mkdir src && echo "fn main() {}" > src/main.rs
+8 -3
View File
@@ -34,6 +34,11 @@ check_network() {
check_docker
check_network
echo "Starting docker-compose..."
docker compose -f ./docker-compose.yml up
echo "Docker-compose started successfully."
echo "Stopping old database containers..."
docker compose -f ./docker-compose.databases.yml down
echo "Starting database containers..."
docker compose -f ./docker-compose.databases.yml up -d
echo "Starting main services..."
docker compose -f ./docker-compose.yml up
+54
View File
@@ -0,0 +1,54 @@
set dotenv-load := true
set shell := ["bash", "-cu"]
CLUSTER_SCRIPT := "docker/entrypoints/app-dev-entrypoint.sh"
up:
bash {{CLUSTER_SCRIPT}}
seed-mongo:
echo "Seeding MongoDB..."
bash ./scripts/mongo/seed-mongo.sh
seed-mongo-auth:
echo "Seeding MongoDB with auth..."
bash ./scripts/mongo/seed-mongo.sh auth
seed-mysql:
echo "Seeding MySQL..."
mysql -h 127.0.0.1 -P "$MYSQL_PORT" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DB" < ./scripts/mysql/seed-mysql.sql
seed-mysql-1gb:
echo "Seeding MySQL (1GB)..."
mysql -h 127.0.0.1 -P "$MYSQL_PORT" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DB" < ./scripts/mysql/seed-1gb.sql
seed-postgres:
echo "Seeding Postgres..."
docker exec -i -e PGPASSWORD="$PG_PASSWORD" "$PG_CONTAINER" \
psql -U "$PG_USER" -d "$PG_DB" < ./scripts/postgres/seed.sql
seed-postgres-1gb:
echo "Seeding Postgres (1GB)..."
docker exec -i -e PGPASSWORD="$PG_PASSWORD" "$PG_CONTAINER" \
psql -U "$PG_USER" -d "$PG_DB" < ./scripts/postgres/seed-1gb.sql
seed-sqlite:
echo "Seeding SQLite..."
bash ./scripts/sqlite/seed.sh
echo "Done"
seed-firebird:
echo "Seeding Firebird..."
docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb < ./scripts/firebird/seed.sql
echo "Verifying Firebird tables..."
docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb -sql "SHOW TABLES;"
seed-all:
just seed-mongo
just seed-mysql
just seed-postgres
just seed-postgres-1gb
just seed-sqlite
just seed-mongo
just seed-firebird
+4
View File
@@ -24,3 +24,7 @@ docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT * FROM users LI
```bash
docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT name FROM sqlite_master WHERE type='table';"
```
```bash
docker compose -f docker-compose.test.yml run agent-test bash -c "cargo clean && cargo test"
```
+11
View File
@@ -0,0 +1,11 @@
CREATE TABLE users (
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
email VARCHAR(255) NOT NULL UNIQUE,
name VARCHAR(255),
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
INSERT INTO users (email, name) VALUES ('alice@example.com', 'Alice');
INSERT INTO users (email, name) VALUES ('bob@example.com', 'Bob');
COMMIT;
+14
View File
@@ -0,0 +1,14 @@
#!/usr/bin/env bash
set -euo pipefail
SEED_VALUE="${SEED:-}"
if [ "$SEED_VALUE" = "big" ]; then
SQLITE_SEED_FILE="./scripts/sqlite/seed-big.sql"
else
SQLITE_SEED_FILE="./scripts/sqlite/seed.sql"
fi
docker exec -u 0 db-sqlite sh -c "chmod -R 777 /workspace/data"
docker exec -u 0 db-sqlite sh -c "touch /workspace/data/app.db"
docker exec -i db-sqlite sh -c "sqlite3 /workspace/data/app.db" < "$SQLITE_SEED_FILE"
-88
View File
@@ -1,88 +0,0 @@
#!/usr/bin/env bash
set -e
POSTGRES_BASE="/usr/local/postgresql"
echo "Detecting OS and architecture..."
OS_TYPE="$(uname -s)"
ARCH="$(uname -m)"
install_pg_binaries() {
echo "Installing PostgreSQL binaries for versions 12-18..."
for v in 12 13 14 15 16 17 18; do
TARGET_DIR="$POSTGRES_BASE/$v/bin"
sudo mkdir -p "$TARGET_DIR"
if [[ "$OS_TYPE" == "Linux" ]]; then
if [[ "$ARCH" == "x86_64" ]]; then
SRC_DIR="./assets/tools/amd64/postgresql/postgresql-$v/bin"
elif [[ "$ARCH" == "aarch64" ]]; then
SRC_DIR="./assets/tools/arm64/postgresql/postgresql-$v/bin"
else
echo "Unsupported architecture: $ARCH"
continue
fi
if [[ -d "$SRC_DIR" ]]; then
echo "Copying PostgreSQL $v binaries from $SRC_DIR to $TARGET_DIR"
sudo cp -r "$SRC_DIR"/* "$TARGET_DIR/"
else
echo "Binaries for PostgreSQL $v not found for Linux, skipping..."
continue
fi
elif [[ "$OS_TYPE" == "Darwin" ]]; then
PG_SRC="$(brew --prefix postgresql@$v)/bin" 2>/dev/null || true
if [[ ! -d "$PG_SRC" ]]; then
echo "PostgreSQL $v not installed via Homebrew. Trying to install..."
if ! brew install postgresql@$v; then
echo "PostgreSQL $v not available, skipping..."
continue
fi
PG_SRC="$(brew --prefix postgresql@$v)/bin"
fi
echo "Copying PostgreSQL $v binaries from $PG_SRC to $TARGET_DIR"
sudo cp -r "$PG_SRC"/* "$TARGET_DIR/"
fi
sudo chown -R "$(whoami)" "$TARGET_DIR"
chmod +x "$TARGET_DIR"/*
done
echo "PostgreSQL binaries installed under $POSTGRES_BASE"
}
if [[ "$OS_TYPE" == "Linux" ]]; then
if command -v apt >/dev/null 2>&1; then
echo "Linux detected with apt. Installing prerequisites..."
sudo apt update
sudo apt install -y wget gnupg lsb-release redis-tools valkey
install_pg_binaries
else
echo "Unsupported Linux distribution. Only apt-based distros are supported."
exit 1
fi
elif [[ "$OS_TYPE" == "Darwin" ]]; then
if command -v brew >/dev/null 2>&1; then
echo "macOS detected. Installing prerequisites..."
brew install redis
brew install valkey
sudo mkdir -p "$POSTGRES_BASE"
sudo chown -R "$(whoami)" "$POSTGRES_BASE"
install_pg_binaries
else
echo "Homebrew not found. Please install Homebrew first: https://brew.sh/"
exit 1
fi
else
echo "Unsupported OS: $OS_TYPE"
exit 1
fi
echo "Tools installation completed successfully."
+15 -8
View File
@@ -4,11 +4,11 @@ use crate::core::context::Context;
use crate::services::backup::BackupService;
use crate::services::config::ConfigService;
use crate::services::cron::CronService;
use crate::services::restore::RestoreService;
use crate::services::status::StatusService;
use crate::utils::common::BackupMethod;
use std::sync::Arc;
use tracing::info;
use crate::services::restore::RestoreService;
pub struct Agent {
ctx: Arc<Context>,
@@ -43,23 +43,30 @@ impl Agent {
let ping_result = self.status_service.ping(&config.databases).await?;
for db in ping_result.databases.iter() {
let database = config.databases.iter().find(|cfg_db|cfg_db.generated_id == db.generated_id).unwrap();
let database = config
.databases
.iter()
.find(|cfg_db| cfg_db.generated_id == db.generated_id)
.unwrap();
info!(
"Generated Id: {} | backup action: {} | restore action: {} | Database Name: {}",
db.generated_id, db.data.backup.action, db.data.restore.action, database.name,
db.generated_id, db.data.backup.action, db.data.restore.action, database.name,
);
let _ = self.cron_service.sync(db).await;
if db.data.backup.action {
let _ = self
.backup_service
.dispatch(&db.generated_id, &config, method.clone(), &db.storages, db.encrypt)
.dispatch(
&db.generated_id,
&config,
method.clone(),
&db.storages,
db.encrypt,
)
.await;
} else if db.data.restore.action {
let _ = self
.restore_service
.dispatch(db, &config)
.await;
let _ = self.restore_service.dispatch(db, &config).await;
}
}
+1 -1
View File
@@ -35,7 +35,7 @@ impl Context {
panic!("Cannot initialize AgentContext due to invalid EDGE_KEY");
}
};
let server_url = format!("{}/api", edge_key.server_url);
let api_client = ApiClient::new(server_url);
+1 -1
View File
@@ -1,2 +1,2 @@
pub mod context;
pub mod agent;
pub mod context;
+11 -8
View File
@@ -4,18 +4,20 @@ use crate::domain::postgres::database::PostgresDatabase;
use crate::domain::postgres::{detect_format_from_file, detect_format_from_size};
use crate::domain::redis::database::RedisDatabase;
use crate::domain::sqlite::database::SqliteDatabase;
use crate::domain::valkey::database::ValkeyDatabase;
use crate::services::config::{DatabaseConfig, DbType};
use anyhow::Result;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use crate::domain::valkey::database::ValkeyDatabase;
use crate::domain::firebird::database::FirebirdDatabase;
use crate::domain::mariadb::database::MariaDBDatabase;
#[async_trait::async_trait]
pub trait Database: Send + Sync {
fn file_extension(&self) -> &'static str;
async fn ping(&self) -> Result<bool>;
async fn backup(&self, backup_dir: &Path, is_test: Option<bool>) -> Result<PathBuf>;
async fn restore(&self, restore_file: &Path, is_test: Option<bool>) -> Result<()>;
async fn backup(&self, backup_dir: &Path) -> Result<PathBuf>;
async fn restore(&self, restore_file: &Path) -> Result<()>;
}
pub struct DatabaseFactory;
@@ -28,11 +30,12 @@ impl DatabaseFactory {
Arc::new(PostgresDatabase::new(cfg, format))
}
DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)),
DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)),
DbType::Mariadb => Arc::new(MariaDBDatabase::new(cfg)),
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
DbType::Redis => Arc::new(RedisDatabase::new(cfg)),
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg))
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)),
DbType::Firebird => Arc::new(FirebirdDatabase::new(cfg)),
}
}
@@ -43,12 +46,12 @@ impl DatabaseFactory {
Arc::new(PostgresDatabase::new(cfg, format))
}
DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)),
DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)),
DbType::Mariadb => Arc::new(MariaDBDatabase::new(cfg)),
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
DbType::Redis => Arc::new(RedisDatabase::new(cfg)),
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg))
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)),
DbType::Firebird => Arc::new(FirebirdDatabase::new(cfg)),
}
}
}
+48
View File
@@ -0,0 +1,48 @@
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
file_extension: &'static str,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
debug!("Starting backup for database {}", cfg.name);
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
let db_path = format!(
"{}/{}:{}",
cfg.host,
cfg.port,
cfg.database
);
info!("Firebird database target: {}", db_path);
info!("Backup file: {}", file_path.display());
let output = Command::new("gbak")
.arg("-b")
.arg("-v")
.arg("-user").arg(&cfg.username)
.arg("-password").arg(&cfg.password)
.arg(db_path)
.arg(&file_path)
.output()
.with_context(|| format!("Failed to run gbak for {}", cfg.name))?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
error!("Firebird backup failed: {}", stderr);
anyhow::bail!("Firebird backup failed for {}: {}", cfg.name, stderr);
}
info!("Firebird backup completed: {}", file_path.display());
Ok(file_path)
})
.await?
}
+48
View File
@@ -0,0 +1,48 @@
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
use anyhow::Result;
use async_trait::async_trait;
use std::path::{Path, PathBuf};
pub struct FirebirdDatabase {
cfg: DatabaseConfig,
}
impl FirebirdDatabase {
pub fn new(cfg: DatabaseConfig) -> Self {
Self { cfg }
}
}
#[async_trait]
impl Database for FirebirdDatabase {
fn file_extension(&self) -> &'static str {
".fbk"
}
async fn ping(&self) -> Result<bool> {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(
self.cfg.clone(),
dir.to_path_buf(),
self.file_extension(),
)
.await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+4
View File
@@ -0,0 +1,4 @@
pub mod database;
mod ping;
mod backup;
mod restore;
+57
View File
@@ -0,0 +1,57 @@
use std::process::Stdio;
use tracing::{error, info};
use tokio::io::AsyncWriteExt;
use crate::services::config::DatabaseConfig;
use tokio::process::Command;
use tokio::time::{Duration, timeout};
pub async fn run(cfg: DatabaseConfig) -> anyhow::Result<bool> {
let db_path = format!("{}/{}:{}", cfg.host, cfg.port, cfg.database);
info!("Running Ping database from {}", db_path);
let mut child = Command::new("isql-fb")
.arg("-q")
.arg("-user")
.arg(&cfg.username)
.arg("-password")
.arg(&cfg.password)
.arg(&db_path)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?;
let query = b"SELECT 1 FROM RDB$DATABASE;\nQUIT;\n";
if let Some(mut stdin) = child.stdin.take() {
stdin.write_all(query).await?;
}
let output = match timeout(Duration::from_secs(5), child.wait_with_output()).await {
Ok(res) => res?,
Err(_) => return Ok(false),
};
if !output.status.success() {
error!("Error output for firebird: {:?}", output);
return Ok(false);
}
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
if stderr.to_lowercase().contains("error") {
error!("Error output for firebird: {:?}", output);
return Ok(false);
}
if stdout.contains("1") {
return Ok(true);
}
info!("stdout {}", stdout);
error!("stderr {}", stderr);
Ok(false)
}
+40
View File
@@ -0,0 +1,40 @@
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
tokio::task::spawn_blocking(move || -> Result<()> {
debug!("Starting Firebird restore for database {}", cfg.name);
let db_path = format!("{}/{}:{}", cfg.host, cfg.port, cfg.database);
info!("Restore source: {}", restore_file.display());
info!("Restore target: {}", db_path);
let output = Command::new("gbak")
.arg("-c")
.arg("-v")
.arg("-replace_database")
.arg("-user")
.arg(&cfg.username)
.arg("-password")
.arg(&cfg.password)
.arg(&restore_file)
.arg(&db_path)
.output()
.with_context(|| format!("Failed to run gbak restore for {}", cfg.name))?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
error!("Firebird restore failed for {}: {}", cfg.name, stderr);
anyhow::bail!("Firebird restore failed for {}: {}", cfg.name, stderr);
}
info!("Firebird restore completed for {}", cfg.name);
Ok(())
})
.await?
}
+66
View File
@@ -0,0 +1,66 @@
use crate::domain::mariadb::connection::{select_mariadb_path, server_version};
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::collections::HashMap;
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
env: HashMap<String, String>,
file_extension: &'static str,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
debug!("Starting backup for database {}", cfg.name);
let version = match futures::executor::block_on(server_version(&cfg)) {
Ok(v) => {
debug!("Mariadb version detected: {}", v);
v
}
Err(e) => {
error!("Failed to get server version for {}: {:?}", cfg.name, e);
return Err(e.into());
}
};
info!("Mariadb version found: {}", version);
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
let mariadb_dump = select_mariadb_path(&version).join("mariadb-dump");
info!("Mariadb dump found: {}", mariadb_dump.display());
let output = Command::new("mariadb-dump")
.arg("--host").arg(&cfg.host)
.arg("--port").arg(cfg.port.to_string())
.arg("--user").arg(&cfg.username)
.arg("--routines")
.arg("--events")
.arg("--triggers")
.arg("--single-transaction")
.arg("--quick")
.arg("--skip-lock-tables")
.arg("--add-drop-database")
.arg("--databases").arg(&cfg.database)
.arg("--compress")
.arg("--max-allowed-packet=512M")
.arg("--net-buffer-length=16K")
.arg("--default-character-set=utf8mb4")
.arg("-r").arg(&file_path)
.envs(env)
.output()
.with_context(|| format!("Failed to run mariadb-dump for {}", cfg.name))?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
anyhow::bail!("Mariadb backup failed for {}: {}", cfg.name, stderr);
}
Ok(file_path)
})
.await?
}
+45
View File
@@ -0,0 +1,45 @@
use std::path::PathBuf;
use crate::services::config::DatabaseConfig;
use anyhow::Result;
use std::process::Command;
pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
let output = Command::new("mariadb")
.arg("--host")
.arg(&cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(&cfg.username)
.arg("-e")
.arg("SELECT VERSION();")
.env("MYSQL_PWD", &cfg.password)
.output()?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
anyhow::bail!("Version query failed: {}", stderr);
}
let version = String::from_utf8_lossy(&output.stdout)
.lines()
.nth(1)
.unwrap_or_default()
.trim()
.to_string();
Ok(version)
}
pub fn select_mariadb_path(version: &str) -> PathBuf {
let mut parts = version.split('.');
let major = parts.next().and_then(|v| v.parse::<u32>().ok()).unwrap_or(10);
let minor = parts.next().and_then(|v| v.parse::<u32>().ok()).unwrap_or(0);
if major < 10 || (major == 10 && minor <= 6) {
"/usr/local/mariadb-10.6/bin".into()
} else {
"/usr/local/mariadb-12.1/bin".into()
}
}
+55
View File
@@ -0,0 +1,55 @@
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
use anyhow::Result;
use async_trait::async_trait;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
pub struct MariaDBDatabase {
cfg: DatabaseConfig,
}
impl MariaDBDatabase {
pub fn new(cfg: DatabaseConfig) -> Self {
Self { cfg }
}
fn build_env(&self) -> HashMap<String, String> {
let mut envs = std::env::vars().collect::<HashMap<_, _>>();
envs.insert("MYSQL_PWD".to_string(), self.cfg.password.to_string());
envs
}
}
#[async_trait]
impl Database for MariaDBDatabase {
fn file_extension(&self) -> &'static str {
".sql"
}
async fn ping(&self) -> Result<bool> {
ping::run(self.cfg.clone(), self.build_env().clone()).await
}
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(
self.cfg.clone(),
dir.to_path_buf(),
self.build_env().clone(),
self.file_extension(),
)
.await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+5
View File
@@ -0,0 +1,5 @@
pub mod backup;
mod connection;
pub mod database;
mod ping;
mod restore;
+28
View File
@@ -0,0 +1,28 @@
use crate::services::config::DatabaseConfig;
use std::collections::HashMap;
use tokio::process::Command;
use tokio::time::{Duration, timeout};
pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::Result<bool> {
let mut cmd = Command::new("mysqladmin");
cmd.arg("--host")
.arg(cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(cfg.username)
.arg("ping")
.envs(env);
let result = timeout(Duration::from_secs(10), cmd.output()).await;
match result {
Ok(output) => {
let output = output?;
Ok(output.status.success())
}
Err(_) => Ok(false),
}
}
+80
View File
@@ -0,0 +1,80 @@
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::fs::File;
use std::io::{Read, Write};
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
let handle = tokio::task::spawn_blocking(move || -> Result<()> {
debug!("Starting restore for database {}", cfg.name);
let mut sql_content = String::new();
let mut file = File::open(&restore_file)
.with_context(|| format!("Failed to open restore file {}", restore_file.display()))?;
file.read_to_string(&mut sql_content)
.with_context(|| format!("Failed to read restore file {}", restore_file.display()))?;
let drop_create_cmd = format!(
"DROP DATABASE IF EXISTS {0}; CREATE DATABASE {0};",
cfg.database
);
let drop_status = Command::new("mariadb")
.arg("--host")
.arg(&cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(&cfg.username)
.arg("-e")
.arg(&drop_create_cmd)
.env("MYSQL_PWD", &cfg.password)
.status()
.with_context(|| format!("Failed to drop/recreate database {}", cfg.name))?;
if !drop_status.success() {
error!("Drop/create database failed for {}", cfg.name);
anyhow::bail!("Failed to drop/recreate database {}", cfg.name);
}
info!("Database {} dropped and recreated", cfg.name);
let mut child = Command::new("mariadb")
.arg("--host")
.arg(&cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(&cfg.username)
.arg(&cfg.database)
.env("MYSQL_PWD", &cfg.password)
.stdin(std::process::Stdio::piped())
.spawn()
.with_context(|| format!("Failed to start MariaDB restore for {}", cfg.name))?;
let mut stdin = child.stdin.take().context("Failed to open child stdin")?;
stdin
.write_all(sql_content.as_bytes())
.context("Failed to write SQL content to MariaDB stdin")?;
stdin.flush()?;
drop(stdin);
let output = child
.wait_with_output()
.with_context(|| format!("Failed to complete MariaDB restore for {}", cfg.name))?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
error!("MariaDB restore failed for {}: {}", cfg.name, stderr);
anyhow::bail!("MariaDB restore failed for {}", cfg.name);
}
info!("Restore finished successfully for database {}", cfg.name);
Ok(())
});
handle.await??;
Ok(())
}
+5 -4
View File
@@ -1,8 +1,9 @@
pub mod factory;
pub mod postgres;
pub mod mysql;
mod mongodb;
mod sqlite;
pub mod mysql;
pub mod postgres;
mod redis;
mod sqlite;
mod valkey;
mod mariadb;
mod firebird;
+1
View File
@@ -15,6 +15,7 @@ pub async fn run(
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
let mongodump = select_mongo_path().join("mongodump");
info!("{:?}", mongodump);
let uri = get_mongo_uri(cfg.clone())?;
let output = Command::new(mongodump)
+4 -4
View File
@@ -16,9 +16,11 @@ pub fn select_mongo_path() -> std::path::PathBuf {
}
pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result<String> {
if cfg.username.is_empty() || cfg.password.is_empty() {
Ok(format!("mongodb://{}:{}/{}", cfg.host, cfg.port, cfg.database))
Ok(format!(
"mongodb://{}:{}/{}",
cfg.host, cfg.port, cfg.database
))
} else {
Ok(format!(
"mongodb://{}:{}@{}:{}/{}?authSource=admin",
@@ -26,5 +28,3 @@ pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result<String> {
))
}
}
+6 -16
View File
@@ -27,27 +27,17 @@ impl Database for MongoDatabase {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
}
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, is_test: Option<bool>) -> Result<()> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
}
async fn restore(&self, file: &Path) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+2 -2
View File
@@ -1,5 +1,5 @@
mod backup;
mod restore;
mod connection;
pub mod database;
mod ping;
mod connection;
mod restore;
+1 -1
View File
@@ -24,7 +24,7 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
error!("Full Error: {}", e);
error!("Check you database network connectivity");
error!("----------------------------------------");
Err(anyhow::anyhow!("Ping failed for {}: {}", cfg.name, e))
Ok(false)
}
}
}
+7 -1
View File
@@ -1,4 +1,4 @@
use crate::domain::mysql::connection::{server_version};
use crate::domain::mysql::connection::server_version;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::collections::HashMap;
@@ -30,6 +30,9 @@ pub async fn run(
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
// let mysql_dump = select_mysql_path(&version).join("mysqldump");
// info!("MySQL dump found: {}", mysql_dump.display());
let output = Command::new("mysqldump")
.arg("--host")
.arg(cfg.host)
@@ -54,9 +57,12 @@ pub async fn run(
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
info!("mysqldump stderr: {}", stderr);
anyhow::bail!("MySQL backup failed for {}: {}", cfg.name, stderr);
}
info!("Output {}", String::from_utf8_lossy(&output.stdout));
Ok(file_path)
})
.await?
+10 -8
View File
@@ -1,14 +1,17 @@
use crate::services::config::DatabaseConfig;
use std::process::Command;
use anyhow::Result;
use std::process::Command;
pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
let output = Command::new("mysql")
.arg("--host").arg(&cfg.host)
.arg("--port").arg(cfg.port.to_string())
.arg("--user").arg(&cfg.username)
.arg("-e").arg("SELECT VERSION();")
.arg("--host")
.arg(&cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(&cfg.username)
.arg("-e")
.arg("SELECT VERSION();")
.env("MYSQL_PWD", &cfg.password)
.output()?;
@@ -19,11 +22,10 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
let version = String::from_utf8_lossy(&output.stdout)
.lines()
.nth(1) // skip column header
.nth(1)
.unwrap_or_default()
.trim()
.to_string();
Ok(version)
}
+18 -26
View File
@@ -1,14 +1,11 @@
use std::collections::HashMap;
use anyhow::Result;
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use super::{
backup,
ping, restore,
};
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
use anyhow::Result;
use async_trait::async_trait;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
pub struct MySQLDatabase {
cfg: DatabaseConfig,
@@ -36,28 +33,23 @@ impl Database for MySQLDatabase {
ping::run(self.cfg.clone(), self.build_env().clone()).await
}
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
}
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.build_env().clone(), self.file_extension()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(
self.cfg.clone(),
dir.to_path_buf(),
self.build_env().clone(),
self.file_extension(),
)
.await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, is_test: Option<bool>) -> Result<()> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
}
async fn restore(&self, file: &Path) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+2 -2
View File
@@ -1,5 +1,5 @@
pub mod backup;
mod connection;
pub mod database;
mod restore;
mod ping;
mod connection;
mod restore;
+1 -2
View File
@@ -4,8 +4,7 @@ use tokio::process::Command;
use tokio::time::{Duration, timeout};
pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::Result<bool> {
let mut cmd = Command::new("mysqladmin");
let mut cmd = Command::new("mariadb-admin");
cmd.arg("--host")
.arg(cfg.host)
.arg("--port")
+2 -3
View File
@@ -11,7 +11,6 @@ pub async fn run(
cfg: DatabaseConfig,
format: PostgresDumpFormat,
backup_dir: PathBuf,
is_test: Option<bool>
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
debug!("Starting backup for database {}", cfg.name);
@@ -27,8 +26,8 @@ pub async fn run(
}
};
let pg_dump = select_pg_path(&version, is_test).join("pg_dump");
let pg_dump = select_pg_path(&version).join("pg_dump");
debug!("Using pg_dump at {:?}", pg_dump);
match format {
+2 -7
View File
@@ -28,14 +28,9 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
Ok(version)
}
pub fn select_pg_path(version: &str, is_test: Option<bool>) -> std::path::PathBuf {
pub fn select_pg_path(version: &str) -> std::path::PathBuf {
let major = version.split('.').next().unwrap_or("17");
if is_test.unwrap_or(false) {
format!("/usr/local/postgresql/{}/bin", major).into()
} else {
format!("/usr/lib/postgresql/{}/bin", major).into()
}
format!("/usr/lib/postgresql/{}/bin", major).into()
}
pub async fn terminate_connections(cfg: &DatabaseConfig) -> Result<()> {
+8 -18
View File
@@ -31,27 +31,17 @@ impl Database for PostgresDatabase {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
}
let res = backup::run(self.cfg.clone(), self.format, dir.to_path_buf(), is_test).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), self.format, dir.to_path_buf()).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, is_test: Option<bool>) -> Result<()> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
}
let res = restore::run(self.cfg.clone(), self.format, file.to_path_buf(), is_test).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
async fn restore(&self, file: &Path) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), self.format, file.to_path_buf()).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+1 -1
View File
@@ -2,4 +2,4 @@
pub enum PostgresDumpFormat {
Fc,
Fd,
}
}
+3 -3
View File
@@ -1,8 +1,8 @@
pub mod backup;
pub mod database;
mod restore;
mod connection;
pub mod database;
mod format;
mod ping;
mod restore;
pub use connection::{detect_format_from_size, detect_format_from_file};
pub use connection::{detect_format_from_file, detect_format_from_size};
+2 -4
View File
@@ -1,8 +1,6 @@
use super::connection::connect;
use crate::services::config::DatabaseConfig;
pub async fn run(
cfg: DatabaseConfig,
) -> anyhow::Result<bool> {
pub async fn run(cfg: DatabaseConfig) -> anyhow::Result<bool> {
Ok(connect(&cfg).await.is_ok())
}
}
+1 -2
View File
@@ -11,7 +11,6 @@ pub async fn run(
cfg: DatabaseConfig,
format: PostgresDumpFormat,
restore_file: PathBuf,
is_test: Option<bool>,
) -> Result<()> {
tokio::task::spawn_blocking(move || -> Result<()> {
debug!("Starting restore for database {}", cfg.name);
@@ -27,7 +26,7 @@ pub async fn run(
}
};
let pg_restore = select_pg_path(&version, is_test).join("pg_restore");
let pg_restore = select_pg_path(&version).join("pg_restore");
debug!("Using pg_restore at {:?}", pg_restore);
+4 -9
View File
@@ -27,19 +27,14 @@ impl Database for RedisDatabase {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
}
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, _file: &Path, _is_test: Option<bool>) -> Result<()> {
async fn restore(&self, _file: &Path) -> Result<()> {
bail!("Restore not supported for Redis databases")
}
}
+8 -11
View File
@@ -1,8 +1,8 @@
use tracing::{debug, info};
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use tokio::process::Command;
use tokio::time::{timeout, Duration};
use anyhow::{Result, Context};
use tokio::time::{Duration, timeout};
use tracing::{debug, error, info};
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
let mut cmd = Command::new("redis-cli");
@@ -21,8 +21,7 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
cmd.arg("PING");
debug!("Command Ping: {:?}", cmd);
debug!("Command Ping Redis: {:?}", cmd);
let result = timeout(Duration::from_secs(10), cmd.output()).await;
@@ -31,17 +30,15 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
let output = output.context("Failed to execute redis-cli")?;
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
info!("Redis stdout: {}", stdout);
info!("Redis stderr: {}", stderr);
if stderr.contains("NOAUTH") {
info!("Redis authentication failed (NOAUTH required)");
error!("Redis authentication failed (NOAUTH required)");
return Ok(false);
}
if !output.status.success() {
info!("Redis command failed with status: {:?}", output.status);
error!("Redis command failed with status: {:?}", output.status);
return Ok(false);
}
@@ -52,4 +49,4 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
Ok(false)
}
}
}
}
+4 -3
View File
@@ -19,6 +19,7 @@ pub async fn run(
};
let db_path = PathBuf::from(db_path_str);
info!("database path: {}", db_path.display());
if !db_path.exists() {
anyhow::bail!("SQLite database file not found: {}", db_path.display());
@@ -31,7 +32,7 @@ pub async fn run(
.arg(format!(".backup '{}'", file_path.display()))
.output()
.context("SQLite backup command failed to start")?;
info!("Backup successful: {:?}", output);
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
error!("SQLite backup failed for {}: {}", cfg.name, stderr);
@@ -41,5 +42,5 @@ pub async fn run(
info!("SQLite backup completed for {}", cfg.name);
Ok(file_path)
})
.await?
}
.await?
}
+6 -16
View File
@@ -27,26 +27,16 @@ impl Database for SqliteDatabase {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
}
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, is_test: Option<bool>) -> Result<()> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
}
async fn restore(&self, file: &Path) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+2 -2
View File
@@ -1,4 +1,4 @@
mod backup;
mod restore;
pub mod database;
mod ping;
pub mod database;
mod restore;
+4 -9
View File
@@ -27,19 +27,14 @@ impl Database for ValkeyDatabase {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, is_test: Option<bool>) -> Result<PathBuf> {
let test_mode = is_test.unwrap_or(false);
if !test_mode {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
}
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
if !test_mode {
FileLock::release(&self.cfg.generated_id).await?;
}
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, _file: &Path, _is_test: Option<bool>) -> Result<()> {
async fn restore(&self, _file: &Path) -> Result<()> {
bail!("Restore not supported for Valkey databases")
}
}
+10 -14
View File
@@ -1,8 +1,8 @@
use tracing::{debug, info};
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use tokio::process::Command;
use tokio::time::{timeout, Duration};
use anyhow::{Result, Context};
use tokio::time::{Duration, timeout};
use tracing::{debug, info, error};
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
let mut cmd = Command::new("valkey-cli");
@@ -21,35 +21,31 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
cmd.arg("PING");
debug!("Command Ping: {:?}", cmd);
debug!("Command Ping Valkey: {:?}", cmd);
let result = timeout(Duration::from_secs(10), cmd.output()).await;
match result {
Ok(output) => {
let output = output.context("Failed to execute redis-cli")?;
let output = output.context("Failed to execute valkey-cli")?;
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
info!("Redis stdout: {}", stdout);
info!("Redis stderr: {}", stderr);
if stderr.contains("NOAUTH") {
info!("Redis authentication failed (NOAUTH required)");
error!("Valkey authentication failed (NOAUTH required)");
return Ok(false);
}
if !output.status.success() {
info!("Redis command failed with status: {:?}", output.status);
error!("Valkey command failed with status: {:?}", output.status);
return Ok(false);
}
Ok(stdout.contains("PONG"))
}
Err(_) => {
info!("Timeout connecting to Redis at {}:{}", cfg.host, cfg.port);
info!("Timeout connecting to Valkey at {}:{}", cfg.host, cfg.port);
Ok(false)
}
}
}
}
+2 -3
View File
@@ -3,19 +3,18 @@ mod domain;
mod services;
mod settings;
mod tasks;
mod utils;
#[cfg(test)]
mod tests;
mod utils;
use crate::tasks::ping::ping_server;
use crate::utils::locks::FileLock;
use crate::utils::logging;
use utils::redis_client;
use utils::task_manager::scheduler;
use crate::utils::logging;
#[tokio::main]
async fn main() {
logging::init_logger();
// Remove all locks on startup
+2 -7
View File
@@ -1,9 +1,9 @@
#![allow(dead_code)]
use crate::services::api::ApiError;
use reqwest::{Client, Method};
use serde::de::DeserializeOwned;
use std::time::Duration;
use crate::services::api::ApiError;
#[derive(Clone, Debug)]
pub struct ApiClient {
@@ -57,12 +57,7 @@ impl ApiClient {
{
let url = format!("{}{}", self.base_url, path);
let res = self
.http
.request(method, &url)
.json(body)
.send()
.await?;
let res = self.http.request(method, &url).json(body).send().await?;
let status = res.status();
let body_text = res.text().await.unwrap_or_default();
@@ -23,7 +23,6 @@ pub struct BackupUpdateRequest {
pub generated_id: String,
}
impl ApiClient {
pub async fn backup_create(
&self,
@@ -1,8 +1,8 @@
use crate::services::api::models::agent::backup::BackupUploadResponse;
use crate::services::api::{ApiClient, ApiError};
use anyhow::Result;
use reqwest::Method;
use serde::Serialize;
use crate::services::api::models::agent::backup::BackupUploadResponse;
#[derive(Serialize)]
pub struct InitUploadRequest {
@@ -1,2 +1,2 @@
pub mod init;
pub mod status;
pub mod status;
+2 -2
View File
@@ -1,3 +1,3 @@
pub mod status;
pub mod backup;
pub mod restore;
pub mod restore;
pub mod status;
+5 -2
View File
@@ -11,6 +11,8 @@ pub struct DatabasePayload<'a> {
pub dbms: &'a str,
#[serde(rename = "generatedId")]
pub generated_id: &'a str,
#[serde(rename = "pingStatus")]
pub ping_status: bool
}
#[derive(Serialize)]
@@ -31,6 +33,7 @@ impl ApiClient {
let agent_id = agent_id.into();
let path = format!("/agent/{}/status", agent_id);
self.request_with_body(Method::POST, path.as_str(), &body).await
self.request_with_body(Method::POST, path.as_str(), &body)
.await
}
}
}
+1 -1
View File
@@ -1,3 +1,3 @@
pub mod agent;
pub use agent::status;
pub use agent::status;
+1 -4
View File
@@ -11,10 +11,7 @@ pub enum ApiError {
Serialization(#[from] serde_json::Error),
#[error("api error: status={status}, body={body}")]
HttpResponse {
status: StatusCode,
body: String,
},
HttpResponse { status: StatusCode, body: String },
#[error("api returned unexpected response")]
UnexpectedResponse,
+1 -2
View File
@@ -1,8 +1,7 @@
pub mod client;
pub mod endpoints;
pub mod error;
pub mod models;
pub mod endpoints;
pub use client::ApiClient;
pub use error::ApiError;
+2 -2
View File
@@ -1,3 +1,3 @@
pub mod status;
pub mod backup;
pub mod restore;
pub mod restore;
pub mod status;
+1 -1
View File
@@ -4,4 +4,4 @@ use serde::{Deserialize, Serialize};
pub struct ResultRestoreResponse {
pub message: String,
pub status: bool,
}
}
+1 -1
View File
@@ -1,8 +1,8 @@
#![allow(dead_code)]
use crate::utils::deserializer::deserialize_snake_case;
use serde::{Deserialize, Serialize};
use toml::Value;
use crate::utils::deserializer::deserialize_snake_case;
#[derive(Debug, Deserialize)]
pub struct PingResult {
-1
View File
@@ -1,2 +1 @@
pub mod agent;
+4 -10
View File
@@ -1,20 +1,14 @@
use super::service::BackupService;
use crate::utils::compress::compress_to_tar_gz_large;
use std::path::PathBuf;
use anyhow::Result;
use std::path::PathBuf;
impl BackupService {
pub async fn compress_backup(
&self,
backup_file: Option<PathBuf>,
) -> Result<PathBuf> {
let file = backup_file
.ok_or_else(|| anyhow::anyhow!("No backup file generated"))?;
pub async fn compress_backup(&self, backup_file: Option<PathBuf>) -> Result<PathBuf> {
let file = backup_file.ok_or_else(|| anyhow::anyhow!("No backup file generated"))?;
let compression = compress_to_tar_gz_large(&file).await?;
Ok(compression.compressed_path)
}
}
}
+2 -4
View File
@@ -1,11 +1,10 @@
use super::service::BackupService;
use crate::services::config::DatabasesConfig;
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::config::DatabasesConfig;
use crate::utils::common::BackupMethod;
use tracing::error;
impl BackupService {
pub async fn dispatch(
&self,
generated_id: &String,
@@ -14,7 +13,6 @@ impl BackupService {
storages: &Vec<DatabaseStorage>,
encrypt: bool,
) {
let Some(cfg) = config
.databases
.iter()
@@ -41,4 +39,4 @@ impl BackupService {
}
});
}
}
}
+4 -6
View File
@@ -1,14 +1,13 @@
use super::service::BackupService;
use crate::services::config::DatabaseConfig;
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::config::DatabaseConfig;
use crate::utils::common::BackupMethod;
use crate::utils::locks::FileLock;
use tempfile::TempDir;
use anyhow::Result;
use tempfile::TempDir;
impl BackupService {
pub async fn execute_backup(
&self,
generated_id: String,
@@ -17,11 +16,10 @@ impl BackupService {
storages: Vec<DatabaseStorage>,
encrypt: bool,
) -> Result<()> {
if FileLock::is_locked(&generated_id).await? {
anyhow::bail!("backup already running");
}
let backup = self.create_backup_record(&generated_id, &method).await?;
let backup_id = backup.backup.id;
@@ -46,4 +44,4 @@ impl BackupService {
Ok(())
}
}
}
+2 -4
View File
@@ -1,16 +1,14 @@
use super::service::BackupService;
use crate::services::api::models::agent::backup::BackupResponse;
use crate::utils::common::BackupMethod;
use anyhow::{Result, anyhow};
use crate::services::api::models::agent::backup::BackupResponse;
impl BackupService {
pub async fn create_backup_record(
&self,
generated_id: &str,
method: &BackupMethod,
) -> Result<BackupResponse> {
let response = self
.ctx
.api
@@ -23,4 +21,4 @@ impl BackupService {
response.ok_or_else(|| anyhow!("backup_create returned empty response"))
}
}
}
+6 -6
View File
@@ -1,11 +1,11 @@
pub mod service;
pub mod compressor;
pub mod dispatcher;
pub mod executor;
pub mod compressor;
pub mod uploader;
pub mod result;
pub mod models;
pub mod helpers;
pub mod models;
pub mod result;
pub mod runner;
pub mod service;
pub mod uploader;
pub use service::BackupService;
pub use service::BackupService;
+2 -2
View File
@@ -1,7 +1,7 @@
#![allow(dead_code)]
use std::path::PathBuf;
use crate::services::config::DbType;
use std::path::PathBuf;
#[derive(Debug, Clone)]
pub struct BackupResult {
@@ -19,4 +19,4 @@ pub struct UploadResult {
pub error: Option<String>,
pub remote_file_path: Option<String>,
pub total_size: Option<u64>,
}
}
+3 -9
View File
@@ -9,12 +9,7 @@ use std::path::Path;
use tracing::{error, info};
impl BackupService {
pub async fn run(
cfg: DatabaseConfig,
tmp_path: &Path,
) -> Result<BackupResult> {
pub async fn run(cfg: DatabaseConfig, tmp_path: &Path) -> Result<BackupResult> {
let db = DatabaseFactory::create_for_backup(cfg.clone()).await;
let generated_id = cfg.generated_id.clone();
@@ -40,8 +35,7 @@ impl BackupService {
});
}
match db.backup(tmp_path, Some(false)).await {
match db.backup(tmp_path).await {
Ok(file) => Ok(BackupResult {
generated_id,
db_type,
@@ -67,4 +61,4 @@ impl BackupService {
}),
}
}
}
}
+2 -2
View File
@@ -1,5 +1,5 @@
use std::sync::Arc;
use crate::core::context::Context as CoreContext;
use std::sync::Arc;
pub struct BackupService {
pub ctx: Arc<CoreContext>,
@@ -9,4 +9,4 @@ impl BackupService {
pub fn new(ctx: Arc<CoreContext>) -> Self {
Self { ctx }
}
}
}
+44 -41
View File
@@ -1,16 +1,15 @@
use super::service::BackupService;
use super::models::{BackupResult, UploadResult};
use super::service::BackupService;
use crate::services::storage;
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::storage;
use crate::utils::common::BackupMethod;
use futures::future::join_all;
use anyhow::{Result, bail};
use tracing::{info, error};
use futures::future::join_all;
use tracing::{error, info};
impl BackupService {
pub async fn upload(
&self,
result: BackupResult,
@@ -19,7 +18,6 @@ impl BackupService {
encrypt: bool,
backup_id: &String,
) -> Result<Vec<UploadResult>> {
if result.code.as_deref() == Some("backup_already_in_progress") {
info!("Skipping send: backup already in progress");
bail!("backup_already_in_progress");
@@ -28,7 +26,6 @@ impl BackupService {
let ctx = self.ctx.clone();
let futures = storages.into_iter().map(|storage| {
let ctx_clone = ctx.clone();
let result_clone = result.clone();
let provider = storage::get_provider(&storage);
@@ -37,13 +34,16 @@ impl BackupService {
let generated_id = result_clone.generated_id.clone();
async move {
info!("Uploading storage -> {:?} for {:?}", storage.provider, storage_id);
info!(
"Uploading storage -> {:?} for {:?}",
storage.provider, storage_id
);
/*
INIT STEP
*/
let init = match ctx_clone.api
let init = match ctx_clone
.api
.backup_upload_init(
ctx_clone.edge_key.agent_id.clone(),
generated_id.clone(),
@@ -107,7 +107,11 @@ impl BackupService {
)
.await;
let status = if upload_result.success { "success" } else { "failed" };
let status = if upload_result.success {
"success"
} else {
"failed"
};
if status != "success" {
return upload_result;
@@ -115,49 +119,48 @@ impl BackupService {
info!(
"Storage {} uploaded to remote path {:?}",
storage_id,
upload_result.remote_file_path
storage_id, upload_result.remote_file_path
);
/*
METADATA VALIDATION
*/
let (remote_path, total_size) = match (
&upload_result.remote_file_path,
upload_result.total_size,
) {
(Some(path), Some(size)) => (path.clone(), size),
_ => {
return UploadResult {
storage_id,
success: false,
error: Some("remote_file_path or total_size missing".into()),
remote_file_path: None,
total_size: None,
};
}
};
let (remote_path, total_size) =
match (&upload_result.remote_file_path, upload_result.total_size) {
(Some(path), Some(size)) => (path.clone(), size),
_ => {
return UploadResult {
storage_id,
success: false,
error: Some("remote_file_path or total_size missing".into()),
remote_file_path: None,
total_size: None,
};
}
};
/*
STATUS UPDATE
*/
match ctx_clone.api.backup_upload_status(
ctx_clone.edge_key.agent_id.clone(),
generated_id,
backup_storage_id,
status,
remote_path,
total_size,
backup_id,
).await {
match ctx_clone
.api
.backup_upload_status(
ctx_clone.edge_key.agent_id.clone(),
generated_id,
backup_storage_id,
status,
remote_path,
total_size,
backup_id,
)
.await
{
Ok(_) => upload_result,
Err(err) => {
error!(
"backup_upload_status failed (storage_id={}): {}",
storage_id,
err
storage_id, err
);
UploadResult {
@@ -178,4 +181,4 @@ impl BackupService {
Ok(results)
}
}
}

Some files were not shown because too many files have changed in this diff Show More