Compare commits

...

85 Commits

Author SHA1 Message Date
github-actions[bot] 8bf6503f1f chore: release 1.7.0 2026-03-28 15:47:43 +00:00
Charles GTE fdf35228d4 feat: add-heathcheck-database (#48)
feat: add-heathcheck-database
2026-03-28 16:45:13 +01:00
Charles GTE 1855c5120a fix: docker-compose.yml 2026-03-28 16:44:07 +01:00
Charles GTE 3d5a7eeded feat: add health check ping for databases 2026-03-28 16:07:37 +01:00
Charles GTE 517f3779cd feat: sending database ping result for each databases to portabase server 2026-03-27 23:07:06 +01:00
github-actions[bot] a1e18754b1 chore: release 1.6.3 2026-03-27 21:27:23 +00:00
Charles GTE 725312e1d5 fix: tests (#47)
* fix: s3 port type mismatch, add string_or_number_to_string deserializer

* fix: adding some tests

* fix: adding some tests

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-27 22:24:59 +01:00
github-actions[bot] 8e8ebb5920 chore: release 1.6.2 2026-03-21 07:53:35 +00:00
Charles GTE 8489a6f6e3 fix: adding retry config for s3 provider (#46) 2026-03-21 08:51:15 +01:00
github-actions[bot] 3f5b9c91cc chore: release 1.6.1 2026-03-18 18:42:23 +00:00
Charles GTE 1c52f298d2 fix: Enable Link-Time Optimization (LTO) and codegen-units = 1 for Release builds (#45) 2026-03-18 19:39:49 +01:00
github-actions[bot] e602c3540f chore: release 1.6.0 2026-03-16 13:38:02 +00:00
Charles GTE b5e8ccac1f feat: add-valkey #43
feat: add-valkey
2026-03-16 14:35:31 +01:00
charles-gauthereau f14a6dc318 fix: docker-compose.yml 2026-03-16 14:32:33 +01:00
charles-gauthereau e1bc77e481 fix: valkey.rs 2026-03-16 14:31:11 +01:00
charles-gauthereau cbc7517603 fix: README.md with Valkey content 2026-03-16 14:28:24 +01:00
charles-gauthereau fc4e4e3881 fix: log in backup.rs 2026-03-16 12:32:57 +01:00
charles-gauthereau 5ec7bc9f28 fix: docker-compose.yml volume for valkey 2026-03-16 12:32:12 +01:00
charles-gauthereau 45c33dd031 feat: add valkey test sequences 2026-03-16 12:30:22 +01:00
charles-gauthereau cddb9d78fa fix: entrypoint.sh 2026-03-16 12:16:03 +01:00
charles-gauthereau cf5de3115c feat: add valkey database engine for backup 2026-03-16 12:11:01 +01:00
charles-gauthereau d6d2ef18f8 fix: internal redis port 2026-03-16 11:24:30 +01:00
github-actions[bot] 987c630c73 chore: release 1.5.1 2026-03-16 07:59:25 +00:00
Charles GTE e9147d6c60 fix: s3 port type mismatch, add string_or_number_to_string deserializer (#42)
Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-16 08:57:00 +01:00
github-actions[bot] 174ead72a3 chore: release 1.5.0 2026-03-15 11:06:29 +00:00
Charles GTE f2a67b3490 feat: add-redis #39
feat: add-redis
2026-03-15 12:04:05 +01:00
charlesgauthereau d4afefbc1e fix: requirements.sh for tests tools 2026-03-14 20:08:26 +01:00
charlesgauthereau be6e147df2 fix: requirements.sh for tests tools 2026-03-14 19:54:15 +01:00
charlesgauthereau 91b6658852 fix: requirements.sh for tests tools 2026-03-14 19:40:01 +01:00
charlesgauthereau f7dcb6b2bb fix: requirements.sh for tests tools 2026-03-14 18:26:26 +01:00
charlesgauthereau ab14f0dcf0 fix: requirements.sh for tests tools 2026-03-14 18:17:43 +01:00
charlesgauthereau 58a0b92710 fix: pipeline codecov.yml 2026-03-14 18:05:29 +01:00
charlesgauthereau 2f24291c0e fix: pipeline codecov.yml 2026-03-14 18:02:06 +01:00
charlesgauthereau 1fdea31430 fix: postgres and redis tests 2026-03-14 17:59:44 +01:00
charlesgauthereau d1d633fd00 feat: add redis database support for backup. 2026-03-13 22:44:46 +01:00
Charles GTE b1c14f098b fix: codecov.yml (#38)
* feat: add codecov.yml

* fix: add codecov.yml

* fix: codecov.yml

---------

Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-13 16:58:51 +01:00
github-actions[bot] 3abb955e24 chore: release 1.4.1 2026-03-13 15:19:42 +00:00
Charles GTE 6d22be9b50 fix: add codecov.yml (#37) [skip-release]
Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-13 16:17:09 +01:00
Charles GTE c2df4ba71e fix: readme badge [skip-release] #36
fix: readme badge  [skip-release]
2026-03-13 15:57:16 +01:00
charles-gauthereau 46b3f2466a fix: readme badge 2026-03-13 13:50:30 +01:00
github-actions[bot] a2be03751f chore: release 1.4.0 2026-03-12 18:23:42 +00:00
Charles GTE 7df9df1605 feat : test coverage for utils functions #35
feat : test coverage for utils functions
2026-03-12 19:21:09 +01:00
charlesgauthereau df5b16a153 fix: docker-compose.yml 2026-03-12 19:16:12 +01:00
charlesgauthereau 824a5d52a3 feat: add test for util functions and start working on database providers tests procedures 2026-03-12 18:56:07 +01:00
charlesgauthereau 1496208db4 Merge branch 'main' into dev 2026-03-09 16:46:00 +01:00
charlesgauthereau 454e1b442f Merge remote-tracking branch 'origin/main' 2026-03-09 16:45:52 +01:00
github-actions[bot] 54bca7452a chore: release 1.3.0 2026-03-09 15:43:44 +00:00
Charles GTE a96cef6cb5 feat: helm chart (#34)
* fix: docker.yml

* feat: add helm chart config

* fix: helm config

* feat: add .github workflow for helm chart

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-09 16:41:14 +01:00
charlesgauthereau fd9f183166 Merge branch 'main' into dev 2026-03-08 22:02:36 +01:00
charlesgauthereau 94fb2535ec Merge remote-tracking branch 'origin/main' 2026-03-08 21:59:02 +01:00
github-actions[bot] 189ad866de chore: release 1.2.5 2026-03-08 20:50:53 +00:00
Charles GTE 55bbd4e724 fix: new-workflow-ci (#32)
* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* fix: docker.yml

* fix: docker.yml

* fix: docker.yml

* fix: docker.yml

* fix: docker.yml

* fix: docker.yml

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-08 21:48:13 +01:00
charlesgauthereau 7fabc9c98c Merge remote-tracking branch 'origin/main' 2026-03-08 21:47:15 +01:00
github-actions[bot] 4f67a57681 chore: release 1.2.4 2026-03-08 20:20:58 +00:00
Charles GTE 45a1118f64 fix: docker build (#31)
* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* fix: docker.yml

* fix: docker.yml

* fix: docker.yml

* fix: docker.yml

* fix: docker.yml

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-08 21:18:27 +01:00
charlesgauthereau 063b2e1c2c Merge remote-tracking branch 'origin/main' 2026-03-08 21:12:09 +01:00
github-actions[bot] f2275d5aca chore: release 1.2.3 2026-03-08 20:00:47 +00:00
Charles GTE b136140b55 fix: new-workflow-ci (#30)
* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* fix: docker.yml

* fix: docker.yml

* fix: docker.yml

* fix: docker.yml

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-08 20:58:19 +01:00
charlesgauthereau 924177a4fe Merge remote-tracking branch 'origin/main' 2026-03-08 20:52:36 +01:00
github-actions[bot] 773077f66b chore: release 1.2.2 2026-03-08 19:48:25 +00:00
Charles GTE 46ff086cf9 fix: new-workflow-ci (#29)
* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* fix: docker.yml

* fix: docker.yml

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-08 20:47:41 +01:00
charlesgauthereau 678df8e2bc fix: docker.yml 2026-03-08 20:46:47 +01:00
github-actions[bot] f563c187db chore: release 1.2.1 2026-03-08 19:43:01 +00:00
Charles GTE df0dffa47b fix: new-workflow-ci (#28)
* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-08 20:42:16 +01:00
github-actions[bot] 864396422b chore: release 1.2.0 2026-03-08 19:38:10 +00:00
Charles GTE e280c1d2f8 fix: workflow (#27)
* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-08 20:31:41 +01:00
Charles GTE 91a9d0654b feat: new workflow CI (#26)
* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-08 20:28:26 +01:00
Charles GTE bc857a4c18 feat: new workflow CI
* feat: new workflow CI

* feat: new workflow CI

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-08 20:24:26 +01:00
Charles GTE 2454915866 feat: new workflow CI (#24)
Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-08 20:22:45 +01:00
Charles GTE a4a7a9a2c4 feat: new workflow CI
feat: new workflow CI
2026-03-08 20:12:43 +01:00
charlesgauthereau 997ba2afa2 feat: new workflow CI 2026-03-08 20:12:00 +01:00
charlesgauthereau 7e0c7c0688 Merge branch 'main' into dev
# Conflicts:
#	docker-compose.yml
2026-03-08 19:48:00 +01:00
Charles GTE f973bc1f52 fix: refactoring for backup and restore service
* fix: docker-compose.yml

* fix: refactoring for backup and restore service

* fix: cargo update

* fix: refactoring of the backup and restore services

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-08 19:47:04 +01:00
charlesgauthereau b9b9fcf6ee Merge branch 'main' into dev 2026-03-08 10:02:34 +01:00
charlesgauthereau 0224605c59 fix: docker-compose.yml 2026-03-08 10:02:17 +01:00
Charles GTE 6ea9fdf8bc Merge pull request #21 from Portabase/dev
chore: add coderabbitai config file
2026-03-05 11:21:45 +01:00
Charles GTE ad68be8a5a fix: coderrabbit.yaml 2026-03-05 11:19:47 +01:00
Charles GTE 434c131afb chore: add coderabbitai config file 2026-03-05 11:10:20 +01:00
Charles GTE 675ac5976d Merge pull request #20 from Portabase/dev
chore: add trailing newlines to README.md
2026-03-05 10:59:26 +01:00
Charles GTE 70543fe032 fix: README.md 2026-03-05 10:45:55 +01:00
Charles GTE 77f90308b4 Update README.md 2026-03-02 08:46:32 +01:00
charlesgauthereau 63cb254abd chore: Cargo.lock 2026-02-24 08:45:50 +01:00
charlesgauthereau 7738745def chore(release): 1.1.6 2026-02-24 08:45:26 +01:00
charlesgauthereau 82659f681c feat: add local test db for sqlite 2026-02-23 22:44:57 +01:00
charlesgauthereau eee3bebd50 feat: add local test db for sqlite 2026-02-23 22:43:44 +01:00
159 changed files with 4746 additions and 2141 deletions
+23
View File
@@ -0,0 +1,23 @@
reviews:
auto_review:
enabled: true
ignore_usernames: ["dependabot[bot]"]
labels: ["!wip", "!draft"]
auto_apply_labels: true
suggested_reviewers: true
auto_assign_reviewers: true
issue_enrichment:
auto_enrich:
enabled: true
labeling:
auto_apply_labels: true
labeling_instructions:
- label: bug
instructions: "Error, crash, or incorrect behavior reports"
- label: enhancement
instructions: "Feature requests or improvements"
- label: documentation
instructions: "Docs updates or missing documentation"
- label: good first issue
instructions: "Beginner-friendly tasks suitable for new contributors"
+172
View File
@@ -0,0 +1,172 @@
#name: Codecov Rust
#
#on:
# push:
# branches: ["main"]
# pull_request:
# branches: ["main"]
#
#env:
# CARGO_TERM_COLOR: always
#
#jobs:
# coverage:
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v4
#
# - name: Install Rust toolchain
# uses: dtolnay/rust-toolchain@stable
# with:
# components: llvm-tools-preview
#
# - name: Build test image (with grcov included)
# run: docker compose -f docker-compose.test.yml build agent-test
#
# - name: Run tests in container
# env:
# CARGO_TARGET_DIR: /app/target
# CARGO_INCREMENTAL: 0
# RUSTFLAGS: "-C instrument-coverage -C link-dead-code"
# LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw"
# run: |
# docker compose -f docker-compose.test.yml run \
# -e CARGO_TARGET_DIR \
# -e CARGO_INCREMENTAL \
# -e RUSTFLAGS \
# -e LLVM_PROFILE_FILE \
# agent-test bash -c "cargo clean && cargo test --verbose && sync"
#
# - name: Verify profraw files exist
# run: |
# docker compose -f docker-compose.test.yml run agent-test \
# find /app/coverage -type f -name "*.profraw" | wc -l || true
#
# - name: Generate coverage report inside container
# run: |
# docker compose -f docker-compose.test.yml run agent-test bash -c "
# rustup component add llvm-tools &&
# grcov /app/coverage \
# --binary-path /app/target/debug \
# -s /app \
# --llvm \
# -t lcov \
# --branch \
# --ignore-not-existing \
# --ignore '/app/target/*' \
# --ignore '/*' \
# -o /app/lcov.info
# "
#
# - name: Copy lcov.info from container to host
# run: |
# docker compose -f docker-compose.test.yml cp agent-test:/app/lcov.info ./lcov.info
#
# - name: Remove container
# run: |
# docker rm agent-test-run
#
# - name: Show basic coverage report info (debug)
# run: |
# echo "lcov.info size:" $(wc -c ./lcov.info | awk '{print $1}')
# head -n 30 ./lcov.info || true
#
# - name: Upload coverage to Codecov
# uses: codecov/codecov-action@v5
# with:
# files: ./lcov.info
# flags: unittests
# name: rust-unit-coverage
# verbose: true
# fail_ci_if_error: true
# env:
# CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
name: Codecov Rust
on:
push:
branches: ["main"]
pull_request:
branches: ["main"]
env:
CARGO_TERM_COLOR: always
CARGO_TARGET_DIR: /app/target
CARGO_INCREMENTAL: 0
RUSTFLAGS: "-C instrument-coverage -C link-dead-code"
LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw"
jobs:
coverage:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
components: llvm-tools-preview
- name: Build test image (with grcov included)
run: docker compose -f docker-compose.test.yml build agent-test
- name: Start agent-test container
run: docker compose -f docker-compose.test.yml up -d agent-test
- name: Run tests inside container
run: |
docker compose -f docker-compose.test.yml exec \
-e CARGO_TARGET_DIR \
-e CARGO_INCREMENTAL \
-e RUSTFLAGS \
-e LLVM_PROFILE_FILE \
agent-test bash -c "
cargo clean &&
cargo test --verbose &&
sync
"
- name: Verify profraw files exist
run: |
docker compose -f docker-compose.test.yml exec \
-e LLVM_PROFILE_FILE \
agent-test find /app/coverage -type f -name "*.profraw" | wc -l || true
- name: Generate coverage report inside container
run: |
docker compose -f docker-compose.test.yml exec \
agent-test bash -c "
rustup component add llvm-tools &&
grcov /app/coverage \
--binary-path /app/target/debug \
-s /app \
--llvm \
-t lcov \
--branch \
--ignore-not-existing \
--ignore '/app/target/*' \
--ignore '/*' \
-o /app/lcov.info
"
- name: Copy lcov.info from container to host
run: docker compose -f docker-compose.test.yml cp agent-test:/app/lcov.info ./lcov.info
- name: Show basic coverage report info (debug)
run: |
echo "lcov.info size:" $(wc -c ./lcov.info | awk '{print $1}')
head -n 30 ./lcov.info || true
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v5
with:
files: ./lcov.info
flags: unittests
name: rust-unit-coverage
verbose: true
fail_ci_if_error: true
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
- name: Stop and remove container
run: docker compose -f docker-compose.test.yml down
+73
View File
@@ -0,0 +1,73 @@
name: Discord Notification
on:
workflow_call:
inputs:
release_tag:
required: true
type: string
discord_title:
required: true
type: string
discord_color:
required: true
type: number
discord_footer:
required: true
type: string
secrets:
DISCORD_WEBHOOK:
required: true
GH_TOKEN:
required: true
jobs:
notify-discord:
runs-on: ubuntu-latest
steps:
- name: Send Discord Notification
env:
DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
GH_TOKEN: ${{ secrets.GH_TOKEN }}
run: |
RELEASE_INFO=$(gh release view "${{ inputs.release_tag }}" -R ${{ github.repository }} --json name,url,body,author)
RELEASE_TITLE=$(echo "$RELEASE_INFO" | jq -r .name)
if [ -z "$RELEASE_TITLE" ] || [ "$RELEASE_TITLE" = "null" ]; then RELEASE_TITLE="${{ inputs.release_tag }}"; fi
RELEASE_URL=$(echo "$RELEASE_INFO" | jq -r .url)
RELEASE_BODY=$(echo "$RELEASE_INFO" | jq -r .body)
AUTHOR_NAME="Portabase"
AUTHOR_ICON="https://github.com/Portabase.png"
PAYLOAD=$(jq -n \
--arg title "$RELEASE_TITLE" \
--arg description "$RELEASE_BODY" \
--arg url "$RELEASE_URL" \
--arg author "$AUTHOR_NAME" \
--arg icon "$AUTHOR_ICON" \
--arg discord_title "${{ inputs.discord_title }}" \
--arg discord_footer "${{ inputs.discord_footer }}" \
--argjson discord_color ${{ inputs.discord_color }} \
'{
content: $discord_title,
embeds: [{
title: $title,
url: $url,
description: $description,
color: $discord_color,
author: {
name: $author,
icon_url: $icon
},
footer: {
text: $discord_footer
}
}]
}'
)
curl -H "Content-Type: application/json" \
-d "$PAYLOAD" \
"$DISCORD_WEBHOOK"
+67 -45
View File
@@ -3,10 +3,16 @@ name: Docker Publish
on:
workflow_call:
inputs:
version:
required: true
type: string
ref:
required: true
type: string
image_name:
required: false
type: string
default: 'portabase/agent'
default: "portabase/agent"
add_latest:
required: false
type: boolean
@@ -14,11 +20,11 @@ on:
target:
required: false
type: string
default: 'prod'
default: "prod"
dockerfile:
required: false
type: string
default: './docker/Dockerfile'
default: "./docker/Dockerfile"
secrets:
DOCKER_USERNAME:
required: true
@@ -27,32 +33,34 @@ on:
jobs:
build:
name: Build and push Docker images
runs-on: ${{ matrix.platform == 'linux/amd64' && 'ubuntu-latest' || matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' }}
name: Build architectures
runs-on: ${{ matrix.platform == 'linux/amd64' && 'ubuntu-latest' || 'ubuntu-24.04-arm' }}
strategy:
fail-fast: false
matrix:
platform: [linux/amd64, linux/arm64]
platform: [ linux/amd64, linux/arm64 ]
steps:
- name: Checkout
uses: actions/checkout@v4
- uses: actions/checkout@v4
with:
ref: ${{ inputs.ref }}
fetch-depth: 0
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Set image tag
id: set-tags
- name: Prepare Image Tags
id: prep
run: |
REF_NAME=${GITHUB_REF#refs/tags/}
if [ "${{ matrix.platform }}" = "linux/amd64" ]; then
IMAGE="${{ inputs.image_name }}:$REF_NAME-amd64"
else
IMAGE="${{ inputs.image_name }}:$REF_NAME-arm64"
fi
echo "image=$IMAGE" >> $GITHUB_OUTPUT
ARCH=${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
echo "image=${{ inputs.image_name }}:${{inputs.version}}-$ARCH" >> $GITHUB_OUTPUT
echo "safe_platform=${{ matrix.platform == 'linux/amd64' && 'linux-amd64' || 'linux-arm64' }}" >> $GITHUB_OUTPUT
- name: Build and push image
uses: docker/build-push-action@v6
@@ -61,57 +69,71 @@ jobs:
file: ${{ inputs.dockerfile }}
platforms: ${{ matrix.platform }}
push: true
tags: ${{ steps.set-tags.outputs.image }}
tags: ${{ steps.prep.outputs.image }}
target: ${{ inputs.target }}
- name: Prepare artifact name
id: artifact
run: |
platform=${{ matrix.platform }}
echo "safe_platform=${platform//\//-}" >> $GITHUB_OUTPUT
echo "${{ steps.set-tags.outputs.image }}" > image.txt
- name: Save image name for manifest
run: echo "${{ steps.prep.outputs.image }}" > image.txt
- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
- uses: actions/upload-artifact@v4
with:
name: image-${{ steps.artifact.outputs.safe_platform }}
name: image-${{ steps.prep.outputs.safe_platform }}
path: image.txt
if-no-files-found: warn
compression-level: 6
overwrite: false
include-hidden-files: false
retention-days: 1
create-manifest:
name: Create multi-arch Docker manifest
name: Create multi-arch manifest
runs-on: ubuntu-latest
needs: build
steps:
- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131
- uses: actions/download-artifact@v4
with:
name: image-linux-amd64
path: /tmp/digests/amd64
- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131
- uses: actions/download-artifact@v4
with:
name: image-linux-arm64
path: /tmp/digests/arm64
- name: Login to Docker
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Generate semantic tags
id: tags
run: |
VERSION="${{ inputs.version }}"
IFS='.' read -r MAJOR MINOR PATCH <<< "$VERSION"
echo "VERSION_TAG=$VERSION" >> $GITHUB_OUTPUT
if [ -n "$MINOR" ]; then
echo "MINOR_TAG=$MAJOR.$MINOR" >> $GITHUB_OUTPUT
fi
if [ -n "$MAJOR" ]; then
echo "MAJOR_TAG=$MAJOR" >> $GITHUB_OUTPUT
fi
if [ "${{ inputs.add_latest }}" = "true" ]; then
echo "LATEST_TAG=latest" >> $GITHUB_OUTPUT
fi
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ inputs.image_name }}
tags: |
type=raw,value=${{ steps.tags.outputs.VERSION_TAG }}
type=raw,value=${{ steps.tags.outputs.MINOR_TAG }}
type=raw,value=${{ steps.tags.outputs.MAJOR_TAG }}
type=raw,value=${{ steps.tags.outputs.LATEST_TAG }}
- name: Create and push manifest list
working-directory: /tmp/digests
run: |
DOCKER_IMAGES="$(cat amd64/image.txt) $(cat arm64/image.txt)"
REF_NAME=${GITHUB_REF#refs/tags/}
MANIFEST_IMAGE="${{ inputs.image_name }}:$REF_NAME"
docker buildx imagetools create $DOCKER_IMAGES -t $MANIFEST_IMAGE
docker buildx imagetools inspect $MANIFEST_IMAGE
if [ "${{ inputs.add_latest }}" = "true" ]; then
docker buildx imagetools create $DOCKER_IMAGES -t ${{ inputs.image_name }}:latest
docker buildx imagetools inspect ${{ inputs.image_name }}:latest
fi
TAG_ARGS=$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON")
echo $TAG_ARGS
docker buildx imagetools create $TAG_ARGS $DOCKER_IMAGES
-128
View File
@@ -1,128 +0,0 @@
name: GitHub Release
on:
workflow_call:
inputs:
prerelease:
required: false
type: boolean
default: false
make_latest:
required: false
type: boolean
default: false
discord_title:
required: true
type: string
discord_color:
required: true
type: number
discord_footer:
required: true
type: string
secrets:
DISCORD_WEBHOOK:
required: true
GH_TOKEN:
required: true
jobs:
create-release:
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Check out the repo
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Build Changelog
id: build_changelog
uses: mikepenz/release-changelog-builder-action@v5
with:
mode: "COMMIT"
configurationJson: |
{
"template": "#{{CHANGELOG}}",
"categories": [
{
"title": "## Feature",
"labels": ["feat", "feature"]
},
{
"title": "## Fix",
"labels": ["fix", "bug"]
},
{
"title": "## Other",
"labels": []
}
],
"label_extractor": [
{
"pattern": "^(build|chore|ci|docs|feat|fix|perf|refactor|revert|style|test){1}(\([\w\-\.]+\))?(!)?: ([\w ])+([\s\S]*)",
"on_property": "title",
"target": "$1"
}
]
}
env:
GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
- name: Create GitHub Release
uses: softprops/action-gh-release@v2
with:
generate_release_notes: false
body: ${{ steps.build_changelog.outputs.changelog }}
prerelease: ${{ inputs.prerelease }}
make_latest: ${{ inputs.make_latest }}
env:
GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
- name: Send Discord Notification
env:
DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
GH_TOKEN: ${{ secrets.GH_TOKEN }}
run: |
RELEASE_INFO=$(gh release view "${{ github.ref_name }}" -R ${{ github.repository }} --json name,url,body,author)
RELEASE_TITLE=$(echo "$RELEASE_INFO" | jq -r .name)
if [ -z "$RELEASE_TITLE" ] || [ "$RELEASE_TITLE" = "null" ]; then RELEASE_TITLE="${{ github.ref_name }}"; fi
RELEASE_URL=$(echo "$RELEASE_INFO" | jq -r .url)
RELEASE_BODY=$(echo "$RELEASE_INFO" | jq -r .body)
AUTHOR_NAME="Portabase"
AUTHOR_ICON="https://github.com/Portabase.png"
PAYLOAD=$(jq -n \
--arg title "$RELEASE_TITLE" \
--arg description "$RELEASE_BODY" \
--arg url "$RELEASE_URL" \
--arg author "$AUTHOR_NAME" \
--arg icon "$AUTHOR_ICON" \
--arg discord_title "${{ inputs.discord_title }}" \
--arg discord_footer "${{ inputs.discord_footer }}" \
--argjson discord_color ${{ inputs.discord_color }} \
'{
content: $discord_title,
embeds: [{
title: $title,
url: $url,
description: $description,
color: $discord_color,
author: {
name: $author,
icon_url: $icon
},
footer: {
text: $discord_footer
}
}]
}'
)
curl -H "Content-Type: application/json" \
-d "$PAYLOAD" \
"$DISCORD_WEBHOOK"
+37
View File
@@ -0,0 +1,37 @@
name: Publish Helm Chart
on:
workflow_call:
inputs:
version:
required: true
type: string
secrets:
GH_TOKEN:
required: true
jobs:
publish-helm:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Helm
uses: azure/setup-helm@v4
- name: Package Helm chart
run: |
mkdir -p ./helm-packages
helm package helm \
--version ${{ inputs.version }} \
--app-version ${{ inputs.version }} \
--destination ./helm-packages
- name: Authenticate to GitHub Packages
run: |
echo "${{ secrets.GH_TOKEN }}" | helm registry login ghcr.io -u ${{ github.actor }} --password-stdin
- name: Push Helm chart to GitHub Packages (OCI)
run: |
helm push ./helm-packages/portabase-agent-${{ inputs.version }}.tgz oci://ghcr.io/portabase/charts
+128 -12
View File
@@ -1,32 +1,148 @@
name: Publish Docker image for release
name: Auto Release & Publish
on:
push:
tags:
- '*.*.*'
- '!*-*'
pull_request:
types: [ closed ]
branches:
- main
permissions:
contents: write
packages: write
jobs:
docker_publish:
check-skip:
runs-on: ubuntu-latest
outputs:
skip: ${{ steps.set-skip.outputs.skip }}
steps:
- name: Determine if release should be skipped
id: set-skip
run: |
TITLE="${{ github.event.pull_request.title }}"
echo "PR title: $TITLE"
if [[ "$TITLE" == *"[skip-release]"* ]]; then
echo "PR title contains [skip-release], skipping release jobs."
echo "skip=true" >> $GITHUB_OUTPUT
else
echo "skip=false" >> $GITHUB_OUTPUT
fi
create-release:
needs: check-skip
if: ${{ needs.check-skip.outputs.skip == 'false' }}
runs-on: ubuntu-latest
outputs:
draft_tag: ${{ steps.release_step.outputs.draft_tag }}
version: ${{ steps.release_step.outputs.version }}
steps:
- uses: actions/create-github-app-token@v1
id: app-token
with:
app-id: ${{ vars.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
- uses: actions/checkout@v4
with:
fetch-depth: 0
token: ${{ steps.app-token.outputs.token }}
ref: main
- name: Setup Rust
uses: dtolnay/rust-toolchain@stable
- name: Install cargo-edit
run: cargo install cargo-edit
- uses: actions/setup-node@v4
with:
node-version: "lts/*"
- name: Install release-it globally
run: |
npm install -g release-it
npm install -g @release-it/conventional-changelog
npm install -g @release-it/bumper
- run: |
git config --global user.name 'github-actions[bot]'
git config --global user.email 'github-actions[bot]@users.noreply.github.com'
- name: Run release-it
id: release_step
run: |
git pull origin main
VERSION=$(release-it --ci --release-version)
echo $VERSION
echo "version=$VERSION" >> $GITHUB_OUTPUT
OUTPUT=$(release-it --ci)
echo "$OUTPUT"
DRAFT_TAG=$(echo "$OUTPUT" | grep -oE 'untagged-[a-z0-9]+')
echo $DRAFT_TAG
echo "draft_tag=$DRAFT_TAG" >> $GITHUB_OUTPUT
env:
GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}
publish-docker:
needs: create-release
if: ${{ needs.create-release.result == 'success' }}
uses: ./.github/workflows/docker.yml
with:
version: ${{ needs.create-release.outputs.version }}
ref: ${{ needs.create-release.outputs.version }}
add_latest: true
secrets:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
github_release:
needs: docker_publish
uses: ./.github/workflows/github.yml
publish-helm:
needs: create-release
if: ${{ needs.create-release.result == 'success' }}
uses: ./.github/workflows/helm.yml
with:
prerelease: false
make_latest: true
version: ${{ needs.create-release.outputs.version }}
secrets:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
finalize-release:
needs:
- create-release
- publish-docker
- publish-helm
runs-on: ubuntu-latest
outputs:
release_tag: ${{ steps.publish_release_step.outputs.release_tag }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Publish GitHub Release
id: publish_release_step
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
OUTPUT=$(gh release edit ${{ needs.create-release.outputs.draft_tag }} --draft=false)
echo "$OUTPUT"
RELEASE_TAG=$(echo "$OUTPUT" | sed -E 's|.*/releases/tag/||')
echo "release_tag=$RELEASE_TAG" >> $GITHUB_OUTPUT
notify-discord:
needs:
- publish-docker
- publish-helm
- create-release
- finalize-release
uses: ./.github/workflows/discord.yml
with:
release_tag: ${{ needs.create-release.outputs.version }}
discord_title: "||@release-agent|| New release published"
discord_color: 5814783
discord_color: 3066993
discord_footer: "Portabase"
secrets:
DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
+62
View File
@@ -0,0 +1,62 @@
{
"github": {
"release": true,
"draft": true,
"tokenRef": "GITHUB_TOKEN"
},
"git": {
"commit": true,
"commitMessage": "chore: release ${version}",
"requireCleanWorkingDir": true,
"tag": true,
"tagName": "${version}",
"push": true
},
"hooks": {
"before:bump": "cargo set-version ${version}"
},
"plugins": {
"@release-it/conventional-changelog": {
"preset": {
"name": "conventionalcommits",
"types": [
{
"type": "feat",
"section": "✨ Features"
},
{
"type": "fix",
"section": "🐛 Bug Fixes"
},
{
"type": "perf",
"section": "⚡️ Performance Improvements"
},
{
"type": "revert",
"section": "⏪️ Reverts"
},
{
"type": "docs",
"section": "📝 Documentation",
"hidden": true
},
{
"type": "chore",
"section": "🔧 Chores",
"hidden": true
}
]
}
},
"@release-it/bumper": {
"out": [
{
"file": "CITATION.cff",
"path": "version",
"type": "text/yaml"
}
]
}
}
}
+9 -4
View File
@@ -1,6 +1,6 @@
cff-version: 1.2.0
title: Portabase Agent (Rust)
message: "If you use this software, please cite it as below."
message: If you use this software, please cite it as below.
type: software
authors:
- family-names: Gauthereau
@@ -9,7 +9,12 @@ authors:
given-names: Killian
repository-code: https://github.com/Portabase/agent-rust
url: https://portabase.io
abstract: "Portabase is a free, open-source, self-hosted solution for database administration, providing backup and restore capabilities, scheduling, retention policies, notifications, and support for multiple storage backends. Its headless agent architecture enables connection to multiple database instances securely and efficiently."
abstract: >-
Portabase is a free, open-source, self-hosted solution for database
administration, providing backup and restore capabilities, scheduling,
retention policies, notifications, and support for multiple storage backends.
Its headless agent architecture enables connection to multiple database
instances securely and efficiently.
keywords:
- database
- administration
@@ -22,5 +27,5 @@ keywords:
- self-hosted
- portabase
license: Apache-2.0
version: 1.1.6-rc.1
date-released: "2026-02-23"
version: 1.7.0
date-released: '2026-02-24'
Generated
+1144 -907
View File
File diff suppressed because it is too large Load Diff
+17 -3
View File
@@ -1,6 +1,6 @@
[package]
name = "portabase-agent"
version = "1.1.6-rc.1"
version = "1.7.0"
edition = "2024"
[dependencies]
@@ -30,7 +30,7 @@ tokio-postgres = "0.7.15"
futures = "0.3.31"
tracing-appender = "0.2.4"
time = { version = "0.3.44", features = ["macros"] }
mongodb = "3.5.0"
mongodb = "3.5.1"
rand = "0.9.2"
bytes = "1.11.0"
async-stream = "0.3.6"
@@ -42,14 +42,28 @@ async-compression = { version = "0.4.37", features = ["tokio", "gzip"] }
tokio-tar = "0.3.1"
oauth2 = "5.0.0"
hyper = "1.8.1"
async-http-client = "0.2.0"
aes-gcm = "0.11.0-rc.3"
generic-array = "0.14.7"
futures-util = "0.3.31"
tokio-stream = "0.1.18"
aes = "0.9.0-rc.4"
typenum = "1.19.0"
testcontainers = "0.27.1"
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "valkey", "mysql", "mariadb", "mongo"] }
postgres = "0.19.12"
url = "2.5.8"
[dev-dependencies]
tokio = { version = "1", features = ["full"] }
mockall = "0.13"
testcontainers = "0.27.1"
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "mysql", "mariadb", "mongo"] }
wiremock = "0.6"
[profile.release]
lto = true
codegen-units = 1
strip = "debuginfo"
[[bin]]
name = "app"
+4
View File
@@ -1,8 +1,12 @@
include .env
export $(shell sed 's/=.*//' .env)
CLUSTER_SCRIPT=docker/entrypoints/app-dev-entrypoint.sh
.PHONY: seed-mongo seed-mysql seed-postgres
up:
@bash $(CLUSTER_SCRIPT)
seed-mongo:
@echo "Seeding MongoDB..."
bash ./scripts/mongo/seed-mongo.sh
+5 -1
View File
@@ -16,9 +16,11 @@
[![Platform](https://img.shields.io/badge/platform-linux%20%7C%20macos%20%7C%20windows-lightgrey)](https://github.com/Portabase/portabase)
[![Support Portabase](https://img.shields.io/badge/Support-Portabase-orange)](https://www.buymeacoffee.com/portabase)
[![PostgreSQL](https://img.shields.io/badge/python-3.13-blue.svg)](https://www.python.org/downloads/release/python-3120/)
[![PostgreSQL](https://img.shields.io/badge/PostgreSQL-336791?logo=postgresql&logoColor=white)](https://www.postgresql.org/)
[![MySQL](https://img.shields.io/badge/MySQL-4479A1?logo=mysql&logoColor=white)](https://www.mysql.com/)
[![SQLite](https://img.shields.io/badge/-SQLite-blue?logo=sqlite&logoColor=white)](https://sqlite.org/)
[![Redis](https://img.shields.io/badge/Redis-DC382D?style=flat&logo=Redis&logoColor=white)](https://redis.io/)
[![Valkey](https://img.shields.io/badge/Valkey-6284fc?style=flat&logo=Valkey&logoColor=white)](https://valkey.io/)
[![MariaDB](https://img.shields.io/badge/MariaDB-003545?logo=mariadb&logoColor=white)](https://mariadb.org/)
[![MongoDB](https://img.shields.io/badge/-MongoDB-13aa52?logo=mongodb&logoColor=white)](https://www.mongodb.com/)
[![Self Hosted](https://img.shields.io/badge/self--hosted-yes-brightgreen)](https://github.com/Portabase/portabase)
@@ -79,3 +81,5 @@ Distributed under the Apache License. See `LICENSE.txt` for more information.
[Docker-url]: https://www.docker.com/
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
+38
View File
@@ -43,6 +43,44 @@
"type": "sqlite",
"path": "/sqlite-data/workspace/data/app.db",
"generated_id": "16678178-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 7 - SQLite DB",
"type": "sqlite",
"path": "/sqlite-data-2/workspace/data/app.db",
"generated_id": "16678179-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 8 - Redis",
"type": "redis",
"port": 6379,
"host": "db-redis",
"generated_id": "16678166-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 9 - Redis Auth",
"type": "redis",
"password": "supersecurepassword",
"port": 6379,
"username": "default",
"host": "db-redis-auth",
"generated_id": "16678160-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 10 - Valkey",
"type": "valkey",
"port": 6379,
"host": "db-valkey",
"generated_id": "16678560-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 11 - Valkey Auth",
"type": "valkey",
"password": "supersecurepassword",
"port": 6379,
"username": "default",
"host": "db-valkey-auth",
"generated_id": "16678561-ff7e-4c97-8c83-0adeff214681"
}
]
}
+2 -6
View File
@@ -5,23 +5,20 @@ services:
# dockerfile: docker/Dockerfile
# target: prod
image: portabase/agent:latest
# platform: linux/arm64
container_name: rust-prod
volumes:
- ./databases.json:/config/config.json
environment:
# APP_ENV: production
LOG: info
TZ: "Europe/Paris"
# DATABASES_CONFIG_FILE: "config.toml"
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiOGZmMDE4NTQtYjJhMS00ZTE0LTkwMjctZTJiOWIxZjQ1YzdlIiwicHVibGljS2V5IjoiLS0tLS1CRUdJTiBSU0EgUFVCTElDIEtFWS0tLS0tXG5NSUlCQ2dLQ0FRRUE5TWV4M2pmdnVLdFB5YU1ERnh2Ulp2dmd3YkRJQ2JzQi81Wll5NDNSVVRBaXZRYjJiSDdYXG5qRHBQd1lJeCs4UFBrbHlRbDVMQzV1UWZEaCs4SVd4OG1LZ3FvMXpWMkdiZXdGbEdEWFYxVEdyU1ZEU25aSWR4XG52bWdYc29EeXhVMlJvWUFUMS9YMWxuc2YxenZKdkFMTkhXdEhRdk42SjVDZTFSMmFsendVRGFEVXlJNzRmSldQXG5tNTh0SDMrYklXL0VVTXdjaWNxM0oySWw3Vm9KNkZNUHJQL1ZSOWEvdFF1SU1qa200MXpFY2NscExPa2luRkxuXG54NmVUWkFSZUpya2UrbnRvZ2t4TGEyRWV5a1lUNzB4V3hKNWp5ZExBVnRvNkkyQlVLVVJoTkowTUFaU29NYUtvXG5iMGJRcnY1UzExZWllMnMrT2I3aTYzSFpkVUx0UmV1MVJ3SURBUUFCXG4tLS0tLUVORCBSU0EgUFVCTElDIEtFWS0tLS0tXG4ifQ"
#DATABASES_CONFIG_FILE: "config.toml"
EDGE_KEY: "<to complete>"
extra_hosts:
- "localhost:host-gateway"
networks:
- portabase
db-mongodb-auth:
container_name: db-mongodb-auth
image: mongo:latest
@@ -63,7 +60,6 @@ volumes:
mongodb-data:
mongodb-data-auth:
networks:
portabase:
name: portabase_network
+22
View File
@@ -0,0 +1,22 @@
services:
agent-test:
build:
context: .
dockerfile: docker/Dockerfile
target: dev
working_dir: /app
container_name: agent-test
volumes:
- .:/app
- cargo-registry:/usr/local/cargo/registry
- cargo-git:/usr/local/cargo/git
- /var/run/docker.sock:/var/run/docker.sock
environment:
APP_ENV: test
LOG: debug
TZ: Europe/Paris
EDGE_KEY: ""
volumes:
cargo-registry:
cargo-git:
+112 -60
View File
@@ -11,13 +11,16 @@ services:
# - ./databases.toml:/config/config.toml
- cargo-registry:/usr/local/cargo/registry
- cargo-git:/usr/local/cargo/git
# - cargo-target:/app/target
- sqlite-data:/sqlite-data/workspace/data
- /var/run/docker.sock:/var/run/docker.sock
# - cargo-target:/app/target
# - sqlite-data:/sqlite-data/workspace/data
# - ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data
environment:
APP_ENV: development
LOG: debug
TZ: "Europe/Paris"
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiZjg4Y2E0MDMtNDgwOS00NGM4LTlkZjItY2VkNWYwYzhkNTM2IiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ=="
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiNGI1OTM2MGItNTNkMi00ZTZmLWE1ODctODcyMmQ1NDc1MTNmIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ=="
#POOLING: 1
#DATABASES_CONFIG_FILE: "config.toml"
extra_hosts:
@@ -38,85 +41,134 @@ services:
- POSTGRES_PASSWORD=changeme
networks:
- portabase
#
# db-mariadb:
# container_name: db-mariadb
# image: mariadb:latest
# ports:
# - "3311:3306"
# environment:
# - MYSQL_DATABASE=mariadb
# - MYSQL_USER=mariadb
# - MYSQL_PASSWORD=changeme
# - MYSQL_RANDOM_ROOT_PASSWORD=yes
# volumes:
# - mariadb-data:/var/lib/mysql
# networks:
# - portabase
#
#
# db-mongodb-auth:
# container_name: db-mongodb-auth
# image: mongo:latest
# ports:
# - "27082:27017"
# environment:
# MONGO_INITDB_ROOT_USERNAME: root
# MONGO_INITDB_ROOT_PASSWORD: rootpassword
# MONGO_INITDB_DATABASE: testdbauth
# command: mongod --auth
# networks:
# - portabase
# volumes:
# - mongodb-data-auth:/data/db
# healthcheck:
# test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ]
# interval: 5s
# timeout: 5s
# retries: 10
#
# db-mongodb:
# container_name: db-mongodb
# image: mongo:latest
# ports:
# - "27083:27017"
# volumes:
# - mongodb-data:/data/db
# healthcheck:
# test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ]
# interval: 5s
# timeout: 5s
# retries: 10
# environment:
# MONGO_INITDB_DATABASE: testdb
# networks:
# - portabase
db-mariadb:
container_name: db-mariadb
image: mariadb:latest
# sqlite:
# container_name: db-sqlite
# image: keinos/sqlite3
# volumes:
# - sqlite-data:/workspace/data
# working_dir: /workspace
# command: tail -f /dev/null
# stdin_open: true
# tty: true
db-redis:
image: redis:latest
container_name: db-redis
ports:
- "3311:3306"
environment:
- MYSQL_DATABASE=mariadb
- MYSQL_USER=mariadb
- MYSQL_PASSWORD=changeme
- MYSQL_RANDOM_ROOT_PASSWORD=yes
- "6379:6379"
volumes:
- mariadb-data:/var/lib/mysql
- redis-data:/data
command: [ "redis-server", "--appendonly", "yes" ]
networks:
- portabase
db-mongodb-auth:
container_name: db-mongodb-auth
image: mongo:latest
db-redis-auth:
image: redis:latest
container_name: db-redis-auth
ports:
- "27082:27017"
environment:
MONGO_INITDB_ROOT_USERNAME: root
MONGO_INITDB_ROOT_PASSWORD: rootpassword
MONGO_INITDB_DATABASE: testdbauth
command: mongod --auth
networks:
- portabase
- "6380:6379"
volumes:
- mongodb-data-auth:/data/db
healthcheck:
test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ]
interval: 5s
timeout: 5s
retries: 10
db-mongodb:
container_name: db-mongodb
image: mongo:latest
ports:
- "27083:27017"
volumes:
- mongodb-data:/data/db
healthcheck:
test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ]
interval: 5s
timeout: 5s
retries: 10
- redis-data-auth:/data
environment:
MONGO_INITDB_DATABASE: testdb
- REDIS_PASSWORD=supersecurepassword
command: [ "redis-server", "--requirepass", "supersecurepassword", "--appendonly", "yes" ]
networks:
- portabase
sqlite:
container_name: db-sqlite
image: keinos/sqlite3
db-valkey:
image: valkey/valkey
container_name: db-valkey
environment:
- ALLOW_EMPTY_PASSWORD=yes
ports:
- '6381:6379'
volumes:
- sqlite-data:/workspace/data
working_dir: /workspace
command: tail -f /dev/null
stdin_open: true
tty: true
- valkey-data:/data
networks:
- portabase
db-valkey-auth:
image: valkey/valkey
container_name: db-valkey-auth
command: >
--requirepass "supersecurepassword"
ports:
- '6382:6379'
volumes:
- valkey-data-auth:/data
networks:
- portabase
volumes:
cargo-registry:
cargo-git:
# cargo-target:
# cargo-target:
postgres-data:
mariadb-data:
mongodb-data:
mongodb-data-auth:
sqlite-data:
# mariadb-data:
# mongodb-data:
# mongodb-data-auth:
# sqlite-data:
redis-data:
redis-data-auth:
valkey-data:
valkey-data-auth:
networks:
portabase:
name: portabase_network
external: true
# docker network create portabase_network
+6 -1
View File
@@ -1,7 +1,7 @@
# =========================
# Base image (shared)
# =========================
FROM rust:1.92.0 AS base
FROM rust:1.94.0 AS base
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
pkg-config \
@@ -16,6 +16,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
curl \
mariadb-client \
sqlite3 \
redis-tools \
valkey \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
@@ -58,6 +60,7 @@ WORKDIR /app
FROM base AS dev
RUN cargo install cargo-watch
RUN cargo install grcov --locked
COPY Cargo.toml Cargo.lock ./
RUN mkdir src && echo "fn main() {}" > src/main.rs
@@ -94,6 +97,8 @@ RUN apt-get update && apt-get install -y \
zlib1g \
mariadb-client \
sqlite3 \
redis-tools \
valkey \
&& rm -rf /var/lib/apt/lists/*
+39
View File
@@ -0,0 +1,39 @@
#!/bin/bash
set -euo pipefail
check_docker() {
if ! docker info > /dev/null 2>&1; then
echo "Docker is not running. Attempting to start Docker..."
if [[ "$OSTYPE" == "darwin"* ]]; then
open -a Docker
echo "Waiting for Docker to start..."
until docker info > /dev/null 2>&1; do
sleep 2
done
elif command -v systemctl >/dev/null 2>&1; then
sudo systemctl start docker
else
echo "Cannot start Docker automatically. Please start Docker manually."
exit 1
fi
else
echo "Docker is running."
fi
}
check_network() {
local network_name="portabase_network"
if ! docker network ls --format '{{.Name}}' | grep -q "^${network_name}$"; then
echo "Docker network '${network_name}' not found. Creating..."
docker network create "${network_name}"
else
echo "Docker network '${network_name}' already exists."
fi
}
check_docker
check_network
echo "Starting docker-compose..."
docker compose -f ./docker-compose.yml up
echo "Docker-compose started successfully."
+10 -3
View File
@@ -34,16 +34,23 @@ if [ -n "$TZ" ]; then
fi
fi
REDIS_PORT=65515
echo "[entrypoint] APP_ENV=$APP_ENV"
echo "[entrypoint] Starting Redis..."
redis-server --daemonize yes
redis-server --port $REDIS_PORT --daemonize yes
echo "[entrypoint] Waiting for Redis to be ready..."
until redis-cli ping >/dev/null 2>&1; do
MAX_RETRIES=20
COUNT=0
until redis-cli -h localhost -p "$REDIS_PORT" ping >/dev/null 2>&1 ; do
COUNT=$((COUNT+1))
if [ $COUNT -ge $MAX_RETRIES ]; then
echo "[ERROR] Redis did not start after $MAX_RETRIES attempts"
exit 1
fi
echo "[entrypoint] Redis not ready, sleeping 1s..."
sleep 1
done
echo "[entrypoint] Redis is ready"
+17
View File
@@ -0,0 +1,17 @@
.DS_Store
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
*.swp
*.bak
*.tmp
*.orig
*~
.project
.idea/
*.tmproj
.vscode/
+29
View File
@@ -0,0 +1,29 @@
apiVersion: v2
name: portabase-agent
description: Helm chart for Portabase Agent
type: application
version: 0.0.0
appVersion: "latest"
keywords:
- postgresql
- mariadb
- mongodb
- mysql
- sqlite
- backup
- database
- restore
- agent
home: https://github.com/Portabase/agent
sources:
- https://github.com/Portabase/agent
- https://github.com/Portabase/agent/tree/main/helm
maintainers:
- name: Charles Gauthereau
url: https://github.com/RambokDev
- name: Killian Larcher
url: https://github.com/KillianLarcher
icon: https://raw.githubusercontent.com/Portabase/agent/main/.github/assets/logo.png
+59
View File
@@ -0,0 +1,59 @@
# Development Notes
## Check that Kubernetes is reachable locally
```bash
kubectl get nodes
```
## Install the local Portabase Agent Helm chart
```bash
helm install portabase-agent . \
--set env.EDGE_KEY=<your-edge-key>
```
## Check the pods
```bash
kubectl get pods
```
## Check the services
```bash
kubectl get svc
```
## To update .env variables or JSON config:
```bash
kubectl rollout restart deployment portabase-agent
```
## Install or upgrade the Helm chart
```bash
helm upgrade portabase-agent . \
--reuse-values \
--set env.EDGE_KEY="NEW_EDGE_KEY"
```
## Rollout to restart
```bash
kubectl rollout restart deployment portabase-agent
```
## List pods to get the pod name
```bash
kubectl get pods -l app=portabase-agent
```
## Get logs for the pod
```bash
kubectl logs portabase-agent-6f7d4f5c6b-abc12
```
## Uninstall Agent
``` bash
helm uninstall portabase-agent
```
+7
View File
@@ -0,0 +1,7 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: portabase-agent-config
data:
config.json: |
{{ .Values.volume.configFile.content | nindent 4 }}
+62
View File
@@ -0,0 +1,62 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: portabase-agent
labels:
app: portabase-agent
spec:
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
app: portabase-agent
template:
metadata:
labels:
app: portabase-agent
spec:
hostAliases:
{{- range .Values.network.hostAliases }}
- ip: {{ .ip }}
hostnames:
{{- range .hostnames }}
- {{ . }}
{{- end }}
{{- end }}
containers:
- name: portabase-agent
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
envFrom:
- configMapRef:
name: portabase-agent-env
volumeMounts:
{{- if .Values.volume.configFile.enabled }}
{{- if .Values.volume.configFile.hostPath }}
- name: config
mountPath: /config/config.json
subPath: config.json
readOnly: true
# uses hostPath
{{- else }}
- name: config
mountPath: /config/config.json
subPath: config.json
# uses ConfigMap content
{{- end }}
{{- end }}
volumes:
{{- if .Values.volume.configFile.enabled }}
{{- if .Values.volume.configFile.hostPath }}
- name: config
hostPath:
path: {{ .Values.volume.configFile.hostPath }}
type: File
{{- else }}
- name: config
configMap:
name: portabase-agent-config
items:
- key: config.json
path: config.json
{{- end }}
{{- end }}
+10
View File
@@ -0,0 +1,10 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: portabase-agent-env
data:
EDGE_KEY: {{ .Values.env.EDGE_KEY | quote }}
TZ: {{ .Values.env.TZ | quote }}
POLLING: {{ .Values.env.POLLING | quote }}
APP_ENV: {{ .Values.env.APP_ENV | quote }}
LOG: {{ .Values.env.LOG | quote }}
+48
View File
@@ -0,0 +1,48 @@
replicaCount: 1
image:
repository: portabase/agent
tag: latest
pullPolicy: IfNotPresent
env:
EDGE_KEY: "your_edge_key_here"
TZ: "UTC"
POLLING: "5"
APP_ENV: "production"
LOG: "info"
resources:
limits:
cpu: 500m
memory: 512Mi
requests:
cpu: 250m
memory: 256Mi
volume:
configFile:
enabled: true
hostPath: "" # Use host file if set, otherwise use `content`
content: | # JSON content for config.json if no hostPath
{
"databases": [
{
"name": "my-site-prod (readable name)",
"database": "devdb",
"type": "postgresql",
"host": "localhost",
"port": 5432,
"username": "admin_prod",
"password": "super_secure_password",
"generated_id": "550e8400-e29b-41d4-a716-446655440000"
}
]
}
network:
hostAliases:
- ip: "127.0.0.1"
hostnames:
- "localhost"
-99
View File
@@ -1,99 +0,0 @@
#!/bin/bash
set -e
if [ -z "$1" ]; then
echo "Usage: ./release <version>"
echo "Example: ./release v1.0.0"
exit 1
fi
VERSION=$1
CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
if [ "$CURRENT_BRANCH" = "main" ]; then
if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo "Error: On 'main' branch, only release tags (X.Y.Z) are allowed."
exit 1
fi
else
if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+$ ]]; then
echo "Error: On branch '$CURRENT_BRANCH', only RC tags matching X.Y.Z-rc.W are allowed (e.g., 1.0.0-rc.1)."
exit 1
fi
fi
CLEAN_VERSION=${VERSION#v}
CURRENT_DATE=$(date +%Y-%m-%d)
echo "Preparing release $VERSION..."
# package.json
if [ -f package.json ]; then
echo "Updating package.json..."
if sed --version >/dev/null 2>&1; then
sed -i "s/\"version\": \".*\"/\"version\": \"$CLEAN_VERSION\"/" package.json
else
sed -i '' "s/\"version\": \".*\"/\"version\": \"$CLEAN_VERSION\"/" package.json
fi
fi
# pyproject.toml
if [ -f pyproject.toml ]; then
echo "Updating pyproject.toml..."
if sed --version >/dev/null 2>&1; then
sed -i "s/^version = \".*\"/version = \"$CLEAN_VERSION\"/" pyproject.toml
else
sed -i '' "s/^version = \".*\"/version = \"$CLEAN_VERSION\"/" pyproject.toml
fi
fi
# Cargo.toml
if [ -f Cargo.toml ]; then
echo "Updating Cargo.toml..."
if sed --version >/dev/null 2>&1; then
sed -i "s/^version = \".*\"/version = \"$CLEAN_VERSION\"/" Cargo.toml
else
sed -i '' "s/^version = \".*\"/version = \"$CLEAN_VERSION\"/" Cargo.toml
fi
fi
# CITATION.cff
if [ -f CITATION.cff ]; then
echo "Updating CITATION.cff..."
if sed --version >/dev/null 2>&1; then
sed -i "s/^version: .*/version: $CLEAN_VERSION/" CITATION.cff
sed -i "s/^date-released: .*/date-released: \"$CURRENT_DATE\"/" CITATION.cff
else
sed -i '' "s/^version: .*/version: $CLEAN_VERSION/" CITATION.cff
sed -i '' "s/^date-released: .*/date-released: \"$CURRENT_DATE\"/" CITATION.cff
fi
fi
git add .
if ! git diff-index --quiet HEAD --; then
echo "Committing changes..."
git commit -m "chore(release): $VERSION"
else
echo "No changes to commit. Proceeding to tag..."
fi
if git rev-parse "$VERSION" >/dev/null 2>&1; then
echo "Tag $VERSION already exists. Aborting."
exit 1
fi
echo "Creating tag $VERSION..."
git tag -a "$VERSION" -m "Release $VERSION"
echo "Pushing changes and tags to remote..."
git push
git push origin "$VERSION"
echo "Successfully released $VERSION!"
+4
View File
@@ -24,3 +24,7 @@ docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT * FROM users LI
```bash
docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT name FROM sqlite_master WHERE type='table';"
```
```bash
docker compose -f docker-compose.test.yml run agent-test bash -c "cargo clean && cargo test"
```
+1 -1
View File
@@ -60,7 +60,7 @@ SELECT
)
FROM users u
JOIN generate_series(1, 30) AS p(post_no)
ON u.id <= 300000;
ON u.id <= 500000;
-- ============================================================
-- OPTIONAL: FORCE DISK MATERIALIZATION
Binary file not shown.
+16 -8
View File
@@ -4,11 +4,11 @@ use crate::core::context::Context;
use crate::services::backup::BackupService;
use crate::services::config::ConfigService;
use crate::services::cron::CronService;
use crate::services::restore::RestoreService;
use crate::services::status::StatusService;
use crate::utils::common::BackupMethod;
use std::sync::Arc;
use tracing::info;
use crate::services::restore::RestoreService;
pub struct Agent {
ctx: Arc<Context>,
@@ -43,22 +43,30 @@ impl Agent {
let ping_result = self.status_service.ping(&config.databases).await?;
for db in ping_result.databases.iter() {
let database = config
.databases
.iter()
.find(|cfg_db| cfg_db.generated_id == db.generated_id)
.unwrap();
info!(
"Generated Id: {} | backup action: {} | restore action: {}",
db.generated_id, db.data.backup.action, db.data.restore.action
"Generated Id: {} | backup action: {} | restore action: {} | Database Name: {}",
db.generated_id, db.data.backup.action, db.data.restore.action, database.name,
);
let _ = self.cron_service.sync(db).await;
if db.data.backup.action {
let _ = self
.backup_service
.dispatch(&db.generated_id, &config, method.clone(), &db.storages, db.encrypt)
.dispatch(
&db.generated_id,
&config,
method.clone(),
&db.storages,
db.encrypt,
)
.await;
} else if db.data.restore.action {
let _ = self
.restore_service
.dispatch(db, &config)
.await;
let _ = self.restore_service.dispatch(db, &config).await;
}
}
+1 -1
View File
@@ -35,7 +35,7 @@ impl Context {
panic!("Cannot initialize AgentContext due to invalid EDGE_KEY");
}
};
let server_url = format!("{}/api", edge_key.server_url);
let api_client = ApiClient::new(server_url);
+1 -1
View File
@@ -1,2 +1,2 @@
pub mod context;
pub mod agent;
pub mod context;
+7 -1
View File
@@ -2,11 +2,13 @@ use crate::domain::mongodb::database::MongoDatabase;
use crate::domain::mysql::database::MySQLDatabase;
use crate::domain::postgres::database::PostgresDatabase;
use crate::domain::postgres::{detect_format_from_file, detect_format_from_size};
use crate::domain::redis::database::RedisDatabase;
use crate::domain::sqlite::database::SqliteDatabase;
use crate::domain::valkey::database::ValkeyDatabase;
use crate::services::config::{DatabaseConfig, DbType};
use anyhow::Result;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use crate::domain::sqlite::database::SqliteDatabase;
#[async_trait::async_trait]
pub trait Database: Send + Sync {
@@ -29,6 +31,8 @@ impl DatabaseFactory {
DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)),
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
DbType::Redis => Arc::new(RedisDatabase::new(cfg)),
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)),
}
}
@@ -42,6 +46,8 @@ impl DatabaseFactory {
DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)),
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
DbType::Redis => Arc::new(RedisDatabase::new(cfg)),
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)),
}
}
}
+4 -3
View File
@@ -1,6 +1,7 @@
pub mod factory;
pub mod postgres;
pub mod mysql;
mod mongodb;
pub mod mysql;
pub mod postgres;
mod redis;
mod sqlite;
mod valkey;
+1
View File
@@ -15,6 +15,7 @@ pub async fn run(
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
let mongodump = select_mongo_path().join("mongodump");
info!("{:?}", mongodump);
let uri = get_mongo_uri(cfg.clone())?;
let output = Command::new(mongodump)
+5 -5
View File
@@ -16,9 +16,11 @@ pub fn select_mongo_path() -> std::path::PathBuf {
}
pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result<String> {
if cfg.username.is_empty() && cfg.password.is_empty() {
Ok(format!("mongodb://{}:{}/{}", cfg.host, cfg.port, cfg.database))
if cfg.username.is_empty() || cfg.password.is_empty() {
Ok(format!(
"mongodb://{}:{}/{}",
cfg.host, cfg.port, cfg.database
))
} else {
Ok(format!(
"mongodb://{}:{}@{}:{}/{}?authSource=admin",
@@ -26,5 +28,3 @@ pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result<String> {
))
}
}
+1 -6
View File
@@ -29,12 +29,7 @@ impl Database for MongoDatabase {
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(
self.cfg.clone(),
dir.to_path_buf(),
self.file_extension(),
)
.await;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
+2 -2
View File
@@ -1,5 +1,5 @@
mod backup;
mod restore;
mod connection;
pub mod database;
mod ping;
mod connection;
mod restore;
+1 -1
View File
@@ -24,7 +24,7 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
error!("Full Error: {}", e);
error!("Check you database network connectivity");
error!("----------------------------------------");
Err(anyhow::anyhow!("Ping failed for {}: {}", cfg.name, e))
Ok(false)
}
}
}
+1 -1
View File
@@ -1,4 +1,4 @@
use crate::domain::mysql::connection::{server_version};
use crate::domain::mysql::connection::server_version;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::collections::HashMap;
+10 -8
View File
@@ -1,14 +1,17 @@
use crate::services::config::DatabaseConfig;
use std::process::Command;
use anyhow::Result;
use std::process::Command;
pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
let output = Command::new("mysql")
.arg("--host").arg(&cfg.host)
.arg("--port").arg(cfg.port.to_string())
.arg("--user").arg(&cfg.username)
.arg("-e").arg("SELECT VERSION();")
.arg("--host")
.arg(&cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(&cfg.username)
.arg("-e")
.arg("SELECT VERSION();")
.env("MYSQL_PWD", &cfg.password)
.output()?;
@@ -19,11 +22,10 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
let version = String::from_utf8_lossy(&output.stdout)
.lines()
.nth(1) // skip column header
.nth(1)
.unwrap_or_default()
.trim()
.to_string();
Ok(version)
}
+12 -9
View File
@@ -1,14 +1,11 @@
use std::collections::HashMap;
use anyhow::Result;
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use super::{
backup,
ping, restore,
};
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
use anyhow::Result;
use async_trait::async_trait;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
pub struct MySQLDatabase {
cfg: DatabaseConfig,
@@ -38,7 +35,13 @@ impl Database for MySQLDatabase {
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.build_env().clone(), self.file_extension()).await;
let res = backup::run(
self.cfg.clone(),
dir.to_path_buf(),
self.build_env().clone(),
self.file_extension(),
)
.await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
+2 -2
View File
@@ -1,5 +1,5 @@
pub mod backup;
mod connection;
pub mod database;
mod restore;
mod ping;
mod connection;
mod restore;
-1
View File
@@ -4,7 +4,6 @@ use tokio::process::Command;
use tokio::time::{Duration, timeout};
pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::Result<bool> {
let mut cmd = Command::new("mysqladmin");
cmd.arg("--host")
.arg(cfg.host)
+1
View File
@@ -27,6 +27,7 @@ pub async fn run(
};
let pg_dump = select_pg_path(&version).join("pg_dump");
debug!("Using pg_dump at {:?}", pg_dump);
match format {
+1 -5
View File
@@ -2,11 +2,7 @@ use anyhow::Result;
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use super::{
backup,
format::PostgresDumpFormat,
ping, restore,
};
use super::{backup, format::PostgresDumpFormat, ping, restore};
use crate::domain::factory::Database;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
+1 -1
View File
@@ -2,4 +2,4 @@
pub enum PostgresDumpFormat {
Fc,
Fd,
}
}
+3 -3
View File
@@ -1,8 +1,8 @@
pub mod backup;
pub mod database;
mod restore;
mod connection;
pub mod database;
mod format;
mod ping;
mod restore;
pub use connection::{detect_format_from_size, detect_format_from_file};
pub use connection::{detect_format_from_file, detect_format_from_size};
+2 -4
View File
@@ -1,8 +1,6 @@
use super::connection::connect;
use crate::services::config::DatabaseConfig;
pub async fn run(
cfg: DatabaseConfig,
) -> anyhow::Result<bool> {
pub async fn run(cfg: DatabaseConfig) -> anyhow::Result<bool> {
Ok(connect(&cfg).await.is_ok())
}
}
+1
View File
@@ -27,6 +27,7 @@ pub async fn run(
};
let pg_restore = select_pg_path(&version).join("pg_restore");
debug!("Using pg_restore at {:?}", pg_restore);
if let Err(e) = futures::executor::block_on(terminate_connections(&cfg)) {
+65
View File
@@ -0,0 +1,65 @@
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
file_extension: &'static str,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
debug!("Starting Redis backup for database {}", cfg.name);
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
let mut cmd = Command::new("redis-cli");
cmd.arg("-h")
.arg(&cfg.host)
.arg("-p")
.arg(cfg.port.to_string());
if !cfg.username.is_empty() {
cmd.arg("--user").arg(&cfg.username);
}
if !cfg.password.is_empty() {
cmd.arg("-a").arg(&cfg.password);
}
cmd.arg("--rdb").arg(&file_path);
debug!("Command Backup: {:?}", cmd);
let output = cmd.output().context("Redis backup command failed")?;
let stderr = String::from_utf8_lossy(&output.stderr);
let stdout = String::from_utf8_lossy(&output.stdout);
if !output.status.success() {
if stderr.contains("NOAUTH") {
error!(
"Redis backup failed for {}: Authentication required (NOAUTH)",
cfg.name
);
anyhow::bail!(
"Redis backup failed for {}: Authentication required",
cfg.name
);
} else {
error!("Redis backup failed for {}: {}", cfg.name, stderr);
anyhow::bail!("Redis backup failed for {}: {}", cfg.name, stderr);
}
}
info!(
"Redis backup completed for {}. Output: {}",
cfg.name, stdout
);
Ok(file_path)
})
.await?
}
+40
View File
@@ -0,0 +1,40 @@
use anyhow::{Result, bail};
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use crate::domain::factory::Database;
use crate::domain::redis::{backup, ping};
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
pub struct RedisDatabase {
cfg: DatabaseConfig,
}
impl RedisDatabase {
pub fn new(cfg: DatabaseConfig) -> Self {
Self { cfg }
}
}
#[async_trait]
impl Database for RedisDatabase {
fn file_extension(&self) -> &'static str {
".rdb"
}
async fn ping(&self) -> Result<bool> {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, _file: &Path) -> Result<()> {
bail!("Restore not supported for Redis databases")
}
}
+3
View File
@@ -0,0 +1,3 @@
mod backup;
pub mod database;
mod ping;
+55
View File
@@ -0,0 +1,55 @@
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use tokio::process::Command;
use tokio::time::{Duration, timeout};
use tracing::{debug, error, info};
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
let mut cmd = Command::new("redis-cli");
cmd.arg("-h")
.arg(&cfg.host)
.arg("-p")
.arg(cfg.port.to_string());
if !cfg.username.is_empty() {
cmd.arg("--user").arg(&cfg.username);
}
if !cfg.password.is_empty() {
cmd.arg("-a").arg(&cfg.password);
}
cmd.arg("PING");
debug!("Command Ping Redis: {:?}", cmd);
let result = timeout(Duration::from_secs(10), cmd.output()).await;
match result {
Ok(output) => {
let output = output.context("Failed to execute redis-cli")?;
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
if !stdout.is_empty() {
error!("Redis stderr: {}", stderr);
}
if stderr.contains("NOAUTH") {
error!("Redis authentication failed (NOAUTH required)");
return Ok(false);
}
if !output.status.success() {
error!("Redis command failed with status: {:?}", output.status);
return Ok(false);
}
Ok(stdout.contains("PONG"))
}
Err(_) => {
info!("Timeout connecting to Redis at {}:{}", cfg.host, cfg.port);
Ok(false)
}
}
}
+2 -2
View File
@@ -41,5 +41,5 @@ pub async fn run(
info!("SQLite backup completed for {}", cfg.name);
Ok(file_path)
})
.await?
}
.await?
}
-1
View File
@@ -33,7 +33,6 @@ impl Database for SqliteDatabase {
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
+2 -2
View File
@@ -1,4 +1,4 @@
mod backup;
mod restore;
pub mod database;
mod ping;
pub mod database;
mod restore;
+65
View File
@@ -0,0 +1,65 @@
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
file_extension: &'static str,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
debug!("Starting Valkey backup for database {}", cfg.name);
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
let mut cmd = Command::new("valkey-cli");
cmd.arg("-h")
.arg(&cfg.host)
.arg("-p")
.arg(cfg.port.to_string());
if !cfg.username.is_empty() {
cmd.arg("--user").arg(&cfg.username);
}
if !cfg.password.is_empty() {
cmd.arg("-a").arg(&cfg.password);
}
cmd.arg("--rdb").arg(&file_path);
debug!("Command Backup: {:?}", cmd);
let output = cmd.output().context("Valkey backup command failed")?;
let stderr = String::from_utf8_lossy(&output.stderr);
let stdout = String::from_utf8_lossy(&output.stdout);
if !output.status.success() {
if stderr.contains("NOAUTH") {
error!(
"Valkey backup failed for {}: Authentication required (NOAUTH)",
cfg.name
);
anyhow::bail!(
"Valkey backup failed for {}: Authentication required",
cfg.name
);
} else {
error!("Valkey backup failed for {}: {}", cfg.name, stderr);
anyhow::bail!("Valkey backup failed for {}: {}", cfg.name, stderr);
}
}
info!(
"Valkey backup completed for {}. Output: {}",
cfg.name, stdout
);
Ok(file_path)
})
.await?
}
+40
View File
@@ -0,0 +1,40 @@
use anyhow::{Result, bail};
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use crate::domain::factory::Database;
use crate::domain::valkey::{backup, ping};
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
pub struct ValkeyDatabase {
cfg: DatabaseConfig,
}
impl ValkeyDatabase {
pub fn new(cfg: DatabaseConfig) -> Self {
Self { cfg }
}
}
#[async_trait]
impl Database for ValkeyDatabase {
fn file_extension(&self) -> &'static str {
".rdb"
}
async fn ping(&self) -> Result<bool> {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, _file: &Path) -> Result<()> {
bail!("Restore not supported for Valkey databases")
}
}
+3
View File
@@ -0,0 +1,3 @@
mod backup;
pub mod database;
mod ping;
+55
View File
@@ -0,0 +1,55 @@
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use tokio::process::Command;
use tokio::time::{Duration, timeout};
use tracing::{debug, info, error};
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
let mut cmd = Command::new("valkey-cli");
cmd.arg("-h")
.arg(&cfg.host)
.arg("-p")
.arg(cfg.port.to_string());
if !cfg.username.is_empty() {
cmd.arg("--user").arg(&cfg.username);
}
if !cfg.password.is_empty() {
cmd.arg("-a").arg(&cfg.password);
}
cmd.arg("PING");
debug!("Command Ping Valkey: {:?}", cmd);
let result = timeout(Duration::from_secs(10), cmd.output()).await;
match result {
Ok(output) => {
let output = output.context("Failed to execute valkey-cli")?;
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
if !stdout.is_empty() {
error!("Valkey stderr: {}", stderr);
}
if stderr.contains("NOAUTH") {
error!("Valkey authentication failed (NOAUTH required)");
return Ok(false);
}
if !output.status.success() {
error!("Valkey command failed with status: {:?}", output.status);
return Ok(false);
}
Ok(stdout.contains("PONG"))
}
Err(_) => {
info!("Timeout connecting to Valkey at {}:{}", cfg.host, cfg.port);
Ok(false)
}
}
}
+3 -2
View File
@@ -3,17 +3,18 @@ mod domain;
mod services;
mod settings;
mod tasks;
#[cfg(test)]
mod tests;
mod utils;
use crate::tasks::ping::ping_server;
use crate::utils::locks::FileLock;
use crate::utils::logging;
use utils::redis_client;
use utils::task_manager::scheduler;
use crate::utils::logging;
#[tokio::main]
async fn main() {
logging::init_logger();
// Remove all locks on startup
+2 -7
View File
@@ -1,9 +1,9 @@
#![allow(dead_code)]
use crate::services::api::ApiError;
use reqwest::{Client, Method};
use serde::de::DeserializeOwned;
use std::time::Duration;
use crate::services::api::ApiError;
#[derive(Clone, Debug)]
pub struct ApiClient {
@@ -57,12 +57,7 @@ impl ApiClient {
{
let url = format!("{}{}", self.base_url, path);
let res = self
.http
.request(method, &url)
.json(body)
.send()
.await?;
let res = self.http.request(method, &url).json(body).send().await?;
let status = res.status();
let body_text = res.text().await.unwrap_or_default();
@@ -23,7 +23,6 @@ pub struct BackupUpdateRequest {
pub generated_id: String,
}
impl ApiClient {
pub async fn backup_create(
&self,
@@ -1,8 +1,8 @@
use crate::services::api::models::agent::backup::BackupUploadResponse;
use crate::services::api::{ApiClient, ApiError};
use anyhow::Result;
use reqwest::Method;
use serde::Serialize;
use crate::services::api::models::agent::backup::BackupUploadResponse;
#[derive(Serialize)]
pub struct InitUploadRequest {
@@ -1,2 +1,2 @@
pub mod init;
pub mod status;
pub mod status;
+2 -1
View File
@@ -1,2 +1,3 @@
pub mod backup;
pub mod restore;
pub mod status;
pub mod backup;
@@ -0,0 +1,33 @@
use crate::services::api::models::agent::restore::ResultRestoreResponse;
use crate::services::api::{ApiClient, ApiError};
use anyhow::Result;
use reqwest::Method;
use serde::Serialize;
#[derive(Serialize)]
pub struct ResultRestoreRequest {
#[serde(rename = "generatedId")]
pub generated_id: String,
pub status: String,
}
impl ApiClient {
pub async fn restore_result(
&self,
agent_id: impl Into<String>,
generated_id: impl Into<String>,
status: impl Into<String>,
) -> Result<Option<ResultRestoreResponse>, ApiError> {
let body = ResultRestoreRequest {
generated_id: generated_id.into(),
status: status.into(),
};
let agent_id = agent_id.into();
let path = format!("/agent/{}/restore", agent_id);
self.request_with_body(Method::POST, path.as_str(), &body)
.await
}
}
+5 -2
View File
@@ -11,6 +11,8 @@ pub struct DatabasePayload<'a> {
pub dbms: &'a str,
#[serde(rename = "generatedId")]
pub generated_id: &'a str,
#[serde(rename = "pingStatus")]
pub ping_status: bool
}
#[derive(Serialize)]
@@ -31,6 +33,7 @@ impl ApiClient {
let agent_id = agent_id.into();
let path = format!("/agent/{}/status", agent_id);
self.request_with_body(Method::POST, path.as_str(), &body).await
self.request_with_body(Method::POST, path.as_str(), &body)
.await
}
}
}
+1 -1
View File
@@ -1,3 +1,3 @@
pub mod agent;
pub use agent::status;
pub use agent::status;
+1 -4
View File
@@ -11,10 +11,7 @@ pub enum ApiError {
Serialization(#[from] serde_json::Error),
#[error("api error: status={status}, body={body}")]
HttpResponse {
status: StatusCode,
body: String,
},
HttpResponse { status: StatusCode, body: String },
#[error("api returned unexpected response")]
UnexpectedResponse,
+1 -2
View File
@@ -1,8 +1,7 @@
pub mod client;
pub mod endpoints;
pub mod error;
pub mod models;
pub mod endpoints;
pub use client::ApiClient;
pub use error::ApiError;
+2 -1
View File
@@ -1,2 +1,3 @@
pub mod backup;
pub mod restore;
pub mod status;
pub mod backup;
+7
View File
@@ -0,0 +1,7 @@
use serde::{Deserialize, Serialize};
#[derive(Debug, Serialize, Deserialize)]
pub struct ResultRestoreResponse {
pub message: String,
pub status: bool,
}
+1 -1
View File
@@ -1,8 +1,8 @@
#![allow(dead_code)]
use crate::utils::deserializer::deserialize_snake_case;
use serde::{Deserialize, Serialize};
use toml::Value;
use crate::utils::deserializer::deserialize_snake_case;
#[derive(Debug, Deserialize)]
pub struct PingResult {
-1
View File
@@ -1,2 +1 @@
pub mod agent;
-415
View File
@@ -1,415 +0,0 @@
#![allow(dead_code)]
use crate::core::context::Context as CoreContext;
use crate::domain::factory::DatabaseFactory;
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::config::{DatabaseConfig, DatabasesConfig, DbType};
use crate::services::storage;
use crate::utils::common::BackupMethod;
use crate::utils::compress::compress_to_tar_gz_large;
use anyhow::Result;
use futures::future::join_all;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use tempfile::TempDir;
use tracing::{error, info};
use crate::utils::locks::FileLock;
#[derive(Debug, Clone)]
pub struct BackupResult {
pub generated_id: String,
pub db_type: DbType,
pub status: String,
pub backup_file: Option<PathBuf>,
pub code: Option<String>,
}
#[derive(Debug)]
pub struct UploadResult {
pub storage_id: String,
pub success: bool,
pub error: Option<String>,
pub remote_file_path: Option<String>,
pub total_size: Option<u64>,
}
pub struct BackupService {
ctx: Arc<CoreContext>,
}
impl BackupService {
pub fn new(ctx: Arc<CoreContext>) -> Self {
Self { ctx }
}
pub async fn dispatch(
&self,
generated_id: &String,
config: &DatabasesConfig,
method: BackupMethod,
storages: &Vec<DatabaseStorage>,
encrypt: bool,
) {
if let Some(cfg) = config
.databases
.iter()
.find(|c| c.generated_id == generated_id.as_str())
{
let db_cfg = cfg.clone();
let ctx = self.ctx.clone();
let storages_clone = storages.clone();
let generated_id_clone = generated_id.clone();
tokio::spawn(async move {
match TempDir::new() {
Ok(temp_dir) => {
match FileLock::is_locked(&generated_id_clone).await {
Ok(true) => {
error!("Backup already running for {}", &generated_id_clone);
return;
}
Ok(false) => {
match ctx
.api
.backup_create(
method.clone().to_string(),
ctx.edge_key.agent_id.clone(),
&generated_id_clone,
)
.await
{
Ok(backup_created_result) => {
info!("Backup created successfully");
let tmp_path = temp_dir.path().to_path_buf();
info!("Created temp directory {}", tmp_path.display());
match BackupService::run(db_cfg, &tmp_path).await {
Ok(mut result) => {
let backup_id = backup_created_result.unwrap().backup.id;
if result.status == "failed" {
error!("Backup failed early for {}", result.generated_id);
let service = BackupService { ctx: ctx.clone() };
let _ = service
.send_result(result, vec![], &backup_id)
.await;
return;
}
if let Some(backup_file) = result.backup_file.take() {
match compress_to_tar_gz_large(&backup_file).await {
Ok(compression_result) => {
result.backup_file =
Some(compression_result.compressed_path);
let service = BackupService { ctx: ctx.clone() };
match service
.upload(
result.clone(),
method,
storages_clone.clone(),
encrypt,
&backup_id,
)
.await
{
Ok(upload_result) => {
match service
.send_result(
result,
upload_result,
&backup_id,
)
.await
{
Ok(_) => {
return;
}
Err(e) => {
error!(
"Failed to send backup result: {}",
e
);
}
}
}
Err(e) => {
error!(
"Failed to upload backup files: {}",
e
);
}
}
}
Err(e) => {
error!(
"Failed to compress backup file : {}",
e
);
}
}
} else {
error!("No backup file generated");
}
}
Err(e) => error!("BackupService run failed: {}", e),
}
// TempDir is automatically deleted when dropped here
}
Err(e) => error!("Backup creation failed: {}", e),
}
}
Err(e) => error!("An error occurred while checking lock : {}", e),
}
}
Err(e) => error!("Failed to create temp dir: {}", e),
}
});
}
}
pub async fn run(cfg: DatabaseConfig, tmp_path: &Path) -> Result<BackupResult> {
let db_instance = DatabaseFactory::create_for_backup(cfg.clone()).await;
let generated_id = cfg.generated_id.clone();
let db_type = cfg.db_type.clone();
let reachable = match db_instance.ping().await {
Ok(v) => v,
Err(e) => {
error!("Ping failed: {}", e);
return Err(e.into());
}
};
info!("Reachable: {}", reachable);
if !reachable {
return Ok(BackupResult {
generated_id,
db_type,
status: "failed".into(),
backup_file: None,
code: None,
});
}
match db_instance.backup(tmp_path).await {
Ok(file) => Ok(BackupResult {
generated_id,
db_type,
status: "success".into(),
backup_file: Some(file),
code: None,
}),
Err(e) => match e.to_string().as_str() {
"backup_already_in_progress" => Ok(BackupResult {
generated_id,
db_type,
status: "failed".into(),
backup_file: None,
code: Some(e.to_string()),
}),
_ => Ok(BackupResult {
generated_id,
db_type,
status: "failed".into(),
backup_file: None,
code: None,
}),
},
}
}
pub async fn upload(
&self,
result: BackupResult,
method: BackupMethod,
storages: Vec<DatabaseStorage>,
encrypt: bool,
backup_id: &String,
) -> Result<Vec<UploadResult>> {
if result.code.as_deref() == Some("backup_already_in_progress") {
info!("Skipping send: backup already in progress");
anyhow::bail!("backup_already_in_progres");
}
let upload_futures = storages.into_iter().map(|storage| {
info!(
"Uploading storage -> {:?} for {:?}",
storage.provider, storage.id
);
let provider = storage::get_provider(&storage);
let result_clone = result.clone();
let ctx_clone = self.ctx.clone();
let storages_clone = storage.clone();
let storage_id = storages_clone.id;
let generated_id = result_clone.generated_id.clone();
async move {
match self
.ctx
.api
.backup_upload_init(
self.ctx.edge_key.agent_id.clone(),
generated_id.clone(),
storage_id.clone(),
backup_id,
)
.await
{
Ok(upload_init_result) => {
info!("Uploading init result: {:#?}", upload_init_result);
let backup_storage_id = upload_init_result.unwrap().backup_storage.id.clone();
match provider {
Some(provider) => {
let upload_result = provider
.upload(
ctx_clone,
result_clone,
method,
&storage,
Some(encrypt),
)
.await;
let status = if upload_result.success {
"success"
} else {
"failed"
};
if status != "success" {
return upload_result;
}
info!("Storage {} uploaded to remote path {:?}", storage_id, upload_result.remote_file_path);
let (remote_path, total_size) = match (
&upload_result.remote_file_path,
upload_result.total_size,
) {
(Some(path), Some(size)) => (path.clone(), size),
_ => {
return UploadResult {
storage_id: storage_id.clone(),
success: false,
error: Some("remote_file_path or total_size missing".to_string()),
remote_file_path: None,
total_size: None,
}
}
};
match self.ctx.api.backup_upload_status(
self.ctx.edge_key.agent_id.clone(),
generated_id.clone(),
backup_storage_id,
status,
remote_path,
total_size,
backup_id,
).await {
Ok(_) => {
upload_result
}
Err(err) => {
error!(
"backup_upload_status failed (generated_id={}, storage_id={}): {}",
generated_id, storage_id, err
);
UploadResult {
storage_id: storage_id.clone(),
success: false,
error: Some(err.to_string()),
remote_file_path: None,
total_size: None,
}
}
}
}
None => {
error!("Skipping storage due to missing provider");
UploadResult {
storage_id: storage_id.clone(),
success: false,
error: Some(
"Skipping storage due to missing provider".to_string(),
),
remote_file_path: None,
total_size: None,
}
}
}
}
Err(e) => {
error!(
"Unable to create the storage backup on remote server : {}",
e
);
UploadResult {
storage_id: storage_id.clone(),
success: false,
error: Some(
"Unable to create the storage backup on remote server".to_string(),
),
remote_file_path: None,
total_size: None,
}
}
}
}
});
let results: Vec<UploadResult> = join_all(upload_futures).await;
info!("Upload results: {:#?}", results);
Ok(results)
}
pub async fn send_result(
&self,
result: BackupResult,
upload_results: Vec<UploadResult>,
backup_id: &String,
) -> Result<()> {
let status = if upload_results.iter().any(|r| r.success) {
"success"
} else {
"failed"
};
let file_size = if status == "failed" {
None
} else {
let mut sum = 0u64;
let mut count = 0u64;
for size in upload_results.iter().filter_map(|r| r.total_size) {
sum += size;
count += 1;
}
if count == 0 {
None
} else {
Some(sum / count)
}
};
match self
.ctx
.api
.backup_update(self.ctx.edge_key.agent_id.clone(), backup_id, status, file_size, &result.generated_id)
.await
{
Ok(_result) => Ok(()),
Err(e) => {
error!(
"backup_update failed (generated_id={}, backup_id={}): {}",
&result.generated_id, &backup_id, e
);
Err(e.into())
}
}
}
}
+14
View File
@@ -0,0 +1,14 @@
use super::service::BackupService;
use crate::utils::compress::compress_to_tar_gz_large;
use anyhow::Result;
use std::path::PathBuf;
impl BackupService {
pub async fn compress_backup(&self, backup_file: Option<PathBuf>) -> Result<PathBuf> {
let file = backup_file.ok_or_else(|| anyhow::anyhow!("No backup file generated"))?;
let compression = compress_to_tar_gz_large(&file).await?;
Ok(compression.compressed_path)
}
}
+42
View File
@@ -0,0 +1,42 @@
use super::service::BackupService;
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::config::DatabasesConfig;
use crate::utils::common::BackupMethod;
use tracing::error;
impl BackupService {
pub async fn dispatch(
&self,
generated_id: &String,
config: &DatabasesConfig,
method: BackupMethod,
storages: &Vec<DatabaseStorage>,
encrypt: bool,
) {
let Some(cfg) = config
.databases
.iter()
.find(|c| c.generated_id == generated_id.as_str())
else {
error!("Database config not found for {}", generated_id);
return;
};
let service = Self {
ctx: self.ctx.clone(),
};
let db_cfg = cfg.clone();
let storages = storages.clone();
let generated_id = generated_id.clone();
tokio::spawn(async move {
if let Err(e) = service
.execute_backup(generated_id, db_cfg, method, storages, encrypt)
.await
{
error!("Backup execution failed: {}", e);
}
});
}
}
+47
View File
@@ -0,0 +1,47 @@
use super::service::BackupService;
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::config::DatabaseConfig;
use crate::utils::common::BackupMethod;
use crate::utils::locks::FileLock;
use anyhow::Result;
use tempfile::TempDir;
impl BackupService {
pub async fn execute_backup(
&self,
generated_id: String,
db_cfg: DatabaseConfig,
method: BackupMethod,
storages: Vec<DatabaseStorage>,
encrypt: bool,
) -> Result<()> {
if FileLock::is_locked(&generated_id).await? {
anyhow::bail!("backup already running");
}
let backup = self.create_backup_record(&generated_id, &method).await?;
let backup_id = backup.backup.id;
let temp_dir = TempDir::new()?;
let tmp_path = temp_dir.path();
let mut result = Self::run(db_cfg, tmp_path).await?;
if result.status == "failed" {
self.send_result(result, vec![], &backup_id).await?;
return Ok(());
}
let compressed = self.compress_backup(result.backup_file.take()).await?;
result.backup_file = Some(compressed);
let uploads = self
.upload(result.clone(), method, storages, encrypt, &backup_id)
.await?;
self.send_result(result, uploads, &backup_id).await?;
Ok(())
}
}
+24
View File
@@ -0,0 +1,24 @@
use super::service::BackupService;
use crate::services::api::models::agent::backup::BackupResponse;
use crate::utils::common::BackupMethod;
use anyhow::{Result, anyhow};
impl BackupService {
pub async fn create_backup_record(
&self,
generated_id: &str,
method: &BackupMethod,
) -> Result<BackupResponse> {
let response = self
.ctx
.api
.backup_create(
method.to_string(),
self.ctx.edge_key.agent_id.clone(),
generated_id,
)
.await?;
response.ok_or_else(|| anyhow!("backup_create returned empty response"))
}
}
+11
View File
@@ -0,0 +1,11 @@
pub mod compressor;
pub mod dispatcher;
pub mod executor;
pub mod helpers;
pub mod models;
pub mod result;
pub mod runner;
pub mod service;
pub mod uploader;
pub use service::BackupService;

Some files were not shown because too many files have changed in this diff Show More