mirror of
https://github.com/Portabase/agent.git
synced 2026-09-11 02:27:10 +00:00
Compare commits
228 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ff47024d9a | |||
| 6537e9df53 | |||
| b8d869d5a6 | |||
| 90941ea67d | |||
| 046d593e2b | |||
| cf9a59a138 | |||
| 2464f6dfb0 | |||
| 069067ca55 | |||
| 04b654d219 | |||
| d83511cb64 | |||
| ca294e968c | |||
| 1be88ffdb9 | |||
| 9d393a96a4 | |||
| 044bf80633 | |||
| 0a6eb6db22 | |||
| b26ff81889 | |||
| 4e2f29f4ca | |||
| d1c8df4cac | |||
| 30f83bafcf | |||
| de106c835e | |||
| 23d6822ddc | |||
| fe1d74945f | |||
| 424a646385 | |||
| 0ef4bba5d7 | |||
| 6548140eaf | |||
| c9725c381e | |||
| 03695b1897 | |||
| 80b1c0dac0 | |||
| 5ededb3764 | |||
| 54cfd673e4 | |||
| 39dbeb5d30 | |||
| 380cea79fb | |||
| 65e4538c2c | |||
| 86a706bca5 | |||
| e8bda60f16 | |||
| 034912b176 | |||
| 4fd50ccda4 | |||
| 84c6b765c8 | |||
| 9790837200 | |||
| 8827234344 | |||
| b3417304bb | |||
| 5f4006b1ca | |||
| f24b1ec3e4 | |||
| bb1b16d0f2 | |||
| 2aca036ee4 | |||
| 2a72a55e57 | |||
| ae63a0df50 | |||
| 560c5e02f2 | |||
| ca17bf9cb9 | |||
| 9fbaab6feb | |||
| 1289497d98 | |||
| 27c55dbfd8 | |||
| 45e3acd709 | |||
| a93054518a | |||
| baf2edc829 | |||
| f050fa35e0 | |||
| 90d55f7630 | |||
| c186d57105 | |||
| 37b4c1fe6d | |||
| 92a3324ca2 | |||
| 82ffb71496 | |||
| 20b3b58024 | |||
| 6cddc73565 | |||
| fd3b0076c0 | |||
| 334ab51125 | |||
| 4df3fe2d0c | |||
| ec7715ca26 | |||
| 32d4e22196 | |||
| f277c5485f | |||
| 585e0bea72 | |||
| ee97c6056a | |||
| b37a4f2180 | |||
| 1d15f40662 | |||
| 324a2ea3f2 | |||
| 9ec92af6c7 | |||
| 1dafdca2a0 | |||
| d12817a960 | |||
| 608b82a254 | |||
| bbecca92f9 | |||
| b2b2733c4d | |||
| 2c257c5a58 | |||
| c24c0d7058 | |||
| 34f353b68f | |||
| ee51996401 | |||
| 694b463787 | |||
| aed1e86956 | |||
| 6388aff1e3 | |||
| 39a77b18a4 | |||
| e62167182e | |||
| 4be54a614e | |||
| c5eaa5086c | |||
| 610d443afc | |||
| 16a37e033e | |||
| 29ca5a317b | |||
| 01f5d34c96 | |||
| 0b7b7ef59b | |||
| 038c5523b7 | |||
| ea010ce713 | |||
| 8897568281 | |||
| 66cad4e12e | |||
| 221ed4e7e1 | |||
| 84735ac399 | |||
| f7639de096 | |||
| e504d09cb3 | |||
| 0363b300a4 | |||
| bf6e7d41ab | |||
| 2c7065ae95 | |||
| 1fece577cc | |||
| 273475fa3b | |||
| 628c017584 | |||
| f02218708a | |||
| 16152328b0 | |||
| 7328827435 | |||
| 348eaac81b | |||
| 0f6c93ecd0 | |||
| befec0deae | |||
| 2052ab0ff5 | |||
| 461e92d67e | |||
| 23678bf2d6 | |||
| 02105a8171 | |||
| f94656a39b | |||
| b94ff4e987 | |||
| dc32c442f3 | |||
| 1453851555 | |||
| 185a7ee612 | |||
| d8328f60ca | |||
| 82c7122790 | |||
| fba72f5c26 | |||
| b537980c71 | |||
| 0261a07fc2 | |||
| c840f91680 | |||
| 2d52eccbb2 | |||
| f9bf481c79 | |||
| 3c853ee7ca | |||
| 45d791358c | |||
| 0b915d87c9 | |||
| 531a25f292 | |||
| 41e2b197cb | |||
| f149363b12 | |||
| aa3ff39bb3 | |||
| 740ee43038 | |||
| 1fccc0bc19 | |||
| 2c15ea64ae | |||
| 18bfdf3b7e | |||
| db26910679 | |||
| eac2853b83 | |||
| 8690a15888 | |||
| a4bbb37a4e | |||
| ac8f7fd8d8 | |||
| c6baa8fdc3 | |||
| 14068846b9 | |||
| 6edf2890f1 | |||
| 5f579690ff | |||
| bfe63d7fd3 | |||
| 0a422448b4 | |||
| 14a6c9ab0a | |||
| 5c35375df7 | |||
| 86027fd6bd | |||
| 61c7224104 | |||
| d7958e03b9 | |||
| 8eab97a3d6 | |||
| 25c95108f0 | |||
| f37e4d2de9 | |||
| b269d98d3c | |||
| a1aa66ff3f | |||
| d4abcbcebc | |||
| 2d44b62844 | |||
| 8bf6503f1f | |||
| fdf35228d4 | |||
| 1855c5120a | |||
| 3d5a7eeded | |||
| 517f3779cd | |||
| a1e18754b1 | |||
| 092f760431 | |||
| 725312e1d5 | |||
| 2c5c805308 | |||
| 8e8ebb5920 | |||
| 8489a6f6e3 | |||
| 38a3274c43 | |||
| 3f5b9c91cc | |||
| 1c52f298d2 | |||
| bb45c1961f | |||
| e602c3540f | |||
| ee7016d1fa | |||
| b5e8ccac1f | |||
| f14a6dc318 | |||
| e1bc77e481 | |||
| cbc7517603 | |||
| fc4e4e3881 | |||
| 5ec7bc9f28 | |||
| 45c33dd031 | |||
| cddb9d78fa | |||
| cf5de3115c | |||
| d6d2ef18f8 | |||
| 987c630c73 | |||
| e9147d6c60 | |||
| c46c301371 | |||
| 174ead72a3 | |||
| f2a67b3490 | |||
| d4afefbc1e | |||
| be6e147df2 | |||
| 91b6658852 | |||
| f7dcb6b2bb | |||
| ab14f0dcf0 | |||
| 58a0b92710 | |||
| 2f24291c0e | |||
| 1fdea31430 | |||
| d1d633fd00 | |||
| b1c14f098b | |||
| 3abb955e24 | |||
| 6d22be9b50 | |||
| c2df4ba71e | |||
| 46b3f2466a | |||
| a2be03751f | |||
| 7df9df1605 | |||
| df5b16a153 | |||
| 824a5d52a3 | |||
| 1496208db4 | |||
| 454e1b442f | |||
| fd9f183166 | |||
| 94fb2535ec | |||
| 7fabc9c98c | |||
| 063b2e1c2c | |||
| 924177a4fe | |||
| 678df8e2bc | |||
| 7e0c7c0688 | |||
| b9b9fcf6ee | |||
| 0224605c59 |
@@ -60,7 +60,7 @@ representative at an online or offline event.
|
|||||||
|
|
||||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
||||||
reported to the community leaders responsible for enforcement at
|
reported to the community leaders responsible for enforcement at
|
||||||
contact@soluce-technologies.com.
|
contact@portabase.io.
|
||||||
All complaints will be reviewed and investigated promptly and fairly.
|
All complaints will be reviewed and investigated promptly and fairly.
|
||||||
|
|
||||||
All community leaders are obligated to respect the privacy and security of the
|
All community leaders are obligated to respect the privacy and security of the
|
||||||
|
|||||||
+1
-1
@@ -18,7 +18,7 @@ We take security seriously and aim to support the following versions of the proj
|
|||||||
If you discover a security vulnerability in this project, we appreciate your help in disclosing it responsibly.
|
If you discover a security vulnerability in this project, we appreciate your help in disclosing it responsibly.
|
||||||
|
|
||||||
1. **Contact Us**
|
1. **Contact Us**
|
||||||
Please report the vulnerability by emailing **[contact@soluce-technologies.com](mailto:contact@soluce-technologies.com)**. Include the following details:
|
Please report the vulnerability by emailing **[contact@portabase.io](mailto:contact@portabase.io)**. Include the following details:
|
||||||
- A detailed description of the issue.
|
- A detailed description of the issue.
|
||||||
- Steps to reproduce the vulnerability (if applicable).
|
- Steps to reproduce the vulnerability (if applicable).
|
||||||
- Any potential impacts or risks.
|
- Any potential impacts or risks.
|
||||||
|
|||||||
@@ -0,0 +1,90 @@
|
|||||||
|
name: Codecov Rust
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: ["main"]
|
||||||
|
pull_request:
|
||||||
|
branches: ["main"]
|
||||||
|
|
||||||
|
env:
|
||||||
|
CARGO_TERM_COLOR: always
|
||||||
|
CARGO_TARGET_DIR: /app/target
|
||||||
|
CARGO_INCREMENTAL: 0
|
||||||
|
RUSTFLAGS: "-C instrument-coverage -C link-dead-code"
|
||||||
|
LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw"
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
coverage:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Build test image
|
||||||
|
run: docker compose -f docker-compose.test.yml build agent-test
|
||||||
|
|
||||||
|
- name: Start agent-test container
|
||||||
|
run: docker compose -f docker-compose.test.yml up -d agent-test
|
||||||
|
|
||||||
|
- name: Run tests inside container
|
||||||
|
run: |
|
||||||
|
docker compose -f docker-compose.test.yml exec -T \
|
||||||
|
-e CARGO_TARGET_DIR \
|
||||||
|
-e CARGO_INCREMENTAL \
|
||||||
|
-e RUSTFLAGS \
|
||||||
|
-e LLVM_PROFILE_FILE \
|
||||||
|
agent-test bash -c "
|
||||||
|
mkdir -p /app/coverage &&
|
||||||
|
rm -rf /app/target/* /app/coverage/* &&
|
||||||
|
cargo test --verbose -- --test-threads=2 &&
|
||||||
|
sync
|
||||||
|
"
|
||||||
|
|
||||||
|
- name: Verify profraw files exist
|
||||||
|
run: |
|
||||||
|
docker compose -f docker-compose.test.yml exec -T \
|
||||||
|
agent-test bash -c '
|
||||||
|
count=$(find /app/coverage -type f -name "*.profraw" | wc -l)
|
||||||
|
echo "profraw files: $count"
|
||||||
|
test "$count" -gt 0
|
||||||
|
'
|
||||||
|
|
||||||
|
- name: Generate coverage report inside container
|
||||||
|
run: |
|
||||||
|
docker compose -f docker-compose.test.yml exec -T \
|
||||||
|
agent-test bash -c "
|
||||||
|
rustup component add llvm-tools-preview &&
|
||||||
|
grcov /app/coverage \
|
||||||
|
--binary-path /app/target/debug \
|
||||||
|
-s /app \
|
||||||
|
--llvm \
|
||||||
|
-t lcov \
|
||||||
|
--branch \
|
||||||
|
--ignore-not-existing \
|
||||||
|
--ignore '/app/target/*' \
|
||||||
|
-o /app/lcov.info
|
||||||
|
"
|
||||||
|
|
||||||
|
- name: Copy lcov.info from container to host
|
||||||
|
run: docker compose -f docker-compose.test.yml cp agent-test:/app/lcov.info ./lcov.info
|
||||||
|
|
||||||
|
- name: Show basic coverage report info
|
||||||
|
run: |
|
||||||
|
echo "lcov.info size:" $(wc -c ./lcov.info | awk '{print $1}')
|
||||||
|
head -n 30 ./lcov.info || true
|
||||||
|
test -s ./lcov.info
|
||||||
|
|
||||||
|
- name: Upload coverage to Codecov
|
||||||
|
uses: codecov/codecov-action@v5
|
||||||
|
with:
|
||||||
|
files: ./lcov.info
|
||||||
|
flags: unittests
|
||||||
|
name: rust-unit-coverage
|
||||||
|
verbose: true
|
||||||
|
fail_ci_if_error: true
|
||||||
|
token: ${{ secrets.CODECOV_TOKEN }}
|
||||||
|
use_pypi: true
|
||||||
|
|
||||||
|
- name: Stop and remove container
|
||||||
|
if: always()
|
||||||
|
run: docker compose -f docker-compose.test.yml down --volumes
|
||||||
@@ -0,0 +1,134 @@
|
|||||||
|
name: GHCR Publish
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_call:
|
||||||
|
inputs:
|
||||||
|
version:
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
ref:
|
||||||
|
required: true
|
||||||
|
type: string
|
||||||
|
add_latest:
|
||||||
|
required: false
|
||||||
|
type: boolean
|
||||||
|
default: false
|
||||||
|
dockerfile:
|
||||||
|
required: false
|
||||||
|
type: string
|
||||||
|
default: "./docker/Dockerfile"
|
||||||
|
target:
|
||||||
|
required: false
|
||||||
|
type: string
|
||||||
|
default: "prod"
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
packages: write
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
name: Build ${{ matrix.platform }}
|
||||||
|
runs-on: ${{ matrix.platform == 'linux/amd64' && 'ubuntu-latest' || 'ubuntu-24.04-arm' }}
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
platform: [ linux/amd64, linux/arm64 ]
|
||||||
|
steps:
|
||||||
|
- name: Prepare vars
|
||||||
|
id: prep
|
||||||
|
run: |
|
||||||
|
ARCH="${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}"
|
||||||
|
echo "arch=$ARCH" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "image=ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/agent" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
ref: ${{ inputs.ref }}
|
||||||
|
fetch-depth: 1
|
||||||
|
|
||||||
|
- name: Set up Docker Buildx
|
||||||
|
uses: docker/setup-buildx-action@v3
|
||||||
|
|
||||||
|
- name: Log in to GHCR
|
||||||
|
uses: docker/login-action@v3
|
||||||
|
with:
|
||||||
|
registry: ghcr.io
|
||||||
|
username: ${{ github.actor }}
|
||||||
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
|
- name: Build and push by digest
|
||||||
|
id: build
|
||||||
|
uses: docker/build-push-action@v6
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: ${{ inputs.dockerfile }}
|
||||||
|
platforms: ${{ matrix.platform }}
|
||||||
|
target: ${{ inputs.target }}
|
||||||
|
provenance: false
|
||||||
|
outputs: type=image,name=${{ steps.prep.outputs.image }},push-by-digest=true,name-canonical=true,push=true
|
||||||
|
cache-from: type=gha,scope=ghcr-${{ steps.prep.outputs.arch }}
|
||||||
|
cache-to: type=gha,mode=max,scope=ghcr-${{ steps.prep.outputs.arch }},ignore-error=true
|
||||||
|
|
||||||
|
- name: Export digest
|
||||||
|
env:
|
||||||
|
DIGEST: ${{ steps.build.outputs.digest }}
|
||||||
|
run: |
|
||||||
|
mkdir -p /tmp/digests
|
||||||
|
touch "/tmp/digests/${DIGEST#sha256:}"
|
||||||
|
|
||||||
|
- name: Upload digest
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: digest-${{ steps.prep.outputs.arch }}
|
||||||
|
path: /tmp/digests/*
|
||||||
|
if-no-files-found: error
|
||||||
|
retention-days: 1
|
||||||
|
|
||||||
|
merge:
|
||||||
|
name: Create multi-arch manifest
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs: build
|
||||||
|
steps:
|
||||||
|
- name: Prepare vars
|
||||||
|
id: prep
|
||||||
|
run: echo "image=ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/agent" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
- name: Download digests
|
||||||
|
uses: actions/download-artifact@v4
|
||||||
|
with:
|
||||||
|
path: /tmp/digests
|
||||||
|
pattern: digest-*
|
||||||
|
merge-multiple: true
|
||||||
|
|
||||||
|
- name: Set up Docker Buildx
|
||||||
|
uses: docker/setup-buildx-action@v3
|
||||||
|
|
||||||
|
- name: Log in to GHCR
|
||||||
|
uses: docker/login-action@v3
|
||||||
|
with:
|
||||||
|
registry: ghcr.io
|
||||||
|
username: ${{ github.actor }}
|
||||||
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
|
- name: Extract Docker metadata
|
||||||
|
id: meta
|
||||||
|
uses: docker/metadata-action@v5
|
||||||
|
with:
|
||||||
|
images: ${{ steps.prep.outputs.image }}
|
||||||
|
tags: |
|
||||||
|
type=semver,pattern={{version}},value=${{ inputs.version }}
|
||||||
|
type=semver,pattern={{major}}.{{minor}},value=${{ inputs.version }}
|
||||||
|
type=semver,pattern={{major}},value=${{ inputs.version }}
|
||||||
|
type=raw,value=latest,enable=${{ inputs.add_latest }}
|
||||||
|
|
||||||
|
- name: Create and push manifest list
|
||||||
|
working-directory: /tmp/digests
|
||||||
|
run: |
|
||||||
|
docker buildx imagetools create \
|
||||||
|
$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
|
||||||
|
$(printf '${{ steps.prep.outputs.image }}@sha256:%s ' *)
|
||||||
|
|
||||||
|
- name: Inspect
|
||||||
|
run: docker buildx imagetools inspect ${{ steps.prep.outputs.image }}:${{ inputs.version }}
|
||||||
@@ -100,6 +100,15 @@ jobs:
|
|||||||
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
|
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
|
||||||
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
|
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
|
||||||
|
|
||||||
|
publish-docker-ghcr:
|
||||||
|
needs: create-release
|
||||||
|
if: ${{ needs.create-release.result == 'success' }}
|
||||||
|
uses: ./.github/workflows/ghcr.yml
|
||||||
|
with:
|
||||||
|
version: ${{ needs.create-release.outputs.version }}
|
||||||
|
ref: ${{ needs.create-release.outputs.version }}
|
||||||
|
add_latest: true
|
||||||
|
|
||||||
publish-helm:
|
publish-helm:
|
||||||
needs: create-release
|
needs: create-release
|
||||||
if: ${{ needs.create-release.result == 'success' }}
|
if: ${{ needs.create-release.result == 'success' }}
|
||||||
@@ -109,11 +118,24 @@ jobs:
|
|||||||
secrets:
|
secrets:
|
||||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
|
build-windows:
|
||||||
|
needs: create-release
|
||||||
|
if: ${{ needs.create-release.result == 'success' }}
|
||||||
|
uses: ./.github/workflows/windows-release.yml
|
||||||
|
with:
|
||||||
|
version: ${{ needs.create-release.outputs.version }}
|
||||||
|
ref: ${{ needs.create-release.outputs.version }}
|
||||||
|
draft_tag: ${{ needs.create-release.outputs.draft_tag }}
|
||||||
|
secrets:
|
||||||
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
||||||
finalize-release:
|
finalize-release:
|
||||||
needs:
|
needs:
|
||||||
- create-release
|
- create-release
|
||||||
- publish-docker
|
- publish-docker
|
||||||
|
- publish-docker-ghcr
|
||||||
- publish-helm
|
- publish-helm
|
||||||
|
- build-windows
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
outputs:
|
outputs:
|
||||||
release_tag: ${{ steps.publish_release_step.outputs.release_tag }}
|
release_tag: ${{ steps.publish_release_step.outputs.release_tag }}
|
||||||
|
|||||||
@@ -0,0 +1,100 @@
|
|||||||
|
name: Build Windows release
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_call:
|
||||||
|
inputs:
|
||||||
|
version:
|
||||||
|
description: 'Release version (git tag), e.g. 1.18.4'
|
||||||
|
type: string
|
||||||
|
required: false
|
||||||
|
ref:
|
||||||
|
description: 'Git ref to check out and build'
|
||||||
|
type: string
|
||||||
|
required: false
|
||||||
|
draft_tag:
|
||||||
|
description: 'Draft GitHub release tag to attach the asset to (e.g. untagged-xxxx). Empty = skip upload.'
|
||||||
|
type: string
|
||||||
|
required: false
|
||||||
|
secrets:
|
||||||
|
GH_TOKEN:
|
||||||
|
required: false
|
||||||
|
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
ref:
|
||||||
|
description: 'Git ref to check out and build'
|
||||||
|
type: string
|
||||||
|
required: false
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build-windows:
|
||||||
|
runs-on: windows-latest
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
ref: ${{ inputs.ref || github.ref }}
|
||||||
|
|
||||||
|
- name: Set up Rust toolchain (MSVC)
|
||||||
|
uses: dtolnay/rust-toolchain@stable
|
||||||
|
with:
|
||||||
|
targets: x86_64-pc-windows-msvc
|
||||||
|
|
||||||
|
- name: Cache cargo build
|
||||||
|
uses: Swatinem/rust-cache@v2
|
||||||
|
|
||||||
|
- name: Cache vcpkg installed packages
|
||||||
|
uses: actions/cache@v4
|
||||||
|
with:
|
||||||
|
path: C:\vcpkg\installed
|
||||||
|
key: vcpkg-openssl-x64-windows-v1
|
||||||
|
|
||||||
|
- name: Install OpenSSL (x64) via vcpkg
|
||||||
|
shell: pwsh
|
||||||
|
run: |
|
||||||
|
# windows-latest ships vcpkg preinstalled; the install is a no-op when the
|
||||||
|
# package is restored from cache.
|
||||||
|
& "$env:VCPKG_INSTALLATION_ROOT\vcpkg.exe" install openssl:x64-windows
|
||||||
|
'VCPKG_ROOT=C:\vcpkg' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||||
|
'OPENSSL_DIR=C:\vcpkg\installed\x64-windows' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||||
|
|
||||||
|
- name: Build (cargo release)
|
||||||
|
shell: pwsh
|
||||||
|
run: cargo build --release --bin app
|
||||||
|
|
||||||
|
- name: Prepare artifact zip
|
||||||
|
id: prepare_artifact
|
||||||
|
shell: pwsh
|
||||||
|
env:
|
||||||
|
RELEASE_VERSION: ${{ inputs.version }}
|
||||||
|
run: |
|
||||||
|
$tag = $env:RELEASE_VERSION
|
||||||
|
if (-not $tag) { $tag = $env:GITHUB_SHA }
|
||||||
|
|
||||||
|
$exe = "target\release\app.exe"
|
||||||
|
if (-not (Test-Path $exe)) { Write-Error "Built binary $exe not found in target/release"; exit 1 }
|
||||||
|
|
||||||
|
$outDir = "artifact"
|
||||||
|
New-Item -ItemType Directory -Path $outDir -Force | Out-Null
|
||||||
|
# Ship under the package name, not the internal bin name "app".
|
||||||
|
Copy-Item -Path $exe -Destination "$outDir\portabase-agent.exe"
|
||||||
|
|
||||||
|
$zipName = "windows-release-$tag.zip"
|
||||||
|
if (Test-Path $zipName) { Remove-Item $zipName }
|
||||||
|
Compress-Archive -Path "$outDir\*" -DestinationPath $zipName -Force
|
||||||
|
"zip=$zipName" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
|
||||||
|
|
||||||
|
- name: Upload build artifact
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: windows-release
|
||||||
|
path: ${{ steps.prepare_artifact.outputs.zip }}
|
||||||
|
|
||||||
|
- name: Attach asset to draft release
|
||||||
|
if: ${{ inputs.draft_tag != '' }}
|
||||||
|
shell: pwsh
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ secrets.GH_TOKEN }}
|
||||||
|
run: |
|
||||||
|
gh release upload "${{ inputs.draft_tag }}" "${{ steps.prepare_artifact.outputs.zip }}" --clobber
|
||||||
+5
-1
@@ -3,4 +3,8 @@
|
|||||||
/src/data/
|
/src/data/
|
||||||
|
|
||||||
.DS_Store
|
.DS_Store
|
||||||
.env
|
.env
|
||||||
|
|
||||||
|
.claude
|
||||||
|
|
||||||
|
/docs
|
||||||
|
|||||||
+1
-1
@@ -27,5 +27,5 @@ keywords:
|
|||||||
- self-hosted
|
- self-hosted
|
||||||
- portabase
|
- portabase
|
||||||
license: Apache-2.0
|
license: Apache-2.0
|
||||||
version: 1.3.0
|
version: 1.19.0
|
||||||
date-released: '2026-02-24'
|
date-released: '2026-02-24'
|
||||||
|
|||||||
Generated
+1654
-769
File diff suppressed because it is too large
Load Diff
+25
-4
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "portabase-agent"
|
name = "portabase-agent"
|
||||||
version = "1.3.0"
|
version = "1.19.0"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
@@ -30,26 +30,47 @@ tokio-postgres = "0.7.15"
|
|||||||
futures = "0.3.31"
|
futures = "0.3.31"
|
||||||
tracing-appender = "0.2.4"
|
tracing-appender = "0.2.4"
|
||||||
time = { version = "0.3.44", features = ["macros"] }
|
time = { version = "0.3.44", features = ["macros"] }
|
||||||
mongodb = "3.5.0"
|
mongodb = "3.5.1"
|
||||||
rand = "0.9.2"
|
rand = "0.9.2"
|
||||||
bytes = "1.11.0"
|
bytes = "1.11.0"
|
||||||
async-stream = "0.3.6"
|
async-stream = "0.3.6"
|
||||||
uuid = { version = "1.20.0", features = ["v4"] }
|
uuid = { version = "1.20.0", features = ["v4"] }
|
||||||
tokio-util = "0.7.18"
|
tokio-util = { version = "0.7.18", features = ["compat", "io"] }
|
||||||
|
tiberius = { version = "0.12", default-features = false, features = ["rustls", "chrono"] }
|
||||||
aws-config = "1.8.13"
|
aws-config = "1.8.13"
|
||||||
aws-sdk-s3 = { version = "1.122.0", features = ["behavior-version-latest"] }
|
aws-sdk-s3 = { version = "1.122.0", features = ["behavior-version-latest"] }
|
||||||
|
azure_core = "1.0.0"
|
||||||
|
azure_storage_blob = "1.0.0"
|
||||||
|
google-cloud-storage = "1.15"
|
||||||
|
google-cloud-auth = "1.13"
|
||||||
async-compression = { version = "0.4.37", features = ["tokio", "gzip"] }
|
async-compression = { version = "0.4.37", features = ["tokio", "gzip"] }
|
||||||
tokio-tar = "0.3.1"
|
tokio-tar = "0.3.1"
|
||||||
oauth2 = "5.0.0"
|
oauth2 = "5.0.0"
|
||||||
hyper = "1.8.1"
|
hyper = "1.8.1"
|
||||||
async-http-client = "0.2.0"
|
|
||||||
aes-gcm = "0.11.0-rc.3"
|
aes-gcm = "0.11.0-rc.3"
|
||||||
generic-array = "0.14.7"
|
generic-array = "0.14.7"
|
||||||
futures-util = "0.3.31"
|
futures-util = "0.3.31"
|
||||||
tokio-stream = "0.1.18"
|
tokio-stream = "0.1.18"
|
||||||
aes = "0.9.0-rc.4"
|
aes = "0.9.0-rc.4"
|
||||||
typenum = "1.19.0"
|
typenum = "1.19.0"
|
||||||
|
testcontainers = "0.27.1"
|
||||||
|
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "valkey", "mysql", "mariadb", "mongo"] }
|
||||||
|
postgres = "0.19.12"
|
||||||
|
url = "2.5.8"
|
||||||
|
percent-encoding = "2.3.2"
|
||||||
|
bollard = "0.20.0"
|
||||||
|
|
||||||
|
[dev-dependencies]
|
||||||
|
tokio = { version = "1", features = ["full"] }
|
||||||
|
mockall = "0.13"
|
||||||
|
testcontainers = "0.27.1"
|
||||||
|
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "mysql", "mariadb", "mongo"] }
|
||||||
|
wiremock = "0.6"
|
||||||
|
|
||||||
|
[profile.release]
|
||||||
|
lto = true
|
||||||
|
codegen-units = 1
|
||||||
|
strip = "debuginfo"
|
||||||
|
|
||||||
[[bin]]
|
[[bin]]
|
||||||
name = "app"
|
name = "app"
|
||||||
|
|||||||
@@ -1,50 +0,0 @@
|
|||||||
include .env
|
|
||||||
export $(shell sed 's/=.*//' .env)
|
|
||||||
|
|
||||||
.PHONY: seed-mongo seed-mysql seed-postgres
|
|
||||||
|
|
||||||
seed-mongo:
|
|
||||||
@echo "Seeding MongoDB..."
|
|
||||||
bash ./scripts/mongo/seed-mongo.sh
|
|
||||||
|
|
||||||
seed-mongo-auth:
|
|
||||||
@echo "Seeding MongoDB with auth..."
|
|
||||||
bash ./scripts/mongo/seed-mongo.sh auth
|
|
||||||
|
|
||||||
seed-mysql:
|
|
||||||
@echo "Seeding MySQL..."
|
|
||||||
mysql -h 127.0.0.1 -P "$$MYSQL_PORT" -u "$$MYSQL_USER" -p"$$MYSQL_PASSWORD" "$$MYSQL_DB" < ./scripts/mysql/seed-mysql.sql
|
|
||||||
|
|
||||||
seed-mysql-1gb:
|
|
||||||
@echo "Seeding MySQL..."
|
|
||||||
mysql -h 127.0.0.1 -P "$$MYSQL_PORT" -u "$$MYSQL_USER" -p"$$MYSQL_PASSWORD" "$$MYSQL_DB" < ./scripts/mysql/seed-1gb.sql
|
|
||||||
|
|
||||||
|
|
||||||
seed-postgres:
|
|
||||||
@echo "Seeding Postgres..."
|
|
||||||
docker exec -i -e PGPASSWORD=$$PG_PASSWORD $$PG_CONTAINER \
|
|
||||||
psql -U $$PG_USER -d $$PG_DB < ./scripts/postgres/seed.sql
|
|
||||||
|
|
||||||
seed-postgres-1gb:
|
|
||||||
@echo "Seeding Postgres..."
|
|
||||||
docker exec -i -e PGPASSWORD=$$PG_PASSWORD $$PG_CONTAINER \
|
|
||||||
psql -U $$PG_USER -d $$PG_DB < ./scripts/postgres/seed-1gb.sql
|
|
||||||
|
|
||||||
|
|
||||||
SQLITE_SEED_FILE := $(if $(filter big,$(SEED)),./scripts/sqlite/seed-big.sql,./scripts/sqlite/seed.sql)
|
|
||||||
|
|
||||||
seed-sqlite:
|
|
||||||
@echo "Seeding Sqlite..."
|
|
||||||
@echo "Run as root to fix permissions inside the volume"
|
|
||||||
docker exec -u 0 -it db-sqlite sh -c "chmod -R 777 /workspace/data"
|
|
||||||
@echo "Create the database file (if it doesn’t exist)"
|
|
||||||
docker exec -u 0 -it db-sqlite sh -c "touch /workspace/data/app.db"
|
|
||||||
@echo "Seed the database"
|
|
||||||
docker exec -i db-sqlite sh -c "sqlite3 /workspace/data/app.db" < $(SQLITE_SEED_FILE)
|
|
||||||
@echo "Verify"
|
|
||||||
docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT name FROM sqlite_master WHERE type='table';"
|
|
||||||
@echo "Done"
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
seed-all: seed-mongo seed-mysql seed-postgres seed-postgres-1gb
|
|
||||||
@@ -13,15 +13,20 @@
|
|||||||
|
|
||||||
[](LICENSE)
|
[](LICENSE)
|
||||||
[](https://hub.docker.com/r/portabase/agent)
|
[](https://hub.docker.com/r/portabase/agent)
|
||||||
|
[](https://github.com/Portabase/agent/pkgs/container/charts%2Fportabase-agent)
|
||||||
[](https://github.com/Portabase/portabase)
|
[](https://github.com/Portabase/portabase)
|
||||||
[](https://www.buymeacoffee.com/portabase)
|
[](https://www.buymeacoffee.com/portabase)
|
||||||
|
|
||||||
[](https://www.python.org/downloads/release/python-3120/)
|
|
||||||
[](https://www.postgresql.org/)
|
[](https://www.postgresql.org/)
|
||||||
[](https://www.mysql.com/)
|
[](https://www.mysql.com/)
|
||||||
[](https://sqlite.org/)
|
[](https://sqlite.org/)
|
||||||
|
[](https://redis.io/)
|
||||||
[](https://mariadb.org/)
|
[](https://mariadb.org/)
|
||||||
[](https://www.mongodb.com/)
|
[](https://www.mongodb.com/)
|
||||||
|
[](https://valkey.io/)
|
||||||
|
[](https://firebirdsql.org/)
|
||||||
|
|
||||||
|
|
||||||
[](https://github.com/Portabase/portabase)
|
[](https://github.com/Portabase/portabase)
|
||||||
[](https://github.com/Portabase/portabase)
|
[](https://github.com/Portabase/portabase)
|
||||||
|
|
||||||
@@ -49,7 +54,7 @@ You have 4 ways to install Portabase Agent:
|
|||||||
|
|
||||||
- Automated CLI (recommended) - [details](https://portabase.io/docs/agent/setup#cli)
|
- Automated CLI (recommended) - [details](https://portabase.io/docs/agent/setup#cli)
|
||||||
- Docker Compose setup - [details](https://portabase.io/docs/agent/setup#docker)
|
- Docker Compose setup - [details](https://portabase.io/docs/agent/setup#docker)
|
||||||
- Kubernetes with Helm (soon)
|
- Kubernetes with Helm [details](https://portabase.io/docs/agent/setup#helm)
|
||||||
- Development setup - [details](https://portabase.io/docs/agent/setup#development)
|
- Development setup - [details](https://portabase.io/docs/agent/setup#development)
|
||||||
|
|
||||||
**Ensure Docker is installed on your machine before getting started.**
|
**Ensure Docker is installed on your machine before getting started.**
|
||||||
|
|||||||
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
Regular → Executable
+87
-5
@@ -10,10 +10,23 @@
|
|||||||
"host": "db-postgres",
|
"host": "db-postgres",
|
||||||
"generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"
|
"generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "Test database 1 - PostgreSQL - BIS",
|
||||||
|
"database": "devdb2",
|
||||||
|
"type": "postgresql",
|
||||||
|
"username": "devuser2",
|
||||||
|
"password": "changeme2",
|
||||||
|
"port": 5432,
|
||||||
|
"host": "db-postgres-2",
|
||||||
|
"generated_id": "16678159-ff7e-5697-8c83-0adeff214681",
|
||||||
|
"options": {
|
||||||
|
"keep_ownership": true
|
||||||
|
}
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "Test database 2 - MariaDB",
|
"name": "Test database 2 - MariaDB",
|
||||||
"database": "mariadb",
|
"database": "mariadb",
|
||||||
"type": "mysql",
|
"type": "mariadb",
|
||||||
"username": "mariadb",
|
"username": "mariadb",
|
||||||
"password": "changeme",
|
"password": "changeme",
|
||||||
"port": 3306,
|
"port": 3306,
|
||||||
@@ -21,7 +34,7 @@
|
|||||||
"generated_id": "16678124-ff7e-4c97-8c83-0adeff214681"
|
"generated_id": "16678124-ff7e-4c97-8c83-0adeff214681"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "Test database 4 - MongoDB Auth",
|
"name": "Test database 3 - MongoDB Auth",
|
||||||
"database": "testdbauth",
|
"database": "testdbauth",
|
||||||
"type": "mongodb",
|
"type": "mongodb",
|
||||||
"username": "root",
|
"username": "root",
|
||||||
@@ -31,7 +44,7 @@
|
|||||||
"generated_id": "16678145-ff7e-4c97-8c83-0adeff214681"
|
"generated_id": "16678145-ff7e-4c97-8c83-0adeff214681"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "Test database 5 - MongoDB",
|
"name": "Test database 4 - MongoDB",
|
||||||
"database": "testdb",
|
"database": "testdb",
|
||||||
"type": "mongodb",
|
"type": "mongodb",
|
||||||
"port": 27017,
|
"port": 27017,
|
||||||
@@ -39,16 +52,85 @@
|
|||||||
"generated_id": "16678147-ff7e-4c97-8c83-0adeff214681"
|
"generated_id": "16678147-ff7e-4c97-8c83-0adeff214681"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "Test database 6 - SQLite DB",
|
"name": "Test database 5 - SQLite DB",
|
||||||
"type": "sqlite",
|
"type": "sqlite",
|
||||||
"path": "/sqlite-data/workspace/data/app.db",
|
"path": "/sqlite-data/workspace/data/app.db",
|
||||||
"generated_id": "16678178-ff7e-4c97-8c83-0adeff214681"
|
"generated_id": "16678178-ff7e-4c97-8c83-0adeff214681"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "Test database 7 - SQLite DB",
|
"name": "Test database 6 - SQLite DB",
|
||||||
"type": "sqlite",
|
"type": "sqlite",
|
||||||
"path": "/sqlite-data-2/workspace/data/app.db",
|
"path": "/sqlite-data-2/workspace/data/app.db",
|
||||||
"generated_id": "16678179-ff7e-4c97-8c83-0adeff214681"
|
"generated_id": "16678179-ff7e-4c97-8c83-0adeff214681"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Test database 7 - Redis",
|
||||||
|
"type": "redis",
|
||||||
|
"port": 6379,
|
||||||
|
"host": "db-redis",
|
||||||
|
"generated_id": "16678166-ff7e-4c97-8c83-0adeff214681"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Test database 8 - Redis Auth",
|
||||||
|
"type": "redis",
|
||||||
|
"password": "supersecurepassword",
|
||||||
|
"port": 6379,
|
||||||
|
"username": "default",
|
||||||
|
"host": "db-redis-auth",
|
||||||
|
"generated_id": "16678160-ff7e-4c97-8c83-0adeff214681"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Test database 9 - Valkey",
|
||||||
|
"type": "valkey",
|
||||||
|
"port": 6379,
|
||||||
|
"host": "db-valkey",
|
||||||
|
"generated_id": "16678560-ff7e-4c97-8c83-0adeff214681"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Test database 10 - Valkey Auth",
|
||||||
|
"type": "valkey",
|
||||||
|
"password": "supersecurepassword",
|
||||||
|
"port": 6379,
|
||||||
|
"username": "default",
|
||||||
|
"host": "db-valkey-auth",
|
||||||
|
"generated_id": "16678561-ff7e-4c97-8c83-0adeff214681"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Test database 11 - Mysql",
|
||||||
|
"database": "mysqldb",
|
||||||
|
"type": "mysql",
|
||||||
|
"username": "mysqldb",
|
||||||
|
"password": "changeme",
|
||||||
|
"port": 3306,
|
||||||
|
"host": "db-mysql",
|
||||||
|
"generated_id": "16656124-ff7e-4c97-8c83-0adeff214681"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Test database 12 - Firebird",
|
||||||
|
"database": "/var/lib/firebird/data/mirror.fdb",
|
||||||
|
"type": "firebird",
|
||||||
|
"username": "alice",
|
||||||
|
"password": "fake_password",
|
||||||
|
"port": 3050,
|
||||||
|
"host": "db-firebird",
|
||||||
|
"generated_id": "16706124-ff7e-4c97-8c83-0adeff214681"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Test database 13 - MsSQL",
|
||||||
|
"database": "myappdb",
|
||||||
|
"type": "mssql",
|
||||||
|
"username": "sa",
|
||||||
|
"password": "Portabase!Strong1",
|
||||||
|
"port": 1433,
|
||||||
|
"host": "db-mssql",
|
||||||
|
"generated_id": "16706125-ff7e-4c97-8c83-0adeff214682"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Test database 14 - Docker Volume",
|
||||||
|
"type": "docker-volume",
|
||||||
|
"volume_name": "databases_sqlite-data",
|
||||||
|
"generated_id": "16706126-ff7e-4c97-8c83-0adeff214690",
|
||||||
|
"container_name": "db-sqlite"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
+77
-3
@@ -29,11 +29,85 @@ host = "db-mongodb-auth"
|
|||||||
generated_id = "16678145-ff7e-4c97-8c83-0adeff214681"
|
generated_id = "16678145-ff7e-4c97-8c83-0adeff214681"
|
||||||
|
|
||||||
[[databases]]
|
[[databases]]
|
||||||
name = "Test database 3 - MongoDB"
|
name = "Test database 4 - MongoDB"
|
||||||
database = "testdb"
|
database = "testdb"
|
||||||
type = "mongodb"
|
type = "mongodb"
|
||||||
username = ""
|
|
||||||
password = ""
|
|
||||||
port = 27017
|
port = 27017
|
||||||
host = "db-mongodb"
|
host = "db-mongodb"
|
||||||
generated_id = "16678147-ff7e-4c97-8c83-0adeff214681"
|
generated_id = "16678147-ff7e-4c97-8c83-0adeff214681"
|
||||||
|
|
||||||
|
[[databases]]
|
||||||
|
name = "Test database 5 - SQLite DB"
|
||||||
|
type = "sqlite"
|
||||||
|
path = "/sqlite-data/workspace/data/app.db"
|
||||||
|
generated_id = "16678178-ff7e-4c97-8c83-0adeff214681"
|
||||||
|
|
||||||
|
[[databases]]
|
||||||
|
name = "Test database 6 - SQLite DB 2"
|
||||||
|
type = "sqlite"
|
||||||
|
path = "/sqlite-data-2/workspace/data/app.db"
|
||||||
|
generated_id = "16678179-ff7e-4c97-8c83-0adeff214681"
|
||||||
|
|
||||||
|
[[databases]]
|
||||||
|
name = "Test database 7 - Redis"
|
||||||
|
type = "redis"
|
||||||
|
port = 6379
|
||||||
|
host = "db-redis"
|
||||||
|
generated_id = "16678166-ff7e-4c97-8c83-0adeff214681"
|
||||||
|
|
||||||
|
[[databases]]
|
||||||
|
name = "Test database 8 - Redis Auth"
|
||||||
|
type = "redis"
|
||||||
|
password = "supersecurepassword"
|
||||||
|
username = "default"
|
||||||
|
port = 6379
|
||||||
|
host = "db-redis-auth"
|
||||||
|
generated_id = "16678160-ff7e-4c97-8c83-0adeff214681"
|
||||||
|
|
||||||
|
[[databases]]
|
||||||
|
name = "Test database 9 - Valkey"
|
||||||
|
type = "valkey"
|
||||||
|
port = 6379
|
||||||
|
host = "db-valkey"
|
||||||
|
generated_id = "16678560-ff7e-4c97-8c83-0adeff214681"
|
||||||
|
|
||||||
|
[[databases]]
|
||||||
|
name = "Test database 10 - Valkey Auth"
|
||||||
|
type = "valkey"
|
||||||
|
password = "supersecurepassword"
|
||||||
|
username = "default"
|
||||||
|
port = 6379
|
||||||
|
host = "db-valkey-auth"
|
||||||
|
generated_id = "16678561-ff7e-4c97-8c83-0adeff214681"
|
||||||
|
|
||||||
|
[[databases]]
|
||||||
|
name = "Test database 11 - Mysql"
|
||||||
|
database = "mysqldb"
|
||||||
|
type = "mysql"
|
||||||
|
password = "changeme"
|
||||||
|
username = "mysqldb"
|
||||||
|
port = 3306
|
||||||
|
host = "db-mysql"
|
||||||
|
generated_id = "16656124-ff7e-4c97-8c83-0adeff214681"
|
||||||
|
|
||||||
|
[[databases]]
|
||||||
|
name = "Test database 12 - Firebird"
|
||||||
|
database = "/var/lib/firebird/data/mirror.fdb"
|
||||||
|
type = "firebird"
|
||||||
|
password = "fake_password"
|
||||||
|
username = "alice"
|
||||||
|
port = 3050
|
||||||
|
host = "db-firebird"
|
||||||
|
generated_id = "16706124-ff7e-4c97-8c83-0adeff214681"
|
||||||
|
|
||||||
|
[[databases]]
|
||||||
|
name = "Test database 13 - MSSQL"
|
||||||
|
database = "myappdb"
|
||||||
|
type = "mssql"
|
||||||
|
username = "sa"
|
||||||
|
password = "Portabase!Strong1"
|
||||||
|
port = 1433
|
||||||
|
host = "db-mssql"
|
||||||
|
generated_id = "16706125-ff7e-4c97-8c83-0adeff214682"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,213 @@
|
|||||||
|
name: databases
|
||||||
|
|
||||||
|
services:
|
||||||
|
db-postgres:
|
||||||
|
container_name: db-postgres
|
||||||
|
image: postgres:17-alpine
|
||||||
|
ports:
|
||||||
|
- "5436:5432"
|
||||||
|
volumes:
|
||||||
|
- postgres-data:/var/lib/postgresql/data
|
||||||
|
environment:
|
||||||
|
- POSTGRES_DB=devdb
|
||||||
|
- POSTGRES_USER=devuser
|
||||||
|
- POSTGRES_PASSWORD=changeme
|
||||||
|
networks:
|
||||||
|
- portabase
|
||||||
|
|
||||||
|
db-postgres-2:
|
||||||
|
container_name: db-postgres-2
|
||||||
|
image: postgres:17-alpine
|
||||||
|
ports:
|
||||||
|
- "5438:5432"
|
||||||
|
volumes:
|
||||||
|
- postgres-data-2:/var/lib/postgresql/data
|
||||||
|
environment:
|
||||||
|
- POSTGRES_DB=devdb2
|
||||||
|
- POSTGRES_USER=devuser2
|
||||||
|
- POSTGRES_PASSWORD=changeme2
|
||||||
|
networks:
|
||||||
|
- portabase
|
||||||
|
|
||||||
|
db-mariadb:
|
||||||
|
container_name: db-mariadb
|
||||||
|
image: mariadb:latest
|
||||||
|
ports:
|
||||||
|
- "3311:3306"
|
||||||
|
environment:
|
||||||
|
- MYSQL_DATABASE=mariadb
|
||||||
|
- MYSQL_USER=mariadb
|
||||||
|
- MYSQL_PASSWORD=changeme
|
||||||
|
- MYSQL_RANDOM_ROOT_PASSWORD=yes
|
||||||
|
volumes:
|
||||||
|
- mariadb-data:/var/lib/mysql
|
||||||
|
networks:
|
||||||
|
- portabase
|
||||||
|
|
||||||
|
db-mysql:
|
||||||
|
container_name: db-mysql
|
||||||
|
image: mysql:9.5
|
||||||
|
ports:
|
||||||
|
- "3312:3306"
|
||||||
|
environment:
|
||||||
|
- MYSQL_DATABASE=mysqldb
|
||||||
|
- MYSQL_USER=mysqldb
|
||||||
|
- MYSQL_PASSWORD=changeme
|
||||||
|
- MYSQL_RANDOM_ROOT_PASSWORD=yes
|
||||||
|
volumes:
|
||||||
|
- mysql-data:/var/lib/mysql
|
||||||
|
networks:
|
||||||
|
- portabase
|
||||||
|
|
||||||
|
|
||||||
|
db-mongodb-auth:
|
||||||
|
container_name: db-mongodb-auth
|
||||||
|
image: mongo:latest
|
||||||
|
ports:
|
||||||
|
- "27082:27017"
|
||||||
|
environment:
|
||||||
|
MONGO_INITDB_ROOT_USERNAME: root
|
||||||
|
MONGO_INITDB_ROOT_PASSWORD: rootpassword
|
||||||
|
MONGO_INITDB_DATABASE: testdbauth
|
||||||
|
command: mongod --auth
|
||||||
|
networks:
|
||||||
|
- portabase
|
||||||
|
volumes:
|
||||||
|
- mongodb-data-auth:/data/db
|
||||||
|
healthcheck:
|
||||||
|
test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 10
|
||||||
|
|
||||||
|
db-mongodb:
|
||||||
|
container_name: db-mongodb
|
||||||
|
image: mongo:latest
|
||||||
|
ports:
|
||||||
|
- "27083:27017"
|
||||||
|
volumes:
|
||||||
|
- mongodb-data:/data/db
|
||||||
|
healthcheck:
|
||||||
|
test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 10
|
||||||
|
environment:
|
||||||
|
MONGO_INITDB_DATABASE: testdb
|
||||||
|
networks:
|
||||||
|
- portabase
|
||||||
|
|
||||||
|
sqlite:
|
||||||
|
container_name: db-sqlite
|
||||||
|
image: keinos/sqlite3
|
||||||
|
volumes:
|
||||||
|
- sqlite-data:/workspace/data
|
||||||
|
working_dir: /workspace
|
||||||
|
command: tail -f /dev/null
|
||||||
|
stdin_open: true
|
||||||
|
tty: true
|
||||||
|
|
||||||
|
db-redis:
|
||||||
|
image: redis:latest
|
||||||
|
container_name: db-redis
|
||||||
|
ports:
|
||||||
|
- "6378:6379"
|
||||||
|
volumes:
|
||||||
|
- redis-data:/data
|
||||||
|
command: [ "redis-server", "--appendonly", "yes" ]
|
||||||
|
networks:
|
||||||
|
- portabase
|
||||||
|
|
||||||
|
db-redis-auth:
|
||||||
|
image: redis:latest
|
||||||
|
container_name: db-redis-auth
|
||||||
|
ports:
|
||||||
|
- "6380:6379"
|
||||||
|
volumes:
|
||||||
|
- redis-data-auth:/data
|
||||||
|
environment:
|
||||||
|
- REDIS_PASSWORD=supersecurepassword
|
||||||
|
command: [ "redis-server", "--requirepass", "supersecurepassword", "--appendonly", "yes" ]
|
||||||
|
networks:
|
||||||
|
- portabase
|
||||||
|
|
||||||
|
db-valkey:
|
||||||
|
image: valkey/valkey
|
||||||
|
container_name: db-valkey
|
||||||
|
environment:
|
||||||
|
- ALLOW_EMPTY_PASSWORD=yes
|
||||||
|
ports:
|
||||||
|
- '6381:6379'
|
||||||
|
volumes:
|
||||||
|
- valkey-data:/data
|
||||||
|
networks:
|
||||||
|
- portabase
|
||||||
|
|
||||||
|
db-valkey-auth:
|
||||||
|
image: valkey/valkey
|
||||||
|
container_name: db-valkey-auth
|
||||||
|
command: >
|
||||||
|
--requirepass "supersecurepassword"
|
||||||
|
ports:
|
||||||
|
- '6382:6379'
|
||||||
|
volumes:
|
||||||
|
- valkey-data-auth:/data
|
||||||
|
networks:
|
||||||
|
- portabase
|
||||||
|
|
||||||
|
db-firebird:
|
||||||
|
image: firebirdsql/firebird
|
||||||
|
container_name: db-firebird
|
||||||
|
restart: always
|
||||||
|
environment:
|
||||||
|
- FIREBIRD_ROOT_PASSWORD=fake_root_password
|
||||||
|
- FIREBIRD_USER=alice
|
||||||
|
- FIREBIRD_PASSWORD=fake_password
|
||||||
|
- FIREBIRD_DATABASE=mirror.fdb
|
||||||
|
- FIREBIRD_DATABASE_DEFAULT_CHARSET=UTF8
|
||||||
|
volumes:
|
||||||
|
- firebird-data:/var/lib/firebird/data
|
||||||
|
ports:
|
||||||
|
- "3060:3050"
|
||||||
|
|
||||||
|
networks:
|
||||||
|
- portabase
|
||||||
|
|
||||||
|
db-mssql:
|
||||||
|
container_name: db-mssql
|
||||||
|
image: mcr.microsoft.com/azure-sql-edge:latest
|
||||||
|
ports:
|
||||||
|
- "1433:1433"
|
||||||
|
environment:
|
||||||
|
ACCEPT_EULA: "Y"
|
||||||
|
MSSQL_SA_PASSWORD: "Portabase!Strong1"
|
||||||
|
volumes:
|
||||||
|
- mssql-data:/var/opt/mssql
|
||||||
|
networks:
|
||||||
|
- portabase
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "cat /proc/net/tcp6 | grep -q '059901' || exit 1"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 20
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
postgres-data:
|
||||||
|
postgres-data-2:
|
||||||
|
mariadb-data:
|
||||||
|
mysql-data:
|
||||||
|
mongodb-data:
|
||||||
|
mongodb-data-auth:
|
||||||
|
sqlite-data:
|
||||||
|
redis-data:
|
||||||
|
redis-data-auth:
|
||||||
|
valkey-data:
|
||||||
|
valkey-data-auth:
|
||||||
|
firebird-data:
|
||||||
|
mssql-data:
|
||||||
|
|
||||||
|
networks:
|
||||||
|
portabase:
|
||||||
|
name: portabase_network
|
||||||
|
external: true
|
||||||
|
|
||||||
+8
-53
@@ -1,69 +1,24 @@
|
|||||||
services:
|
services:
|
||||||
rust-app:
|
rust-app:
|
||||||
# build:
|
build:
|
||||||
# context: .
|
context: .
|
||||||
# dockerfile: docker/Dockerfile
|
dockerfile: docker/Dockerfile
|
||||||
# target: prod
|
target: prod
|
||||||
image: portabase/agent:latest
|
# image: portabase/agent:latest
|
||||||
# platform: linux/arm64
|
|
||||||
container_name: rust-prod
|
container_name: rust-prod
|
||||||
volumes:
|
volumes:
|
||||||
- ./databases.json:/config/config.json
|
- ./databases.json:/config/config.json
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock
|
||||||
environment:
|
environment:
|
||||||
# APP_ENV: production
|
|
||||||
LOG: info
|
LOG: info
|
||||||
TZ: "Europe/Paris"
|
TZ: "Europe/Paris"
|
||||||
# DATABASES_CONFIG_FILE: "config.toml"
|
#DATABASES_CONFIG_FILE: "config.toml"
|
||||||
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiOGZmMDE4NTQtYjJhMS00ZTE0LTkwMjctZTJiOWIxZjQ1YzdlIiwicHVibGljS2V5IjoiLS0tLS1CRUdJTiBSU0EgUFVCTElDIEtFWS0tLS0tXG5NSUlCQ2dLQ0FRRUE5TWV4M2pmdnVLdFB5YU1ERnh2Ulp2dmd3YkRJQ2JzQi81Wll5NDNSVVRBaXZRYjJiSDdYXG5qRHBQd1lJeCs4UFBrbHlRbDVMQzV1UWZEaCs4SVd4OG1LZ3FvMXpWMkdiZXdGbEdEWFYxVEdyU1ZEU25aSWR4XG52bWdYc29EeXhVMlJvWUFUMS9YMWxuc2YxenZKdkFMTkhXdEhRdk42SjVDZTFSMmFsendVRGFEVXlJNzRmSldQXG5tNTh0SDMrYklXL0VVTXdjaWNxM0oySWw3Vm9KNkZNUHJQL1ZSOWEvdFF1SU1qa200MXpFY2NscExPa2luRkxuXG54NmVUWkFSZUpya2UrbnRvZ2t4TGEyRWV5a1lUNzB4V3hKNWp5ZExBVnRvNkkyQlVLVVJoTkowTUFaU29NYUtvXG5iMGJRcnY1UzExZWllMnMrT2I3aTYzSFpkVUx0UmV1MVJ3SURBUUFCXG4tLS0tLUVORCBSU0EgUFVCTElDIEtFWS0tLS0tXG4ifQ"
|
EDGE_KEY: "<to complete>"
|
||||||
extra_hosts:
|
extra_hosts:
|
||||||
- "localhost:host-gateway"
|
- "localhost:host-gateway"
|
||||||
networks:
|
networks:
|
||||||
- portabase
|
- portabase
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
db-mongodb-auth:
|
|
||||||
container_name: db-mongodb-auth
|
|
||||||
image: mongo:latest
|
|
||||||
ports:
|
|
||||||
- "27082:27017"
|
|
||||||
environment:
|
|
||||||
MONGO_INITDB_ROOT_USERNAME: root
|
|
||||||
MONGO_INITDB_ROOT_PASSWORD: rootpassword
|
|
||||||
MONGO_INITDB_DATABASE: testdbauth
|
|
||||||
command: mongod --auth
|
|
||||||
networks:
|
|
||||||
- portabase
|
|
||||||
volumes:
|
|
||||||
- mongodb-data-auth:/data/db
|
|
||||||
healthcheck:
|
|
||||||
test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ]
|
|
||||||
interval: 5s
|
|
||||||
timeout: 5s
|
|
||||||
retries: 10
|
|
||||||
|
|
||||||
db-mongodb:
|
|
||||||
container_name: db-mongodb
|
|
||||||
image: mongo:latest
|
|
||||||
ports:
|
|
||||||
- "27083:27017"
|
|
||||||
volumes:
|
|
||||||
- mongodb-data:/data/db
|
|
||||||
healthcheck:
|
|
||||||
test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ]
|
|
||||||
interval: 5s
|
|
||||||
timeout: 5s
|
|
||||||
retries: 10
|
|
||||||
environment:
|
|
||||||
MONGO_INITDB_DATABASE: testdb
|
|
||||||
networks:
|
|
||||||
- portabase
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
mongodb-data:
|
|
||||||
mongodb-data-auth:
|
|
||||||
|
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
portabase:
|
portabase:
|
||||||
name: portabase_network
|
name: portabase_network
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
services:
|
||||||
|
agent-test:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: docker/Dockerfile
|
||||||
|
target: dev
|
||||||
|
working_dir: /app
|
||||||
|
container_name: agent-test
|
||||||
|
volumes:
|
||||||
|
- .:/app
|
||||||
|
- cargo-registry:/usr/local/cargo/registry
|
||||||
|
- cargo-git:/usr/local/cargo/git
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock
|
||||||
|
environment:
|
||||||
|
APP_ENV: test
|
||||||
|
LOG: debug
|
||||||
|
TZ: Europe/Paris
|
||||||
|
EDGE_KEY: ""
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
cargo-registry:
|
||||||
|
cargo-git:
|
||||||
+17
-91
@@ -7,117 +7,43 @@ services:
|
|||||||
container_name: rust-dev
|
container_name: rust-dev
|
||||||
volumes:
|
volumes:
|
||||||
- .:/app
|
- .:/app
|
||||||
- ./databases.json:/config/config.json
|
|
||||||
# - ./databases.toml:/config/config.toml
|
|
||||||
- cargo-registry:/usr/local/cargo/registry
|
- cargo-registry:/usr/local/cargo/registry
|
||||||
- cargo-git:/usr/local/cargo/git
|
- cargo-git:/usr/local/cargo/git
|
||||||
# - cargo-target:/app/target
|
- ./databases.json:/config/config.json
|
||||||
# - sqlite-data:/sqlite-data/workspace/data
|
#- ./databases.toml:/config/config.toml
|
||||||
# - ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data
|
- /var/run/docker.sock:/var/run/docker.sock
|
||||||
|
# - cargo-target:/app/target
|
||||||
|
- databases_sqlite-data:/sqlite-data/workspace/data
|
||||||
|
- ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data
|
||||||
|
# - /bigdisk:/scratch
|
||||||
environment:
|
environment:
|
||||||
APP_ENV: development
|
APP_ENV: development
|
||||||
LOG: debug
|
LOG: debug
|
||||||
TZ: "Europe/Paris"
|
TZ: "Europe/Paris"
|
||||||
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiNjI1MDQzY2YtN2MwMC00M2M4LWJjYzktZDM1MTk5ODk2ZGNkIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ=="
|
# TMPDIR: /scratch
|
||||||
|
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiZDY4MzU2MTQtNzE2NC00OTQ4LWJlZjMtMTlkZDc5NGQzYmRhIiwibWFzdGVyS2V5QjY0IjoiV2NiM0pQQkVTaFBjRjg5UXZwRVJuamU4NGZmak1kNm4vS2dJOUpjMCtmVT0ifQ=="
|
||||||
|
#CHUNK_SIZE_MB: "1"
|
||||||
#POOLING: 1
|
#POOLING: 1
|
||||||
#DATABASES_CONFIG_FILE: "config.toml"
|
#DATABASES_CONFIG_FILE: "config.toml"
|
||||||
extra_hosts:
|
extra_hosts:
|
||||||
- "localhost:host-gateway"
|
- "localhost:host-gateway"
|
||||||
networks:
|
networks:
|
||||||
- portabase
|
- portabase
|
||||||
|
cpus: "1.50"
|
||||||
db-postgres:
|
mem_limit: 4g
|
||||||
container_name: db-postgres
|
memswap_limit: 4g
|
||||||
image: postgres:17-alpine
|
pids_limit: 512
|
||||||
ports:
|
|
||||||
- "5436:5432"
|
|
||||||
volumes:
|
|
||||||
- postgres-data:/var/lib/postgresql/data
|
|
||||||
environment:
|
|
||||||
- POSTGRES_DB=devdb
|
|
||||||
- POSTGRES_USER=devuser
|
|
||||||
- POSTGRES_PASSWORD=changeme
|
|
||||||
networks:
|
|
||||||
- portabase
|
|
||||||
#
|
|
||||||
# db-mariadb:
|
|
||||||
# container_name: db-mariadb
|
|
||||||
# image: mariadb:latest
|
|
||||||
# ports:
|
|
||||||
# - "3311:3306"
|
|
||||||
# environment:
|
|
||||||
# - MYSQL_DATABASE=mariadb
|
|
||||||
# - MYSQL_USER=mariadb
|
|
||||||
# - MYSQL_PASSWORD=changeme
|
|
||||||
# - MYSQL_RANDOM_ROOT_PASSWORD=yes
|
|
||||||
# volumes:
|
|
||||||
# - mariadb-data:/var/lib/mysql
|
|
||||||
# networks:
|
|
||||||
# - portabase
|
|
||||||
#
|
|
||||||
#
|
|
||||||
# db-mongodb-auth:
|
|
||||||
# container_name: db-mongodb-auth
|
|
||||||
# image: mongo:latest
|
|
||||||
# ports:
|
|
||||||
# - "27082:27017"
|
|
||||||
# environment:
|
|
||||||
# MONGO_INITDB_ROOT_USERNAME: root
|
|
||||||
# MONGO_INITDB_ROOT_PASSWORD: rootpassword
|
|
||||||
# MONGO_INITDB_DATABASE: testdbauth
|
|
||||||
# command: mongod --auth
|
|
||||||
# networks:
|
|
||||||
# - portabase
|
|
||||||
# volumes:
|
|
||||||
# - mongodb-data-auth:/data/db
|
|
||||||
# healthcheck:
|
|
||||||
# test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ]
|
|
||||||
# interval: 5s
|
|
||||||
# timeout: 5s
|
|
||||||
# retries: 10
|
|
||||||
#
|
|
||||||
# db-mongodb:
|
|
||||||
# container_name: db-mongodb
|
|
||||||
# image: mongo:latest
|
|
||||||
# ports:
|
|
||||||
# - "27083:27017"
|
|
||||||
# volumes:
|
|
||||||
# - mongodb-data:/data/db
|
|
||||||
# healthcheck:
|
|
||||||
# test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ]
|
|
||||||
# interval: 5s
|
|
||||||
# timeout: 5s
|
|
||||||
# retries: 10
|
|
||||||
# environment:
|
|
||||||
# MONGO_INITDB_DATABASE: testdb
|
|
||||||
# networks:
|
|
||||||
# - portabase
|
|
||||||
|
|
||||||
# sqlite:
|
|
||||||
# container_name: db-sqlite
|
|
||||||
# image: keinos/sqlite3
|
|
||||||
# volumes:
|
|
||||||
# - sqlite-data:/workspace/data
|
|
||||||
# working_dir: /workspace
|
|
||||||
# command: tail -f /dev/null
|
|
||||||
# stdin_open: true
|
|
||||||
# tty: true
|
|
||||||
|
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
cargo-registry:
|
cargo-registry:
|
||||||
cargo-git:
|
cargo-git:
|
||||||
# cargo-target:
|
# cargo-target:
|
||||||
|
databases_sqlite-data:
|
||||||
postgres-data:
|
external: true
|
||||||
# mariadb-data:
|
|
||||||
# mongodb-data:
|
|
||||||
# mongodb-data-auth:
|
|
||||||
# sqlite-data:
|
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
portabase:
|
portabase:
|
||||||
name: portabase_network
|
name: portabase_network
|
||||||
external: true
|
external: true
|
||||||
|
|
||||||
# docker network create portabase_network
|
|
||||||
+49
-2
@@ -1,7 +1,15 @@
|
|||||||
|
# =========================
|
||||||
|
# MySQL client tools
|
||||||
|
# =========================
|
||||||
|
FROM mysql:8.4 AS mysql-client-tools
|
||||||
|
RUN mkdir -p /mysql-exports/bin /mysql-exports/lib \
|
||||||
|
&& cp /usr/bin/mysqldump /mysql-exports/bin/ \
|
||||||
|
&& find /usr/lib -name "libmysqlclient.so.21*" -exec cp {} /mysql-exports/lib/ \;
|
||||||
|
|
||||||
# =========================
|
# =========================
|
||||||
# Base image (shared)
|
# Base image (shared)
|
||||||
# =========================
|
# =========================
|
||||||
FROM rust:1.92.0 AS base
|
FROM rust:1.94.0 AS base
|
||||||
|
|
||||||
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||||
pkg-config \
|
pkg-config \
|
||||||
@@ -16,9 +24,26 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
|||||||
curl \
|
curl \
|
||||||
mariadb-client \
|
mariadb-client \
|
||||||
sqlite3 \
|
sqlite3 \
|
||||||
|
redis-tools \
|
||||||
|
valkey \
|
||||||
|
firebird3.0-utils \
|
||||||
&& apt-get clean \
|
&& apt-get clean \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
ENV DOTNET_ROOT=/usr/local/dotnet
|
||||||
|
RUN curl -sSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh \
|
||||||
|
&& chmod +x /tmp/dotnet-install.sh \
|
||||||
|
&& /tmp/dotnet-install.sh --channel 8.0 --install-dir /usr/local/dotnet \
|
||||||
|
&& rm /tmp/dotnet-install.sh \
|
||||||
|
&& /usr/local/dotnet/dotnet tool install --global microsoft.sqlpackage
|
||||||
|
|
||||||
|
ENV PATH="$PATH:/usr/local/dotnet:/root/.dotnet/tools"
|
||||||
|
|
||||||
|
RUN ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') \
|
||||||
|
&& curl -sSL "https://github.com/microsoft/go-sqlcmd/releases/latest/download/sqlcmd-linux-${ARCH}.tar.bz2" \
|
||||||
|
| tar -xjf - -C /usr/local/bin sqlcmd \
|
||||||
|
&& chmod +x /usr/local/bin/sqlcmd
|
||||||
|
|
||||||
ARG TARGETARCH
|
ARG TARGETARCH
|
||||||
|
|
||||||
# =========================
|
# =========================
|
||||||
@@ -44,12 +69,20 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \
|
|||||||
rm -rf /tmp/pg-x64 /tmp/pg-arm && \
|
rm -rf /tmp/pg-x64 /tmp/pg-arm && \
|
||||||
chmod +x /usr/lib/postgresql/*/bin/*
|
chmod +x /usr/lib/postgresql/*/bin/*
|
||||||
|
|
||||||
|
|
||||||
# =========================
|
# =========================
|
||||||
# MongoDB client binaries
|
# MongoDB client binaries
|
||||||
# =========================
|
# =========================
|
||||||
COPY assets/tools/${TARGETARCH}/mongodb/ /usr/local/mongodb/
|
COPY assets/tools/${TARGETARCH}/mongodb/ /usr/local/mongodb/
|
||||||
RUN chmod +x /usr/local/mongodb/bin/*
|
RUN chmod +x /usr/local/mongodb/bin/*
|
||||||
|
|
||||||
|
# =========================
|
||||||
|
# MySQL real mysqldump binary
|
||||||
|
# =========================
|
||||||
|
COPY --from=mysql-client-tools /mysql-exports/bin/mysqldump /usr/local/bin/mysqldump
|
||||||
|
COPY --from=mysql-client-tools /mysql-exports/lib/ /usr/local/lib/
|
||||||
|
RUN chmod +x /usr/local/bin/mysqldump && ldconfig
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
# =========================
|
# =========================
|
||||||
@@ -58,6 +91,7 @@ WORKDIR /app
|
|||||||
FROM base AS dev
|
FROM base AS dev
|
||||||
|
|
||||||
RUN cargo install cargo-watch
|
RUN cargo install cargo-watch
|
||||||
|
RUN cargo install grcov --locked
|
||||||
|
|
||||||
COPY Cargo.toml Cargo.lock ./
|
COPY Cargo.toml Cargo.lock ./
|
||||||
RUN mkdir src && echo "fn main() {}" > src/main.rs
|
RUN mkdir src && echo "fn main() {}" > src/main.rs
|
||||||
@@ -92,10 +126,19 @@ RUN apt-get update && apt-get install -y \
|
|||||||
libreadline8 \
|
libreadline8 \
|
||||||
libncurses6 \
|
libncurses6 \
|
||||||
zlib1g \
|
zlib1g \
|
||||||
|
curl \
|
||||||
mariadb-client \
|
mariadb-client \
|
||||||
sqlite3 \
|
sqlite3 \
|
||||||
|
redis-tools \
|
||||||
|
valkey \
|
||||||
|
firebird3.0-utils \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
ENV DOTNET_ROOT=/usr/local/dotnet
|
||||||
|
RUN curl -sSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh \
|
||||||
|
&& chmod +x /tmp/dotnet-install.sh \
|
||||||
|
&& /tmp/dotnet-install.sh --channel 8.0 --runtime dotnet --install-dir /usr/local/dotnet \
|
||||||
|
&& rm /tmp/dotnet-install.sh
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
@@ -106,8 +149,12 @@ RUN chmod +x /entrypoint.sh
|
|||||||
|
|
||||||
COPY --from=base /usr/lib/postgresql/ /usr/lib/postgresql/
|
COPY --from=base /usr/lib/postgresql/ /usr/lib/postgresql/
|
||||||
COPY --from=base /usr/local/mongodb/bin/ /usr/local/mongodb/bin/
|
COPY --from=base /usr/local/mongodb/bin/ /usr/local/mongodb/bin/
|
||||||
|
COPY --from=base /root/.dotnet/tools/ /root/.dotnet/tools/
|
||||||
|
COPY --from=mysql-client-tools /mysql-exports/bin/mysqldump /usr/local/bin/mysqldump
|
||||||
|
COPY --from=mysql-client-tools /mysql-exports/lib/ /usr/local/lib/
|
||||||
|
RUN chmod +x /usr/local/bin/mysqldump && ldconfig
|
||||||
|
|
||||||
|
ENV PATH="$PATH:/usr/local/dotnet:/root/.dotnet/tools"
|
||||||
ENV APP_ENV=production
|
ENV APP_ENV=production
|
||||||
|
|
||||||
CMD ["/entrypoint.sh"]
|
CMD ["/entrypoint.sh"]
|
||||||
|
|||||||
Executable
+44
@@ -0,0 +1,44 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
check_docker() {
|
||||||
|
if ! docker info > /dev/null 2>&1; then
|
||||||
|
echo "Docker is not running. Attempting to start Docker..."
|
||||||
|
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||||
|
open -a Docker
|
||||||
|
echo "Waiting for Docker to start..."
|
||||||
|
until docker info > /dev/null 2>&1; do
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
elif command -v systemctl >/dev/null 2>&1; then
|
||||||
|
sudo systemctl start docker
|
||||||
|
else
|
||||||
|
echo "Cannot start Docker automatically. Please start Docker manually."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo "Docker is running."
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
check_network() {
|
||||||
|
local network_name="portabase_network"
|
||||||
|
if ! docker network ls --format '{{.Name}}' | grep -q "^${network_name}$"; then
|
||||||
|
echo "Docker network '${network_name}' not found. Creating..."
|
||||||
|
docker network create "${network_name}"
|
||||||
|
else
|
||||||
|
echo "Docker network '${network_name}' already exists."
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
check_docker
|
||||||
|
check_network
|
||||||
|
|
||||||
|
echo "Stopping old database containers..."
|
||||||
|
docker compose -f ./docker-compose.databases.yml down
|
||||||
|
|
||||||
|
echo "Starting database containers..."
|
||||||
|
docker compose -f ./docker-compose.databases.yml up -d
|
||||||
|
|
||||||
|
echo "Starting main services..."
|
||||||
|
docker compose -f ./docker-compose.yml up
|
||||||
+10
-3
@@ -34,16 +34,23 @@ if [ -n "$TZ" ]; then
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
REDIS_PORT=65515
|
||||||
echo "[entrypoint] APP_ENV=$APP_ENV"
|
echo "[entrypoint] APP_ENV=$APP_ENV"
|
||||||
echo "[entrypoint] Starting Redis..."
|
echo "[entrypoint] Starting Redis..."
|
||||||
redis-server --daemonize yes
|
redis-server --port $REDIS_PORT --daemonize yes
|
||||||
|
|
||||||
echo "[entrypoint] Waiting for Redis to be ready..."
|
echo "[entrypoint] Waiting for Redis to be ready..."
|
||||||
until redis-cli ping >/dev/null 2>&1; do
|
MAX_RETRIES=20
|
||||||
|
COUNT=0
|
||||||
|
until redis-cli -h localhost -p "$REDIS_PORT" ping >/dev/null 2>&1 ; do
|
||||||
|
COUNT=$((COUNT+1))
|
||||||
|
if [ $COUNT -ge $MAX_RETRIES ]; then
|
||||||
|
echo "[ERROR] Redis did not start after $MAX_RETRIES attempts"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
echo "[entrypoint] Redis not ready, sleeping 1s..."
|
echo "[entrypoint] Redis not ready, sleeping 1s..."
|
||||||
sleep 1
|
sleep 1
|
||||||
done
|
done
|
||||||
|
|
||||||
echo "[entrypoint] Redis is ready"
|
echo "[entrypoint] Redis is ready"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,79 @@
|
|||||||
|
set dotenv-load := true
|
||||||
|
set shell := ["bash", "-cu"]
|
||||||
|
|
||||||
|
CLUSTER_SCRIPT := "docker/entrypoints/app-dev-entrypoint.sh"
|
||||||
|
|
||||||
|
up:
|
||||||
|
bash {{CLUSTER_SCRIPT}}
|
||||||
|
|
||||||
|
seed-mongo:
|
||||||
|
echo "Seeding MongoDB..."
|
||||||
|
bash ./scripts/mongo/seed-mongo.sh
|
||||||
|
|
||||||
|
seed-mongo-auth:
|
||||||
|
echo "Seeding MongoDB with auth..."
|
||||||
|
bash ./scripts/mongo/seed-mongo.sh auth
|
||||||
|
|
||||||
|
seed-mysql:
|
||||||
|
echo "Seeding MySQL..."
|
||||||
|
mysql -h 127.0.0.1 -P "$MYSQL_PORT" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DB" < ./scripts/mysql/seed-mysql.sql
|
||||||
|
|
||||||
|
seed-mysql-1gb:
|
||||||
|
echo "Seeding MySQL (1GB)..."
|
||||||
|
mysql -h 127.0.0.1 -P "$MYSQL_PORT" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DB" < ./scripts/mysql/seed-1gb.sql
|
||||||
|
|
||||||
|
seed-postgres:
|
||||||
|
echo "Seeding Postgres..."
|
||||||
|
docker exec -i -e PGPASSWORD="$PG_PASSWORD" "$PG_CONTAINER" \
|
||||||
|
psql -U "$PG_USER" -d "$PG_DB" < ./scripts/postgres/seed.sql
|
||||||
|
|
||||||
|
seed-postgres-1gb:
|
||||||
|
echo "Seeding Postgres (1GB)..."
|
||||||
|
docker exec -i -e PGPASSWORD="$PG_PASSWORD" "$PG_CONTAINER" \
|
||||||
|
psql -U "$PG_USER" -d "$PG_DB" < ./scripts/postgres/seed-1gb.sql
|
||||||
|
|
||||||
|
seed-sqlite:
|
||||||
|
echo "Seeding SQLite..."
|
||||||
|
bash ./scripts/sqlite/seed.sh
|
||||||
|
echo "Done"
|
||||||
|
|
||||||
|
seed-firebird:
|
||||||
|
echo "Seeding Firebird..."
|
||||||
|
docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb < ./scripts/firebird/seed.sql
|
||||||
|
|
||||||
|
echo "Verifying Firebird tables..."
|
||||||
|
echo "SELECT RDB\$RELATION_NAME FROM RDB\$RELATIONS WHERE RDB\$SYSTEM_FLAG = 0 AND RDB\$VIEW_BLR IS NULL;" \
|
||||||
|
| docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb
|
||||||
|
|
||||||
|
seed-firebird-large:
|
||||||
|
echo "Seeding Firebird..."
|
||||||
|
docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb < ./scripts/firebird/seed-large.sql
|
||||||
|
|
||||||
|
echo "Verifying Firebird tables..."
|
||||||
|
echo "SELECT RDB\$RELATION_NAME FROM RDB\$RELATIONS WHERE RDB\$SYSTEM_FLAG = 0 AND RDB\$VIEW_BLR IS NULL;" \
|
||||||
|
| docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb
|
||||||
|
|
||||||
|
seed-mssql:
|
||||||
|
echo "Seeding MSSQL..."
|
||||||
|
docker exec -i rust-dev sqlcmd -S "db-mssql,1433" -U sa -P "$MSSQL_SA_PASSWORD" -N disable -i /app/scripts/mssql/seed.sql
|
||||||
|
echo "Done"
|
||||||
|
|
||||||
|
seed-all:
|
||||||
|
just seed-mongo
|
||||||
|
just seed-mysql
|
||||||
|
just seed-postgres
|
||||||
|
just seed-postgres-1gb
|
||||||
|
just seed-sqlite
|
||||||
|
just seed-mongo
|
||||||
|
just seed-firebird
|
||||||
|
just seed-mssql
|
||||||
|
|
||||||
|
test:
|
||||||
|
echo "Build test image (with grcov included)"
|
||||||
|
docker compose -f docker-compose.test.yml build agent-test
|
||||||
|
echo "Start agent-test container"
|
||||||
|
docker compose -f docker-compose.test.yml up -d agent-test
|
||||||
|
echo "Run tests inside container"
|
||||||
|
docker compose -f docker-compose.test.yml exec -e CARGO_INCREMENTAL -e RUSTFLAGS -e LLVM_PROFILE_FILE agent-test bash -c "cargo test --verbose && sync"
|
||||||
|
echo "Down volumes tests databases"
|
||||||
|
docker compose -f docker-compose.test.yml down --volumes
|
||||||
@@ -24,3 +24,7 @@ docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT * FROM users LI
|
|||||||
```bash
|
```bash
|
||||||
docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT name FROM sqlite_master WHERE type='table';"
|
docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT name FROM sqlite_master WHERE type='table';"
|
||||||
```
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose -f docker-compose.test.yml run agent-test bash -c "cargo clean && cargo test"
|
||||||
|
```
|
||||||
@@ -0,0 +1,259 @@
|
|||||||
|
SET SQL DIALECT 3;
|
||||||
|
SET BAIL ON;
|
||||||
|
SET AUTODDL OFF;
|
||||||
|
|
||||||
|
CREATE TABLE users (
|
||||||
|
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||||
|
email VARCHAR(255) NOT NULL UNIQUE,
|
||||||
|
name VARCHAR(255),
|
||||||
|
payload BLOB SUB_TYPE TEXT,
|
||||||
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
INSERT INTO users (email, name, payload)
|
||||||
|
VALUES ('alice@example.com', 'Alice', 'Alice seed data');
|
||||||
|
|
||||||
|
INSERT INTO users (email, name, payload)
|
||||||
|
VALUES ('bob@example.com', 'Bob', 'Bob seed data');
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Each procedure call generates approximately 128 MiB:
|
||||||
|
*
|
||||||
|
* 128 rows
|
||||||
|
* × 128 chunks per row
|
||||||
|
* × 8191 bytes per chunk
|
||||||
|
* = approximately 128 MiB
|
||||||
|
*
|
||||||
|
* 40 calls = approximately 5 GiB.
|
||||||
|
*/
|
||||||
|
|
||||||
|
SET TERM ^;
|
||||||
|
|
||||||
|
CREATE PROCEDURE seed_users_batch (
|
||||||
|
p_rows INTEGER,
|
||||||
|
p_chunks_per_row INTEGER
|
||||||
|
)
|
||||||
|
AS
|
||||||
|
DECLARE VARIABLE v_row_index INTEGER;
|
||||||
|
DECLARE VARIABLE v_chunk_index INTEGER;
|
||||||
|
DECLARE VARIABLE v_uuid VARCHAR(36);
|
||||||
|
DECLARE VARIABLE v_chunk VARCHAR(8191);
|
||||||
|
DECLARE VARIABLE v_payload BLOB SUB_TYPE TEXT;
|
||||||
|
BEGIN
|
||||||
|
v_row_index = 0;
|
||||||
|
|
||||||
|
WHILE (v_row_index < p_rows) DO
|
||||||
|
BEGIN
|
||||||
|
v_payload = NULL;
|
||||||
|
v_chunk_index = 0;
|
||||||
|
|
||||||
|
WHILE (v_chunk_index < p_chunks_per_row) DO
|
||||||
|
BEGIN
|
||||||
|
/*
|
||||||
|
* Generate a different chunk to avoid producing a completely
|
||||||
|
* uniform BLOB.
|
||||||
|
*/
|
||||||
|
v_chunk = RPAD(
|
||||||
|
UUID_TO_CHAR(GEN_UUID()),
|
||||||
|
8191,
|
||||||
|
UUID_TO_CHAR(GEN_UUID())
|
||||||
|
);
|
||||||
|
|
||||||
|
v_payload = BLOB_APPEND(v_payload, v_chunk);
|
||||||
|
v_chunk_index = v_chunk_index + 1;
|
||||||
|
END
|
||||||
|
|
||||||
|
v_uuid = UUID_TO_CHAR(GEN_UUID());
|
||||||
|
|
||||||
|
INSERT INTO users (
|
||||||
|
email,
|
||||||
|
name,
|
||||||
|
payload
|
||||||
|
)
|
||||||
|
VALUES (
|
||||||
|
:v_uuid || '@example.test',
|
||||||
|
'Seed User ' || :v_uuid,
|
||||||
|
:v_payload
|
||||||
|
);
|
||||||
|
|
||||||
|
v_row_index = v_row_index + 1;
|
||||||
|
END
|
||||||
|
END^
|
||||||
|
|
||||||
|
SET TERM ;^
|
||||||
|
|
||||||
|
/* Batch 01 — approximately 128 MiB */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 02 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 03 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 04 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 05 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 06 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 07 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 08 — approximately 1 GiB total */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 09 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 10 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 11 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 12 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 13 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 14 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 15 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 16 — approximately 2 GiB total */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 17 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 18 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 19 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 20 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 21 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 22 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 23 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 24 — approximately 3 GiB total */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 25 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 26 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 27 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 28 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 29 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 30 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 31 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 32 — approximately 4 GiB total */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 33 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 34 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 35 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 36 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 37 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 38 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 39 */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
/* Batch 40 — approximately 5 GiB total */
|
||||||
|
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
DROP PROCEDURE seed_users_batch;
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
|
|
||||||
|
SELECT
|
||||||
|
COUNT(*) AS user_count,
|
||||||
|
CAST(SUM(OCTET_LENGTH(payload)) / 1073741824.0 AS DECIMAL(18, 2))
|
||||||
|
AS payload_size_gib
|
||||||
|
FROM users;
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
CREATE TABLE users (
|
||||||
|
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||||
|
email VARCHAR(255) NOT NULL UNIQUE,
|
||||||
|
name VARCHAR(255),
|
||||||
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
INSERT INTO users (email, name) VALUES ('alice@example.com', 'Alice');
|
||||||
|
INSERT INTO users (email, name) VALUES ('bob@example.com', 'Bob');
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
IF NOT EXISTS (SELECT name FROM sys.databases WHERE name = N'myappdb')
|
||||||
|
CREATE DATABASE [myappdb];
|
||||||
|
GO
|
||||||
|
|
||||||
|
USE [myappdb];
|
||||||
|
GO
|
||||||
|
|
||||||
|
IF OBJECT_ID('users', 'U') IS NULL
|
||||||
|
CREATE TABLE users (
|
||||||
|
id INT IDENTITY(1,1) PRIMARY KEY,
|
||||||
|
email NVARCHAR(255) NOT NULL UNIQUE,
|
||||||
|
name NVARCHAR(255),
|
||||||
|
created_at DATETIME DEFAULT GETDATE()
|
||||||
|
);
|
||||||
|
GO
|
||||||
|
|
||||||
|
INSERT INTO users (email, name) VALUES ('alice@example.com', 'Alice');
|
||||||
|
INSERT INTO users (email, name) VALUES ('bob@example.com', 'Bob');
|
||||||
|
INSERT INTO users (email, name) VALUES ('charlie@example.com', 'Charlie');
|
||||||
|
GO
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SEED_VALUE="${SEED:-}"
|
||||||
|
|
||||||
|
if [ "$SEED_VALUE" = "big" ]; then
|
||||||
|
SQLITE_SEED_FILE="./scripts/sqlite/seed-big.sql"
|
||||||
|
else
|
||||||
|
SQLITE_SEED_FILE="./scripts/sqlite/seed.sql"
|
||||||
|
fi
|
||||||
|
|
||||||
|
docker exec -u 0 db-sqlite sh -c "chmod -R 777 /workspace/data"
|
||||||
|
docker exec -u 0 db-sqlite sh -c "touch /workspace/data/app.db"
|
||||||
|
docker exec -i db-sqlite sh -c "sqlite3 /workspace/data/app.db" < "$SQLITE_SEED_FILE"
|
||||||
+44
-14
@@ -2,13 +2,16 @@
|
|||||||
|
|
||||||
use crate::core::context::Context;
|
use crate::core::context::Context;
|
||||||
use crate::services::backup::BackupService;
|
use crate::services::backup::BackupService;
|
||||||
use crate::services::config::ConfigService;
|
use crate::services::config::{ConfigService, DatabaseConfig};
|
||||||
use crate::services::cron::CronService;
|
use crate::services::cron::CronService;
|
||||||
use crate::services::status::StatusService;
|
use crate::services::dashboard_config::{collect_configs, load_cache, merge, persist_cache};
|
||||||
use crate::utils::common::BackupMethod;
|
|
||||||
use std::sync::Arc;
|
|
||||||
use tracing::info;
|
|
||||||
use crate::services::restore::RestoreService;
|
use crate::services::restore::RestoreService;
|
||||||
|
use crate::services::status::StatusService;
|
||||||
|
use crate::settings::CONFIG;
|
||||||
|
use crate::utils::common::BackupMethod;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use tracing::{error, info, warn};
|
||||||
|
|
||||||
pub struct Agent {
|
pub struct Agent {
|
||||||
ctx: Arc<Context>,
|
ctx: Arc<Context>,
|
||||||
@@ -17,6 +20,8 @@ pub struct Agent {
|
|||||||
cron_service: CronService,
|
cron_service: CronService,
|
||||||
backup_service: BackupService,
|
backup_service: BackupService,
|
||||||
restore_service: RestoreService,
|
restore_service: RestoreService,
|
||||||
|
dashboard_cache: Vec<DatabaseConfig>,
|
||||||
|
cache_path: PathBuf,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Agent {
|
impl Agent {
|
||||||
@@ -28,6 +33,9 @@ impl Agent {
|
|||||||
let backup_service = BackupService::new(ctx.clone());
|
let backup_service = BackupService::new(ctx.clone());
|
||||||
let restore_service = RestoreService::new(ctx.clone());
|
let restore_service = RestoreService::new(ctx.clone());
|
||||||
|
|
||||||
|
let cache_path = PathBuf::from(&CONFIG.data_path).join("dashboard_databases.json");
|
||||||
|
let dashboard_cache = load_cache(&cache_path);
|
||||||
|
|
||||||
Agent {
|
Agent {
|
||||||
ctx,
|
ctx,
|
||||||
config_service,
|
config_service,
|
||||||
@@ -35,30 +43,52 @@ impl Agent {
|
|||||||
cron_service,
|
cron_service,
|
||||||
backup_service,
|
backup_service,
|
||||||
restore_service,
|
restore_service,
|
||||||
|
dashboard_cache,
|
||||||
|
cache_path,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn run(&mut self, method: BackupMethod) -> Result<(), Box<dyn std::error::Error>> {
|
pub async fn run(&mut self, method: BackupMethod) -> Result<(), Box<dyn std::error::Error>> {
|
||||||
let config = self.config_service.load(None)?;
|
let local = self.config_service.load_optional(None);
|
||||||
let ping_result = self.status_service.ping(&config.databases).await?;
|
|
||||||
|
let merged_in = merge(&local.databases, &self.dashboard_cache);
|
||||||
|
let ping_result = self.status_service.ping(&merged_in.databases).await?;
|
||||||
|
|
||||||
|
self.dashboard_cache = collect_configs(&ping_result);
|
||||||
|
if let Err(e) = persist_cache(&self.cache_path, &self.dashboard_cache) {
|
||||||
|
error!("Failed to persist dashboard cache: {e}");
|
||||||
|
}
|
||||||
|
|
||||||
|
let merged = merge(&local.databases, &self.dashboard_cache);
|
||||||
|
|
||||||
for db in ping_result.databases.iter() {
|
for db in ping_result.databases.iter() {
|
||||||
|
let Some(database) = merged
|
||||||
|
.databases
|
||||||
|
.iter()
|
||||||
|
.find(|cfg_db| cfg_db.generated_id == db.generated_id)
|
||||||
|
else {
|
||||||
|
warn!("No config for returned database {}; skipping", db.generated_id);
|
||||||
|
continue;
|
||||||
|
};
|
||||||
info!(
|
info!(
|
||||||
"Generated Id: {} | backup action: {} | restore action: {}",
|
"Generated Id: {} | backup action: {} | restore action: {} | Database Name: {}",
|
||||||
db.generated_id, db.data.backup.action, db.data.restore.action
|
db.generated_id, db.data.backup.action, db.data.restore.action, database.name,
|
||||||
);
|
);
|
||||||
let _ = self.cron_service.sync(db).await;
|
let _ = self.cron_service.sync(db).await;
|
||||||
|
|
||||||
if db.data.backup.action {
|
if db.data.backup.action {
|
||||||
let _ = self
|
let _ = self
|
||||||
.backup_service
|
.backup_service
|
||||||
.dispatch(&db.generated_id, &config, method.clone(), &db.storages, db.encrypt)
|
.dispatch(
|
||||||
|
&db.generated_id,
|
||||||
|
&merged,
|
||||||
|
method.clone(),
|
||||||
|
&db.storages,
|
||||||
|
db.encrypt,
|
||||||
|
)
|
||||||
.await;
|
.await;
|
||||||
} else if db.data.restore.action {
|
} else if db.data.restore.action {
|
||||||
let _ = self
|
let _ = self.restore_service.dispatch(db, &merged).await;
|
||||||
.restore_service
|
|
||||||
.dispatch(db, &config)
|
|
||||||
.await;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -35,7 +35,7 @@ impl Context {
|
|||||||
panic!("Cannot initialize AgentContext due to invalid EDGE_KEY");
|
panic!("Cannot initialize AgentContext due to invalid EDGE_KEY");
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let server_url = format!("{}/api", edge_key.server_url);
|
let server_url = format!("{}/api", edge_key.server_url);
|
||||||
let api_client = ApiClient::new(server_url);
|
let api_client = ApiClient::new(server_url);
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -1,2 +1,2 @@
|
|||||||
pub mod context;
|
|
||||||
pub mod agent;
|
pub mod agent;
|
||||||
|
pub mod context;
|
||||||
|
|||||||
@@ -0,0 +1,68 @@
|
|||||||
|
use crate::domain::docker_volume::docker::{
|
||||||
|
client, create_helper, remove_helper, resolve_helper_image, start_container, stop_container,
|
||||||
|
};
|
||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
use anyhow::{Context, Result};
|
||||||
|
use bollard::query_parameters::DownloadFromContainerOptions;
|
||||||
|
use futures_util::StreamExt;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Instant;
|
||||||
|
use tokio::fs::File;
|
||||||
|
use tokio::io::AsyncWriteExt;
|
||||||
|
|
||||||
|
pub async fn run(cfg: DatabaseConfig, backup_dir: PathBuf, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||||
|
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||||
|
futures::executor::block_on(async move {
|
||||||
|
logger.log("info", format!("Starting docker-volume backup for {}", cfg.name));
|
||||||
|
|
||||||
|
let docker = client()?;
|
||||||
|
let image = resolve_helper_image(&docker).await?;
|
||||||
|
logger.log("debug", format!("Helper image: {image}"));
|
||||||
|
|
||||||
|
if let Some(name) = &cfg.container_name {
|
||||||
|
logger.log("info", format!("Stopping container {name} for consistent backup"));
|
||||||
|
stop_container(&docker, name).await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let result = async {
|
||||||
|
let helper = create_helper(&docker, &image, &cfg.volume_name, &cfg.generated_id, true, None).await?;
|
||||||
|
|
||||||
|
let file_path = backup_dir.join(format!("{}.tar", cfg.generated_id));
|
||||||
|
let start = Instant::now();
|
||||||
|
|
||||||
|
let dl_opts = DownloadFromContainerOptions { path: "/vol".to_string() };
|
||||||
|
let mut stream = docker.download_from_container(&helper.id, Some(dl_opts));
|
||||||
|
|
||||||
|
let mut out = File::create(&file_path)
|
||||||
|
.await
|
||||||
|
.with_context(|| format!("Failed to create backup file {}", file_path.display()))?;
|
||||||
|
let mut bytes_written: u64 = 0;
|
||||||
|
while let Some(chunk) = stream.next().await {
|
||||||
|
let chunk = chunk.context("Error streaming volume archive from Docker")?;
|
||||||
|
bytes_written += chunk.len() as u64;
|
||||||
|
out.write_all(&chunk).await?;
|
||||||
|
}
|
||||||
|
out.flush().await?;
|
||||||
|
|
||||||
|
let duration_ms = start.elapsed().as_millis() as f64;
|
||||||
|
logger.log_command("docker download_from_container", None, Some(0), Some(duration_ms));
|
||||||
|
logger.log("info", format!("Volume backup wrote {bytes_written} bytes to {}", file_path.display()));
|
||||||
|
|
||||||
|
remove_helper(&docker, &helper.id).await;
|
||||||
|
anyhow::Ok(file_path)
|
||||||
|
}
|
||||||
|
.await;
|
||||||
|
|
||||||
|
if let Some(name) = &cfg.container_name {
|
||||||
|
if let Err(e) = start_container(&docker, name).await {
|
||||||
|
logger.log("error", format!("Failed to restart container {name}: {e}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
result
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.await?
|
||||||
|
}
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
use anyhow::Result;
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use super::{backup, ping, restore};
|
||||||
|
use crate::domain::factory::Database;
|
||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
use crate::utils::locks::{DbOpLock, FileLock};
|
||||||
|
|
||||||
|
pub struct DockerVolumeDatabase {
|
||||||
|
cfg: DatabaseConfig,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl DockerVolumeDatabase {
|
||||||
|
pub fn new(cfg: DatabaseConfig) -> Self {
|
||||||
|
Self { cfg }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl Database for DockerVolumeDatabase {
|
||||||
|
fn file_extension(&self) -> &'static str {
|
||||||
|
".tar"
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn ping(&self) -> Result<bool> {
|
||||||
|
ping::run(self.cfg.clone()).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||||
|
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||||
|
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), logger).await;
|
||||||
|
FileLock::release(&self.cfg.generated_id).await?;
|
||||||
|
res
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
|
||||||
|
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||||
|
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
|
||||||
|
FileLock::release(&self.cfg.generated_id).await?;
|
||||||
|
res
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,169 @@
|
|||||||
|
#![allow(dead_code)]
|
||||||
|
|
||||||
|
use anyhow::{Context, Result};
|
||||||
|
use bollard::Docker;
|
||||||
|
use bollard::models::{ContainerCreateBody, HostConfig};
|
||||||
|
use bollard::query_parameters::{
|
||||||
|
CreateContainerOptions, InspectContainerOptions, ListContainersOptions,
|
||||||
|
RemoveContainerOptions, StartContainerOptions, StopContainerOptions,
|
||||||
|
};
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use tracing::{info, warn};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
pub const EPHEMERAL_LABEL: &str = "io.portabase.ephemeral";
|
||||||
|
const HELPER_MOUNT: &str = "/vol";
|
||||||
|
|
||||||
|
pub fn client() -> Result<Docker> {
|
||||||
|
Docker::connect_with_defaults().context("Failed to connect to Docker daemon socket")
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn parse_container_id(mountinfo: &str, cgroup: &str) -> Option<String> {
|
||||||
|
for src in [mountinfo, cgroup] {
|
||||||
|
for line in src.lines() {
|
||||||
|
for marker in ["/containers/", "/docker/"] {
|
||||||
|
if let Some(idx) = line.find(marker) {
|
||||||
|
let rest = &line[idx + marker.len()..];
|
||||||
|
let id: String = rest.chars().take_while(|c| c.is_ascii_hexdigit()).collect();
|
||||||
|
if id.len() >= 64 {
|
||||||
|
return Some(id[..64].to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn resolve_helper_image(docker: &Docker) -> Result<String> {
|
||||||
|
if let Ok(img) = std::env::var("PORTABASE_HELPER_IMAGE") {
|
||||||
|
if !img.trim().is_empty() {
|
||||||
|
return Ok(img);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mountinfo = std::fs::read_to_string("/proc/self/mountinfo").unwrap_or_default();
|
||||||
|
let cgroup = std::fs::read_to_string("/proc/self/cgroup").unwrap_or_default();
|
||||||
|
let id = parse_container_id(&mountinfo, &cgroup).context(
|
||||||
|
"Could not determine own container id; set PORTABASE_HELPER_IMAGE to a locally-present image",
|
||||||
|
)?;
|
||||||
|
let info = docker
|
||||||
|
.inspect_container(&id, None::<InspectContainerOptions>)
|
||||||
|
.await
|
||||||
|
.with_context(|| format!("Failed to inspect self container {id}"))?;
|
||||||
|
info.image
|
||||||
|
.context("Self container inspection returned no image reference")
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct Helper {
|
||||||
|
pub id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn create_helper(
|
||||||
|
docker: &Docker,
|
||||||
|
image: &str,
|
||||||
|
volume_name: &str,
|
||||||
|
generated_id: &str,
|
||||||
|
read_only: bool,
|
||||||
|
cmd: Option<Vec<String>>,
|
||||||
|
) -> Result<Helper> {
|
||||||
|
let bind = format!(
|
||||||
|
"{volume_name}:{HELPER_MOUNT}{}",
|
||||||
|
if read_only { ":ro" } else { "" }
|
||||||
|
);
|
||||||
|
let mut labels = HashMap::new();
|
||||||
|
labels.insert(EPHEMERAL_LABEL.to_string(), "true".to_string());
|
||||||
|
labels.insert("com.docker.compose.project".to_string(), String::new());
|
||||||
|
labels.insert("com.docker.compose.service".to_string(), String::new());
|
||||||
|
labels.insert("com.docker.compose.oneoff".to_string(), String::new());
|
||||||
|
|
||||||
|
let name = format!(
|
||||||
|
"portabase-vol-{generated_id}-{}",
|
||||||
|
&Uuid::new_v4().to_string()[..8]
|
||||||
|
);
|
||||||
|
|
||||||
|
let body = ContainerCreateBody {
|
||||||
|
image: Some(image.to_string()),
|
||||||
|
cmd,
|
||||||
|
labels: Some(labels),
|
||||||
|
host_config: Some(HostConfig {
|
||||||
|
binds: Some(vec![bind]),
|
||||||
|
auto_remove: Some(false),
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let opts = CreateContainerOptions {
|
||||||
|
name: Some(name),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let res = docker
|
||||||
|
.create_container(Some(opts), body)
|
||||||
|
.await
|
||||||
|
.with_context(|| format!("Failed to create helper container for volume {volume_name}"))?;
|
||||||
|
|
||||||
|
Ok(Helper { id: res.id })
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
pub async fn remove_helper(docker: &Docker, id: &str) {
|
||||||
|
let stop_opts = StopContainerOptions {
|
||||||
|
t: Some(2),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let _ = docker.stop_container(id, Some(stop_opts)).await;
|
||||||
|
|
||||||
|
if let Ok(info) = docker
|
||||||
|
.inspect_container(id, None::<InspectContainerOptions>)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
let name = info.name.unwrap_or_default();
|
||||||
|
let name = name.trim_start_matches('/');
|
||||||
|
let code = info.state.and_then(|s| s.exit_code).unwrap_or_default();
|
||||||
|
info!("Helper container {name} exited with code {code}");
|
||||||
|
}
|
||||||
|
|
||||||
|
let opts = RemoveContainerOptions {
|
||||||
|
force: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
if let Err(e) = docker.remove_container(id, Some(opts)).await {
|
||||||
|
warn!("Failed to remove helper container {id}: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn stop_container(docker: &Docker, name: &str) -> Result<()> {
|
||||||
|
docker
|
||||||
|
.stop_container(name, None::<StopContainerOptions>)
|
||||||
|
.await
|
||||||
|
.with_context(|| format!("Failed to stop container {name}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn start_container(docker: &Docker, name: &str) -> Result<()> {
|
||||||
|
docker
|
||||||
|
.start_container(name, None::<StartContainerOptions>)
|
||||||
|
.await
|
||||||
|
.with_context(|| format!("Failed to start container {name}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn sweep_ephemeral(docker: &Docker) -> Result<usize> {
|
||||||
|
let mut filters = HashMap::new();
|
||||||
|
filters.insert("label".to_string(), vec![format!("{EPHEMERAL_LABEL}=true")]);
|
||||||
|
|
||||||
|
let opts = ListContainersOptions {
|
||||||
|
all: true,
|
||||||
|
filters: Some(filters),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let list = docker.list_containers(Some(opts)).await?;
|
||||||
|
let mut removed = 0;
|
||||||
|
for c in list {
|
||||||
|
if let Some(id) = c.id {
|
||||||
|
remove_helper(docker, &id).await;
|
||||||
|
removed += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(removed)
|
||||||
|
}
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
pub mod backup;
|
||||||
|
pub mod database;
|
||||||
|
pub mod docker;
|
||||||
|
pub mod ping;
|
||||||
|
pub mod restore;
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
use crate::domain::docker_volume::docker::client;
|
||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
use anyhow::Result;
|
||||||
|
|
||||||
|
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
|
||||||
|
let docker = client()?;
|
||||||
|
match docker.inspect_volume(&cfg.volume_name).await {
|
||||||
|
Ok(_) => Ok(true),
|
||||||
|
Err(bollard::errors::Error::DockerResponseServerError { status_code: 404, .. }) => Ok(false),
|
||||||
|
Err(e) => Err(e.into()),
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,102 @@
|
|||||||
|
use crate::domain::docker_volume::docker::{
|
||||||
|
client, create_helper, remove_helper, resolve_helper_image, start_container, stop_container,
|
||||||
|
};
|
||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
use anyhow::{Context, Result};
|
||||||
|
use bollard::exec::StartExecResults;
|
||||||
|
use bollard::models::ExecConfig;
|
||||||
|
use bollard::query_parameters::UploadToContainerOptions;
|
||||||
|
use futures_util::StreamExt;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
pub async fn run(cfg: DatabaseConfig, archive: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
|
||||||
|
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||||
|
futures::executor::block_on(async move {
|
||||||
|
logger.log("info", format!("Starting docker-volume restore for {}", cfg.name));
|
||||||
|
|
||||||
|
let docker = client()?;
|
||||||
|
let image = resolve_helper_image(&docker).await?;
|
||||||
|
|
||||||
|
logger.log("debug", format!("Restore archive: {}", archive.display()));
|
||||||
|
|
||||||
|
if let Some(name) = &cfg.container_name {
|
||||||
|
logger.log("info", format!("Stopping container {name} for restore"));
|
||||||
|
stop_container(&docker, name).await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let result = async {
|
||||||
|
let helper = create_helper(
|
||||||
|
&docker,
|
||||||
|
&image,
|
||||||
|
&cfg.volume_name,
|
||||||
|
&cfg.generated_id,
|
||||||
|
false,
|
||||||
|
Some(vec![
|
||||||
|
"sh".into(),
|
||||||
|
"-c".into(),
|
||||||
|
"trap 'exit 0' TERM; sleep 2147483647 & wait".into(),
|
||||||
|
]),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
start_container(&docker, &helper.id).await?;
|
||||||
|
|
||||||
|
|
||||||
|
let exec = docker
|
||||||
|
.create_exec(
|
||||||
|
&helper.id,
|
||||||
|
ExecConfig {
|
||||||
|
cmd: Some(vec![
|
||||||
|
"sh".to_string(),
|
||||||
|
"-c".to_string(),
|
||||||
|
"rm -rf /vol/* /vol/.[!.]* 2>/dev/null || true".to_string(),
|
||||||
|
]),
|
||||||
|
attach_stdout: Some(true),
|
||||||
|
attach_stderr: Some(true),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.context("Failed to create wipe exec")?;
|
||||||
|
|
||||||
|
if let StartExecResults::Attached { mut output, .. } =
|
||||||
|
docker.start_exec(&exec.id, None).await.context("Failed to run wipe exec")?
|
||||||
|
{
|
||||||
|
while output.next().await.is_some() {}
|
||||||
|
}
|
||||||
|
|
||||||
|
let start = Instant::now();
|
||||||
|
|
||||||
|
let file = tokio::fs::File::open(&archive)
|
||||||
|
.await
|
||||||
|
.with_context(|| format!("Failed to open {}", archive.display()))?;
|
||||||
|
let stream = tokio_util::io::ReaderStream::new(file);
|
||||||
|
|
||||||
|
let up_opts = UploadToContainerOptions { path: "/".to_string(), ..Default::default() };
|
||||||
|
docker
|
||||||
|
.upload_to_container(&helper.id, Some(up_opts), bollard::body_try_stream(stream))
|
||||||
|
.await
|
||||||
|
.context("Failed to upload volume archive")?;
|
||||||
|
|
||||||
|
let duration_ms = start.elapsed().as_millis() as f64;
|
||||||
|
logger.log_command("docker upload_to_container", None, Some(0), Some(duration_ms));
|
||||||
|
|
||||||
|
remove_helper(&docker, &helper.id).await;
|
||||||
|
logger.log("info", format!("Volume restore completed for {}", cfg.name));
|
||||||
|
anyhow::Ok(())
|
||||||
|
}
|
||||||
|
.await;
|
||||||
|
|
||||||
|
if let Some(name) = &cfg.container_name {
|
||||||
|
if let Err(e) = start_container(&docker, name).await {
|
||||||
|
logger.log("error", format!("Failed to restart container {name}: {e}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
result
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.await?
|
||||||
|
}
|
||||||
+25
-5
@@ -1,19 +1,27 @@
|
|||||||
|
use crate::domain::docker_volume::database::DockerVolumeDatabase;
|
||||||
use crate::domain::mongodb::database::MongoDatabase;
|
use crate::domain::mongodb::database::MongoDatabase;
|
||||||
use crate::domain::mysql::database::MySQLDatabase;
|
use crate::domain::mysql::database::MySQLDatabase;
|
||||||
|
use crate::domain::postgres::cluster::database::PostgresClusterDatabase;
|
||||||
use crate::domain::postgres::database::PostgresDatabase;
|
use crate::domain::postgres::database::PostgresDatabase;
|
||||||
use crate::domain::postgres::{detect_format_from_file, detect_format_from_size};
|
use crate::domain::postgres::{detect_format_from_file, detect_format_from_size};
|
||||||
|
use crate::domain::redis::database::RedisDatabase;
|
||||||
|
use crate::domain::sqlite::database::SqliteDatabase;
|
||||||
|
use crate::domain::valkey::database::ValkeyDatabase;
|
||||||
|
use crate::domain::firebird::database::FirebirdDatabase;
|
||||||
|
use crate::domain::mariadb::database::MariaDBDatabase;
|
||||||
|
use crate::domain::mssql::database::MssqlDatabase;
|
||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
use crate::services::config::{DatabaseConfig, DbType};
|
use crate::services::config::{DatabaseConfig, DbType};
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use crate::domain::sqlite::database::SqliteDatabase;
|
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait]
|
||||||
pub trait Database: Send + Sync {
|
pub trait Database: Send + Sync {
|
||||||
fn file_extension(&self) -> &'static str;
|
fn file_extension(&self) -> &'static str;
|
||||||
async fn ping(&self) -> Result<bool>;
|
async fn ping(&self) -> Result<bool>;
|
||||||
async fn backup(&self, backup_dir: &Path) -> Result<PathBuf>;
|
async fn backup(&self, backup_dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf>;
|
||||||
async fn restore(&self, restore_file: &Path) -> Result<()>;
|
async fn restore(&self, restore_file: &Path, logger: Arc<JobLogger>) -> Result<()>;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct DatabaseFactory;
|
pub struct DatabaseFactory;
|
||||||
@@ -25,10 +33,16 @@ impl DatabaseFactory {
|
|||||||
let format = detect_format_from_size(&cfg).await;
|
let format = detect_format_from_size(&cfg).await;
|
||||||
Arc::new(PostgresDatabase::new(cfg, format))
|
Arc::new(PostgresDatabase::new(cfg, format))
|
||||||
}
|
}
|
||||||
|
DbType::PostgresqlCluster => Arc::new(PostgresClusterDatabase::new(cfg)),
|
||||||
DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)),
|
DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)),
|
||||||
DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)),
|
DbType::Mariadb => Arc::new(MariaDBDatabase::new(cfg)),
|
||||||
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
|
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
|
||||||
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
|
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
|
||||||
|
DbType::Redis => Arc::new(RedisDatabase::new(cfg)),
|
||||||
|
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)),
|
||||||
|
DbType::Firebird => Arc::new(FirebirdDatabase::new(cfg)),
|
||||||
|
DbType::Mssql => Arc::new(MssqlDatabase::new(cfg)),
|
||||||
|
DbType::DockerVolume => Arc::new(DockerVolumeDatabase::new(cfg)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -38,10 +52,16 @@ impl DatabaseFactory {
|
|||||||
let format = detect_format_from_file(restore_file);
|
let format = detect_format_from_file(restore_file);
|
||||||
Arc::new(PostgresDatabase::new(cfg, format))
|
Arc::new(PostgresDatabase::new(cfg, format))
|
||||||
}
|
}
|
||||||
|
DbType::PostgresqlCluster => Arc::new(PostgresClusterDatabase::new(cfg)),
|
||||||
DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)),
|
DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)),
|
||||||
DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)),
|
DbType::Mariadb => Arc::new(MariaDBDatabase::new(cfg)),
|
||||||
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
|
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
|
||||||
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
|
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
|
||||||
|
DbType::Redis => Arc::new(RedisDatabase::new(cfg)),
|
||||||
|
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)),
|
||||||
|
DbType::Firebird => Arc::new(FirebirdDatabase::new(cfg)),
|
||||||
|
DbType::Mssql => Arc::new(MssqlDatabase::new(cfg)),
|
||||||
|
DbType::DockerVolume => Arc::new(DockerVolumeDatabase::new(cfg)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,55 @@
|
|||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
use anyhow::{Context, Result};
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::process::Command;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
pub async fn run(
|
||||||
|
cfg: DatabaseConfig,
|
||||||
|
backup_dir: PathBuf,
|
||||||
|
file_extension: &'static str,
|
||||||
|
logger: Arc<JobLogger>,
|
||||||
|
) -> Result<PathBuf> {
|
||||||
|
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||||
|
logger.log("info", format!("Starting Firebird backup for database: {}", cfg.name));
|
||||||
|
|
||||||
|
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
|
||||||
|
let db_path = format!("{}/{}:{}", cfg.host, cfg.port, cfg.database);
|
||||||
|
|
||||||
|
logger.log("info", format!("Firebird target: {} → {}", db_path, file_path.display()));
|
||||||
|
|
||||||
|
let start = Instant::now();
|
||||||
|
let output = Command::new("gbak")
|
||||||
|
.arg("-b")
|
||||||
|
.arg("-g")
|
||||||
|
.arg("-v")
|
||||||
|
.arg("-user").arg(&cfg.username)
|
||||||
|
.arg("-password").arg(&cfg.password)
|
||||||
|
.arg(&db_path)
|
||||||
|
.arg(&file_path)
|
||||||
|
.output()
|
||||||
|
.with_context(|| format!("Failed to run gbak for {}", cfg.name))?;
|
||||||
|
let duration_ms = start.elapsed().as_millis() as f64;
|
||||||
|
let exit_code = output.status.code().unwrap_or(-1);
|
||||||
|
|
||||||
|
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||||
|
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
|
||||||
|
let combined_output = if stderr.is_empty() && stdout.is_empty() {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(format!("{}{}", stdout, stderr).trim().to_string())
|
||||||
|
};
|
||||||
|
|
||||||
|
if !output.status.success() {
|
||||||
|
logger.log_command("gbak", combined_output, Some(exit_code), Some(duration_ms));
|
||||||
|
anyhow::bail!("Firebird backup failed for {}: {}", cfg.name, stderr);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.log_command("gbak", combined_output, Some(0), Some(duration_ms));
|
||||||
|
logger.log("info", format!("Firebird backup completed for {}", cfg.name));
|
||||||
|
Ok(file_path)
|
||||||
|
})
|
||||||
|
.await?
|
||||||
|
}
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
use super::{backup, ping, restore};
|
||||||
|
use crate::domain::factory::Database;
|
||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
use crate::utils::locks::{DbOpLock, FileLock};
|
||||||
|
use anyhow::Result;
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
pub struct FirebirdDatabase {
|
||||||
|
cfg: DatabaseConfig,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FirebirdDatabase {
|
||||||
|
pub fn new(cfg: DatabaseConfig) -> Self {
|
||||||
|
Self { cfg }
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl Database for FirebirdDatabase {
|
||||||
|
fn file_extension(&self) -> &'static str {
|
||||||
|
".fbk"
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn ping(&self) -> Result<bool> {
|
||||||
|
ping::run(self.cfg.clone()).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||||
|
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||||
|
let res = backup::run(
|
||||||
|
self.cfg.clone(),
|
||||||
|
dir.to_path_buf(),
|
||||||
|
self.file_extension(),
|
||||||
|
logger,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
FileLock::release(&self.cfg.generated_id).await?;
|
||||||
|
res
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
|
||||||
|
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||||
|
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
|
||||||
|
FileLock::release(&self.cfg.generated_id).await?;
|
||||||
|
res
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
pub mod database;
|
||||||
|
mod ping;
|
||||||
|
mod backup;
|
||||||
|
mod restore;
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
use std::process::Stdio;
|
||||||
|
use tracing::{error, info};
|
||||||
|
use tokio::io::AsyncWriteExt;
|
||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
use tokio::process::Command;
|
||||||
|
use tokio::time::{Duration, timeout};
|
||||||
|
|
||||||
|
pub async fn run(cfg: DatabaseConfig) -> anyhow::Result<bool> {
|
||||||
|
let db_path = format!("{}/{}:{}", cfg.host, cfg.port, cfg.database);
|
||||||
|
|
||||||
|
info!("Running Ping database from {}", db_path);
|
||||||
|
|
||||||
|
let mut child = Command::new("isql-fb")
|
||||||
|
.arg("-q")
|
||||||
|
.arg("-user")
|
||||||
|
.arg(&cfg.username)
|
||||||
|
.arg("-password")
|
||||||
|
.arg(&cfg.password)
|
||||||
|
.arg(&db_path)
|
||||||
|
.stdin(Stdio::piped())
|
||||||
|
.stdout(Stdio::piped())
|
||||||
|
.stderr(Stdio::piped())
|
||||||
|
.spawn()?;
|
||||||
|
|
||||||
|
let query = b"SELECT 1 FROM RDB$DATABASE;\nQUIT;\n";
|
||||||
|
|
||||||
|
if let Some(mut stdin) = child.stdin.take() {
|
||||||
|
stdin.write_all(query).await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let output = match timeout(Duration::from_secs(5), child.wait_with_output()).await {
|
||||||
|
Ok(res) => res?,
|
||||||
|
Err(_) => return Ok(false),
|
||||||
|
};
|
||||||
|
|
||||||
|
if !output.status.success() {
|
||||||
|
error!("Error output for firebird: {:?}", output);
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||||
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||||
|
|
||||||
|
if stderr.to_lowercase().contains("error") {
|
||||||
|
error!("Error output for firebird: {:?}", output);
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
if stdout.contains("1") {
|
||||||
|
return Ok(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
info!("stdout {}", stdout);
|
||||||
|
error!("stderr {}", stderr);
|
||||||
|
|
||||||
|
Ok(false)
|
||||||
|
}
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
use anyhow::{Context, Result};
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::process::Command;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
|
||||||
|
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||||
|
logger.log("debug", format!("Starting Firebird restore for database {}", cfg.name));
|
||||||
|
|
||||||
|
let db_path = format!("{}/{}:{}", cfg.host, cfg.port, cfg.database);
|
||||||
|
|
||||||
|
logger.log("info", format!("Restore source: {}", restore_file.display()));
|
||||||
|
logger.log("info", format!("Restore target: {}", db_path));
|
||||||
|
|
||||||
|
let start = Instant::now();
|
||||||
|
let output = Command::new("gbak")
|
||||||
|
.arg("-c")
|
||||||
|
.arg("-v")
|
||||||
|
.arg("-replace_database")
|
||||||
|
.arg("-user")
|
||||||
|
.arg(&cfg.username)
|
||||||
|
.arg("-password")
|
||||||
|
.arg(&cfg.password)
|
||||||
|
.arg(&restore_file)
|
||||||
|
.arg(&db_path)
|
||||||
|
.output()
|
||||||
|
.with_context(|| format!("Failed to run gbak restore for {}", cfg.name))?;
|
||||||
|
|
||||||
|
let duration_ms = start.elapsed().as_millis() as f64;
|
||||||
|
let exit_code = output.status.code().unwrap_or(-1);
|
||||||
|
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||||
|
|
||||||
|
if !output.status.success() {
|
||||||
|
logger.log_command("gbak", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
|
||||||
|
logger.log("error", format!("Firebird restore failed for {}: {}", cfg.name, stderr));
|
||||||
|
anyhow::bail!("Firebird restore failed for {}: {}", cfg.name, stderr);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.log_command("gbak", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
|
||||||
|
logger.log("info", format!("Firebird restore completed for {}", cfg.name));
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.await?
|
||||||
|
}
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
use crate::domain::mariadb::connection::{select_mariadb_path, server_version};
|
||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
|
use crate::domain::mysql::connection::connection_args;
|
||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
use anyhow::{Context, Result};
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::process::Command;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
pub async fn run(
|
||||||
|
cfg: DatabaseConfig,
|
||||||
|
backup_dir: PathBuf,
|
||||||
|
env: HashMap<String, String>,
|
||||||
|
file_extension: &'static str,
|
||||||
|
logger: Arc<JobLogger>,
|
||||||
|
) -> Result<PathBuf> {
|
||||||
|
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||||
|
logger.log("info", format!("Starting backup for database {}", cfg.name));
|
||||||
|
|
||||||
|
let version = match futures::executor::block_on(server_version(&cfg)) {
|
||||||
|
Ok(v) => {
|
||||||
|
logger.log("info", format!("MariaDB version detected: {}", v));
|
||||||
|
v
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
logger.log("error", format!("Failed to get server version: {}", e));
|
||||||
|
return Err(e.into());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
|
||||||
|
let _mariadb_dump = select_mariadb_path(&version).join("mariadb-dump");
|
||||||
|
|
||||||
|
logger.log("debug", format!("Using mariadb-dump at {}", _mariadb_dump.display()));
|
||||||
|
|
||||||
|
if let Ok(out) = Command::new("mariadb-dump").arg("--version").output() {
|
||||||
|
logger.log("debug", format!("mariadb-dump client: {}", String::from_utf8_lossy(&out.stdout).trim()));
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.log("info", format!("Running mariadb-dump for {}", cfg.name));
|
||||||
|
|
||||||
|
let start = Instant::now();
|
||||||
|
let output = Command::new("mariadb-dump")
|
||||||
|
.args(connection_args(&cfg))
|
||||||
|
.arg("--routines")
|
||||||
|
.arg("--events")
|
||||||
|
.arg("--triggers")
|
||||||
|
.arg("--single-transaction")
|
||||||
|
.arg("--quick")
|
||||||
|
.arg("--skip-lock-tables")
|
||||||
|
.arg("--no-create-db")
|
||||||
|
.arg("--skip-add-drop-table")
|
||||||
|
.arg("--compress")
|
||||||
|
.arg("--verbose")
|
||||||
|
.arg(format!("--max-allowed-packet={}", cfg.max_packet_size))
|
||||||
|
.arg("--net-buffer-length=16K")
|
||||||
|
.arg("--default-character-set=utf8mb4")
|
||||||
|
.arg(&cfg.database)
|
||||||
|
.arg("-r").arg(&file_path)
|
||||||
|
.envs(env)
|
||||||
|
.output()
|
||||||
|
.with_context(|| format!("Failed to run mariadb-dump for {}", cfg.name))?;
|
||||||
|
let duration_ms = start.elapsed().as_millis() as f64;
|
||||||
|
let exit_code = output.status.code().unwrap_or(-1);
|
||||||
|
|
||||||
|
if !output.status.success() {
|
||||||
|
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||||
|
logger.log_command("mariadb-dump", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
|
||||||
|
anyhow::bail!("Mariadb backup failed for {}: {}", cfg.name, stderr);
|
||||||
|
}
|
||||||
|
|
||||||
|
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||||
|
logger.log_command("mariadb-dump", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
|
||||||
|
logger.log("info", format!("mariadb-dump completed for {}", cfg.name));
|
||||||
|
Ok(file_path)
|
||||||
|
})
|
||||||
|
.await?
|
||||||
|
}
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
use std::path::PathBuf;
|
||||||
|
use crate::domain::mysql::connection::connection_args;
|
||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
use anyhow::Result;
|
||||||
|
use std::process::Command;
|
||||||
|
|
||||||
|
pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
|
||||||
|
let output = Command::new("mariadb")
|
||||||
|
.args(connection_args(cfg))
|
||||||
|
.arg("-e")
|
||||||
|
.arg("SELECT VERSION();")
|
||||||
|
.env("MYSQL_PWD", &cfg.password)
|
||||||
|
.output()?;
|
||||||
|
|
||||||
|
if !output.status.success() {
|
||||||
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||||
|
anyhow::bail!("Version query failed: {}", stderr);
|
||||||
|
}
|
||||||
|
|
||||||
|
let version = String::from_utf8_lossy(&output.stdout)
|
||||||
|
.lines()
|
||||||
|
.nth(1)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.trim()
|
||||||
|
.to_string();
|
||||||
|
|
||||||
|
Ok(version)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
pub fn select_mariadb_path(version: &str) -> PathBuf {
|
||||||
|
let mut parts = version.split('.');
|
||||||
|
let major = parts.next().and_then(|v| v.parse::<u32>().ok()).unwrap_or(10);
|
||||||
|
let minor = parts.next().and_then(|v| v.parse::<u32>().ok()).unwrap_or(0);
|
||||||
|
|
||||||
|
if major < 10 || (major == 10 && minor <= 6) {
|
||||||
|
"/usr/local/mariadb-10.6/bin".into()
|
||||||
|
} else {
|
||||||
|
"/usr/local/mariadb-12.1/bin".into()
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
use super::{backup, ping, restore};
|
||||||
|
use crate::domain::factory::Database;
|
||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
use crate::utils::locks::{DbOpLock, FileLock};
|
||||||
|
use anyhow::Result;
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
pub struct MariaDBDatabase {
|
||||||
|
cfg: DatabaseConfig,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MariaDBDatabase {
|
||||||
|
pub fn new(cfg: DatabaseConfig) -> Self {
|
||||||
|
Self { cfg }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_env(&self) -> HashMap<String, String> {
|
||||||
|
let mut envs = std::env::vars().collect::<HashMap<_, _>>();
|
||||||
|
envs.insert("MYSQL_PWD".to_string(), self.cfg.password.to_string());
|
||||||
|
envs
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl Database for MariaDBDatabase {
|
||||||
|
fn file_extension(&self) -> &'static str {
|
||||||
|
".sql"
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn ping(&self) -> Result<bool> {
|
||||||
|
ping::run(self.cfg.clone(), self.build_env().clone()).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||||
|
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||||
|
let res = backup::run(
|
||||||
|
self.cfg.clone(),
|
||||||
|
dir.to_path_buf(),
|
||||||
|
self.build_env().clone(),
|
||||||
|
self.file_extension(),
|
||||||
|
logger,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
FileLock::release(&self.cfg.generated_id).await?;
|
||||||
|
res
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
|
||||||
|
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||||
|
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
|
||||||
|
FileLock::release(&self.cfg.generated_id).await?;
|
||||||
|
res
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
pub mod backup;
|
||||||
|
mod connection;
|
||||||
|
pub mod database;
|
||||||
|
mod ping;
|
||||||
|
mod restore;
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
use crate::domain::mysql::connection::connection_args;
|
||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use tokio::process::Command;
|
||||||
|
use tokio::time::{Duration, timeout};
|
||||||
|
|
||||||
|
pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::Result<bool> {
|
||||||
|
let mut cmd = Command::new("mysqladmin");
|
||||||
|
cmd.args(connection_args(&cfg))
|
||||||
|
.arg("ping")
|
||||||
|
.envs(env);
|
||||||
|
|
||||||
|
let result = timeout(Duration::from_secs(10), cmd.output()).await;
|
||||||
|
|
||||||
|
match result {
|
||||||
|
Ok(output) => {
|
||||||
|
let output = output?;
|
||||||
|
Ok(output.status.success())
|
||||||
|
}
|
||||||
|
Err(_) => Ok(false),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
|
use crate::domain::mysql::connection::connection_args;
|
||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
use anyhow::{Context, Result};
|
||||||
|
use std::fs::File;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::process::Command;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
|
||||||
|
let handle = tokio::task::spawn_blocking(move || -> Result<()> {
|
||||||
|
logger.log("info", format!("Starting restore for database {}", cfg.name));
|
||||||
|
|
||||||
|
let mut file = File::open(&restore_file)
|
||||||
|
.with_context(|| format!("Failed to open restore file {}", restore_file.display()))?;
|
||||||
|
|
||||||
|
let drop_create_cmd = format!(
|
||||||
|
"DROP DATABASE IF EXISTS `{0}`; CREATE DATABASE `{0}`;",
|
||||||
|
cfg.database
|
||||||
|
);
|
||||||
|
|
||||||
|
let drop_start = Instant::now();
|
||||||
|
let drop_output = Command::new("mariadb")
|
||||||
|
.args(connection_args(&cfg))
|
||||||
|
.arg("-e")
|
||||||
|
.arg(&drop_create_cmd)
|
||||||
|
.env("MYSQL_PWD", &cfg.password)
|
||||||
|
.output()
|
||||||
|
.with_context(|| format!("Failed to drop/recreate database {}", cfg.name))?;
|
||||||
|
|
||||||
|
let drop_duration_ms = drop_start.elapsed().as_millis() as f64;
|
||||||
|
let drop_exit_code = drop_output.status.code().unwrap_or(-1);
|
||||||
|
let drop_stderr = String::from_utf8_lossy(&drop_output.stderr).to_string();
|
||||||
|
|
||||||
|
if !drop_output.status.success() {
|
||||||
|
logger.log_command("mariadb", Some(drop_stderr.clone()), Some(drop_exit_code), Some(drop_duration_ms));
|
||||||
|
logger.log("error", format!("Drop/create database failed for {}: {}", cfg.name, drop_stderr));
|
||||||
|
anyhow::bail!("Failed to drop/recreate database {}", cfg.name);
|
||||||
|
}
|
||||||
|
logger.log_command("mariadb", if drop_stderr.is_empty() { None } else { Some(drop_stderr) }, Some(0), Some(drop_duration_ms));
|
||||||
|
logger.log("info", format!("Database {} dropped and recreated", cfg.name));
|
||||||
|
|
||||||
|
let start = Instant::now();
|
||||||
|
|
||||||
|
let mut child = Command::new("mariadb")
|
||||||
|
.args(connection_args(&cfg))
|
||||||
|
.arg("--database")
|
||||||
|
.arg(&cfg.database)
|
||||||
|
.env("MYSQL_PWD", &cfg.password)
|
||||||
|
.stdin(std::process::Stdio::piped())
|
||||||
|
.spawn()
|
||||||
|
.with_context(|| format!("Failed to start MariaDB restore for {}", cfg.name))?;
|
||||||
|
|
||||||
|
let mut stdin = child.stdin.take().context("Failed to open child stdin")?;
|
||||||
|
std::io::copy(&mut file, &mut stdin)
|
||||||
|
.context("Failed to stream SQL content to MariaDB stdin")?;
|
||||||
|
drop(stdin);
|
||||||
|
|
||||||
|
let output = child
|
||||||
|
.wait_with_output()
|
||||||
|
.with_context(|| format!("Failed to complete MariaDB restore for {}", cfg.name))?;
|
||||||
|
|
||||||
|
let duration_ms = start.elapsed().as_millis() as f64;
|
||||||
|
let exit_code = output.status.code().unwrap_or(-1);
|
||||||
|
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||||
|
|
||||||
|
if !output.status.success() {
|
||||||
|
logger.log_command("mariadb", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
|
||||||
|
logger.log("error", format!("MariaDB restore failed for {}: {}", cfg.name, stderr));
|
||||||
|
anyhow::bail!("MariaDB restore failed for {}", cfg.name);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.log_command("mariadb", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
|
||||||
|
logger.log("info", format!("Restore finished successfully for database {}", cfg.name));
|
||||||
|
Ok(())
|
||||||
|
});
|
||||||
|
|
||||||
|
handle.await??;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
+8
-3
@@ -1,6 +1,11 @@
|
|||||||
|
pub mod docker_volume;
|
||||||
pub mod factory;
|
pub mod factory;
|
||||||
pub mod postgres;
|
|
||||||
pub mod mysql;
|
|
||||||
mod mongodb;
|
mod mongodb;
|
||||||
|
pub mod mysql;
|
||||||
|
pub mod postgres;
|
||||||
|
mod redis;
|
||||||
mod sqlite;
|
mod sqlite;
|
||||||
|
mod valkey;
|
||||||
|
mod mariadb;
|
||||||
|
mod firebird;
|
||||||
|
pub mod mssql;
|
||||||
|
|||||||
@@ -1,35 +1,48 @@
|
|||||||
use crate::domain::mongodb::connection::{get_mongo_uri, select_mongo_path};
|
use crate::domain::mongodb::connection::{get_mongo_uri, select_mongo_path};
|
||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
use crate::services::config::DatabaseConfig;
|
use crate::services::config::DatabaseConfig;
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::process::Command;
|
use std::process::Command;
|
||||||
use tracing::{debug, error, info};
|
use std::sync::Arc;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
pub async fn run(
|
pub async fn run(
|
||||||
cfg: DatabaseConfig,
|
cfg: DatabaseConfig,
|
||||||
backup_dir: PathBuf,
|
backup_dir: PathBuf,
|
||||||
file_extension: &'static str,
|
file_extension: &'static str,
|
||||||
|
logger: Arc<JobLogger>,
|
||||||
) -> Result<PathBuf> {
|
) -> Result<PathBuf> {
|
||||||
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||||
debug!("Starting MongoDB backup for database {}", cfg.name);
|
logger.log("info", format!("Starting MongoDB backup for database {}", cfg.name));
|
||||||
|
|
||||||
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
|
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
|
||||||
let mongodump = select_mongo_path().join("mongodump");
|
let mongodump = select_mongo_path().join("mongodump");
|
||||||
let uri = get_mongo_uri(cfg.clone())?;
|
let uri = get_mongo_uri(cfg.clone())?;
|
||||||
|
|
||||||
|
logger.log("info", format!("Running mongodump for {}", cfg.name));
|
||||||
|
|
||||||
|
let start = Instant::now();
|
||||||
let output = Command::new(mongodump)
|
let output = Command::new(mongodump)
|
||||||
.arg(format!("--uri={}", uri))
|
.arg(format!("--uri={}", uri))
|
||||||
.arg(format!("--archive={}", file_path.display()))
|
.arg(format!("--archive={}", file_path.display()))
|
||||||
.arg("--gzip")
|
.arg("--gzip")
|
||||||
|
.arg("--verbose")
|
||||||
.output()
|
.output()
|
||||||
.context("MongoDB backup failed")?;
|
.context("MongoDB backup failed")?;
|
||||||
|
let duration_ms = start.elapsed().as_millis() as f64;
|
||||||
|
let exit_code = output.status.code().unwrap_or(-1);
|
||||||
|
|
||||||
|
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||||
|
|
||||||
if !output.status.success() {
|
if !output.status.success() {
|
||||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
logger.log("error", format!("MongoDB backup failed for {}: {}", cfg.name, stderr));
|
||||||
error!("MongoDB backup failed for {}: {}", cfg.name, stderr);
|
logger.log_command("mongodump", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
|
||||||
anyhow::bail!("MongoDB backup failed for {}: {}", cfg.name, stderr);
|
anyhow::bail!("MongoDB backup failed for {}: {}", cfg.name, stderr);
|
||||||
}
|
}
|
||||||
info!("MongoDB backup completed for {}", cfg.name);
|
|
||||||
|
logger.log_command("mongodump", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
|
||||||
|
logger.log("info", format!("MongoDB backup completed for {}", cfg.name));
|
||||||
Ok(file_path)
|
Ok(file_path)
|
||||||
})
|
})
|
||||||
.await?
|
.await?
|
||||||
|
|||||||
@@ -1,6 +1,13 @@
|
|||||||
use crate::services::config::DatabaseConfig;
|
use crate::services::config::DatabaseConfig;
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use mongodb::Client;
|
use mongodb::Client;
|
||||||
|
use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
|
||||||
|
|
||||||
|
const USERINFO_ENCODE: &AsciiSet = &NON_ALPHANUMERIC
|
||||||
|
.remove(b'-')
|
||||||
|
.remove(b'_')
|
||||||
|
.remove(b'.')
|
||||||
|
.remove(b'~');
|
||||||
|
|
||||||
pub async fn connect(cfg: DatabaseConfig) -> Result<Client> {
|
pub async fn connect(cfg: DatabaseConfig) -> Result<Client> {
|
||||||
let uri = get_mongo_uri(cfg)?;
|
let uri = get_mongo_uri(cfg)?;
|
||||||
@@ -16,15 +23,118 @@ pub fn select_mongo_path() -> std::path::PathBuf {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result<String> {
|
pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result<String> {
|
||||||
|
Ok(build_mongo_uri(&cfg, true))
|
||||||
if cfg.username.is_empty() || cfg.password.is_empty() {
|
|
||||||
Ok(format!("mongodb://{}:{}/{}", cfg.host, cfg.port, cfg.database))
|
|
||||||
} else {
|
|
||||||
Ok(format!(
|
|
||||||
"mongodb://{}:{}@{}:{}/{}?authSource=admin",
|
|
||||||
cfg.username, cfg.password, cfg.host, cfg.port, cfg.database
|
|
||||||
))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn build_mongo_uri(cfg: &DatabaseConfig, include_db: bool) -> String {
|
||||||
|
let is_srv = cfg.port == 0;
|
||||||
|
let scheme = if is_srv { "mongodb+srv" } else { "mongodb" };
|
||||||
|
let has_auth = !cfg.username.is_empty() && !cfg.password.is_empty();
|
||||||
|
|
||||||
|
let credentials = if has_auth {
|
||||||
|
format!(
|
||||||
|
"{}:{}@",
|
||||||
|
utf8_percent_encode(&cfg.username, USERINFO_ENCODE),
|
||||||
|
utf8_percent_encode(&cfg.password, USERINFO_ENCODE)
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
String::new()
|
||||||
|
};
|
||||||
|
|
||||||
|
let authority = if is_srv {
|
||||||
|
cfg.host.clone()
|
||||||
|
} else {
|
||||||
|
format!("{}:{}", cfg.host, cfg.port)
|
||||||
|
};
|
||||||
|
|
||||||
|
let path = if include_db {
|
||||||
|
format!("/{}", cfg.database)
|
||||||
|
} else {
|
||||||
|
"/".to_string()
|
||||||
|
};
|
||||||
|
|
||||||
|
let query = if has_auth { "?authSource=admin" } else { "" };
|
||||||
|
|
||||||
|
format!("{}://{}{}{}{}", scheme, credentials, authority, path, query)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn extract_db_name(dry_output: &str) -> Option<String> {
|
||||||
|
let mut dbs = std::collections::HashSet::new();
|
||||||
|
for line in dry_output.lines() {
|
||||||
|
if let Some(pos) = line.find("archive prelude ") {
|
||||||
|
let rest = &line[pos + "archive prelude ".len()..];
|
||||||
|
if let Some(dot) = rest.find('.') {
|
||||||
|
let db = &rest[..dot];
|
||||||
|
dbs.insert(db.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dbs.into_iter().next()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::services::config::{DatabaseConfig, DbType};
|
||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
fn cfg(host: &str, port: u16, user: &str, pass: &str) -> DatabaseConfig {
|
||||||
|
DatabaseConfig {
|
||||||
|
name: "t".into(),
|
||||||
|
database: "mydb".into(),
|
||||||
|
db_type: DbType::MongoDB,
|
||||||
|
username: user.into(),
|
||||||
|
password: pass.into(),
|
||||||
|
port,
|
||||||
|
host: host.into(),
|
||||||
|
generated_id: "id".into(),
|
||||||
|
path: String::new(),
|
||||||
|
max_packet_size: String::new(),
|
||||||
|
volume_name: String::new(),
|
||||||
|
container_name: None,
|
||||||
|
options: HashMap::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn standard_with_auth() {
|
||||||
|
let c = cfg("localhost", 27017, "user", "pass");
|
||||||
|
assert_eq!(
|
||||||
|
build_mongo_uri(&c, true),
|
||||||
|
"mongodb://user:pass@localhost:27017/mydb?authSource=admin"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn standard_no_auth() {
|
||||||
|
let c = cfg("localhost", 27017, "", "");
|
||||||
|
assert_eq!(build_mongo_uri(&c, true), "mongodb://localhost:27017/mydb");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn srv_with_auth() {
|
||||||
|
let c = cfg("cluster.example.mongodb.net", 0, "user", "pass");
|
||||||
|
assert_eq!(
|
||||||
|
build_mongo_uri(&c, true),
|
||||||
|
"mongodb+srv://user:pass@cluster.example.mongodb.net/mydb?authSource=admin"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn srv_no_db_for_dryrun() {
|
||||||
|
let c = cfg("cluster.example.mongodb.net", 0, "user", "pass");
|
||||||
|
assert_eq!(
|
||||||
|
build_mongo_uri(&c, false),
|
||||||
|
"mongodb+srv://user:pass@cluster.example.mongodb.net/?authSource=admin"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn encodes_special_chars_in_credentials() {
|
||||||
|
let c = cfg("cluster.example.mongodb.net", 0, "user", "p@ss:w/rd?");
|
||||||
|
assert_eq!(
|
||||||
|
build_mongo_uri(&c, true),
|
||||||
|
"mongodb+srv://user:p%40ss%3Aw%2Frd%3F@cluster.example.mongodb.net/mydb?authSource=admin"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,9 +1,11 @@
|
|||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
use super::{backup, ping, restore};
|
use super::{backup, ping, restore};
|
||||||
use crate::domain::factory::Database;
|
use crate::domain::factory::Database;
|
||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
use crate::services::config::DatabaseConfig;
|
use crate::services::config::DatabaseConfig;
|
||||||
use crate::utils::locks::{DbOpLock, FileLock};
|
use crate::utils::locks::{DbOpLock, FileLock};
|
||||||
|
|
||||||
@@ -27,21 +29,16 @@ impl Database for MongoDatabase {
|
|||||||
ping::run(self.cfg.clone()).await
|
ping::run(self.cfg.clone()).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
|
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||||
let res = backup::run(
|
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension(), logger).await;
|
||||||
self.cfg.clone(),
|
|
||||||
dir.to_path_buf(),
|
|
||||||
self.file_extension(),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
FileLock::release(&self.cfg.generated_id).await?;
|
FileLock::release(&self.cfg.generated_id).await?;
|
||||||
res
|
res
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn restore(&self, file: &Path) -> Result<()> {
|
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
|
||||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||||
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
|
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
|
||||||
FileLock::release(&self.cfg.generated_id).await?;
|
FileLock::release(&self.cfg.generated_id).await?;
|
||||||
res
|
res
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
mod backup;
|
mod backup;
|
||||||
mod restore;
|
mod connection;
|
||||||
pub mod database;
|
pub mod database;
|
||||||
mod ping;
|
mod ping;
|
||||||
mod connection;
|
mod restore;
|
||||||
|
|||||||
@@ -19,12 +19,16 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
|
|||||||
Ok(_) => Ok(true),
|
Ok(_) => Ok(true),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
error!("--- MongoDB Connection Error Details ---");
|
error!("--- MongoDB Connection Error Details ---");
|
||||||
error!("Target Host: {}:{}", cfg.host, cfg.port);
|
if cfg.port == 0 {
|
||||||
|
error!("Target Host: {} (srv)", cfg.host);
|
||||||
|
} else {
|
||||||
|
error!("Target Host: {}:{}", cfg.host, cfg.port);
|
||||||
|
}
|
||||||
error!("Error Kind: {:?}", e.kind);
|
error!("Error Kind: {:?}", e.kind);
|
||||||
error!("Full Error: {}", e);
|
error!("Full Error: {}", e);
|
||||||
error!("Check you database network connectivity");
|
error!("Check you database network connectivity");
|
||||||
error!("----------------------------------------");
|
error!("----------------------------------------");
|
||||||
Err(anyhow::anyhow!("Ping failed for {}: {}", cfg.name, e))
|
Ok(false)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,32 +1,70 @@
|
|||||||
use crate::domain::mongodb::connection::{get_mongo_uri, select_mongo_path};
|
use crate::domain::mongodb::connection::{
|
||||||
|
build_mongo_uri, extract_db_name, get_mongo_uri, select_mongo_path,
|
||||||
|
};
|
||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
use crate::services::config::DatabaseConfig;
|
use crate::services::config::DatabaseConfig;
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::process::Command;
|
use std::process::Command;
|
||||||
use tracing::{debug, error, info};
|
use std::sync::Arc;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
|
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
|
||||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||||
debug!("Starting MongoDB restore for database {}", cfg.name);
|
logger.log("debug", format!("Starting MongoDB restore for database {}", cfg.name));
|
||||||
|
|
||||||
let mongorestore = select_mongo_path().join("mongorestore");
|
let mongorestore = select_mongo_path().join("mongorestore");
|
||||||
let uri = get_mongo_uri(cfg.clone())?;
|
let uri = get_mongo_uri(cfg.clone())?;
|
||||||
|
|
||||||
let output = Command::new(mongorestore)
|
let dry_start = Instant::now();
|
||||||
|
let dry_run = Command::new(&mongorestore)
|
||||||
|
.arg(format!("--uri={}", build_mongo_uri(&cfg, false)))
|
||||||
|
.arg(format!("--archive={}", restore_file.display()))
|
||||||
|
.arg("--gzip")
|
||||||
|
.arg("--dryRun")
|
||||||
|
.arg("--verbose")
|
||||||
|
.output()?;
|
||||||
|
|
||||||
|
let dry_duration_ms = dry_start.elapsed().as_millis() as f64;
|
||||||
|
let dry_exit_code = dry_run.status.code().unwrap_or(-1);
|
||||||
|
let dry_output = String::from_utf8_lossy(&dry_run.stderr);
|
||||||
|
logger.log_command(
|
||||||
|
"mongorestore --dryRun",
|
||||||
|
if dry_output.is_empty() { None } else { Some(dry_output.to_string()) },
|
||||||
|
Some(dry_exit_code),
|
||||||
|
Some(dry_duration_ms),
|
||||||
|
);
|
||||||
|
let source_db = extract_db_name(&dry_output).unwrap_or_else(|| {
|
||||||
|
logger.log("info", format!("Could not detect source database from archive, falling back to configured database: {}", cfg.database));
|
||||||
|
cfg.database.clone()
|
||||||
|
});
|
||||||
|
|
||||||
|
logger.log("info", format!("Using source database in archive: {}", source_db));
|
||||||
|
|
||||||
|
let start = Instant::now();
|
||||||
|
let output = Command::new(&mongorestore)
|
||||||
.arg(format!("--uri={}", uri))
|
.arg(format!("--uri={}", uri))
|
||||||
.arg(format!("--archive={}", restore_file.display()))
|
.arg(format!("--archive={}", restore_file.display()))
|
||||||
.arg("--gzip")
|
.arg("--gzip")
|
||||||
.arg("--drop")
|
.arg("--drop")
|
||||||
|
.arg(format!("--nsInclude={}.*", source_db))
|
||||||
|
.arg(format!("--nsFrom={}.*", source_db))
|
||||||
|
.arg(format!("--nsTo={}.*", cfg.database))
|
||||||
.output()
|
.output()
|
||||||
.with_context(|| format!("Failed to run mongorestore for {}", cfg.name))?;
|
.with_context(|| format!("Failed to run mongorestore for {}", cfg.name))?;
|
||||||
|
|
||||||
|
let duration_ms = start.elapsed().as_millis() as f64;
|
||||||
|
let exit_code = output.status.code().unwrap_or(-1);
|
||||||
|
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||||
|
|
||||||
if !output.status.success() {
|
if !output.status.success() {
|
||||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
logger.log_command("mongorestore", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
|
||||||
error!("MongoDB restore failed for {}: {}", cfg.name, stderr);
|
logger.log("error", format!("MongoDB restore failed for {}: {}", cfg.name, stderr));
|
||||||
anyhow::bail!("MongoDB restore failed for : {}", cfg.name);
|
anyhow::bail!("MongoDB restore failed for: {}", cfg.name);
|
||||||
}
|
}
|
||||||
|
|
||||||
info!("MongoDB restore completed for {}", cfg.name);
|
logger.log_command("mongorestore", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
|
||||||
|
logger.log("info", format!("MongoDB restore completed for {}", cfg.name));
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
.await?
|
.await?
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
use anyhow::{Context, Result};
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::process::Command;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
pub async fn run(
|
||||||
|
cfg: DatabaseConfig,
|
||||||
|
backup_dir: PathBuf,
|
||||||
|
file_extension: &'static str,
|
||||||
|
logger: Arc<JobLogger>,
|
||||||
|
) -> Result<PathBuf> {
|
||||||
|
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||||
|
logger.log("info", format!("Starting MSSQL backup for database {}", cfg.name));
|
||||||
|
|
||||||
|
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
|
||||||
|
let connection_string = format!(
|
||||||
|
"Server=tcp:{},{};Database={};User Id={};Password={};TrustServerCertificate=True;Encrypt=False",
|
||||||
|
cfg.host, cfg.port, cfg.database, cfg.username, cfg.password
|
||||||
|
);
|
||||||
|
|
||||||
|
logger.log("info", format!("MSSQL backup: {}:{}/{} → {}", cfg.host, cfg.port, cfg.database, file_path.display()));
|
||||||
|
|
||||||
|
let start = Instant::now();
|
||||||
|
let output = Command::new("sqlpackage")
|
||||||
|
.arg("/a:Export")
|
||||||
|
.arg(format!("/scs:{}", connection_string))
|
||||||
|
.arg(format!("/tf:{}", file_path.display()))
|
||||||
|
.output()
|
||||||
|
.with_context(|| format!("Failed to run sqlpackage for {}", cfg.name))?;
|
||||||
|
let duration_ms = start.elapsed().as_millis() as f64;
|
||||||
|
let exit_code = output.status.code().unwrap_or(-1);
|
||||||
|
|
||||||
|
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||||
|
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
|
||||||
|
|
||||||
|
if !output.status.success() {
|
||||||
|
logger.log("error", format!("MSSQL backup failed — stderr: {} stdout: {}", stderr, stdout));
|
||||||
|
let out = format!("stderr: {} stdout: {}", stderr, stdout);
|
||||||
|
logger.log_command("sqlpackage", Some(out), Some(exit_code), Some(duration_ms));
|
||||||
|
anyhow::bail!("MSSQL backup failed for {}: {}", cfg.name, stderr);
|
||||||
|
}
|
||||||
|
|
||||||
|
let combined = if stdout.is_empty() && stderr.is_empty() {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(format!("{}{}", stdout, stderr).trim().to_string())
|
||||||
|
};
|
||||||
|
logger.log_command("sqlpackage", combined, Some(0), Some(duration_ms));
|
||||||
|
logger.log("info", format!("MSSQL backup completed for {}", cfg.name));
|
||||||
|
Ok(file_path)
|
||||||
|
})
|
||||||
|
.await?
|
||||||
|
}
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
use anyhow::Result;
|
||||||
|
use tiberius::{AuthMethod, Client, Config};
|
||||||
|
use tokio::net::TcpStream;
|
||||||
|
use tokio_util::compat::{Compat, TokioAsyncWriteCompatExt};
|
||||||
|
|
||||||
|
pub async fn build_client(cfg: &DatabaseConfig) -> Result<Client<Compat<TcpStream>>> {
|
||||||
|
let mut config = Config::new();
|
||||||
|
config.host(&cfg.host);
|
||||||
|
config.port(cfg.port);
|
||||||
|
config.authentication(AuthMethod::sql_server(&cfg.username, &cfg.password));
|
||||||
|
config.trust_cert();
|
||||||
|
|
||||||
|
let tcp = TcpStream::connect(config.get_addr()).await?;
|
||||||
|
tcp.set_nodelay(true)?;
|
||||||
|
let client = Client::connect(config, tcp.compat_write()).await?;
|
||||||
|
Ok(client)
|
||||||
|
}
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
use super::{backup, ping, restore};
|
||||||
|
use crate::domain::factory::Database;
|
||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
use crate::utils::locks::{DbOpLock, FileLock};
|
||||||
|
use anyhow::Result;
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
pub struct MssqlDatabase {
|
||||||
|
cfg: DatabaseConfig,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MssqlDatabase {
|
||||||
|
pub fn new(cfg: DatabaseConfig) -> Self {
|
||||||
|
Self { cfg }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl Database for MssqlDatabase {
|
||||||
|
fn file_extension(&self) -> &'static str {
|
||||||
|
".bacpac"
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn ping(&self) -> Result<bool> {
|
||||||
|
ping::run(self.cfg.clone()).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||||
|
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||||
|
let res = backup::run(
|
||||||
|
self.cfg.clone(),
|
||||||
|
dir.to_path_buf(),
|
||||||
|
self.file_extension(),
|
||||||
|
logger,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
FileLock::release(&self.cfg.generated_id).await?;
|
||||||
|
res
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
|
||||||
|
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||||
|
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
|
||||||
|
FileLock::release(&self.cfg.generated_id).await?;
|
||||||
|
res
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
pub mod database;
|
||||||
|
mod connection;
|
||||||
|
mod ping;
|
||||||
|
mod backup;
|
||||||
|
mod restore;
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
use anyhow::Result;
|
||||||
|
use tracing::{error, info};
|
||||||
|
|
||||||
|
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
|
||||||
|
info!("Running ping for MSSQL database {}", cfg.name);
|
||||||
|
|
||||||
|
match super::connection::build_client(&cfg).await {
|
||||||
|
Ok(mut client) => match client.simple_query("SELECT 1").await {
|
||||||
|
Ok(_) => {
|
||||||
|
info!("MSSQL ping succeeded for {}", cfg.name);
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("MSSQL ping query failed for {}: {:?}", cfg.name, e);
|
||||||
|
Ok(false)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
Err(e) => {
|
||||||
|
error!("MSSQL connection failed for {}: {:?}", cfg.name, e);
|
||||||
|
Ok(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
use anyhow::{Context, Result};
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::process::Command;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
|
||||||
|
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||||
|
logger.log("debug", format!("Starting MSSQL restore for database {}", cfg.name));
|
||||||
|
|
||||||
|
let connection_string = format!(
|
||||||
|
"Server=tcp:{},{};Database={};User Id={};Password={};TrustServerCertificate=True;Encrypt=False",
|
||||||
|
cfg.host, cfg.port, cfg.database, cfg.username, cfg.password
|
||||||
|
);
|
||||||
|
|
||||||
|
logger.log("info", format!(
|
||||||
|
"MSSQL restore: {} → {}:{}/{}",
|
||||||
|
restore_file.display(),
|
||||||
|
cfg.host,
|
||||||
|
cfg.port,
|
||||||
|
cfg.database
|
||||||
|
));
|
||||||
|
|
||||||
|
let start = Instant::now();
|
||||||
|
let output = Command::new("sqlpackage")
|
||||||
|
.arg("/a:Import")
|
||||||
|
.arg(format!("/tcs:{}", connection_string))
|
||||||
|
.arg(format!("/sf:{}", restore_file.display()))
|
||||||
|
.output()
|
||||||
|
.with_context(|| format!("Failed to run sqlpackage restore for {}", cfg.name))?;
|
||||||
|
|
||||||
|
let duration_ms = start.elapsed().as_millis() as f64;
|
||||||
|
let exit_code = output.status.code().unwrap_or(-1);
|
||||||
|
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||||
|
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
|
||||||
|
let combined = format!("{}{}", stdout, stderr);
|
||||||
|
|
||||||
|
if !output.status.success() {
|
||||||
|
logger.log_command("sqlpackage", if combined.is_empty() { None } else { Some(combined.clone()) }, Some(exit_code), Some(duration_ms));
|
||||||
|
logger.log("error", format!(
|
||||||
|
"MSSQL restore failed for {} — stderr: {} stdout: {}",
|
||||||
|
cfg.name, stderr, stdout
|
||||||
|
));
|
||||||
|
anyhow::bail!("MSSQL restore failed for {}: {}", cfg.name, stderr);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.log_command("sqlpackage", if combined.is_empty() { None } else { Some(combined) }, Some(0), Some(duration_ms));
|
||||||
|
logger.log("info", format!("MSSQL restore completed for {}", cfg.name));
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.await?
|
||||||
|
}
|
||||||
+36
-20
@@ -1,62 +1,78 @@
|
|||||||
use crate::domain::mysql::connection::{server_version};
|
use crate::domain::mysql::connection::{connection_args, server_version};
|
||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
use crate::services::config::DatabaseConfig;
|
use crate::services::config::DatabaseConfig;
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::process::Command;
|
use std::process::Command;
|
||||||
use tracing::{debug, error, info};
|
use std::sync::Arc;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
pub async fn run(
|
pub async fn run(
|
||||||
cfg: DatabaseConfig,
|
cfg: DatabaseConfig,
|
||||||
backup_dir: PathBuf,
|
backup_dir: PathBuf,
|
||||||
env: HashMap<String, String>,
|
env: HashMap<String, String>,
|
||||||
file_extension: &'static str,
|
file_extension: &'static str,
|
||||||
|
logger: Arc<JobLogger>,
|
||||||
) -> Result<PathBuf> {
|
) -> Result<PathBuf> {
|
||||||
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||||
debug!("Starting backup for database {}", cfg.name);
|
logger.log("info", format!("Starting backup for database {}", cfg.name));
|
||||||
|
|
||||||
let version = match futures::executor::block_on(server_version(&cfg)) {
|
let _version = match futures::executor::block_on(server_version(&cfg)) {
|
||||||
Ok(v) => {
|
Ok(v) => {
|
||||||
debug!("Mysql version detected: {}", v);
|
logger.log("debug", format!("MySQL version detected: {}", v));
|
||||||
v
|
v
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
error!("Failed to get server version for {}: {:?}", cfg.name, e);
|
logger.log("error", format!("Failed to get server version: {}", e));
|
||||||
return Err(e.into());
|
return Err(e.into());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
info!("Mysql version found: {}", version);
|
|
||||||
|
|
||||||
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
|
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
|
||||||
|
|
||||||
|
if let Ok(out) = Command::new("mysqldump").arg("--version").output() {
|
||||||
|
logger.log("debug", format!("mysqldump client: {}", String::from_utf8_lossy(&out.stdout).trim()));
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.log("info", format!("Running mysqldump for {}", cfg.name));
|
||||||
|
|
||||||
|
let start = Instant::now();
|
||||||
let output = Command::new("mysqldump")
|
let output = Command::new("mysqldump")
|
||||||
.arg("--host")
|
.args(connection_args(&cfg))
|
||||||
.arg(cfg.host)
|
|
||||||
.arg("--port")
|
|
||||||
.arg(cfg.port.to_string())
|
|
||||||
.arg("--user")
|
|
||||||
.arg(cfg.username)
|
|
||||||
.arg("--routines")
|
.arg("--routines")
|
||||||
.arg("--events")
|
.arg("--events")
|
||||||
.arg("--triggers")
|
.arg("--triggers")
|
||||||
.arg("--verbose")
|
.arg("--verbose")
|
||||||
.arg("--single-transaction")
|
.arg("--single-transaction")
|
||||||
|
.arg("--set-gtid-purged=OFF")
|
||||||
|
.arg("--no-tablespaces")
|
||||||
.arg("--quick")
|
.arg("--quick")
|
||||||
.arg("--add-drop-database")
|
.arg("--skip-lock-tables")
|
||||||
.arg("--databases")
|
.arg("--skip-add-drop-table")
|
||||||
.arg(cfg.database)
|
.arg("--no-create-db")
|
||||||
.arg("-r")
|
.arg("--default-character-set=utf8mb4")
|
||||||
.arg(&file_path)
|
.arg("--network-timeout")
|
||||||
|
.arg(format!("--max-allowed-packet={}", cfg.max_packet_size))
|
||||||
|
.arg(&cfg.database)
|
||||||
|
.arg("-r").arg(&file_path)
|
||||||
.envs(env)
|
.envs(env)
|
||||||
.output()
|
.output()
|
||||||
.with_context(|| format!("Failed to run mysqldump for {}", cfg.name))?;
|
.with_context(|| format!("Failed to run mysqldump for {}", cfg.name))?;
|
||||||
|
let duration_ms = start.elapsed().as_millis() as f64;
|
||||||
|
let exit_code = output.status.code().unwrap_or(-1);
|
||||||
|
|
||||||
|
let _stdout = String::from_utf8_lossy(&output.stdout).to_string();
|
||||||
|
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||||
|
|
||||||
if !output.status.success() {
|
if !output.status.success() {
|
||||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
logger.log_command("mysqldump", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
|
||||||
anyhow::bail!("MySQL backup failed for {}: {}", cfg.name, stderr);
|
anyhow::bail!("MySQL backup failed for {}: {}", cfg.name, stderr);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
logger.log_command("mysqldump", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
|
||||||
|
logger.log("info", format!("mysqldump completed for {}", cfg.name));
|
||||||
|
|
||||||
Ok(file_path)
|
Ok(file_path)
|
||||||
})
|
})
|
||||||
.await?
|
.await?
|
||||||
|
|||||||
@@ -1,14 +1,36 @@
|
|||||||
use crate::services::config::DatabaseConfig;
|
use crate::services::config::DatabaseConfig;
|
||||||
use std::process::Command;
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
|
use std::process::Command;
|
||||||
|
|
||||||
|
pub fn connection_args(cfg: &DatabaseConfig) -> Vec<String> {
|
||||||
|
let protocol = cfg
|
||||||
|
.options
|
||||||
|
.get("protocol")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("tcp");
|
||||||
|
|
||||||
|
let mut args = vec![
|
||||||
|
format!("--protocol={}", protocol),
|
||||||
|
"--host".to_string(),
|
||||||
|
cfg.host.clone(),
|
||||||
|
"--port".to_string(),
|
||||||
|
cfg.port.to_string(),
|
||||||
|
"--user".to_string(),
|
||||||
|
cfg.username.clone(),
|
||||||
|
];
|
||||||
|
|
||||||
|
if let Some(socket) = cfg.options.get("socket").and_then(|v| v.as_str()) {
|
||||||
|
args.push(format!("--socket={}", socket));
|
||||||
|
}
|
||||||
|
|
||||||
|
args
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
|
pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
|
||||||
|
|
||||||
let output = Command::new("mysql")
|
let output = Command::new("mysql")
|
||||||
.arg("--host").arg(&cfg.host)
|
.args(connection_args(cfg))
|
||||||
.arg("--port").arg(cfg.port.to_string())
|
.arg("-e")
|
||||||
.arg("--user").arg(&cfg.username)
|
.arg("SELECT VERSION();")
|
||||||
.arg("-e").arg("SELECT VERSION();")
|
|
||||||
.env("MYSQL_PWD", &cfg.password)
|
.env("MYSQL_PWD", &cfg.password)
|
||||||
.output()?;
|
.output()?;
|
||||||
|
|
||||||
@@ -19,11 +41,10 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
|
|||||||
|
|
||||||
let version = String::from_utf8_lossy(&output.stdout)
|
let version = String::from_utf8_lossy(&output.stdout)
|
||||||
.lines()
|
.lines()
|
||||||
.nth(1) // skip column header
|
.nth(1)
|
||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
.trim()
|
.trim()
|
||||||
.to_string();
|
.to_string();
|
||||||
|
|
||||||
Ok(version)
|
Ok(version)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,14 +1,13 @@
|
|||||||
use std::collections::HashMap;
|
use super::{backup, ping, restore};
|
||||||
use anyhow::Result;
|
|
||||||
use async_trait::async_trait;
|
|
||||||
use std::path::{Path, PathBuf};
|
|
||||||
use super::{
|
|
||||||
backup,
|
|
||||||
ping, restore,
|
|
||||||
};
|
|
||||||
use crate::domain::factory::Database;
|
use crate::domain::factory::Database;
|
||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
use crate::services::config::DatabaseConfig;
|
use crate::services::config::DatabaseConfig;
|
||||||
use crate::utils::locks::{DbOpLock, FileLock};
|
use crate::utils::locks::{DbOpLock, FileLock};
|
||||||
|
use anyhow::Result;
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
pub struct MySQLDatabase {
|
pub struct MySQLDatabase {
|
||||||
cfg: DatabaseConfig,
|
cfg: DatabaseConfig,
|
||||||
@@ -36,16 +35,23 @@ impl Database for MySQLDatabase {
|
|||||||
ping::run(self.cfg.clone(), self.build_env().clone()).await
|
ping::run(self.cfg.clone(), self.build_env().clone()).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
|
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||||
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.build_env().clone(), self.file_extension()).await;
|
let res = backup::run(
|
||||||
|
self.cfg.clone(),
|
||||||
|
dir.to_path_buf(),
|
||||||
|
self.build_env().clone(),
|
||||||
|
self.file_extension(),
|
||||||
|
logger,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
FileLock::release(&self.cfg.generated_id).await?;
|
FileLock::release(&self.cfg.generated_id).await?;
|
||||||
res
|
res
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn restore(&self, file: &Path) -> Result<()> {
|
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
|
||||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||||
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
|
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
|
||||||
FileLock::release(&self.cfg.generated_id).await?;
|
FileLock::release(&self.cfg.generated_id).await?;
|
||||||
res
|
res
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
pub mod backup;
|
pub mod backup;
|
||||||
|
pub mod connection;
|
||||||
pub mod database;
|
pub mod database;
|
||||||
mod restore;
|
|
||||||
mod ping;
|
mod ping;
|
||||||
mod connection;
|
mod restore;
|
||||||
|
|||||||
@@ -1,17 +1,12 @@
|
|||||||
|
use crate::domain::mysql::connection::connection_args;
|
||||||
use crate::services::config::DatabaseConfig;
|
use crate::services::config::DatabaseConfig;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use tokio::process::Command;
|
use tokio::process::Command;
|
||||||
use tokio::time::{Duration, timeout};
|
use tokio::time::{Duration, timeout};
|
||||||
|
|
||||||
pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::Result<bool> {
|
pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::Result<bool> {
|
||||||
|
let mut cmd = Command::new("mariadb-admin");
|
||||||
let mut cmd = Command::new("mysqladmin");
|
cmd.args(connection_args(&cfg))
|
||||||
cmd.arg("--host")
|
|
||||||
.arg(cfg.host)
|
|
||||||
.arg("--port")
|
|
||||||
.arg(cfg.port.to_string())
|
|
||||||
.arg("--user")
|
|
||||||
.arg(cfg.username)
|
|
||||||
.arg("ping")
|
.arg("ping")
|
||||||
.envs(env);
|
.envs(env);
|
||||||
|
|
||||||
|
|||||||
+34
-32
@@ -1,52 +1,51 @@
|
|||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
|
use crate::domain::mysql::connection::connection_args;
|
||||||
use crate::services::config::DatabaseConfig;
|
use crate::services::config::DatabaseConfig;
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
use std::io::{Read, Write};
|
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::process::Command;
|
use std::process::Command;
|
||||||
use tracing::{debug, error, info};
|
use std::sync::Arc;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
|
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
|
||||||
let handle = tokio::task::spawn_blocking(move || -> Result<()> {
|
let handle = tokio::task::spawn_blocking(move || -> Result<()> {
|
||||||
debug!("Starting restore for database {}", cfg.name);
|
logger.log("info", format!("Starting restore for database {}", cfg.name));
|
||||||
|
|
||||||
let mut sql_content = String::new();
|
|
||||||
let mut file = File::open(&restore_file)
|
let mut file = File::open(&restore_file)
|
||||||
.with_context(|| format!("Failed to open restore file {}", restore_file.display()))?;
|
.with_context(|| format!("Failed to open restore file {}", restore_file.display()))?;
|
||||||
file.read_to_string(&mut sql_content)
|
|
||||||
.with_context(|| format!("Failed to read restore file {}", restore_file.display()))?;
|
|
||||||
|
|
||||||
let drop_create_cmd = format!(
|
let drop_create_cmd = format!(
|
||||||
"DROP DATABASE IF EXISTS {0}; CREATE DATABASE {0};",
|
"DROP DATABASE IF EXISTS `{0}`; CREATE DATABASE `{0}`;",
|
||||||
cfg.database
|
cfg.database
|
||||||
);
|
);
|
||||||
|
|
||||||
let drop_status = Command::new("mysql")
|
let drop_start = Instant::now();
|
||||||
.arg("--host")
|
let drop_output = Command::new("mysql")
|
||||||
.arg(&cfg.host)
|
.args(connection_args(&cfg))
|
||||||
.arg("--port")
|
|
||||||
.arg(cfg.port.to_string())
|
|
||||||
.arg("--user")
|
|
||||||
.arg(&cfg.username)
|
|
||||||
.arg("-e")
|
.arg("-e")
|
||||||
.arg(&drop_create_cmd)
|
.arg(&drop_create_cmd)
|
||||||
.env("MYSQL_PWD", &cfg.password)
|
.env("MYSQL_PWD", &cfg.password)
|
||||||
.status()
|
.output()
|
||||||
.with_context(|| format!("Failed to drop/recreate database {}", cfg.name))?;
|
.with_context(|| format!("Failed to drop/recreate database {}", cfg.name))?;
|
||||||
|
|
||||||
if !drop_status.success() {
|
let drop_duration_ms = drop_start.elapsed().as_millis() as f64;
|
||||||
error!("Drop/create database failed for {}", cfg.name);
|
let drop_exit_code = drop_output.status.code().unwrap_or(-1);
|
||||||
|
let drop_stderr = String::from_utf8_lossy(&drop_output.stderr).to_string();
|
||||||
|
|
||||||
|
if !drop_output.status.success() {
|
||||||
|
logger.log_command("mysql", Some(drop_stderr.clone()), Some(drop_exit_code), Some(drop_duration_ms));
|
||||||
|
logger.log("error", format!("Drop/create database failed for {}: {}", cfg.name, drop_stderr));
|
||||||
anyhow::bail!("Failed to drop/recreate database {}", cfg.name);
|
anyhow::bail!("Failed to drop/recreate database {}", cfg.name);
|
||||||
}
|
}
|
||||||
info!("Database {} dropped and recreated", cfg.name);
|
logger.log_command("mysql", if drop_stderr.is_empty() { None } else { Some(drop_stderr) }, Some(0), Some(drop_duration_ms));
|
||||||
|
logger.log("info", format!("Database {} dropped and recreated", cfg.name));
|
||||||
|
|
||||||
|
let start = Instant::now();
|
||||||
|
|
||||||
let mut child = Command::new("mysql")
|
let mut child = Command::new("mysql")
|
||||||
.arg("--host")
|
.args(connection_args(&cfg))
|
||||||
.arg(&cfg.host)
|
.arg("--database")
|
||||||
.arg("--port")
|
|
||||||
.arg(cfg.port.to_string())
|
|
||||||
.arg("--user")
|
|
||||||
.arg(&cfg.username)
|
|
||||||
.arg(&cfg.database)
|
.arg(&cfg.database)
|
||||||
.env("MYSQL_PWD", &cfg.password)
|
.env("MYSQL_PWD", &cfg.password)
|
||||||
.stdin(std::process::Stdio::piped())
|
.stdin(std::process::Stdio::piped())
|
||||||
@@ -54,23 +53,26 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
|
|||||||
.with_context(|| format!("Failed to start mysql restore for {}", cfg.name))?;
|
.with_context(|| format!("Failed to start mysql restore for {}", cfg.name))?;
|
||||||
|
|
||||||
let mut stdin = child.stdin.take().context("Failed to open child stdin")?;
|
let mut stdin = child.stdin.take().context("Failed to open child stdin")?;
|
||||||
stdin
|
std::io::copy(&mut file, &mut stdin)
|
||||||
.write_all(sql_content.as_bytes())
|
.context("Failed to stream SQL content to mysql stdin")?;
|
||||||
.context("Failed to write SQL content to mysql stdin")?;
|
|
||||||
stdin.flush()?;
|
|
||||||
drop(stdin);
|
drop(stdin);
|
||||||
|
|
||||||
let output = child
|
let output = child
|
||||||
.wait_with_output()
|
.wait_with_output()
|
||||||
.with_context(|| format!("Failed to complete mysql restore for {}", cfg.name))?;
|
.with_context(|| format!("Failed to complete mysql restore for {}", cfg.name))?;
|
||||||
|
|
||||||
|
let duration_ms = start.elapsed().as_millis() as f64;
|
||||||
|
let exit_code = output.status.code().unwrap_or(-1);
|
||||||
|
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||||
|
|
||||||
if !output.status.success() {
|
if !output.status.success() {
|
||||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
logger.log_command("mysql", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
|
||||||
error!("MySQL restore failed for {}: {}", cfg.name, stderr);
|
logger.log("error", format!("MySQL restore failed for {}: {}", cfg.name, stderr));
|
||||||
anyhow::bail!("MySQL restore failed for {}", cfg.name);
|
anyhow::bail!("MySQL restore failed for {}", cfg.name);
|
||||||
}
|
}
|
||||||
|
|
||||||
info!("Restore finished successfully for database {}", cfg.name);
|
logger.log_command("mysql", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
|
||||||
|
logger.log("info", format!("Restore finished successfully for database {}", cfg.name));
|
||||||
Ok(())
|
Ok(())
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -1,141 +1,150 @@
|
|||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
|
use std::collections::HashMap;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::process::Command;
|
use std::process::Command;
|
||||||
use tracing::{debug, error, info};
|
use std::sync::Arc;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
use super::connection::{select_pg_path, server_version};
|
use super::connection::{select_pg_path, server_version};
|
||||||
use super::format::PostgresDumpFormat;
|
use super::format::PostgresDumpFormat;
|
||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
use crate::services::config::DatabaseConfig;
|
use crate::services::config::DatabaseConfig;
|
||||||
|
|
||||||
pub async fn run(
|
pub async fn run(
|
||||||
cfg: DatabaseConfig,
|
cfg: DatabaseConfig,
|
||||||
format: PostgresDumpFormat,
|
format: PostgresDumpFormat,
|
||||||
backup_dir: PathBuf,
|
backup_dir: PathBuf,
|
||||||
|
env: HashMap<String, String>,
|
||||||
|
logger: Arc<JobLogger>,
|
||||||
) -> Result<PathBuf> {
|
) -> Result<PathBuf> {
|
||||||
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||||
debug!("Starting backup for database {}", cfg.name);
|
logger.log("info", format!("Starting backup for database {}", cfg.name));
|
||||||
|
|
||||||
let version = match futures::executor::block_on(server_version(&cfg)) {
|
let version = match futures::executor::block_on(server_version(&cfg)) {
|
||||||
Ok(v) => {
|
Ok(v) => {
|
||||||
debug!("Postgres version detected: {}", v);
|
logger.log("debug", format!("Postgres version detected: {}", v));
|
||||||
v
|
v
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
error!("Failed to get server version for {}: {:?}", cfg.name, e);
|
logger.log("error", format!("Failed to get server version: {}", e));
|
||||||
return Err(e.into());
|
return Err(e.into());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let pg_dump = select_pg_path(&version).join("pg_dump");
|
let pg_dump = select_pg_path(&version).join("pg_dump");
|
||||||
debug!("Using pg_dump at {:?}", pg_dump);
|
logger.log("debug", format!("Using pg_dump at {:?}", pg_dump));
|
||||||
|
|
||||||
match format {
|
match format {
|
||||||
PostgresDumpFormat::Fc => {
|
PostgresDumpFormat::Fc => {
|
||||||
info!("Running FC backup for {}", cfg.name);
|
logger.log("info", format!("Running FC backup for {}", cfg.name));
|
||||||
let file_path = backup_dir.join(format!("{}.dump", cfg.generated_id));
|
|
||||||
let url = format!(
|
|
||||||
"postgresql://{}:{}@{}:{}/{}",
|
|
||||||
cfg.username, cfg.password, cfg.host, cfg.port, cfg.database
|
|
||||||
);
|
|
||||||
|
|
||||||
let status = Command::new(&pg_dump)
|
let file_path = backup_dir.join(format!("{}.dump", cfg.generated_id));
|
||||||
.arg("--dbname")
|
|
||||||
.arg(&url)
|
let start = Instant::now();
|
||||||
|
let output = Command::new(&pg_dump)
|
||||||
|
.arg("--host").arg(&cfg.host)
|
||||||
|
.arg("--port").arg(cfg.port.to_string())
|
||||||
|
.arg("--username").arg(&cfg.username)
|
||||||
|
.arg("--dbname").arg(&cfg.database)
|
||||||
.arg("-Fc")
|
.arg("-Fc")
|
||||||
.arg("-f")
|
.arg("-f").arg(&file_path)
|
||||||
.arg(&file_path)
|
|
||||||
.arg("-v")
|
.arg("-v")
|
||||||
.arg("--compress=3")
|
.arg("--compress=3")
|
||||||
.status();
|
.envs(env)
|
||||||
|
.output();
|
||||||
|
let duration_ms = start.elapsed().as_millis() as f64;
|
||||||
|
|
||||||
match status {
|
match output {
|
||||||
Ok(s) if s.success() => info!(
|
Ok(o) => {
|
||||||
"FC backup completed successfully for {} at {:?}",
|
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
|
||||||
cfg.name, file_path
|
let exit_code = o.status.code().unwrap_or(-1);
|
||||||
),
|
|
||||||
Ok(s) => {
|
if o.status.success() {
|
||||||
error!("FC backup failed with status {:?} for {}", s, cfg.name);
|
logger.log("info", format!("FC backup completed successfully for {} at {:?}", cfg.name, file_path));
|
||||||
anyhow::bail!("Postgres backup failed for {}", cfg.name);
|
logger.log_command("pg_dump", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
|
||||||
|
} else {
|
||||||
|
logger.log("error", format!("FC backup failed with status {:?} for {}", o.status, cfg.name));
|
||||||
|
logger.log_command("pg_dump", Some(stderr), Some(exit_code), Some(duration_ms));
|
||||||
|
anyhow::bail!("Postgres backup failed for {}", cfg.name);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
error!("Error executing pg_dump for {}: {:?}", cfg.name, e);
|
logger.log("error", format!("Error executing pg_dump for {}: {:?}", cfg.name, e));
|
||||||
|
logger.log_command("pg_dump", Some(e.to_string()), Some(-1), Some(duration_ms));
|
||||||
return Err(e.into());
|
return Err(e.into());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
info!("Backup finished for database {}", cfg.name);
|
logger.log("info", format!("Backup finished for database {}", cfg.name));
|
||||||
Ok(file_path)
|
Ok(file_path)
|
||||||
}
|
}
|
||||||
|
|
||||||
PostgresDumpFormat::Fd => {
|
PostgresDumpFormat::Fd => {
|
||||||
info!("Running FD backup for {}", cfg.name);
|
logger.log("info", format!("Running FD backup for {}", cfg.name));
|
||||||
|
|
||||||
let dump_dir = backup_dir.join(format!("{}_dir", cfg.generated_id));
|
let dump_dir = backup_dir.join(format!("{}_dir", cfg.generated_id));
|
||||||
let tar_file = backup_dir.join(format!("{}.tar.gz", cfg.generated_id));
|
let tar_file = backup_dir.join(format!("{}.tar.gz", cfg.generated_id));
|
||||||
|
|
||||||
if let Err(e) = std::fs::create_dir_all(&dump_dir) {
|
if let Err(e) = std::fs::create_dir_all(&dump_dir) {
|
||||||
error!(
|
logger.log("error", format!("Failed to create dump directory {:?} for {}: {:?}", dump_dir, cfg.name, e));
|
||||||
"Failed to create dump directory {:?} for {}: {:?}",
|
|
||||||
dump_dir, cfg.name, e
|
|
||||||
);
|
|
||||||
return Err(e.into());
|
return Err(e.into());
|
||||||
}
|
}
|
||||||
|
|
||||||
let url = format!(
|
let cmd_label = format!("pg_dump -Fd {}@{}:{}/{}", cfg.username, cfg.host, cfg.port, cfg.database);
|
||||||
"postgresql://{}:{}@{}:{}/{}",
|
|
||||||
cfg.username, cfg.password, cfg.host, cfg.port, cfg.database
|
|
||||||
);
|
|
||||||
|
|
||||||
let status = Command::new(&pg_dump)
|
let start = Instant::now();
|
||||||
.arg("--dbname")
|
let output = Command::new(&pg_dump)
|
||||||
.arg(&url)
|
.arg("--host").arg(&cfg.host)
|
||||||
|
.arg("--port").arg(cfg.port.to_string())
|
||||||
|
.arg("--username").arg(&cfg.username)
|
||||||
|
.arg("--dbname").arg(&cfg.database)
|
||||||
.arg("-Fd")
|
.arg("-Fd")
|
||||||
.arg("-j")
|
.arg("-j").arg("4")
|
||||||
.arg("4")
|
.arg("-f").arg(&dump_dir)
|
||||||
.arg("-f")
|
|
||||||
.arg(&dump_dir)
|
|
||||||
.arg("-v")
|
.arg("-v")
|
||||||
.status();
|
.envs(env)
|
||||||
|
.output();
|
||||||
|
let duration_ms = start.elapsed().as_millis() as f64;
|
||||||
|
|
||||||
match status {
|
match output {
|
||||||
Ok(s) if s.success() => {
|
Ok(o) => {
|
||||||
info!("FD backup pg_dump completed successfully for {}", cfg.name)
|
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
|
||||||
}
|
let exit_code = o.status.code().unwrap_or(-1);
|
||||||
Ok(s) => {
|
if o.status.success() {
|
||||||
error!(
|
logger.log("info", format!("FD backup pg_dump completed successfully for {}", cfg.name));
|
||||||
"FD backup pg_dump failed with status {:?} for {}",
|
logger.log_command(cmd_label, if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
|
||||||
s, cfg.name
|
} else {
|
||||||
);
|
logger.log("error", format!("FD backup pg_dump failed with status {:?} for {}", o.status, cfg.name));
|
||||||
anyhow::bail!("Postgres FD backup failed for {}", cfg.name);
|
logger.log_command(cmd_label, Some(stderr), Some(exit_code), Some(duration_ms));
|
||||||
|
anyhow::bail!("Postgres FD backup failed for {}", cfg.name);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
error!("Error executing pg_dump for {}: {:?}", cfg.name, e);
|
logger.log("error", format!("Error executing pg_dump for {}: {:?}", cfg.name, e));
|
||||||
|
logger.log_command(cmd_label, Some(e.to_string()), Some(-1), Some(duration_ms));
|
||||||
return Err(e.into());
|
return Err(e.into());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
match std::fs::File::create(&tar_file) {
|
match std::fs::File::create(&tar_file) {
|
||||||
Ok(tar_gz) => {
|
Ok(tar_gz) => {
|
||||||
let enc =
|
let enc = flate2::write::GzEncoder::new(tar_gz, flate2::Compression::default());
|
||||||
flate2::write::GzEncoder::new(tar_gz, flate2::Compression::default());
|
|
||||||
let mut tar = tar::Builder::new(enc);
|
let mut tar = tar::Builder::new(enc);
|
||||||
if let Err(e) = tar.append_dir_all(".", &dump_dir) {
|
if let Err(e) = tar.append_dir_all(".", &dump_dir) {
|
||||||
error!("Failed to append dump_dir to tar for {}: {:?}", cfg.name, e);
|
logger.log("error", format!("Failed to append dump_dir to tar for {}: {:?}", cfg.name, e));
|
||||||
return Err(e.into());
|
return Err(e.into());
|
||||||
}
|
}
|
||||||
if let Err(e) = tar.finish() {
|
if let Err(e) = tar.finish() {
|
||||||
error!("Failed to finish tar archive for {}: {:?}", cfg.name, e);
|
logger.log("error", format!("Failed to finish tar archive for {}: {:?}", cfg.name, e));
|
||||||
return Err(e.into());
|
return Err(e.into());
|
||||||
}
|
}
|
||||||
info!("FD backup archive created at {:?}", tar_file);
|
logger.log("info", format!("FD backup archive created at {:?}", tar_file));
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
error!(
|
logger.log("error", format!("Failed to create tar.gz file {:?} for {}: {:?}", tar_file, cfg.name, e));
|
||||||
"Failed to create tar.gz file {:?} for {}: {:?}",
|
|
||||||
tar_file, cfg.name, e
|
|
||||||
);
|
|
||||||
return Err(e.into());
|
return Err(e.into());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
info!("Backup finished for database {}", cfg.name);
|
logger.log("info", format!("Backup finished for database {}", cfg.name));
|
||||||
Ok(tar_file)
|
Ok(tar_file)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,25 @@
|
|||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum RestoreCleanMode {
|
||||||
|
None,
|
||||||
|
Clean,
|
||||||
|
DropSchemas,
|
||||||
|
DropDatabase,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RestoreCleanMode {
|
||||||
|
pub fn from_config(cfg: &DatabaseConfig) -> (Self, Option<String>) {
|
||||||
|
match cfg.options.get("clean_mode").and_then(|v| v.as_str()) {
|
||||||
|
None | Some("clean") => (Self::Clean, None),
|
||||||
|
Some("none") => (Self::None, None),
|
||||||
|
Some("drop_schemas") => (Self::DropSchemas, None),
|
||||||
|
Some("drop_database") => (Self::DropDatabase, None),
|
||||||
|
Some(other) => (Self::Clean, Some(other.to_string())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn uses_pg_restore_clean(self) -> bool {
|
||||||
|
matches!(self, Self::Clean)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
use anyhow::Result;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::process::Command;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use super::super::connection::{
|
||||||
|
is_superuser, pg_dumpall_binary_name, select_pg_path, server_version,
|
||||||
|
};
|
||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
|
||||||
|
pub async fn run(
|
||||||
|
cfg: DatabaseConfig,
|
||||||
|
backup_dir: PathBuf,
|
||||||
|
env: HashMap<String, String>,
|
||||||
|
logger: Arc<JobLogger>,
|
||||||
|
) -> Result<PathBuf> {
|
||||||
|
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||||
|
logger.log("info", format!("Starting cluster backup for {}", cfg.name));
|
||||||
|
|
||||||
|
let version = match futures::executor::block_on(server_version(&cfg)) {
|
||||||
|
Ok(v) => v,
|
||||||
|
Err(e) => {
|
||||||
|
logger.log("error", format!("Failed to get server version for {}: {:?}", cfg.name, e));
|
||||||
|
return Err(e.into());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
match futures::executor::block_on(is_superuser(&cfg)) {
|
||||||
|
Ok(true) => {}
|
||||||
|
Ok(false) => {
|
||||||
|
logger.log("error", format!("postgresql-cluster backup requires a superuser role for {}", cfg.name));
|
||||||
|
anyhow::bail!("postgresql-cluster backup requires a superuser role for {}", cfg.name);
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
logger.log("error", format!("Failed to check superuser status for {}: {:?}", cfg.name, e));
|
||||||
|
return Err(e.into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let pg_dumpall = select_pg_path(&version).join(pg_dumpall_binary_name());
|
||||||
|
let file_path = backup_dir.join(format!("{}.sql", cfg.generated_id));
|
||||||
|
|
||||||
|
logger.log("info", format!("Running pg_dumpall for cluster {} via {:?}", cfg.name, pg_dumpall));
|
||||||
|
|
||||||
|
let start = Instant::now();
|
||||||
|
let output = Command::new(&pg_dumpall)
|
||||||
|
.arg("--host").arg(&cfg.host)
|
||||||
|
.arg("--port").arg(cfg.port.to_string())
|
||||||
|
.arg("--username").arg(&cfg.username)
|
||||||
|
.arg("--clean")
|
||||||
|
.arg("--if-exists")
|
||||||
|
.arg("-v")
|
||||||
|
.arg("-f").arg(&file_path)
|
||||||
|
.envs(env)
|
||||||
|
.output();
|
||||||
|
let duration_ms = start.elapsed().as_millis() as f64;
|
||||||
|
|
||||||
|
match output {
|
||||||
|
Ok(o) => {
|
||||||
|
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
|
||||||
|
let exit_code = o.status.code().unwrap_or(-1);
|
||||||
|
if o.status.success() {
|
||||||
|
logger.log_command("pg_dumpall", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
|
||||||
|
logger.log("info", format!("Cluster backup completed for {} at {:?}", cfg.name, file_path));
|
||||||
|
Ok(file_path)
|
||||||
|
} else {
|
||||||
|
logger.log_command("pg_dumpall", Some(stderr), Some(exit_code), Some(duration_ms));
|
||||||
|
anyhow::bail!("Cluster backup (pg_dumpall) failed for {}", cfg.name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
logger.log_command("pg_dumpall", Some(e.to_string()), Some(-1), Some(duration_ms));
|
||||||
|
Err(e.into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await?
|
||||||
|
}
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
use anyhow::Result;
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use super::super::ping;
|
||||||
|
use super::{backup, restore};
|
||||||
|
use crate::domain::factory::Database;
|
||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
use crate::utils::locks::{DbOpLock, FileLock};
|
||||||
|
|
||||||
|
pub struct PostgresClusterDatabase {
|
||||||
|
pub cfg: DatabaseConfig,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PostgresClusterDatabase {
|
||||||
|
pub fn new(cfg: DatabaseConfig) -> Self {
|
||||||
|
Self { cfg }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_env(&self) -> HashMap<String, String> {
|
||||||
|
let mut envs = std::env::vars().collect::<HashMap<_, _>>();
|
||||||
|
envs.insert("PGPASSWORD".to_string(), self.cfg.password.to_string());
|
||||||
|
envs
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl Database for PostgresClusterDatabase {
|
||||||
|
fn file_extension(&self) -> &'static str {
|
||||||
|
".sql"
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn ping(&self) -> Result<bool> {
|
||||||
|
ping::run(self.cfg.clone()).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||||
|
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||||
|
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.build_env(), logger).await;
|
||||||
|
FileLock::release(&self.cfg.generated_id).await?;
|
||||||
|
res
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
|
||||||
|
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||||
|
let res = restore::run(self.cfg.clone(), file.to_path_buf(), self.build_env(), logger).await;
|
||||||
|
FileLock::release(&self.cfg.generated_id).await?;
|
||||||
|
res
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
pub mod backup;
|
||||||
|
pub mod database;
|
||||||
|
pub mod restore;
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
use anyhow::Result;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::process::Command;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use super::super::connection::{is_superuser, psql_binary_name, select_pg_path, server_version, terminate_all_connections};
|
||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
|
||||||
|
pub async fn run(
|
||||||
|
cfg: DatabaseConfig,
|
||||||
|
restore_file: PathBuf,
|
||||||
|
env: HashMap<String, String>,
|
||||||
|
logger: Arc<JobLogger>,
|
||||||
|
) -> Result<()> {
|
||||||
|
let handle = tokio::runtime::Handle::current();
|
||||||
|
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||||
|
logger.log("info", format!("Starting cluster restore for {}", cfg.name));
|
||||||
|
|
||||||
|
let version = match handle.block_on(server_version(&cfg)) {
|
||||||
|
Ok(v) => v,
|
||||||
|
Err(e) => {
|
||||||
|
logger.log("error", format!("Failed to get server version for {}: {:?}", cfg.name, e));
|
||||||
|
return Err(e.into());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
match handle.block_on(is_superuser(&cfg)) {
|
||||||
|
Ok(true) => {}
|
||||||
|
Ok(false) => {
|
||||||
|
logger.log("error", format!("postgresql-cluster restore requires a superuser role for {}", cfg.name));
|
||||||
|
anyhow::bail!("postgresql-cluster restore requires a superuser role for {}", cfg.name);
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
logger.log("error", format!("Failed to check superuser status for {}: {:?}", cfg.name, e));
|
||||||
|
return Err(e.into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let psql = select_pg_path(&version).join(psql_binary_name());
|
||||||
|
|
||||||
|
if let Err(e) = handle.block_on(terminate_all_connections(&cfg)) {
|
||||||
|
logger.log("error", format!("Failed to terminate connections for cluster {}: {:?}", cfg.name, e));
|
||||||
|
return Err(e.into());
|
||||||
|
}
|
||||||
|
logger.log("info", format!("All user database connections terminated for cluster {}", cfg.name));
|
||||||
|
|
||||||
|
logger.log("info", format!("Replaying cluster dump for {} via {:?}", cfg.name, psql));
|
||||||
|
|
||||||
|
let start = Instant::now();
|
||||||
|
let output = Command::new(&psql)
|
||||||
|
.arg("--host").arg(&cfg.host)
|
||||||
|
.arg("--port").arg(cfg.port.to_string())
|
||||||
|
.arg("--username").arg(&cfg.username)
|
||||||
|
.arg("--dbname").arg("postgres")
|
||||||
|
.arg("-f").arg(&restore_file)
|
||||||
|
.envs(env)
|
||||||
|
.output();
|
||||||
|
let duration_ms = start.elapsed().as_millis() as f64;
|
||||||
|
|
||||||
|
match output {
|
||||||
|
Ok(o) => {
|
||||||
|
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
|
||||||
|
let stdout = String::from_utf8_lossy(&o.stdout).to_string();
|
||||||
|
let combined = format!("{}{}", stdout, stderr);
|
||||||
|
let exit_code = o.status.code().unwrap_or(-1);
|
||||||
|
if o.status.success() {
|
||||||
|
logger.log_command("psql", if combined.is_empty() { None } else { Some(combined) }, Some(0), Some(duration_ms));
|
||||||
|
logger.log("info", format!("Cluster restore completed for {}", cfg.name));
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
logger.log_command("psql", if combined.is_empty() { None } else { Some(combined) }, Some(exit_code), Some(duration_ms));
|
||||||
|
anyhow::bail!("Cluster restore (psql) failed for {}", cfg.name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
logger.log_command("psql", Some(e.to_string()), Some(-1), Some(duration_ms));
|
||||||
|
Err(e.into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await?
|
||||||
|
}
|
||||||
@@ -1,18 +1,23 @@
|
|||||||
use crate::domain::postgres::format::PostgresDumpFormat;
|
use crate::domain::postgres::format::PostgresDumpFormat;
|
||||||
use crate::services::config::DatabaseConfig;
|
use crate::services::config::DatabaseConfig;
|
||||||
|
use crate::settings::CONFIG;
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use tokio_postgres::{Client, NoTls};
|
use tokio_postgres::{Client, Config, NoTls};
|
||||||
use tracing::{error, info};
|
use tracing::{error, info};
|
||||||
|
|
||||||
pub async fn connect(cfg: &DatabaseConfig) -> Result<Client> {
|
pub async fn connect(cfg: &DatabaseConfig) -> Result<Client> {
|
||||||
info!("Connecting to postgres database {}:{}", cfg.host, cfg.port);
|
info!("Connecting to postgres database {}:{}", cfg.host, cfg.port);
|
||||||
let dsn = format!(
|
|
||||||
"host={} port={} user={} password={} dbname={}",
|
|
||||||
cfg.host, cfg.port, cfg.username, cfg.password, cfg.database
|
|
||||||
);
|
|
||||||
|
|
||||||
let (client, connection) = tokio_postgres::connect(&dsn, NoTls).await?;
|
let mut config = Config::new();
|
||||||
|
config
|
||||||
|
.host(&cfg.host)
|
||||||
|
.port(cfg.port)
|
||||||
|
.user(&cfg.username)
|
||||||
|
.password(&cfg.password)
|
||||||
|
.dbname(&cfg.database);
|
||||||
|
|
||||||
|
let (client, connection) = config.connect(NoTls).await?;
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
if let Err(e) = connection.await {
|
if let Err(e) = connection.await {
|
||||||
error!("Postgres connection error: {}", e);
|
error!("Postgres connection error: {}", e);
|
||||||
@@ -28,11 +33,133 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
|
|||||||
Ok(version)
|
Ok(version)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn server_version_major(cfg: &DatabaseConfig) -> Result<u32> {
|
||||||
|
let v = server_version(cfg).await?;
|
||||||
|
Ok(v.split(['.', ' '])
|
||||||
|
.next()
|
||||||
|
.and_then(|s| s.parse().ok())
|
||||||
|
.unwrap_or(17))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn is_superuser(cfg: &DatabaseConfig) -> Result<bool> {
|
||||||
|
let client = connect(cfg).await?;
|
||||||
|
let is_super: bool = client
|
||||||
|
.query_one("SELECT current_setting('is_superuser') = 'on';", &[])
|
||||||
|
.await?
|
||||||
|
.get(0);
|
||||||
|
|
||||||
|
Ok(is_super)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn can_drop_database(cfg: &DatabaseConfig) -> Result<bool> {
|
||||||
|
let client = connect(cfg).await?;
|
||||||
|
let row = client
|
||||||
|
.query_one(
|
||||||
|
"SELECT r.rolsuper OR (r.rolcreatedb AND pg_catalog.pg_has_role(current_user, d.datdba, 'USAGE')) \
|
||||||
|
FROM pg_roles r, pg_database d \
|
||||||
|
WHERE r.rolname = current_user AND d.datname = current_database()",
|
||||||
|
&[],
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(row.get(0))
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
pub fn select_pg_path(version: &str) -> std::path::PathBuf {
|
pub fn select_pg_path(version: &str) -> std::path::PathBuf {
|
||||||
|
select_pg_path_with(version, &CONFIG.pg_bin_dir)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn select_pg_path_with(version: &str, pg_bin_dir: &str) -> std::path::PathBuf {
|
||||||
let major = version.split('.').next().unwrap_or("17");
|
let major = version.split('.').next().unwrap_or("17");
|
||||||
|
|
||||||
|
if !pg_bin_dir.is_empty() {
|
||||||
|
return pg_bin_dir.into();
|
||||||
|
}
|
||||||
|
|
||||||
|
let candidates: Vec<std::path::PathBuf> = if cfg!(target_os = "windows") {
|
||||||
|
vec![
|
||||||
|
// Default install path used by the official EDB Windows installer
|
||||||
|
format!(r"C:\Program Files\PostgreSQL\{major}\bin").into(),
|
||||||
|
format!(r"C:\Program Files (x86)\PostgreSQL\{major}\bin").into(),
|
||||||
|
]
|
||||||
|
} else if cfg!(target_os = "macos") {
|
||||||
|
vec![
|
||||||
|
// Homebrew on Apple Silicon
|
||||||
|
format!("/opt/homebrew/opt/postgresql@{major}/bin").into(),
|
||||||
|
// Homebrew on Intel
|
||||||
|
format!("/usr/local/opt/postgresql@{major}/bin").into(),
|
||||||
|
// Postgres.app
|
||||||
|
format!("/Applications/Postgres.app/Contents/Versions/{major}/bin").into(),
|
||||||
|
]
|
||||||
|
} else {
|
||||||
|
vec![
|
||||||
|
// Debian/Ubuntu packages
|
||||||
|
format!("/usr/lib/postgresql/{major}/bin").into(),
|
||||||
|
// Common RPM-based distro layout
|
||||||
|
format!("/usr/pgsql-{major}/bin").into(),
|
||||||
|
]
|
||||||
|
};
|
||||||
|
|
||||||
|
if let Some(found) = candidates.into_iter().find(|p| pg_dump_exists_in(p)) {
|
||||||
|
return found;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(dir) = find_pg_dump_in_path() {
|
||||||
|
return dir;
|
||||||
|
}
|
||||||
|
|
||||||
format!("/usr/lib/postgresql/{}/bin", major).into()
|
format!("/usr/lib/postgresql/{}/bin", major).into()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn pg_dump_binary_name() -> &'static str {
|
||||||
|
if cfg!(target_os = "windows") {
|
||||||
|
"pg_dump.exe"
|
||||||
|
} else {
|
||||||
|
"pg_dump"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn pg_dumpall_binary_name() -> &'static str {
|
||||||
|
if cfg!(target_os = "windows") {
|
||||||
|
"pg_dumpall.exe"
|
||||||
|
} else {
|
||||||
|
"pg_dumpall"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn psql_binary_name() -> &'static str {
|
||||||
|
if cfg!(target_os = "windows") {
|
||||||
|
"psql.exe"
|
||||||
|
} else {
|
||||||
|
"psql"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn pg_restore_binary_name() -> &'static str {
|
||||||
|
if cfg!(target_os = "windows") {
|
||||||
|
"pg_restore.exe"
|
||||||
|
} else {
|
||||||
|
"pg_restore"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn quote_ident(s: &str) -> String {
|
||||||
|
format!("\"{}\"", s.replace('"', "\"\""))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn quote_literal(s: &str) -> String {
|
||||||
|
format!("'{}'", s.replace('\'', "''"))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn pg_dump_exists_in(dir: &std::path::Path) -> bool {
|
||||||
|
dir.join(pg_dump_binary_name()).is_file()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn find_pg_dump_in_path() -> Option<std::path::PathBuf> {
|
||||||
|
let path_var = std::env::var_os("PATH")?;
|
||||||
|
std::env::split_paths(&path_var).find(|dir| pg_dump_exists_in(dir))
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn terminate_connections(cfg: &DatabaseConfig) -> Result<()> {
|
pub async fn terminate_connections(cfg: &DatabaseConfig) -> Result<()> {
|
||||||
let mut admin = cfg.clone();
|
let mut admin = cfg.clone();
|
||||||
admin.database = "postgres".to_string().into();
|
admin.database = "postgres".to_string().into();
|
||||||
@@ -54,6 +181,128 @@ pub async fn terminate_connections(cfg: &DatabaseConfig) -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn terminate_all_connections(cfg: &DatabaseConfig) -> Result<()> {
|
||||||
|
let mut admin = cfg.clone();
|
||||||
|
admin.database = "postgres".to_string().into();
|
||||||
|
|
||||||
|
let client = connect(&admin).await?;
|
||||||
|
|
||||||
|
client
|
||||||
|
.execute(
|
||||||
|
r#"
|
||||||
|
SELECT pg_terminate_backend(pid)
|
||||||
|
FROM pg_stat_activity
|
||||||
|
WHERE datname NOT IN ('postgres', 'template0', 'template1')
|
||||||
|
AND pid <> pg_backend_pid();
|
||||||
|
"#,
|
||||||
|
&[],
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn drop_and_recreate_database(cfg: &DatabaseConfig) -> Result<()> {
|
||||||
|
let mut admin_cfg = cfg.clone();
|
||||||
|
admin_cfg.database = "postgres".to_string();
|
||||||
|
let admin = connect(&admin_cfg).await?;
|
||||||
|
|
||||||
|
let row = admin
|
||||||
|
.query_opt(
|
||||||
|
r#"
|
||||||
|
SELECT pg_encoding_to_char(encoding), datcollate, datctype,
|
||||||
|
pg_get_userbyid(datdba), datistemplate
|
||||||
|
FROM pg_database WHERE datname = $1
|
||||||
|
"#,
|
||||||
|
&[&cfg.database],
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let (encoding, collate, ctype, owner) = match &row {
|
||||||
|
Some(r) => (
|
||||||
|
r.get::<_, String>(0),
|
||||||
|
r.get::<_, String>(1),
|
||||||
|
r.get::<_, String>(2),
|
||||||
|
r.get::<_, String>(3),
|
||||||
|
),
|
||||||
|
None => ("UTF8".into(), "C".into(), "C".into(), cfg.username.clone()),
|
||||||
|
};
|
||||||
|
|
||||||
|
if let Some(r) = &row {
|
||||||
|
if r.get::<_, bool>(4) {
|
||||||
|
anyhow::bail!("Refusing to drop template database {}", cfg.database);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let db = quote_ident(&cfg.database);
|
||||||
|
|
||||||
|
if let Err(e) = admin
|
||||||
|
.batch_execute(&format!("ALTER DATABASE {db} WITH ALLOW_CONNECTIONS false"))
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
tracing::warn!("ALLOW_CONNECTIONS false failed for {}: {e}", cfg.database);
|
||||||
|
}
|
||||||
|
|
||||||
|
let major = server_version_major(&admin_cfg).await?;
|
||||||
|
let drop_stmt = if major >= 13 {
|
||||||
|
format!("DROP DATABASE IF EXISTS {db} WITH (FORCE)")
|
||||||
|
} else {
|
||||||
|
format!("DROP DATABASE IF EXISTS {db}")
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut last_err = None;
|
||||||
|
let mut dropped = false;
|
||||||
|
for _ in 0..3 {
|
||||||
|
let _ = terminate_connections(cfg).await;
|
||||||
|
match admin.batch_execute(&drop_stmt).await {
|
||||||
|
Ok(()) => {
|
||||||
|
dropped = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
last_err = Some(e);
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !dropped {
|
||||||
|
let _ = admin
|
||||||
|
.batch_execute(&format!("ALTER DATABASE {db} WITH ALLOW_CONNECTIONS true"))
|
||||||
|
.await;
|
||||||
|
return Err(last_err
|
||||||
|
.map(anyhow::Error::from)
|
||||||
|
.unwrap_or_else(|| anyhow::anyhow!("DROP DATABASE {} failed", cfg.database)));
|
||||||
|
}
|
||||||
|
|
||||||
|
admin
|
||||||
|
.batch_execute(&format!(
|
||||||
|
"CREATE DATABASE {db} OWNER {} TEMPLATE template0 ENCODING {} LC_COLLATE {} LC_CTYPE {}",
|
||||||
|
quote_ident(&owner),
|
||||||
|
quote_literal(&encoding),
|
||||||
|
quote_literal(&collate),
|
||||||
|
quote_literal(&ctype),
|
||||||
|
))
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn sniff_format(restore_file: &Path) -> Result<PostgresDumpFormat> {
|
||||||
|
use std::io::Read;
|
||||||
|
let mut f = std::fs::File::open(restore_file)?;
|
||||||
|
let mut magic = [0u8; 5];
|
||||||
|
let n = f.read(&mut magic)?;
|
||||||
|
let head = &magic[..n];
|
||||||
|
if head.starts_with(b"PGDMP") {
|
||||||
|
Ok(PostgresDumpFormat::Fc)
|
||||||
|
} else if head.starts_with(&[0x1f, 0x8b]) {
|
||||||
|
Ok(PostgresDumpFormat::Fd)
|
||||||
|
} else {
|
||||||
|
anyhow::bail!("Unrecognized dump format for {:?}", restore_file)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn detect_format_from_file(restore_file: &Path) -> PostgresDumpFormat {
|
pub fn detect_format_from_file(restore_file: &Path) -> PostgresDumpFormat {
|
||||||
match restore_file.extension().and_then(|e| e.to_str()) {
|
match restore_file.extension().and_then(|e| e.to_str()) {
|
||||||
Some("dump") => PostgresDumpFormat::Fc,
|
Some("dump") => PostgresDumpFormat::Fc,
|
||||||
@@ -63,11 +312,50 @@ pub fn detect_format_from_file(restore_file: &Path) -> PostgresDumpFormat {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn drop_all_schemas(cfg: &DatabaseConfig) -> Result<Vec<String>> {
|
||||||
|
let client = connect(cfg).await?;
|
||||||
|
let rows = client
|
||||||
|
.query(
|
||||||
|
r#"
|
||||||
|
SELECT nspname FROM pg_namespace
|
||||||
|
WHERE nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast')
|
||||||
|
AND nspname NOT LIKE 'pg\_temp\_%'
|
||||||
|
AND nspname NOT LIKE 'pg\_toast\_temp\_%'
|
||||||
|
ORDER BY nspname
|
||||||
|
"#,
|
||||||
|
&[],
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let schemas: Vec<String> = rows.iter().map(|r| r.get::<_, String>(0)).collect();
|
||||||
|
for s in &schemas {
|
||||||
|
client
|
||||||
|
.batch_execute(&format!("DROP SCHEMA IF EXISTS {} CASCADE", quote_ident(s)))
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
client
|
||||||
|
.batch_execute("SELECT lo_unlink(oid) FROM pg_largeobject_metadata")
|
||||||
|
.await
|
||||||
|
.ok();
|
||||||
|
Ok(schemas)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn recreate_public_schema(cfg: &DatabaseConfig, owner: &str) -> Result<()> {
|
||||||
|
let client = connect(cfg).await?;
|
||||||
|
client
|
||||||
|
.batch_execute(&format!(
|
||||||
|
"CREATE SCHEMA IF NOT EXISTS public AUTHORIZATION {}; GRANT USAGE ON SCHEMA public TO PUBLIC;",
|
||||||
|
quote_ident(owner)
|
||||||
|
))
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn detect_format_from_size(cfg: &DatabaseConfig) -> PostgresDumpFormat {
|
pub async fn detect_format_from_size(cfg: &DatabaseConfig) -> PostgresDumpFormat {
|
||||||
info!(
|
info!(
|
||||||
"Detecting database format {:?} - {:?}",
|
"Detecting database format {:?} - {:?}",
|
||||||
cfg.name, cfg.generated_id
|
cfg.name, cfg.generated_id
|
||||||
);
|
);
|
||||||
|
|
||||||
let client = match connect(cfg).await {
|
let client = match connect(cfg).await {
|
||||||
Ok(c) => c,
|
Ok(c) => c,
|
||||||
Err(_) => return PostgresDumpFormat::Fc,
|
Err(_) => return PostgresDumpFormat::Fc,
|
||||||
|
|||||||
@@ -1,13 +1,11 @@
|
|||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
|
use std::collections::HashMap;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::Arc;
|
||||||
use super::{
|
use super::{backup, format::PostgresDumpFormat, ping, restore};
|
||||||
backup,
|
|
||||||
format::PostgresDumpFormat,
|
|
||||||
ping, restore,
|
|
||||||
};
|
|
||||||
use crate::domain::factory::Database;
|
use crate::domain::factory::Database;
|
||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
use crate::services::config::DatabaseConfig;
|
use crate::services::config::DatabaseConfig;
|
||||||
use crate::utils::locks::{DbOpLock, FileLock};
|
use crate::utils::locks::{DbOpLock, FileLock};
|
||||||
|
|
||||||
@@ -20,6 +18,12 @@ impl PostgresDatabase {
|
|||||||
pub fn new(cfg: DatabaseConfig, format: PostgresDumpFormat) -> Self {
|
pub fn new(cfg: DatabaseConfig, format: PostgresDumpFormat) -> Self {
|
||||||
Self { cfg, format }
|
Self { cfg, format }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn build_env(&self) -> HashMap<String, String> {
|
||||||
|
let mut envs = std::env::vars().collect::<HashMap<_, _>>();
|
||||||
|
envs.insert("PGPASSWORD".to_string(), self.cfg.password.to_string());
|
||||||
|
envs
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
@@ -35,16 +39,30 @@ impl Database for PostgresDatabase {
|
|||||||
ping::run(self.cfg.clone()).await
|
ping::run(self.cfg.clone()).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
|
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||||
let res = backup::run(self.cfg.clone(), self.format, dir.to_path_buf()).await;
|
let res = backup::run(
|
||||||
|
self.cfg.clone(),
|
||||||
|
self.format,
|
||||||
|
dir.to_path_buf(),
|
||||||
|
self.build_env(),
|
||||||
|
logger,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
FileLock::release(&self.cfg.generated_id).await?;
|
FileLock::release(&self.cfg.generated_id).await?;
|
||||||
res
|
res
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn restore(&self, file: &Path) -> Result<()> {
|
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
|
||||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||||
let res = restore::run(self.cfg.clone(), self.format, file.to_path_buf()).await;
|
let res = restore::run(
|
||||||
|
self.cfg.clone(),
|
||||||
|
self.format,
|
||||||
|
file.to_path_buf(),
|
||||||
|
self.build_env(),
|
||||||
|
logger,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
FileLock::release(&self.cfg.generated_id).await?;
|
FileLock::release(&self.cfg.generated_id).await?;
|
||||||
res
|
res
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
#[derive(Clone, Copy)]
|
#[derive(Clone, Copy, PartialEq, Debug)]
|
||||||
pub enum PostgresDumpFormat {
|
pub enum PostgresDumpFormat {
|
||||||
Fc,
|
Fc,
|
||||||
Fd,
|
Fd,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,8 +1,10 @@
|
|||||||
pub mod backup;
|
pub mod backup;
|
||||||
|
pub(crate) mod cluster;
|
||||||
|
pub(crate) mod clean_mode;
|
||||||
|
pub(crate) mod connection;
|
||||||
pub mod database;
|
pub mod database;
|
||||||
mod restore;
|
pub(crate) mod format;
|
||||||
mod connection;
|
|
||||||
mod format;
|
|
||||||
mod ping;
|
mod ping;
|
||||||
|
pub(crate) mod restore;
|
||||||
|
|
||||||
pub use connection::{detect_format_from_size, detect_format_from_file};
|
pub use connection::{detect_format_from_file, detect_format_from_size};
|
||||||
|
|||||||
@@ -1,8 +1,6 @@
|
|||||||
use super::connection::connect;
|
use super::connection::connect;
|
||||||
use crate::services::config::DatabaseConfig;
|
use crate::services::config::DatabaseConfig;
|
||||||
|
|
||||||
pub async fn run(
|
pub async fn run(cfg: DatabaseConfig) -> anyhow::Result<bool> {
|
||||||
cfg: DatabaseConfig,
|
|
||||||
) -> anyhow::Result<bool> {
|
|
||||||
Ok(connect(&cfg).await.is_ok())
|
Ok(connect(&cfg).await.is_ok())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,172 +0,0 @@
|
|||||||
use anyhow::Result;
|
|
||||||
use std::path::PathBuf;
|
|
||||||
use std::process::Command;
|
|
||||||
use tracing::{debug, error, info};
|
|
||||||
|
|
||||||
use super::connection::{select_pg_path, server_version, terminate_connections};
|
|
||||||
use super::format::PostgresDumpFormat;
|
|
||||||
use crate::services::config::DatabaseConfig;
|
|
||||||
|
|
||||||
pub async fn run(
|
|
||||||
cfg: DatabaseConfig,
|
|
||||||
format: PostgresDumpFormat,
|
|
||||||
restore_file: PathBuf,
|
|
||||||
) -> Result<()> {
|
|
||||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
|
||||||
debug!("Starting restore for database {}", cfg.name);
|
|
||||||
|
|
||||||
let version = match futures::executor::block_on(server_version(&cfg)) {
|
|
||||||
Ok(v) => {
|
|
||||||
debug!("Postgres version detected: {}", v);
|
|
||||||
v
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
error!("Failed to get server version for {}: {:?}", cfg.name, e);
|
|
||||||
return Err(e.into());
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let pg_restore = select_pg_path(&version).join("pg_restore");
|
|
||||||
debug!("Using pg_restore at {:?}", pg_restore);
|
|
||||||
|
|
||||||
if let Err(e) = futures::executor::block_on(terminate_connections(&cfg)) {
|
|
||||||
error!("Failed to terminate connections for {}: {:?}", cfg.name, e);
|
|
||||||
return Err(e.into());
|
|
||||||
}
|
|
||||||
info!("Connections terminated for database {}", cfg.name);
|
|
||||||
|
|
||||||
let url = format!(
|
|
||||||
"postgresql://{}:{}@{}:{}/postgres",
|
|
||||||
cfg.username, cfg.password, cfg.host, cfg.port
|
|
||||||
);
|
|
||||||
|
|
||||||
debug!("Restore URL: {}", url);
|
|
||||||
|
|
||||||
match format {
|
|
||||||
PostgresDumpFormat::Fc => {
|
|
||||||
info!("Running FC restore for {}", cfg.name);
|
|
||||||
let status = Command::new(&pg_restore)
|
|
||||||
.arg("--no-owner")
|
|
||||||
.arg("--no-privileges")
|
|
||||||
.arg("--clean")
|
|
||||||
.arg("--if-exists")
|
|
||||||
.arg("--create")
|
|
||||||
.arg("--dbname")
|
|
||||||
.arg(&url)
|
|
||||||
.arg("-v")
|
|
||||||
.arg(&restore_file)
|
|
||||||
.env("PGPASSWORD", &cfg.password)
|
|
||||||
.status();
|
|
||||||
|
|
||||||
match status {
|
|
||||||
Ok(s) if s.success() => {
|
|
||||||
info!("FC restore completed successfully for {}", cfg.name)
|
|
||||||
}
|
|
||||||
Ok(s) => {
|
|
||||||
error!("FC restore failed with status {:?} for {}", s, cfg.name);
|
|
||||||
anyhow::bail!("Postgres restore failed for {}", cfg.name);
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
error!("Error executing pg_restore for {}: {:?}", cfg.name, e);
|
|
||||||
return Err(e.into());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
PostgresDumpFormat::Fd => {
|
|
||||||
info!("Running FD restore for {}", cfg.name);
|
|
||||||
|
|
||||||
let tar_gz = match std::fs::File::open(&restore_file) {
|
|
||||||
Ok(f) => f,
|
|
||||||
Err(e) => {
|
|
||||||
error!(
|
|
||||||
"Failed to open restore file {:?} for {}: {:?}",
|
|
||||||
restore_file, cfg.name, e
|
|
||||||
);
|
|
||||||
return Err(e.into());
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
info!("tar_gz {:?}", tar_gz);
|
|
||||||
|
|
||||||
let dec = flate2::read::GzDecoder::new(tar_gz);
|
|
||||||
let mut archive = tar::Archive::new(dec);
|
|
||||||
|
|
||||||
let tmp_dir = match tempfile::TempDir::new() {
|
|
||||||
Ok(d) => d,
|
|
||||||
Err(e) => {
|
|
||||||
error!(
|
|
||||||
"Failed to create temporary directory for FD restore of {}: {:?}",
|
|
||||||
cfg.name, e
|
|
||||||
);
|
|
||||||
return Err(e.into());
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
if let Err(e) = archive.unpack(tmp_dir.path()) {
|
|
||||||
error!("Failed to unpack FD archive for {}: {:?}", cfg.name, e);
|
|
||||||
return Err(e.into());
|
|
||||||
}
|
|
||||||
|
|
||||||
debug!("Listing contents of temp dir: {}", tmp_dir.path().display());
|
|
||||||
|
|
||||||
for entry in std::fs::read_dir(tmp_dir.path())? {
|
|
||||||
if let Ok(entry) = entry {
|
|
||||||
let path = entry.path();
|
|
||||||
let file_type = entry.file_type()?;
|
|
||||||
debug!(
|
|
||||||
" - {} | is_dir: {} | is_file: {}",
|
|
||||||
path.display(),
|
|
||||||
file_type.is_dir(),
|
|
||||||
file_type.is_file()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let dump_dir = if tmp_dir.path().join("toc.dat").exists() {
|
|
||||||
tmp_dir.path().to_path_buf()
|
|
||||||
} else {
|
|
||||||
std::fs::read_dir(tmp_dir.path())?
|
|
||||||
.filter_map(|e| e.ok())
|
|
||||||
.find(|entry| entry.path().join("toc.dat").exists())
|
|
||||||
.map(|e| e.path())
|
|
||||||
.ok_or_else(|| anyhow::anyhow!("Invalid FD archive: toc.dat not found"))?
|
|
||||||
};
|
|
||||||
|
|
||||||
let status = Command::new(&pg_restore)
|
|
||||||
.arg("--no-owner")
|
|
||||||
.arg("--no-privileges")
|
|
||||||
.arg("--clean")
|
|
||||||
.arg("--if-exists")
|
|
||||||
.arg("--create")
|
|
||||||
.arg("--dbname")
|
|
||||||
.arg(&url)
|
|
||||||
.arg("-v")
|
|
||||||
.arg("-j")
|
|
||||||
.arg("4")
|
|
||||||
.arg(dump_dir)
|
|
||||||
.env("PGPASSWORD", &cfg.password)
|
|
||||||
.status();
|
|
||||||
|
|
||||||
match status {
|
|
||||||
Ok(s) if s.success() => {
|
|
||||||
info!("FD restore completed successfully for {}", cfg.name)
|
|
||||||
}
|
|
||||||
Ok(s) => {
|
|
||||||
error!("FD restore failed with status {:?} for {}", s, cfg.name);
|
|
||||||
anyhow::bail!("Postgres FD restore failed for {}", cfg.name);
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
error!("Error executing pg_restore for {}: {:?}", cfg.name, e);
|
|
||||||
return Err(e.into());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
info!("Restore finished for database {}", cfg.name);
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
})
|
|
||||||
.await?
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
use anyhow::Result;
|
||||||
|
use std::process::Command;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
|
||||||
|
pub(crate) fn run_pg_restore(
|
||||||
|
mut cmd: Command,
|
||||||
|
logger: &JobLogger,
|
||||||
|
cfg: &DatabaseConfig,
|
||||||
|
) -> Result<()> {
|
||||||
|
let start = Instant::now();
|
||||||
|
let output = cmd.output();
|
||||||
|
let duration_ms = start.elapsed().as_millis() as f64;
|
||||||
|
|
||||||
|
match output {
|
||||||
|
Ok(o) => {
|
||||||
|
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
|
||||||
|
let stdout = String::from_utf8_lossy(&o.stdout).to_string();
|
||||||
|
let combined = format!("{}{}", stdout, stderr);
|
||||||
|
let exit_code = o.status.code().unwrap_or(-1);
|
||||||
|
let payload = if combined.is_empty() { None } else { Some(combined) };
|
||||||
|
|
||||||
|
if o.status.success() {
|
||||||
|
logger.log_command("pg_restore", payload, Some(0), Some(duration_ms));
|
||||||
|
logger.log("info", format!("Restore completed successfully for {}", cfg.name));
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
logger.log_command("pg_restore", payload, Some(exit_code), Some(duration_ms));
|
||||||
|
logger.log("error", format!("Restore failed with status {:?} for {}", o.status, cfg.name));
|
||||||
|
anyhow::bail!("Postgres restore failed for {}", cfg.name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
logger.log_command("pg_restore", Some(e.to_string()), Some(-1), Some(duration_ms));
|
||||||
|
logger.log("error", format!("Error executing pg_restore for {}: {:?}", cfg.name, e));
|
||||||
|
Err(e.into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
mod command;
|
||||||
|
mod prepare;
|
||||||
|
mod run;
|
||||||
|
mod toc;
|
||||||
|
|
||||||
|
pub use run::run;
|
||||||
|
pub(crate) use command::run_pg_restore;
|
||||||
|
pub(crate) use prepare::prepare_archive;
|
||||||
|
pub(crate) use toc::toc_creates_public_schema;
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
use anyhow::Result;
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::process::Command;
|
||||||
|
|
||||||
|
use crate::domain::postgres::connection::sniff_format;
|
||||||
|
use crate::domain::postgres::format::PostgresDumpFormat;
|
||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
|
|
||||||
|
pub(crate) struct PreparedArchive {
|
||||||
|
path: PathBuf,
|
||||||
|
_tmp: Option<tempfile::TempDir>,
|
||||||
|
toc: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PreparedArchive {
|
||||||
|
pub(crate) fn path(&self) -> &Path {
|
||||||
|
&self.path
|
||||||
|
}
|
||||||
|
pub(crate) fn toc(&self) -> &str {
|
||||||
|
&self.toc
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn prepare_archive(
|
||||||
|
format: PostgresDumpFormat,
|
||||||
|
restore_file: &Path,
|
||||||
|
pg_restore: &Path,
|
||||||
|
logger: &JobLogger,
|
||||||
|
) -> Result<PreparedArchive> {
|
||||||
|
let sniffed = sniff_format(restore_file)?;
|
||||||
|
if sniffed != format {
|
||||||
|
logger.log("warn", format!("Declared format {:?} != sniffed {:?}; using sniffed", format, sniffed));
|
||||||
|
}
|
||||||
|
let format = sniffed;
|
||||||
|
|
||||||
|
let (path, tmp) = match format {
|
||||||
|
PostgresDumpFormat::Fc => (restore_file.to_path_buf(), None),
|
||||||
|
PostgresDumpFormat::Fd => {
|
||||||
|
let tar_gz = std::fs::File::open(restore_file)?;
|
||||||
|
let dec = flate2::read::GzDecoder::new(tar_gz);
|
||||||
|
let mut archive = tar::Archive::new(dec);
|
||||||
|
let tmp_dir = tempfile::TempDir::new()?;
|
||||||
|
archive.unpack(tmp_dir.path())?;
|
||||||
|
|
||||||
|
let dump_dir = if tmp_dir.path().join("toc.dat").exists() {
|
||||||
|
tmp_dir.path().to_path_buf()
|
||||||
|
} else {
|
||||||
|
std::fs::read_dir(tmp_dir.path())?
|
||||||
|
.filter_map(|e| e.ok())
|
||||||
|
.find(|entry| entry.path().join("toc.dat").exists())
|
||||||
|
.map(|e| e.path())
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("Invalid FD archive: toc.dat not found"))?
|
||||||
|
};
|
||||||
|
(dump_dir, Some(tmp_dir))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let toc_out = Command::new(pg_restore).arg("-l").arg(&path).output()?;
|
||||||
|
if !toc_out.status.success() {
|
||||||
|
let stderr = String::from_utf8_lossy(&toc_out.stderr).to_string();
|
||||||
|
logger.log("error", format!("pg_restore -l failed: {}", stderr));
|
||||||
|
anyhow::bail!("Archive validation failed (pg_restore -l): {}", stderr);
|
||||||
|
}
|
||||||
|
let toc = String::from_utf8_lossy(&toc_out.stdout).to_string();
|
||||||
|
|
||||||
|
Ok(PreparedArchive { path, _tmp: tmp, toc })
|
||||||
|
}
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
use anyhow::Result;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::process::Command;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use super::{prepare_archive, run_pg_restore, toc_creates_public_schema};
|
||||||
|
use crate::domain::postgres::clean_mode::RestoreCleanMode;
|
||||||
|
use crate::domain::postgres::connection::{
|
||||||
|
can_drop_database, drop_all_schemas, drop_and_recreate_database, pg_restore_binary_name,
|
||||||
|
recreate_public_schema, select_pg_path, server_version, terminate_connections,
|
||||||
|
};
|
||||||
|
use crate::domain::postgres::format::PostgresDumpFormat;
|
||||||
|
use crate::services::backup::logger::JobLogger;
|
||||||
|
use crate::services::config::DatabaseConfig;
|
||||||
|
|
||||||
|
pub async fn run(
|
||||||
|
cfg: DatabaseConfig,
|
||||||
|
format: PostgresDumpFormat,
|
||||||
|
restore_file: PathBuf,
|
||||||
|
env: HashMap<String, String>,
|
||||||
|
logger: Arc<JobLogger>,
|
||||||
|
) -> Result<()> {
|
||||||
|
let handle = tokio::runtime::Handle::current();
|
||||||
|
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||||
|
logger.log("info", format!("Starting restore for database {}", cfg.name));
|
||||||
|
|
||||||
|
let version = match handle.block_on(server_version(&cfg)) {
|
||||||
|
Ok(v) => {
|
||||||
|
logger.log("debug", format!("Postgres version detected: {}", v));
|
||||||
|
v
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
logger.log("error", format!("Failed to get server version for {}: {:?}", cfg.name, e));
|
||||||
|
return Err(e.into());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let pg_restore = select_pg_path(&version).join(pg_restore_binary_name());
|
||||||
|
|
||||||
|
logger.log("debug", format!("Using pg_restore at {:?}", pg_restore));
|
||||||
|
|
||||||
|
let keep_ownership = cfg.options
|
||||||
|
.get("keep_ownership")
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false);
|
||||||
|
|
||||||
|
if keep_ownership {
|
||||||
|
logger.log("info", format!("Restoring ownership and privileges for {}", cfg.name));
|
||||||
|
} else {
|
||||||
|
logger.log("info", format!("Stripping ownership and privileges for {} (--no-owner --no-privileges)", cfg.name));
|
||||||
|
}
|
||||||
|
|
||||||
|
let (mode, bad_value) = RestoreCleanMode::from_config(&cfg);
|
||||||
|
if let Some(v) = bad_value {
|
||||||
|
logger.log("warn", format!("Unknown clean_mode '{}' for {}, falling back to 'clean'", v, cfg.name));
|
||||||
|
}
|
||||||
|
|
||||||
|
let prepared = prepare_archive(format, &restore_file, &pg_restore, &logger)?;
|
||||||
|
|
||||||
|
match mode {
|
||||||
|
RestoreCleanMode::DropSchemas => {
|
||||||
|
handle.block_on(terminate_connections(&cfg))?;
|
||||||
|
let owner = cfg.username.clone();
|
||||||
|
let dropped = handle.block_on(drop_all_schemas(&cfg))?;
|
||||||
|
logger.log("warn", format!("clean_mode=drop_schemas dropped schemas {:?} in {}", dropped, cfg.database));
|
||||||
|
if !toc_creates_public_schema(prepared.toc()) {
|
||||||
|
handle.block_on(recreate_public_schema(&cfg, &owner))?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
RestoreCleanMode::DropDatabase => {
|
||||||
|
if !handle.block_on(can_drop_database(&cfg))? {
|
||||||
|
anyhow::bail!(
|
||||||
|
"clean_mode=drop_database requires CREATEDB + ownership on {}; use clean_mode=drop_schemas instead",
|
||||||
|
cfg.database
|
||||||
|
);
|
||||||
|
}
|
||||||
|
logger.log("warn", format!("clean_mode=drop_database DROPPING database {} before restore", cfg.database));
|
||||||
|
handle.block_on(drop_and_recreate_database(&cfg))?;
|
||||||
|
}
|
||||||
|
RestoreCleanMode::Clean | RestoreCleanMode::None => {
|
||||||
|
handle.block_on(terminate_connections(&cfg))?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut cmd = Command::new(&pg_restore);
|
||||||
|
if !keep_ownership {
|
||||||
|
cmd.args(["--no-owner", "--no-privileges"]);
|
||||||
|
}
|
||||||
|
if mode.uses_pg_restore_clean() {
|
||||||
|
cmd.args(["--clean", "--if-exists"]);
|
||||||
|
}
|
||||||
|
cmd.arg("--host").arg(&cfg.host)
|
||||||
|
.arg("--port").arg(cfg.port.to_string())
|
||||||
|
.arg("--username").arg(&cfg.username)
|
||||||
|
.arg("--dbname").arg(&cfg.database)
|
||||||
|
.arg("-v");
|
||||||
|
if matches!(format, PostgresDumpFormat::Fd) {
|
||||||
|
cmd.arg("-j").arg("4");
|
||||||
|
}
|
||||||
|
cmd.arg(prepared.path()).envs(env);
|
||||||
|
|
||||||
|
run_pg_restore(cmd, &logger, &cfg)?;
|
||||||
|
logger.log("info", format!("Restore finished for database {}", cfg.name));
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.await?
|
||||||
|
}
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
pub(crate) fn toc_creates_public_schema(toc: &str) -> bool {
|
||||||
|
toc.lines().any(|l| {
|
||||||
|
l.split(" SCHEMA - ")
|
||||||
|
.nth(1)
|
||||||
|
.and_then(|rest| rest.split_whitespace().next())
|
||||||
|
== Some("public")
|
||||||
|
})
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user