mirror of
https://github.com/Portabase/agent.git
synced 2026-09-11 02:27:10 +00:00
Compare commits
78 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| cc1703592e | |||
| 5f92297108 | |||
| d1821f7045 | |||
| 0fe4d50cbd | |||
| 1f29466a28 | |||
| b74aaa0bbc | |||
| 7b5e5b2c78 | |||
| 99f1ef2081 | |||
| 42c3c5945a | |||
| db0f87c2e9 | |||
| f95f0aa73a | |||
| 2177dfd44b | |||
| 0a53eec184 | |||
| 87f33af772 | |||
| b6a120fcbf | |||
| 5298d82576 | |||
| b0da2e40a3 | |||
| 55e20d48e7 | |||
| f7f5f7e141 | |||
| 2170f96a72 | |||
| 298d46ba81 | |||
| 6537e9df53 | |||
| b8d869d5a6 | |||
| 90941ea67d | |||
| 046d593e2b | |||
| cf9a59a138 | |||
| 2464f6dfb0 | |||
| 069067ca55 | |||
| 04b654d219 | |||
| d83511cb64 | |||
| ca294e968c | |||
| 1be88ffdb9 | |||
| 9d393a96a4 | |||
| 044bf80633 | |||
| 0a6eb6db22 | |||
| b26ff81889 | |||
| 4e2f29f4ca | |||
| d1c8df4cac | |||
| 30f83bafcf | |||
| de106c835e | |||
| 23d6822ddc | |||
| fe1d74945f | |||
| 424a646385 | |||
| 0ef4bba5d7 | |||
| 6548140eaf | |||
| c9725c381e | |||
| 03695b1897 | |||
| 80b1c0dac0 | |||
| 5ededb3764 | |||
| 54cfd673e4 | |||
| 39dbeb5d30 | |||
| 380cea79fb | |||
| 65e4538c2c | |||
| 86a706bca5 | |||
| e8bda60f16 | |||
| 034912b176 | |||
| 4fd50ccda4 | |||
| 84c6b765c8 | |||
| 9790837200 | |||
| 8827234344 | |||
| b3417304bb | |||
| 5f4006b1ca | |||
| f24b1ec3e4 | |||
| bb1b16d0f2 | |||
| 2aca036ee4 | |||
| 2a72a55e57 | |||
| ae63a0df50 | |||
| 560c5e02f2 | |||
| ca17bf9cb9 | |||
| 9fbaab6feb | |||
| 1289497d98 | |||
| 27c55dbfd8 | |||
| 45e3acd709 | |||
| a93054518a | |||
| baf2edc829 | |||
| c186d57105 | |||
| 37b4c1fe6d | |||
| 92a3324ca2 |
+79
-17
@@ -27,15 +27,80 @@ permissions:
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Build and push to GHCR
|
||||
runs-on: ubuntu-latest
|
||||
build:
|
||||
name: Build ${{ matrix.platform }}
|
||||
runs-on: ${{ matrix.platform == 'linux/amd64' && 'ubuntu-latest' || 'ubuntu-24.04-arm' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
platform: [ linux/amd64, linux/arm64 ]
|
||||
steps:
|
||||
- name: Prepare vars
|
||||
id: prep
|
||||
run: |
|
||||
ARCH="${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}"
|
||||
echo "arch=$ARCH" >> "$GITHUB_OUTPUT"
|
||||
echo "image=ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/agent" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
fetch-depth: 0
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and push by digest
|
||||
id: build
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: ${{ inputs.dockerfile }}
|
||||
platforms: ${{ matrix.platform }}
|
||||
target: ${{ inputs.target }}
|
||||
provenance: false
|
||||
outputs: type=image,name=${{ steps.prep.outputs.image }},push-by-digest=true,name-canonical=true,push=true
|
||||
cache-from: type=gha,scope=ghcr-${{ steps.prep.outputs.arch }}
|
||||
cache-to: type=gha,mode=max,scope=ghcr-${{ steps.prep.outputs.arch }},ignore-error=true
|
||||
|
||||
- name: Export digest
|
||||
env:
|
||||
DIGEST: ${{ steps.build.outputs.digest }}
|
||||
run: |
|
||||
mkdir -p /tmp/digests
|
||||
touch "/tmp/digests/${DIGEST#sha256:}"
|
||||
|
||||
- name: Upload digest
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: digest-${{ steps.prep.outputs.arch }}
|
||||
path: /tmp/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
merge:
|
||||
name: Create multi-arch manifest
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
steps:
|
||||
- name: Prepare vars
|
||||
id: prep
|
||||
run: echo "image=ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/agent" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Download digests
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: /tmp/digests
|
||||
pattern: digest-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
@@ -51,22 +116,19 @@ jobs:
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ghcr.io/${{ github.repository_owner }}/agent
|
||||
images: ${{ steps.prep.outputs.image }}
|
||||
tags: |
|
||||
type=semver,pattern={{version}},value=${{ inputs.version }}
|
||||
type=semver,pattern={{major}}.{{minor}},value=${{ inputs.version }}
|
||||
type=semver,pattern={{major}},value=${{ inputs.version }}
|
||||
type=raw,value=latest,enable=${{ inputs.add_latest }}
|
||||
|
||||
- name: Build and push image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: ${{ inputs.dockerfile }}
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
target: ${{ inputs.target }}
|
||||
provenance: false
|
||||
cache-from: type=gha,scope=ghcr-build
|
||||
cache-to: type=gha,mode=max,scope=ghcr-build,ignore-error=true
|
||||
- name: Create and push manifest list
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
docker buildx imagetools create \
|
||||
$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
|
||||
$(printf '${{ steps.prep.outputs.image }}@sha256:%s ' *)
|
||||
|
||||
- name: Inspect
|
||||
run: docker buildx imagetools inspect ${{ steps.prep.outputs.image }}:${{ inputs.version }}
|
||||
|
||||
@@ -118,11 +118,24 @@ jobs:
|
||||
secrets:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build-windows:
|
||||
needs: create-release
|
||||
if: ${{ needs.create-release.result == 'success' }}
|
||||
uses: ./.github/workflows/windows-release.yml
|
||||
with:
|
||||
version: ${{ needs.create-release.outputs.version }}
|
||||
ref: ${{ needs.create-release.outputs.version }}
|
||||
draft_tag: ${{ needs.create-release.outputs.draft_tag }}
|
||||
secrets:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
finalize-release:
|
||||
needs:
|
||||
- create-release
|
||||
- publish-docker
|
||||
- publish-docker-ghcr
|
||||
- publish-helm
|
||||
- build-windows
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
release_tag: ${{ steps.publish_release_step.outputs.release_tag }}
|
||||
|
||||
@@ -1,14 +1,30 @@
|
||||
name: Build Windows release
|
||||
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
version:
|
||||
description: 'Release version (git tag), e.g. 1.18.4'
|
||||
type: string
|
||||
required: false
|
||||
ref:
|
||||
description: 'Git ref to check out and build'
|
||||
type: string
|
||||
required: false
|
||||
draft_tag:
|
||||
description: 'Draft GitHub release tag to attach the asset to (e.g. untagged-xxxx). Empty = skip upload.'
|
||||
type: string
|
||||
required: false
|
||||
secrets:
|
||||
GH_TOKEN:
|
||||
required: false
|
||||
|
||||
workflow_dispatch:
|
||||
push:
|
||||
tags:
|
||||
- '[0-9]+.[0-9]+.[0-9]+'
|
||||
branches:
|
||||
- main
|
||||
- master
|
||||
inputs:
|
||||
ref:
|
||||
description: 'Git ref to check out and build'
|
||||
type: string
|
||||
required: false
|
||||
|
||||
jobs:
|
||||
build-windows:
|
||||
@@ -17,80 +33,53 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Set up Rust toolchain (MSVC)
|
||||
uses: actions-rs/toolchain@v1
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: stable-x86_64-pc-windows-msvc
|
||||
profile: minimal
|
||||
override: true
|
||||
targets: x86_64-pc-windows-msvc
|
||||
|
||||
- name: Install vcpkg and OpenSSL (x64)
|
||||
shell: pwsh
|
||||
run: |
|
||||
# Install vcpkg and the prebuilt OpenSSL package
|
||||
git clone https://github.com/microsoft/vcpkg C:\vcpkg
|
||||
C:\vcpkg\bootstrap-vcpkg.bat
|
||||
C:\vcpkg\vcpkg install openssl:x64-windows
|
||||
# Export variables for subsequent steps
|
||||
'VCPKG_ROOT=C:\vcpkg' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
'OPENSSL_DIR=C:\vcpkg\installed\x64-windows' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
- name: Cache cargo build
|
||||
uses: Swatinem/rust-cache@v2
|
||||
|
||||
- name: Build (cargo release)
|
||||
shell: pwsh
|
||||
env:
|
||||
# Cargo / openssl-sys will pick up OPENSSL_DIR from the environment
|
||||
OPENSSL_DIR: ${{ env.OPENSSL_DIR }}
|
||||
run: |
|
||||
# Ensure the environment variable is present for this step
|
||||
if (-Not $env:OPENSSL_DIR) { Write-Host "OPENSSL_DIR not set, printing env for debugging"; Get-ChildItem Env: | ForEach-Object { Write-Host $_ } }
|
||||
# Build the declared bin target explicitly (Cargo.toml [[bin]] name = "app")
|
||||
cargo build --release --bin app
|
||||
run: cargo build --release --bin app
|
||||
|
||||
- name: Prepare artifact zip
|
||||
id: prepare_artifact
|
||||
shell: pwsh
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.ref_name }}
|
||||
RELEASE_VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
$tag = $env:RELEASE_TAG
|
||||
$tag = $env:RELEASE_VERSION
|
||||
if (-not $tag) { $tag = $env:GITHUB_SHA }
|
||||
|
||||
# Package the declared bin target deterministically (Cargo.toml [[bin]] name = "app")
|
||||
$exe = "target\release\app.exe"
|
||||
if (-not (Test-Path $exe)) { Write-Error "Built binary $exe not found in target/release"; exit 1 }
|
||||
|
||||
$outDir = "artifact"
|
||||
New-Item -ItemType Directory -Path $outDir -Force | Out-Null
|
||||
# Ship under the package name, not the internal bin name "app"
|
||||
# Ship under the package name, not the internal bin name "app".
|
||||
Copy-Item -Path $exe -Destination "$outDir\portabase-agent.exe"
|
||||
|
||||
$zipName = "windows-release-$tag.zip"
|
||||
if (Test-Path $zipName) { Remove-Item $zipName }
|
||||
Compress-Archive -Path "$outDir\*" -DestinationPath $zipName -Force
|
||||
Write-Host "ZIP=$zipName"
|
||||
Write-Output "zip=$zipName" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
|
||||
"zip=$zipName" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
|
||||
|
||||
- name: Upload build artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: windows-release
|
||||
path: windows-release-*.zip
|
||||
path: ${{ steps.prepare_artifact.outputs.zip }}
|
||||
|
||||
- name: Create GitHub Release
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
id: create_release
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
tag_name: ${{ github.ref_name }}
|
||||
- name: Attach asset to draft release
|
||||
if: ${{ inputs.draft_tag != '' }}
|
||||
shell: pwsh
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload release asset
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: actions/upload-release-asset@v1
|
||||
with:
|
||||
upload_url: ${{ steps.create_release.outputs.upload_url }}
|
||||
asset_path: windows-release-${{ github.ref_name }}.zip
|
||||
asset_name: windows-release-${{ github.ref_name }}.zip
|
||||
asset_content_type: application/zip
|
||||
GH_TOKEN: ${{ secrets.GH_TOKEN }}
|
||||
run: |
|
||||
gh release upload "${{ inputs.draft_tag }}" "${{ steps.prepare_artifact.outputs.zip }}" --clobber
|
||||
|
||||
+1
-1
@@ -27,5 +27,5 @@ keywords:
|
||||
- self-hosted
|
||||
- portabase
|
||||
license: Apache-2.0
|
||||
version: 1.17.1
|
||||
version: 1.20.1
|
||||
date-released: '2026-02-24'
|
||||
|
||||
Generated
+4
-67
@@ -1707,21 +1707,6 @@ version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
|
||||
|
||||
[[package]]
|
||||
name = "foreign-types"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
|
||||
dependencies = [
|
||||
"foreign-types-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "foreign-types-shared"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
|
||||
|
||||
[[package]]
|
||||
name = "form_urlencoded"
|
||||
version = "1.2.2"
|
||||
@@ -3245,32 +3230,6 @@ dependencies = [
|
||||
"portable-atomic",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openssl"
|
||||
version = "0.10.76"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"cfg-if",
|
||||
"foreign-types",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"openssl-macros",
|
||||
"openssl-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openssl-macros"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openssl-probe"
|
||||
version = "0.1.6"
|
||||
@@ -3283,18 +3242,6 @@ version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
|
||||
|
||||
[[package]]
|
||||
name = "openssl-sys"
|
||||
version = "0.9.112"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"pkg-config",
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry"
|
||||
version = "0.32.0"
|
||||
@@ -3478,12 +3425,6 @@ dependencies = [
|
||||
"spki",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
|
||||
|
||||
[[package]]
|
||||
name = "plain"
|
||||
version = "0.2.3"
|
||||
@@ -3503,7 +3444,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "portabase-agent"
|
||||
version = "1.17.1"
|
||||
version = "1.20.1"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"aes-gcm",
|
||||
@@ -3528,19 +3469,21 @@ dependencies = [
|
||||
"google-cloud-auth",
|
||||
"google-cloud-storage",
|
||||
"hex",
|
||||
"hmac 0.12.1",
|
||||
"hyper 1.10.1",
|
||||
"log",
|
||||
"mockall",
|
||||
"mongodb",
|
||||
"oauth2",
|
||||
"once_cell",
|
||||
"openssl",
|
||||
"percent-encoding",
|
||||
"postgres",
|
||||
"rand 0.9.2",
|
||||
"redis",
|
||||
"reqwest 0.13.4",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"tar",
|
||||
"tempfile",
|
||||
"testcontainers",
|
||||
@@ -5566,12 +5509,6 @@ version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
|
||||
|
||||
[[package]]
|
||||
name = "vcpkg"
|
||||
version = "0.2.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.5"
|
||||
|
||||
+4
-2
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "portabase-agent"
|
||||
version = "1.17.1"
|
||||
version = "1.20.1"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
@@ -22,7 +22,8 @@ anyhow = "1.0.100"
|
||||
tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "fs"] }
|
||||
async-trait = "0.1.89"
|
||||
tempfile = "3.24.0"
|
||||
openssl = "0.10.75"
|
||||
hmac = "0.12"
|
||||
sha2 = "0.10"
|
||||
hex = "0.4.3"
|
||||
flate2 = "1.1.5"
|
||||
tar = "0.4.44"
|
||||
@@ -57,6 +58,7 @@ testcontainers = "0.27.1"
|
||||
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "valkey", "mysql", "mariadb", "mongo"] }
|
||||
postgres = "0.19.12"
|
||||
url = "2.5.8"
|
||||
percent-encoding = "2.3.2"
|
||||
bollard = "0.20.0"
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
+6
-47
@@ -1,13 +1,14 @@
|
||||
services:
|
||||
rust-app:
|
||||
# build:
|
||||
# context: .
|
||||
# dockerfile: docker/Dockerfile
|
||||
# target: prod
|
||||
image: portabase/agent:latest
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/Dockerfile
|
||||
target: prod
|
||||
# image: portabase/agent:latest
|
||||
container_name: rust-prod
|
||||
volumes:
|
||||
- ./databases.json:/config/config.json
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
environment:
|
||||
LOG: info
|
||||
TZ: "Europe/Paris"
|
||||
@@ -18,48 +19,6 @@ services:
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
|
||||
db-mongodb-auth:
|
||||
container_name: db-mongodb-auth
|
||||
image: mongo:latest
|
||||
ports:
|
||||
- "27082:27017"
|
||||
environment:
|
||||
MONGO_INITDB_ROOT_USERNAME: root
|
||||
MONGO_INITDB_ROOT_PASSWORD: rootpassword
|
||||
MONGO_INITDB_DATABASE: testdbauth
|
||||
command: mongod --auth
|
||||
networks:
|
||||
- portabase
|
||||
volumes:
|
||||
- mongodb-data-auth:/data/db
|
||||
healthcheck:
|
||||
test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
db-mongodb:
|
||||
container_name: db-mongodb
|
||||
image: mongo:latest
|
||||
ports:
|
||||
- "27083:27017"
|
||||
volumes:
|
||||
- mongodb-data:/data/db
|
||||
healthcheck:
|
||||
test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
environment:
|
||||
MONGO_INITDB_DATABASE: testdb
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
volumes:
|
||||
mongodb-data:
|
||||
mongodb-data-auth:
|
||||
|
||||
networks:
|
||||
portabase:
|
||||
name: portabase_network
|
||||
|
||||
+3
-1
@@ -21,9 +21,11 @@ services:
|
||||
LOG: debug
|
||||
TZ: "Europe/Paris"
|
||||
# TMPDIR: /scratch
|
||||
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiNTljYzRjYTUtOTAyNy00ZThiLTk1NDktMjAzOTI3ZDVjNmUyIiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ=="
|
||||
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiOWMxMzM5NjItMGE5OC00MmRkLTk1NjUtOTA5ZTkyYTI5N2VkIiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ=="
|
||||
#CHUNK_SIZE_MB: "1"
|
||||
#POOLING: 1
|
||||
#RETRY_ATTEMPTS: 3
|
||||
#RETRY_BACKOFF_MS: 1000
|
||||
#DATABASES_CONFIG_FILE: "config.toml"
|
||||
extra_hosts:
|
||||
- "localhost:host-gateway"
|
||||
|
||||
@@ -142,6 +142,8 @@ RUN curl -sSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh \
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN mkdir -p /config
|
||||
|
||||
COPY --from=builder /app/target/release/app /usr/local/bin/app
|
||||
COPY --from=builder /app/version.env /app/version.env
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
|
||||
@@ -7,4 +7,6 @@ data:
|
||||
TZ: {{ .Values.env.TZ | quote }}
|
||||
POLLING: {{ .Values.env.POLLING | quote }}
|
||||
APP_ENV: {{ .Values.env.APP_ENV | quote }}
|
||||
LOG: {{ .Values.env.LOG | quote }}
|
||||
LOG: {{ .Values.env.LOG | quote }}
|
||||
RETRY_ATTEMPTS: {{ .Values.env.RETRY_ATTEMPTS | quote }}
|
||||
RETRY_BACKOFF_MS: {{ .Values.env.RETRY_BACKOFF_MS | quote }}
|
||||
|
||||
@@ -11,6 +11,8 @@ env:
|
||||
POLLING: "5"
|
||||
APP_ENV: "production"
|
||||
LOG: "info"
|
||||
RETRY_ATTEMPTS: "3"
|
||||
RETRY_BACKOFF_MS: "1000"
|
||||
|
||||
resources:
|
||||
limits:
|
||||
|
||||
@@ -45,6 +45,14 @@ seed-firebird:
|
||||
echo "SELECT RDB\$RELATION_NAME FROM RDB\$RELATIONS WHERE RDB\$SYSTEM_FLAG = 0 AND RDB\$VIEW_BLR IS NULL;" \
|
||||
| docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb
|
||||
|
||||
seed-firebird-large:
|
||||
echo "Seeding Firebird..."
|
||||
docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb < ./scripts/firebird/seed-large.sql
|
||||
|
||||
echo "Verifying Firebird tables..."
|
||||
echo "SELECT RDB\$RELATION_NAME FROM RDB\$RELATIONS WHERE RDB\$SYSTEM_FLAG = 0 AND RDB\$VIEW_BLR IS NULL;" \
|
||||
| docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb
|
||||
|
||||
seed-mssql:
|
||||
echo "Seeding MSSQL..."
|
||||
docker exec -i rust-dev sqlcmd -S "db-mssql,1433" -U sa -P "$MSSQL_SA_PASSWORD" -N disable -i /app/scripts/mssql/seed.sql
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
SET SQL DIALECT 3;
|
||||
SET BAIL ON;
|
||||
SET AUTODDL OFF;
|
||||
|
||||
CREATE TABLE users (
|
||||
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
email VARCHAR(255) NOT NULL UNIQUE,
|
||||
name VARCHAR(255),
|
||||
payload BLOB SUB_TYPE TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
COMMIT;
|
||||
|
||||
INSERT INTO users (email, name, payload)
|
||||
VALUES ('alice@example.com', 'Alice', 'Alice seed data');
|
||||
|
||||
INSERT INTO users (email, name, payload)
|
||||
VALUES ('bob@example.com', 'Bob', 'Bob seed data');
|
||||
|
||||
COMMIT;
|
||||
|
||||
/*
|
||||
* Each procedure call generates approximately 128 MiB:
|
||||
*
|
||||
* 128 rows
|
||||
* × 128 chunks per row
|
||||
* × 8191 bytes per chunk
|
||||
* = approximately 128 MiB
|
||||
*
|
||||
* 40 calls = approximately 5 GiB.
|
||||
*/
|
||||
|
||||
SET TERM ^;
|
||||
|
||||
CREATE PROCEDURE seed_users_batch (
|
||||
p_rows INTEGER,
|
||||
p_chunks_per_row INTEGER
|
||||
)
|
||||
AS
|
||||
DECLARE VARIABLE v_row_index INTEGER;
|
||||
DECLARE VARIABLE v_chunk_index INTEGER;
|
||||
DECLARE VARIABLE v_uuid VARCHAR(36);
|
||||
DECLARE VARIABLE v_chunk VARCHAR(8191);
|
||||
DECLARE VARIABLE v_payload BLOB SUB_TYPE TEXT;
|
||||
BEGIN
|
||||
v_row_index = 0;
|
||||
|
||||
WHILE (v_row_index < p_rows) DO
|
||||
BEGIN
|
||||
v_payload = NULL;
|
||||
v_chunk_index = 0;
|
||||
|
||||
WHILE (v_chunk_index < p_chunks_per_row) DO
|
||||
BEGIN
|
||||
/*
|
||||
* Generate a different chunk to avoid producing a completely
|
||||
* uniform BLOB.
|
||||
*/
|
||||
v_chunk = RPAD(
|
||||
UUID_TO_CHAR(GEN_UUID()),
|
||||
8191,
|
||||
UUID_TO_CHAR(GEN_UUID())
|
||||
);
|
||||
|
||||
v_payload = BLOB_APPEND(v_payload, v_chunk);
|
||||
v_chunk_index = v_chunk_index + 1;
|
||||
END
|
||||
|
||||
v_uuid = UUID_TO_CHAR(GEN_UUID());
|
||||
|
||||
INSERT INTO users (
|
||||
email,
|
||||
name,
|
||||
payload
|
||||
)
|
||||
VALUES (
|
||||
:v_uuid || '@example.test',
|
||||
'Seed User ' || :v_uuid,
|
||||
:v_payload
|
||||
);
|
||||
|
||||
v_row_index = v_row_index + 1;
|
||||
END
|
||||
END^
|
||||
|
||||
SET TERM ;^
|
||||
|
||||
/* Batch 01 — approximately 128 MiB */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 02 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 03 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 04 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 05 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 06 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 07 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 08 — approximately 1 GiB total */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 09 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 10 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 11 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 12 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 13 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 14 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 15 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 16 — approximately 2 GiB total */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 17 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 18 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 19 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 20 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 21 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 22 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 23 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 24 — approximately 3 GiB total */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 25 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 26 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 27 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 28 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 29 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 30 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 31 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 32 — approximately 4 GiB total */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 33 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 34 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 35 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 36 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 37 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 38 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 39 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 40 — approximately 5 GiB total */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
DROP PROCEDURE seed_users_batch;
|
||||
|
||||
COMMIT;
|
||||
|
||||
SELECT
|
||||
COUNT(*) AS user_count,
|
||||
CAST(SUM(OCTET_LENGTH(payload)) / 1073741824.0 AS DECIMAL(18, 2))
|
||||
AS payload_size_gib
|
||||
FROM users;
|
||||
|
||||
COMMIT;
|
||||
+30
-8
@@ -2,13 +2,16 @@
|
||||
|
||||
use crate::core::context::Context;
|
||||
use crate::services::backup::BackupService;
|
||||
use crate::services::config::ConfigService;
|
||||
use crate::services::config::{ConfigService, DatabaseConfig};
|
||||
use crate::services::cron::CronService;
|
||||
use crate::services::dashboard_config::{collect_configs, load_cache, merge, persist_cache};
|
||||
use crate::services::restore::RestoreService;
|
||||
use crate::services::status::StatusService;
|
||||
use crate::settings::CONFIG;
|
||||
use crate::utils::common::BackupMethod;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use tracing::info;
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
pub struct Agent {
|
||||
ctx: Arc<Context>,
|
||||
@@ -17,6 +20,8 @@ pub struct Agent {
|
||||
cron_service: CronService,
|
||||
backup_service: BackupService,
|
||||
restore_service: RestoreService,
|
||||
dashboard_cache: Vec<DatabaseConfig>,
|
||||
cache_path: PathBuf,
|
||||
}
|
||||
|
||||
impl Agent {
|
||||
@@ -28,6 +33,9 @@ impl Agent {
|
||||
let backup_service = BackupService::new(ctx.clone());
|
||||
let restore_service = RestoreService::new(ctx.clone());
|
||||
|
||||
let cache_path = PathBuf::from(&CONFIG.data_path).join("dashboard_databases.json");
|
||||
let dashboard_cache = load_cache(&cache_path);
|
||||
|
||||
Agent {
|
||||
ctx,
|
||||
config_service,
|
||||
@@ -35,19 +43,33 @@ impl Agent {
|
||||
cron_service,
|
||||
backup_service,
|
||||
restore_service,
|
||||
dashboard_cache,
|
||||
cache_path,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn run(&mut self, method: BackupMethod) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let config = self.config_service.load(None)?;
|
||||
let ping_result = self.status_service.ping(&config.databases).await?;
|
||||
let local = self.config_service.load_optional(None);
|
||||
|
||||
let merged_in = merge(&local.databases, &self.dashboard_cache);
|
||||
let ping_result = self.status_service.ping(&merged_in.databases).await?;
|
||||
|
||||
self.dashboard_cache = collect_configs(&ping_result);
|
||||
if let Err(e) = persist_cache(&self.cache_path, &self.dashboard_cache) {
|
||||
error!("Failed to persist dashboard cache: {e}");
|
||||
}
|
||||
|
||||
let merged = merge(&local.databases, &self.dashboard_cache);
|
||||
|
||||
for db in ping_result.databases.iter() {
|
||||
let database = config
|
||||
let Some(database) = merged
|
||||
.databases
|
||||
.iter()
|
||||
.find(|cfg_db| cfg_db.generated_id == db.generated_id)
|
||||
.unwrap();
|
||||
else {
|
||||
warn!("No config for returned database {}; skipping", db.generated_id);
|
||||
continue;
|
||||
};
|
||||
info!(
|
||||
"Generated Id: {} | backup action: {} | restore action: {} | Database Name: {}",
|
||||
db.generated_id, db.data.backup.action, db.data.restore.action, database.name,
|
||||
@@ -59,14 +81,14 @@ impl Agent {
|
||||
.backup_service
|
||||
.dispatch(
|
||||
&db.generated_id,
|
||||
&config,
|
||||
&merged,
|
||||
method.clone(),
|
||||
&db.storages,
|
||||
db.encrypt,
|
||||
)
|
||||
.await;
|
||||
} else if db.data.restore.action {
|
||||
let _ = self.restore_service.dispatch(db, &config).await;
|
||||
let _ = self.restore_service.dispatch(db, &merged).await;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ pub const EPHEMERAL_LABEL: &str = "io.portabase.ephemeral";
|
||||
const HELPER_MOUNT: &str = "/vol";
|
||||
|
||||
pub fn client() -> Result<Docker> {
|
||||
Docker::connect_with_unix_defaults().context("Failed to connect to Docker daemon socket")
|
||||
Docker::connect_with_defaults().context("Failed to connect to Docker daemon socket")
|
||||
}
|
||||
|
||||
pub fn parse_container_id(mountinfo: &str, cgroup: &str) -> Option<String> {
|
||||
|
||||
@@ -23,6 +23,7 @@ pub async fn run(
|
||||
let start = Instant::now();
|
||||
let output = Command::new("gbak")
|
||||
.arg("-b")
|
||||
.arg("-g")
|
||||
.arg("-v")
|
||||
.arg("-user").arg(&cfg.username)
|
||||
.arg("-password").arg(&cfg.password)
|
||||
|
||||
@@ -20,6 +20,7 @@ pub async fn run(cfg: DatabaseConfig) -> anyhow::Result<bool> {
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.kill_on_drop(true)
|
||||
.spawn()?;
|
||||
|
||||
let query = b"SELECT 1 FROM RDB$DATABASE;\nQUIT;\n";
|
||||
|
||||
@@ -12,7 +12,8 @@ pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::R
|
||||
.arg("--user")
|
||||
.arg(cfg.username)
|
||||
.arg("ping")
|
||||
.envs(env);
|
||||
.envs(env)
|
||||
.kill_on_drop(true);
|
||||
|
||||
let result = timeout(Duration::from_secs(10), cmd.output()).await;
|
||||
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::Result;
|
||||
use mongodb::Client;
|
||||
use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
|
||||
|
||||
const USERINFO_ENCODE: &AsciiSet = &NON_ALPHANUMERIC
|
||||
.remove(b'-')
|
||||
.remove(b'_')
|
||||
.remove(b'.')
|
||||
.remove(b'~');
|
||||
|
||||
pub async fn connect(cfg: DatabaseConfig) -> Result<Client> {
|
||||
let uri = get_mongo_uri(cfg)?;
|
||||
@@ -16,19 +23,40 @@ pub fn select_mongo_path() -> std::path::PathBuf {
|
||||
}
|
||||
|
||||
pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result<String> {
|
||||
if cfg.username.is_empty() || cfg.password.is_empty() {
|
||||
Ok(format!(
|
||||
"mongodb://{}:{}/{}",
|
||||
cfg.host, cfg.port, cfg.database
|
||||
))
|
||||
} else {
|
||||
Ok(format!(
|
||||
"mongodb://{}:{}@{}:{}/{}?authSource=admin",
|
||||
cfg.username, cfg.password, cfg.host, cfg.port, cfg.database
|
||||
))
|
||||
}
|
||||
Ok(build_mongo_uri(&cfg, true))
|
||||
}
|
||||
|
||||
pub fn build_mongo_uri(cfg: &DatabaseConfig, include_db: bool) -> String {
|
||||
let is_srv = cfg.port == 0;
|
||||
let scheme = if is_srv { "mongodb+srv" } else { "mongodb" };
|
||||
let has_auth = !cfg.username.is_empty() && !cfg.password.is_empty();
|
||||
|
||||
let credentials = if has_auth {
|
||||
format!(
|
||||
"{}:{}@",
|
||||
utf8_percent_encode(&cfg.username, USERINFO_ENCODE),
|
||||
utf8_percent_encode(&cfg.password, USERINFO_ENCODE)
|
||||
)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
|
||||
let authority = if is_srv {
|
||||
cfg.host.clone()
|
||||
} else {
|
||||
format!("{}:{}", cfg.host, cfg.port)
|
||||
};
|
||||
|
||||
let path = if include_db {
|
||||
format!("/{}", cfg.database)
|
||||
} else {
|
||||
"/".to_string()
|
||||
};
|
||||
|
||||
let query = if has_auth { "?authSource=admin" } else { "" };
|
||||
|
||||
format!("{}://{}{}{}{}", scheme, credentials, authority, path, query)
|
||||
}
|
||||
|
||||
pub fn extract_db_name(dry_output: &str) -> Option<String> {
|
||||
let mut dbs = std::collections::HashSet::new();
|
||||
@@ -43,3 +71,70 @@ pub fn extract_db_name(dry_output: &str) -> Option<String> {
|
||||
}
|
||||
dbs.into_iter().next()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::services::config::{DatabaseConfig, DbType};
|
||||
use std::collections::HashMap;
|
||||
|
||||
fn cfg(host: &str, port: u16, user: &str, pass: &str) -> DatabaseConfig {
|
||||
DatabaseConfig {
|
||||
name: "t".into(),
|
||||
database: "mydb".into(),
|
||||
db_type: DbType::MongoDB,
|
||||
username: user.into(),
|
||||
password: pass.into(),
|
||||
port,
|
||||
host: host.into(),
|
||||
generated_id: "id".into(),
|
||||
path: String::new(),
|
||||
max_packet_size: String::new(),
|
||||
volume_name: String::new(),
|
||||
container_name: None,
|
||||
options: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standard_with_auth() {
|
||||
let c = cfg("localhost", 27017, "user", "pass");
|
||||
assert_eq!(
|
||||
build_mongo_uri(&c, true),
|
||||
"mongodb://user:pass@localhost:27017/mydb?authSource=admin"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standard_no_auth() {
|
||||
let c = cfg("localhost", 27017, "", "");
|
||||
assert_eq!(build_mongo_uri(&c, true), "mongodb://localhost:27017/mydb");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn srv_with_auth() {
|
||||
let c = cfg("cluster.example.mongodb.net", 0, "user", "pass");
|
||||
assert_eq!(
|
||||
build_mongo_uri(&c, true),
|
||||
"mongodb+srv://user:pass@cluster.example.mongodb.net/mydb?authSource=admin"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn srv_no_db_for_dryrun() {
|
||||
let c = cfg("cluster.example.mongodb.net", 0, "user", "pass");
|
||||
assert_eq!(
|
||||
build_mongo_uri(&c, false),
|
||||
"mongodb+srv://user:pass@cluster.example.mongodb.net/?authSource=admin"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encodes_special_chars_in_credentials() {
|
||||
let c = cfg("cluster.example.mongodb.net", 0, "user", "p@ss:w/rd?");
|
||||
assert_eq!(
|
||||
build_mongo_uri(&c, true),
|
||||
"mongodb+srv://user:p%40ss%3Aw%2Frd%3F@cluster.example.mongodb.net/mydb?authSource=admin"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,11 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
|
||||
Ok(_) => Ok(true),
|
||||
Err(e) => {
|
||||
error!("--- MongoDB Connection Error Details ---");
|
||||
error!("Target Host: {}:{}", cfg.host, cfg.port);
|
||||
if cfg.port == 0 {
|
||||
error!("Target Host: {} (srv)", cfg.host);
|
||||
} else {
|
||||
error!("Target Host: {}:{}", cfg.host, cfg.port);
|
||||
}
|
||||
error!("Error Kind: {:?}", e.kind);
|
||||
error!("Full Error: {}", e);
|
||||
error!("Check you database network connectivity");
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
use crate::domain::mongodb::connection::{extract_db_name, get_mongo_uri, select_mongo_path};
|
||||
use crate::domain::mongodb::connection::{
|
||||
build_mongo_uri, extract_db_name, get_mongo_uri, select_mongo_path,
|
||||
};
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
@@ -16,13 +18,7 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogg
|
||||
|
||||
let dry_start = Instant::now();
|
||||
let dry_run = Command::new(&mongorestore)
|
||||
.arg(format!(
|
||||
"--uri={}",
|
||||
format!(
|
||||
"mongodb://{}:{}@{}:{}/?authSource=admin",
|
||||
cfg.username, cfg.password, cfg.host, cfg.port
|
||||
)
|
||||
))
|
||||
.arg(format!("--uri={}", build_mongo_uri(&cfg, false)))
|
||||
.arg(format!("--archive={}", restore_file.display()))
|
||||
.arg("--gzip")
|
||||
.arg("--dryRun")
|
||||
|
||||
@@ -12,7 +12,8 @@ pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::R
|
||||
.arg("--user")
|
||||
.arg(cfg.username)
|
||||
.arg("ping")
|
||||
.envs(env);
|
||||
.envs(env)
|
||||
.kill_on_drop(true);
|
||||
|
||||
let result = timeout(Duration::from_secs(10), cmd.output()).await;
|
||||
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
use crate::services::config::DatabaseConfig;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum RestoreCleanMode {
|
||||
None,
|
||||
Clean,
|
||||
DropSchemas,
|
||||
DropDatabase,
|
||||
}
|
||||
|
||||
impl RestoreCleanMode {
|
||||
pub fn from_config(cfg: &DatabaseConfig) -> (Self, Option<String>) {
|
||||
match cfg.options.get("clean_mode").and_then(|v| v.as_str()) {
|
||||
None | Some("clean") => (Self::Clean, None),
|
||||
Some("none") => (Self::None, None),
|
||||
Some("drop_schemas") => (Self::DropSchemas, None),
|
||||
Some("drop_database") => (Self::DropDatabase, None),
|
||||
Some(other) => (Self::Clean, Some(other.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn uses_pg_restore_clean(self) -> bool {
|
||||
matches!(self, Self::Clean)
|
||||
}
|
||||
}
|
||||
@@ -15,10 +15,11 @@ pub async fn run(
|
||||
env: HashMap<String, String>,
|
||||
logger: Arc<JobLogger>,
|
||||
) -> Result<()> {
|
||||
let handle = tokio::runtime::Handle::current();
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
logger.log("info", format!("Starting cluster restore for {}", cfg.name));
|
||||
|
||||
let version = match futures::executor::block_on(server_version(&cfg)) {
|
||||
let version = match handle.block_on(server_version(&cfg)) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
logger.log("error", format!("Failed to get server version for {}: {:?}", cfg.name, e));
|
||||
@@ -26,7 +27,7 @@ pub async fn run(
|
||||
}
|
||||
};
|
||||
|
||||
match futures::executor::block_on(is_superuser(&cfg)) {
|
||||
match handle.block_on(is_superuser(&cfg)) {
|
||||
Ok(true) => {}
|
||||
Ok(false) => {
|
||||
logger.log("error", format!("postgresql-cluster restore requires a superuser role for {}", cfg.name));
|
||||
@@ -40,7 +41,7 @@ pub async fn run(
|
||||
|
||||
let psql = select_pg_path(&version).join(psql_binary_name());
|
||||
|
||||
if let Err(e) = futures::executor::block_on(terminate_all_connections(&cfg)) {
|
||||
if let Err(e) = handle.block_on(terminate_all_connections(&cfg)) {
|
||||
logger.log("error", format!("Failed to terminate connections for cluster {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
|
||||
@@ -33,6 +33,14 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
pub async fn server_version_major(cfg: &DatabaseConfig) -> Result<u32> {
|
||||
let v = server_version(cfg).await?;
|
||||
Ok(v.split(['.', ' '])
|
||||
.next()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(17))
|
||||
}
|
||||
|
||||
pub async fn is_superuser(cfg: &DatabaseConfig) -> Result<bool> {
|
||||
let client = connect(cfg).await?;
|
||||
let is_super: bool = client
|
||||
@@ -43,6 +51,19 @@ pub async fn is_superuser(cfg: &DatabaseConfig) -> Result<bool> {
|
||||
Ok(is_super)
|
||||
}
|
||||
|
||||
pub async fn can_drop_database(cfg: &DatabaseConfig) -> Result<bool> {
|
||||
let client = connect(cfg).await?;
|
||||
let row = client
|
||||
.query_one(
|
||||
"SELECT r.rolsuper OR (r.rolcreatedb AND pg_catalog.pg_has_role(current_user, d.datdba, 'USAGE')) \
|
||||
FROM pg_roles r, pg_database d \
|
||||
WHERE r.rolname = current_user AND d.datname = current_database()",
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
Ok(row.get(0))
|
||||
}
|
||||
|
||||
|
||||
pub fn select_pg_path(version: &str) -> std::path::PathBuf {
|
||||
select_pg_path_with(version, &CONFIG.pg_bin_dir)
|
||||
@@ -114,6 +135,22 @@ pub(crate) fn psql_binary_name() -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn pg_restore_binary_name() -> &'static str {
|
||||
if cfg!(target_os = "windows") {
|
||||
"pg_restore.exe"
|
||||
} else {
|
||||
"pg_restore"
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn quote_ident(s: &str) -> String {
|
||||
format!("\"{}\"", s.replace('"', "\"\""))
|
||||
}
|
||||
|
||||
pub(crate) fn quote_literal(s: &str) -> String {
|
||||
format!("'{}'", s.replace('\'', "''"))
|
||||
}
|
||||
|
||||
pub(crate) fn pg_dump_exists_in(dir: &std::path::Path) -> bool {
|
||||
dir.join(pg_dump_binary_name()).is_file()
|
||||
}
|
||||
@@ -165,6 +202,107 @@ pub async fn terminate_all_connections(cfg: &DatabaseConfig) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn drop_and_recreate_database(cfg: &DatabaseConfig) -> Result<()> {
|
||||
let mut admin_cfg = cfg.clone();
|
||||
admin_cfg.database = "postgres".to_string();
|
||||
let admin = connect(&admin_cfg).await?;
|
||||
|
||||
let row = admin
|
||||
.query_opt(
|
||||
r#"
|
||||
SELECT pg_encoding_to_char(encoding), datcollate, datctype,
|
||||
pg_get_userbyid(datdba), datistemplate
|
||||
FROM pg_database WHERE datname = $1
|
||||
"#,
|
||||
&[&cfg.database],
|
||||
)
|
||||
.await?;
|
||||
|
||||
let (encoding, collate, ctype, owner) = match &row {
|
||||
Some(r) => (
|
||||
r.get::<_, String>(0),
|
||||
r.get::<_, String>(1),
|
||||
r.get::<_, String>(2),
|
||||
r.get::<_, String>(3),
|
||||
),
|
||||
None => ("UTF8".into(), "C".into(), "C".into(), cfg.username.clone()),
|
||||
};
|
||||
|
||||
if let Some(r) = &row {
|
||||
if r.get::<_, bool>(4) {
|
||||
anyhow::bail!("Refusing to drop template database {}", cfg.database);
|
||||
}
|
||||
}
|
||||
|
||||
let db = quote_ident(&cfg.database);
|
||||
|
||||
if let Err(e) = admin
|
||||
.batch_execute(&format!("ALTER DATABASE {db} WITH ALLOW_CONNECTIONS false"))
|
||||
.await
|
||||
{
|
||||
tracing::warn!("ALLOW_CONNECTIONS false failed for {}: {e}", cfg.database);
|
||||
}
|
||||
|
||||
let major = server_version_major(&admin_cfg).await?;
|
||||
let drop_stmt = if major >= 13 {
|
||||
format!("DROP DATABASE IF EXISTS {db} WITH (FORCE)")
|
||||
} else {
|
||||
format!("DROP DATABASE IF EXISTS {db}")
|
||||
};
|
||||
|
||||
let mut last_err = None;
|
||||
let mut dropped = false;
|
||||
for _ in 0..3 {
|
||||
let _ = terminate_connections(cfg).await;
|
||||
match admin.batch_execute(&drop_stmt).await {
|
||||
Ok(()) => {
|
||||
dropped = true;
|
||||
break;
|
||||
}
|
||||
Err(e) => {
|
||||
last_err = Some(e);
|
||||
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !dropped {
|
||||
let _ = admin
|
||||
.batch_execute(&format!("ALTER DATABASE {db} WITH ALLOW_CONNECTIONS true"))
|
||||
.await;
|
||||
return Err(last_err
|
||||
.map(anyhow::Error::from)
|
||||
.unwrap_or_else(|| anyhow::anyhow!("DROP DATABASE {} failed", cfg.database)));
|
||||
}
|
||||
|
||||
admin
|
||||
.batch_execute(&format!(
|
||||
"CREATE DATABASE {db} OWNER {} TEMPLATE template0 ENCODING {} LC_COLLATE {} LC_CTYPE {}",
|
||||
quote_ident(&owner),
|
||||
quote_literal(&encoding),
|
||||
quote_literal(&collate),
|
||||
quote_literal(&ctype),
|
||||
))
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn sniff_format(restore_file: &Path) -> Result<PostgresDumpFormat> {
|
||||
use std::io::Read;
|
||||
let mut f = std::fs::File::open(restore_file)?;
|
||||
let mut magic = [0u8; 5];
|
||||
let n = f.read(&mut magic)?;
|
||||
let head = &magic[..n];
|
||||
if head.starts_with(b"PGDMP") {
|
||||
Ok(PostgresDumpFormat::Fc)
|
||||
} else if head.starts_with(&[0x1f, 0x8b]) {
|
||||
Ok(PostgresDumpFormat::Fd)
|
||||
} else {
|
||||
anyhow::bail!("Unrecognized dump format for {:?}", restore_file)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn detect_format_from_file(restore_file: &Path) -> PostgresDumpFormat {
|
||||
match restore_file.extension().and_then(|e| e.to_str()) {
|
||||
Some("dump") => PostgresDumpFormat::Fc,
|
||||
@@ -174,6 +312,44 @@ pub fn detect_format_from_file(restore_file: &Path) -> PostgresDumpFormat {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn drop_all_schemas(cfg: &DatabaseConfig) -> Result<Vec<String>> {
|
||||
let client = connect(cfg).await?;
|
||||
let rows = client
|
||||
.query(
|
||||
r#"
|
||||
SELECT nspname FROM pg_namespace
|
||||
WHERE nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast')
|
||||
AND nspname NOT LIKE 'pg\_temp\_%'
|
||||
AND nspname NOT LIKE 'pg\_toast\_temp\_%'
|
||||
ORDER BY nspname
|
||||
"#,
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
let schemas: Vec<String> = rows.iter().map(|r| r.get::<_, String>(0)).collect();
|
||||
for s in &schemas {
|
||||
client
|
||||
.batch_execute(&format!("DROP SCHEMA IF EXISTS {} CASCADE", quote_ident(s)))
|
||||
.await?;
|
||||
}
|
||||
client
|
||||
.batch_execute("SELECT lo_unlink(oid) FROM pg_largeobject_metadata")
|
||||
.await
|
||||
.ok();
|
||||
Ok(schemas)
|
||||
}
|
||||
|
||||
pub async fn recreate_public_schema(cfg: &DatabaseConfig, owner: &str) -> Result<()> {
|
||||
let client = connect(cfg).await?;
|
||||
client
|
||||
.batch_execute(&format!(
|
||||
"CREATE SCHEMA IF NOT EXISTS public AUTHORIZATION {}; GRANT USAGE ON SCHEMA public TO PUBLIC;",
|
||||
quote_ident(owner)
|
||||
))
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn detect_format_from_size(cfg: &DatabaseConfig) -> PostgresDumpFormat {
|
||||
info!(
|
||||
"Detecting database format {:?} - {:?}",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#[derive(Clone, Copy)]
|
||||
#[derive(Clone, Copy, PartialEq, Debug)]
|
||||
pub enum PostgresDumpFormat {
|
||||
Fc,
|
||||
Fd,
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
pub mod backup;
|
||||
pub(crate) mod cluster;
|
||||
pub(crate) mod clean_mode;
|
||||
pub(crate) mod connection;
|
||||
pub mod database;
|
||||
mod format;
|
||||
pub(crate) mod format;
|
||||
mod ping;
|
||||
mod restore;
|
||||
pub(crate) mod restore;
|
||||
|
||||
pub use connection::{detect_format_from_file, detect_format_from_size};
|
||||
|
||||
@@ -1,214 +0,0 @@
|
||||
use anyhow::Result;
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use super::connection::{select_pg_path, server_version, terminate_connections};
|
||||
use super::format::PostgresDumpFormat;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
format: PostgresDumpFormat,
|
||||
restore_file: PathBuf,
|
||||
env: HashMap<String, String>,
|
||||
logger: Arc<JobLogger>,
|
||||
) -> Result<()> {
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
logger.log("info", format!("Starting restore for database {}", cfg.name));
|
||||
|
||||
let version = match futures::executor::block_on(server_version(&cfg)) {
|
||||
Ok(v) => {
|
||||
logger.log("debug", format!("Postgres version detected: {}", v));
|
||||
v
|
||||
}
|
||||
Err(e) => {
|
||||
logger.log("error", format!("Failed to get server version for {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
|
||||
let pg_restore = select_pg_path(&version).join("pg_restore");
|
||||
|
||||
logger.log("debug", format!("Using pg_restore at {:?}", pg_restore));
|
||||
|
||||
if let Err(e) = futures::executor::block_on(terminate_connections(&cfg)) {
|
||||
logger.log("error", format!("Failed to terminate connections for {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
logger.log("info", format!("Connections terminated for database {}", cfg.name));
|
||||
|
||||
let keep_ownership = cfg.options
|
||||
.get("keep_ownership")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
|
||||
if keep_ownership {
|
||||
logger.log("info", format!("Restoring ownership and privileges for {}", cfg.name));
|
||||
} else {
|
||||
logger.log("info", format!("Stripping ownership and privileges for {} (--no-owner --no-privileges)", cfg.name));
|
||||
}
|
||||
|
||||
match format {
|
||||
PostgresDumpFormat::Fc => {
|
||||
logger.log("info", format!("Running FC restore for {}", cfg.name));
|
||||
let start = Instant::now();
|
||||
let mut cmd = Command::new(&pg_restore);
|
||||
if !keep_ownership {
|
||||
cmd.arg("--no-owner").arg("--no-privileges");
|
||||
}
|
||||
let output = cmd
|
||||
.arg("--clean")
|
||||
.arg("--if-exists")
|
||||
// .arg("--create")
|
||||
.arg("--host").arg(&cfg.host)
|
||||
.arg("--port").arg(cfg.port.to_string())
|
||||
.arg("--username").arg(&cfg.username)
|
||||
.arg("--dbname").arg(&cfg.database)
|
||||
.arg("-v")
|
||||
.arg(&restore_file)
|
||||
.envs(env)
|
||||
.output();
|
||||
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
|
||||
match output {
|
||||
Ok(o) => {
|
||||
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
|
||||
let stdout = String::from_utf8_lossy(&o.stdout).to_string();
|
||||
let combined = format!("{}{}", stdout, stderr);
|
||||
let exit_code = o.status.code().unwrap_or(-1);
|
||||
|
||||
if o.status.success() {
|
||||
logger.log_command("pg_restore", if combined.is_empty() { None } else { Some(combined) }, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("FC restore completed successfully for {}", cfg.name))
|
||||
} else {
|
||||
logger.log_command("pg_restore", if combined.is_empty() { None } else { Some(combined) }, Some(exit_code), Some(duration_ms));
|
||||
logger.log("error", format!("FC restore failed with status {:?} for {}", o.status, cfg.name));
|
||||
anyhow::bail!("Postgres restore failed for {}", cfg.name);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
logger.log_command("pg_restore", Some(e.to_string()), Some(-1), Some(duration_ms));
|
||||
logger.log("error", format!("Error executing pg_restore for {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
PostgresDumpFormat::Fd => {
|
||||
logger.log("info", format!("Running FD restore for {}", cfg.name));
|
||||
|
||||
let tar_gz = match std::fs::File::open(&restore_file) {
|
||||
Ok(f) => f,
|
||||
Err(e) => {
|
||||
logger.log("error", format!(
|
||||
"Failed to open restore file {:?} for {}: {:?}",
|
||||
restore_file, cfg.name, e
|
||||
));
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
|
||||
logger.log("info", format!("tar_gz {:?}", tar_gz));
|
||||
|
||||
let dec = flate2::read::GzDecoder::new(tar_gz);
|
||||
let mut archive = tar::Archive::new(dec);
|
||||
|
||||
let tmp_dir = match tempfile::TempDir::new() {
|
||||
Ok(d) => d,
|
||||
Err(e) => {
|
||||
logger.log("error", format!(
|
||||
"Failed to create temporary directory for FD restore of {}: {:?}",
|
||||
cfg.name, e
|
||||
));
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(e) = archive.unpack(tmp_dir.path()) {
|
||||
logger.log("error", format!("Failed to unpack FD archive for {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
|
||||
logger.log("debug", format!("Listing contents of temp dir: {}", tmp_dir.path().display()));
|
||||
|
||||
for entry in std::fs::read_dir(tmp_dir.path())? {
|
||||
if let Ok(entry) = entry {
|
||||
let path = entry.path();
|
||||
let file_type = entry.file_type()?;
|
||||
logger.log("debug", format!(
|
||||
" - {} | is_dir: {} | is_file: {}",
|
||||
path.display(),
|
||||
file_type.is_dir(),
|
||||
file_type.is_file()
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let dump_dir = if tmp_dir.path().join("toc.dat").exists() {
|
||||
tmp_dir.path().to_path_buf()
|
||||
} else {
|
||||
std::fs::read_dir(tmp_dir.path())?
|
||||
.filter_map(|e| e.ok())
|
||||
.find(|entry| entry.path().join("toc.dat").exists())
|
||||
.map(|e| e.path())
|
||||
.ok_or_else(|| anyhow::anyhow!("Invalid FD archive: toc.dat not found"))?
|
||||
};
|
||||
|
||||
let start = Instant::now();
|
||||
let mut cmd = Command::new(&pg_restore);
|
||||
if !keep_ownership {
|
||||
cmd.arg("--no-owner").arg("--no-privileges");
|
||||
}
|
||||
let output = cmd
|
||||
.arg("--clean")
|
||||
.arg("--if-exists")
|
||||
// .arg("--create")
|
||||
.arg("--host").arg(&cfg.host)
|
||||
.arg("--port").arg(cfg.port.to_string())
|
||||
.arg("--username").arg(&cfg.username)
|
||||
.arg("--dbname").arg(&cfg.database)
|
||||
.arg("-v")
|
||||
.arg("-j")
|
||||
.arg("4")
|
||||
.arg(dump_dir)
|
||||
.envs(env)
|
||||
.output();
|
||||
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
|
||||
match output {
|
||||
Ok(o) => {
|
||||
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
|
||||
let stdout = String::from_utf8_lossy(&o.stdout).to_string();
|
||||
let combined = format!("{}{}", stdout, stderr);
|
||||
let exit_code = o.status.code().unwrap_or(-1);
|
||||
|
||||
if o.status.success() {
|
||||
logger.log_command("pg_restore", if combined.is_empty() { None } else { Some(combined) }, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("FD restore completed successfully for {}", cfg.name))
|
||||
} else {
|
||||
logger.log_command("pg_restore", if combined.is_empty() { None } else { Some(combined) }, Some(exit_code), Some(duration_ms));
|
||||
logger.log("error", format!("FD restore failed with status {:?} for {}", o.status, cfg.name));
|
||||
anyhow::bail!("Postgres FD restore failed for {}", cfg.name);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
logger.log_command("pg_restore", Some(e.to_string()), Some(-1), Some(duration_ms));
|
||||
logger.log("error", format!("Error executing pg_restore for {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.log("info", format!("Restore finished for database {}", cfg.name));
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.await?
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
use anyhow::Result;
|
||||
use std::process::Command;
|
||||
use std::time::Instant;
|
||||
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
|
||||
pub(crate) fn run_pg_restore(
|
||||
mut cmd: Command,
|
||||
logger: &JobLogger,
|
||||
cfg: &DatabaseConfig,
|
||||
) -> Result<()> {
|
||||
let start = Instant::now();
|
||||
let output = cmd.output();
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
|
||||
match output {
|
||||
Ok(o) => {
|
||||
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
|
||||
let stdout = String::from_utf8_lossy(&o.stdout).to_string();
|
||||
let combined = format!("{}{}", stdout, stderr);
|
||||
let exit_code = o.status.code().unwrap_or(-1);
|
||||
let payload = if combined.is_empty() { None } else { Some(combined) };
|
||||
|
||||
if o.status.success() {
|
||||
logger.log_command("pg_restore", payload, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("Restore completed successfully for {}", cfg.name));
|
||||
Ok(())
|
||||
} else {
|
||||
logger.log_command("pg_restore", payload, Some(exit_code), Some(duration_ms));
|
||||
logger.log("error", format!("Restore failed with status {:?} for {}", o.status, cfg.name));
|
||||
anyhow::bail!("Postgres restore failed for {}", cfg.name);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
logger.log_command("pg_restore", Some(e.to_string()), Some(-1), Some(duration_ms));
|
||||
logger.log("error", format!("Error executing pg_restore for {}: {:?}", cfg.name, e));
|
||||
Err(e.into())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
mod command;
|
||||
mod prepare;
|
||||
mod run;
|
||||
mod toc;
|
||||
|
||||
pub use run::run;
|
||||
pub(crate) use command::run_pg_restore;
|
||||
pub(crate) use prepare::prepare_archive;
|
||||
pub(crate) use toc::toc_creates_public_schema;
|
||||
@@ -0,0 +1,67 @@
|
||||
use anyhow::Result;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
|
||||
use crate::domain::postgres::connection::sniff_format;
|
||||
use crate::domain::postgres::format::PostgresDumpFormat;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
|
||||
pub(crate) struct PreparedArchive {
|
||||
path: PathBuf,
|
||||
_tmp: Option<tempfile::TempDir>,
|
||||
toc: String,
|
||||
}
|
||||
|
||||
impl PreparedArchive {
|
||||
pub(crate) fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
pub(crate) fn toc(&self) -> &str {
|
||||
&self.toc
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_archive(
|
||||
format: PostgresDumpFormat,
|
||||
restore_file: &Path,
|
||||
pg_restore: &Path,
|
||||
logger: &JobLogger,
|
||||
) -> Result<PreparedArchive> {
|
||||
let sniffed = sniff_format(restore_file)?;
|
||||
if sniffed != format {
|
||||
logger.log("warn", format!("Declared format {:?} != sniffed {:?}; using sniffed", format, sniffed));
|
||||
}
|
||||
let format = sniffed;
|
||||
|
||||
let (path, tmp) = match format {
|
||||
PostgresDumpFormat::Fc => (restore_file.to_path_buf(), None),
|
||||
PostgresDumpFormat::Fd => {
|
||||
let tar_gz = std::fs::File::open(restore_file)?;
|
||||
let dec = flate2::read::GzDecoder::new(tar_gz);
|
||||
let mut archive = tar::Archive::new(dec);
|
||||
let tmp_dir = tempfile::TempDir::new()?;
|
||||
archive.unpack(tmp_dir.path())?;
|
||||
|
||||
let dump_dir = if tmp_dir.path().join("toc.dat").exists() {
|
||||
tmp_dir.path().to_path_buf()
|
||||
} else {
|
||||
std::fs::read_dir(tmp_dir.path())?
|
||||
.filter_map(|e| e.ok())
|
||||
.find(|entry| entry.path().join("toc.dat").exists())
|
||||
.map(|e| e.path())
|
||||
.ok_or_else(|| anyhow::anyhow!("Invalid FD archive: toc.dat not found"))?
|
||||
};
|
||||
(dump_dir, Some(tmp_dir))
|
||||
}
|
||||
};
|
||||
|
||||
let toc_out = Command::new(pg_restore).arg("-l").arg(&path).output()?;
|
||||
if !toc_out.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&toc_out.stderr).to_string();
|
||||
logger.log("error", format!("pg_restore -l failed: {}", stderr));
|
||||
anyhow::bail!("Archive validation failed (pg_restore -l): {}", stderr);
|
||||
}
|
||||
let toc = String::from_utf8_lossy(&toc_out.stdout).to_string();
|
||||
|
||||
Ok(PreparedArchive { path, _tmp: tmp, toc })
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
use anyhow::Result;
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::{prepare_archive, run_pg_restore, toc_creates_public_schema};
|
||||
use crate::domain::postgres::clean_mode::RestoreCleanMode;
|
||||
use crate::domain::postgres::connection::{
|
||||
can_drop_database, drop_all_schemas, drop_and_recreate_database, pg_restore_binary_name,
|
||||
recreate_public_schema, select_pg_path, server_version, terminate_connections,
|
||||
};
|
||||
use crate::domain::postgres::format::PostgresDumpFormat;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
format: PostgresDumpFormat,
|
||||
restore_file: PathBuf,
|
||||
env: HashMap<String, String>,
|
||||
logger: Arc<JobLogger>,
|
||||
) -> Result<()> {
|
||||
let handle = tokio::runtime::Handle::current();
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
logger.log("info", format!("Starting restore for database {}", cfg.name));
|
||||
|
||||
let version = match handle.block_on(server_version(&cfg)) {
|
||||
Ok(v) => {
|
||||
logger.log("debug", format!("Postgres version detected: {}", v));
|
||||
v
|
||||
}
|
||||
Err(e) => {
|
||||
logger.log("error", format!("Failed to get server version for {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
|
||||
let pg_restore = select_pg_path(&version).join(pg_restore_binary_name());
|
||||
|
||||
logger.log("debug", format!("Using pg_restore at {:?}", pg_restore));
|
||||
|
||||
let keep_ownership = cfg.options
|
||||
.get("keep_ownership")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
|
||||
if keep_ownership {
|
||||
logger.log("info", format!("Restoring ownership and privileges for {}", cfg.name));
|
||||
} else {
|
||||
logger.log("info", format!("Stripping ownership and privileges for {} (--no-owner --no-privileges)", cfg.name));
|
||||
}
|
||||
|
||||
let (mode, bad_value) = RestoreCleanMode::from_config(&cfg);
|
||||
if let Some(v) = bad_value {
|
||||
logger.log("warn", format!("Unknown clean_mode '{}' for {}, falling back to 'clean'", v, cfg.name));
|
||||
}
|
||||
|
||||
let prepared = prepare_archive(format, &restore_file, &pg_restore, &logger)?;
|
||||
|
||||
match mode {
|
||||
RestoreCleanMode::DropSchemas => {
|
||||
handle.block_on(terminate_connections(&cfg))?;
|
||||
let owner = cfg.username.clone();
|
||||
let dropped = handle.block_on(drop_all_schemas(&cfg))?;
|
||||
logger.log("warn", format!("clean_mode=drop_schemas dropped schemas {:?} in {}", dropped, cfg.database));
|
||||
if !toc_creates_public_schema(prepared.toc()) {
|
||||
handle.block_on(recreate_public_schema(&cfg, &owner))?;
|
||||
}
|
||||
}
|
||||
RestoreCleanMode::DropDatabase => {
|
||||
if !handle.block_on(can_drop_database(&cfg))? {
|
||||
anyhow::bail!(
|
||||
"clean_mode=drop_database requires CREATEDB + ownership on {}; use clean_mode=drop_schemas instead",
|
||||
cfg.database
|
||||
);
|
||||
}
|
||||
logger.log("warn", format!("clean_mode=drop_database DROPPING database {} before restore", cfg.database));
|
||||
handle.block_on(drop_and_recreate_database(&cfg))?;
|
||||
}
|
||||
RestoreCleanMode::Clean | RestoreCleanMode::None => {
|
||||
handle.block_on(terminate_connections(&cfg))?;
|
||||
}
|
||||
}
|
||||
|
||||
let mut cmd = Command::new(&pg_restore);
|
||||
if !keep_ownership {
|
||||
cmd.args(["--no-owner", "--no-privileges"]);
|
||||
}
|
||||
if mode.uses_pg_restore_clean() {
|
||||
cmd.args(["--clean", "--if-exists"]);
|
||||
}
|
||||
cmd.arg("--host").arg(&cfg.host)
|
||||
.arg("--port").arg(cfg.port.to_string())
|
||||
.arg("--username").arg(&cfg.username)
|
||||
.arg("--dbname").arg(&cfg.database)
|
||||
.arg("-v");
|
||||
if matches!(format, PostgresDumpFormat::Fd) {
|
||||
cmd.arg("-j").arg("4");
|
||||
}
|
||||
cmd.arg(prepared.path()).envs(env);
|
||||
|
||||
run_pg_restore(cmd, &logger, &cfg)?;
|
||||
logger.log("info", format!("Restore finished for database {}", cfg.name));
|
||||
Ok(())
|
||||
})
|
||||
.await?
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
pub(crate) fn toc_creates_public_schema(toc: &str) -> bool {
|
||||
toc.lines().any(|l| {
|
||||
l.split(" SCHEMA - ")
|
||||
.nth(1)
|
||||
.and_then(|rest| rest.split_whitespace().next())
|
||||
== Some("public")
|
||||
})
|
||||
}
|
||||
@@ -21,6 +21,8 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
|
||||
|
||||
cmd.arg("PING");
|
||||
|
||||
cmd.kill_on_drop(true);
|
||||
|
||||
debug!("Command Ping Redis: {:?}", cmd);
|
||||
|
||||
let result = timeout(Duration::from_secs(10), cmd.output()).await;
|
||||
|
||||
@@ -20,6 +20,7 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
|
||||
}
|
||||
|
||||
cmd.arg("PING");
|
||||
cmd.kill_on_drop(true);
|
||||
|
||||
debug!("Command Ping Valkey: {:?}", cmd);
|
||||
|
||||
|
||||
@@ -22,7 +22,6 @@ async fn main() {
|
||||
eprintln!("Failed to clean locks on startup: {:?}", e);
|
||||
}
|
||||
|
||||
// Best-effort cleanup of ephemeral helper containers orphaned by a crash.
|
||||
match crate::domain::docker_volume::docker::client() {
|
||||
Ok(docker) => match crate::domain::docker_volume::docker::sweep_ephemeral(&docker).await {
|
||||
Ok(n) if n > 0 => tracing::info!("Removed {n} orphaned ephemeral helper container(s)"),
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::utils::deserializer::{deserialize_snake_case, string_or_number_to_string};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use toml::Value;
|
||||
@@ -39,6 +40,13 @@ pub struct DatabaseStatus {
|
||||
pub storages_encrypted: Option<bool>,
|
||||
#[serde(default)]
|
||||
pub storages_ciphertext: Option<String>,
|
||||
#[serde(default)]
|
||||
pub config_encrypted: Option<bool>,
|
||||
#[serde(default)]
|
||||
pub config_ciphertext: Option<String>,
|
||||
/// Filled in memory after decrypting `config_ciphertext`; never on the wire.
|
||||
#[serde(skip)]
|
||||
pub resolved_config: Option<DatabaseConfig>,
|
||||
pub encrypt: bool,
|
||||
pub data: DatabaseData,
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use crate::services::config::DbType;
|
||||
use std::fmt::{self, Display, Formatter};
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -20,3 +21,9 @@ pub struct UploadResult {
|
||||
pub remote_file_path: Option<String>,
|
||||
pub total_size: Option<u64>,
|
||||
}
|
||||
|
||||
impl Display for UploadResult {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}", self.error.as_deref().unwrap_or("unknown error"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ use super::service::BackupService;
|
||||
|
||||
use crate::domain::factory::DatabaseFactory;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::utils::retry::{RetryPolicy, retry};
|
||||
|
||||
use anyhow::Result;
|
||||
use std::path::Path;
|
||||
@@ -39,7 +40,31 @@ impl BackupService {
|
||||
});
|
||||
}
|
||||
|
||||
match db.backup(tmp_path, Arc::clone(&logger)).await {
|
||||
let policy = RetryPolicy::default();
|
||||
|
||||
let db_ref = &db;
|
||||
let logger_ref = &logger;
|
||||
|
||||
let outcome = retry("Database backup", &logger, &policy, move |attempt| {
|
||||
let dir = tmp_path.join(format!("attempt-{attempt}"));
|
||||
|
||||
async move {
|
||||
if let Err(e) = tokio::fs::create_dir_all(&dir).await {
|
||||
return Err(anyhow::Error::from(e));
|
||||
}
|
||||
|
||||
match db_ref.backup(&dir, Arc::clone(logger_ref)).await {
|
||||
Ok(f) => Ok(f),
|
||||
Err(e) => {
|
||||
let _ = tokio::fs::remove_dir_all(&dir).await;
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
|
||||
match outcome {
|
||||
Ok(file) => Ok(BackupResult {
|
||||
generated_id,
|
||||
db_type,
|
||||
|
||||
@@ -4,6 +4,7 @@ use super::service::BackupService;
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::services::storage;
|
||||
use crate::utils::common::BackupMethod;
|
||||
use crate::utils::retry::{RetryPolicy, retry};
|
||||
use anyhow::{Result, bail};
|
||||
use futures::future::join_all;
|
||||
use std::sync::Arc;
|
||||
@@ -97,16 +98,44 @@ impl BackupService {
|
||||
/*
|
||||
STORAGE UPLOAD
|
||||
*/
|
||||
let upload_result = provider
|
||||
.upload(
|
||||
ctx_clone.clone(),
|
||||
result_clone,
|
||||
method,
|
||||
&storage,
|
||||
Some(encrypt),
|
||||
&backup_storage_id,
|
||||
let policy = RetryPolicy::default();
|
||||
|
||||
let attempt_result = if result_clone.backup_file.is_none() {
|
||||
logger_clone.log("error", format!("Missing backup file for storage {}", storage_id));
|
||||
|
||||
Err(UploadResult {
|
||||
storage_id: storage_id.clone(),
|
||||
success: false,
|
||||
error: Some("Missing backup file path".into()),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
})
|
||||
} else {
|
||||
retry(
|
||||
&format!("Upload to storage {storage_id}"),
|
||||
&logger_clone,
|
||||
&policy,
|
||||
|_| async {
|
||||
let r = provider
|
||||
.upload(
|
||||
ctx_clone.clone(),
|
||||
result_clone.clone(),
|
||||
method,
|
||||
&storage,
|
||||
Some(encrypt),
|
||||
&backup_storage_id,
|
||||
)
|
||||
.await;
|
||||
|
||||
if r.success { Ok(r) } else { Err(r) }
|
||||
},
|
||||
)
|
||||
.await;
|
||||
.await
|
||||
};
|
||||
|
||||
let upload_result = match attempt_result {
|
||||
Ok(r) | Err(r) => r,
|
||||
};
|
||||
|
||||
let status = if upload_result.success { "success" } else { "failed" };
|
||||
|
||||
@@ -117,8 +146,6 @@ impl BackupService {
|
||||
upload_result.error.as_deref().unwrap_or("unknown error")
|
||||
));
|
||||
|
||||
// `backup_upload_init` opened a per-storage record; close it as "failed"
|
||||
// so the server is notified of the failure (no path/size on this path).
|
||||
if let Err(err) = ctx_clone
|
||||
.api
|
||||
.backup_upload_status(
|
||||
|
||||
+136
-128
@@ -1,7 +1,7 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use crate::core::context::Context;
|
||||
use serde::Deserialize;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json;
|
||||
use std::collections::HashMap;
|
||||
use std::fs::File;
|
||||
@@ -12,7 +12,7 @@ use toml;
|
||||
use tracing::info;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum DbType {
|
||||
Mysql,
|
||||
@@ -49,7 +49,7 @@ impl DbType {
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
pub struct DatabaseConfig {
|
||||
pub name: String,
|
||||
pub database: String,
|
||||
@@ -68,7 +68,7 @@ pub struct DatabaseConfig {
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
||||
pub struct DatabasesConfig {
|
||||
pub databases: Vec<DatabaseConfig>,
|
||||
}
|
||||
@@ -98,6 +98,106 @@ pub struct InputDatabasesConfig {
|
||||
pub databases: Vec<InputDatabaseConfig>,
|
||||
}
|
||||
|
||||
fn required<T: Clone>(opt: &Option<T>, db_name: &str, field_name: &str) -> Result<T, String> {
|
||||
match opt {
|
||||
Some(v) => Ok(v.clone()),
|
||||
None => Err(format!(
|
||||
"Missing required field '{}' for database '{}'",
|
||||
field_name, db_name
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn optional<T: Clone + Default>(opt: &Option<T>) -> T {
|
||||
opt.clone().unwrap_or_default()
|
||||
}
|
||||
|
||||
pub fn build_config(db: InputDatabaseConfig) -> Result<DatabaseConfig, String> {
|
||||
if Uuid::parse_str(&db.generated_id).is_err() {
|
||||
return Err(format!("Invalid UUID for database '{}'", db.name));
|
||||
}
|
||||
|
||||
let username = match db.db_type {
|
||||
DbType::Postgresql
|
||||
| DbType::PostgresqlCluster
|
||||
| DbType::Mysql
|
||||
| DbType::Mariadb
|
||||
| DbType::Mssql => required(&db.username, &db.name, "username")?,
|
||||
_ => optional(&db.username),
|
||||
};
|
||||
let password = match db.db_type {
|
||||
DbType::Postgresql
|
||||
| DbType::PostgresqlCluster
|
||||
| DbType::Mysql
|
||||
| DbType::Mariadb
|
||||
| DbType::Mssql => required(&db.password, &db.name, "password")?,
|
||||
_ => optional(&db.password),
|
||||
};
|
||||
let host = match db.db_type {
|
||||
DbType::Postgresql
|
||||
| DbType::PostgresqlCluster
|
||||
| DbType::Mysql
|
||||
| DbType::Mariadb
|
||||
| DbType::MongoDB
|
||||
| DbType::Redis
|
||||
| DbType::Firebird
|
||||
| DbType::Valkey
|
||||
| DbType::Mssql => required(&db.host, &db.name, "host")?,
|
||||
DbType::Sqlite | DbType::DockerVolume => optional(&db.host),
|
||||
};
|
||||
|
||||
let port = match db.db_type {
|
||||
DbType::Postgresql
|
||||
| DbType::PostgresqlCluster
|
||||
| DbType::Mysql
|
||||
| DbType::Mariadb
|
||||
| DbType::Redis
|
||||
| DbType::Firebird
|
||||
| DbType::Valkey
|
||||
| DbType::Mssql => required(&db.port, &db.name, "port")?,
|
||||
DbType::MongoDB | DbType::Sqlite | DbType::DockerVolume => db.port.unwrap_or(0),
|
||||
};
|
||||
|
||||
let database_name = match db.db_type {
|
||||
DbType::Sqlite | DbType::Redis | DbType::Valkey | DbType::DockerVolume => {
|
||||
optional(&db.database)
|
||||
}
|
||||
DbType::PostgresqlCluster => db
|
||||
.database
|
||||
.clone()
|
||||
.unwrap_or_else(|| "postgres".to_string()),
|
||||
_ => required(&db.database, &db.name, "database")?,
|
||||
};
|
||||
let path_val = match db.db_type {
|
||||
DbType::Sqlite => required(&db.path, &db.name, "path")?,
|
||||
_ => optional(&db.path),
|
||||
};
|
||||
let max_packet_size = match db.db_type {
|
||||
DbType::Mysql | DbType::Mariadb => db.max_packet_size.unwrap_or_else(|| "512M".to_string()),
|
||||
_ => String::new(),
|
||||
};
|
||||
let volume_name = match db.db_type {
|
||||
DbType::DockerVolume => required(&db.volume_name, &db.name, "volume_name")?,
|
||||
_ => optional(&db.volume_name),
|
||||
};
|
||||
|
||||
Ok(DatabaseConfig {
|
||||
name: db.name,
|
||||
database: database_name,
|
||||
db_type: db.db_type,
|
||||
username,
|
||||
password,
|
||||
host,
|
||||
port,
|
||||
generated_id: db.generated_id,
|
||||
path: path_val,
|
||||
max_packet_size,
|
||||
volume_name,
|
||||
container_name: db.container_name.clone(),
|
||||
options: db.options.unwrap_or_default(),
|
||||
})
|
||||
}
|
||||
|
||||
pub struct ConfigService {
|
||||
ctx: Arc<Context>,
|
||||
}
|
||||
@@ -107,16 +207,19 @@ impl ConfigService {
|
||||
ConfigService { ctx }
|
||||
}
|
||||
|
||||
pub fn load(&self, file_path: Option<&str>) -> Result<DatabasesConfig, String> {
|
||||
let path: String = if let Some(fp) = file_path {
|
||||
fp.to_string()
|
||||
} else {
|
||||
format!(
|
||||
fn resolve_path(file_path: Option<&str>) -> String {
|
||||
match file_path {
|
||||
Some(fp) => fp.to_string(),
|
||||
None => format!(
|
||||
"{}/{}",
|
||||
crate::settings::CONFIG.data_path,
|
||||
crate::settings::CONFIG.databases_config_file
|
||||
)
|
||||
};
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn load(&self, file_path: Option<&str>) -> Result<DatabasesConfig, String> {
|
||||
let path = Self::resolve_path(file_path);
|
||||
|
||||
info!("Loading databases config from: {}", path);
|
||||
|
||||
@@ -150,128 +253,33 @@ impl ConfigService {
|
||||
_ => return Err("Unsupported config file format. Use .json or .toml".to_string()),
|
||||
};
|
||||
|
||||
fn required<T: Clone>(
|
||||
opt: &Option<T>,
|
||||
db_name: &str,
|
||||
field_name: &str,
|
||||
) -> Result<T, String> {
|
||||
match opt {
|
||||
Some(v) => Ok(v.clone()),
|
||||
None => {
|
||||
let msg = format!(
|
||||
"Missing required field '{}' for database '{}'",
|
||||
field_name, db_name
|
||||
);
|
||||
Err(msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn optional<T: Clone>(opt: &Option<T>) -> T
|
||||
where
|
||||
T: Default,
|
||||
{
|
||||
opt.clone().unwrap_or_default()
|
||||
}
|
||||
|
||||
let mut databases = Vec::with_capacity(input_config.databases.len());
|
||||
|
||||
for db in input_config.databases {
|
||||
if Uuid::parse_str(&db.generated_id).is_err() {
|
||||
return Err(format!("Invalid UUID for database '{}'", db.name));
|
||||
}
|
||||
|
||||
let username = match db.db_type {
|
||||
DbType::Postgresql
|
||||
| DbType::PostgresqlCluster
|
||||
| DbType::Mysql
|
||||
| DbType::Mariadb
|
||||
| DbType::Mssql => required(&db.username, &db.name, "username")?,
|
||||
_ => optional(&db.username),
|
||||
};
|
||||
|
||||
let password = match db.db_type {
|
||||
DbType::Postgresql
|
||||
| DbType::PostgresqlCluster
|
||||
| DbType::Mysql
|
||||
| DbType::Mariadb
|
||||
| DbType::Mssql => required(&db.password, &db.name, "password")?,
|
||||
_ => optional(&db.password),
|
||||
};
|
||||
|
||||
let host = match db.db_type {
|
||||
DbType::Postgresql
|
||||
| DbType::PostgresqlCluster
|
||||
| DbType::Mysql
|
||||
| DbType::Mariadb
|
||||
| DbType::MongoDB
|
||||
| DbType::Redis
|
||||
| DbType::Firebird
|
||||
| DbType::Valkey
|
||||
| DbType::Mssql => required(&db.host, &db.name, "host")?,
|
||||
DbType::Sqlite | DbType::DockerVolume => optional(&db.host),
|
||||
};
|
||||
|
||||
let port = match db.db_type {
|
||||
DbType::Postgresql
|
||||
| DbType::PostgresqlCluster
|
||||
| DbType::Mysql
|
||||
| DbType::Mariadb
|
||||
| DbType::MongoDB
|
||||
| DbType::Redis
|
||||
| DbType::Firebird
|
||||
| DbType::Valkey
|
||||
| DbType::Mssql => required(&db.port, &db.name, "port")?,
|
||||
DbType::Sqlite | DbType::DockerVolume => db.port.unwrap_or(0),
|
||||
};
|
||||
|
||||
let database_name = match db.db_type {
|
||||
DbType::Sqlite | DbType::Redis | DbType::Valkey | DbType::DockerVolume => {
|
||||
optional(&db.database)
|
||||
}
|
||||
DbType::PostgresqlCluster => db
|
||||
.database
|
||||
.clone()
|
||||
.unwrap_or_else(|| "postgres".to_string()),
|
||||
_ => required(&db.database, &db.name, "database")?,
|
||||
};
|
||||
|
||||
let path_val = match db.db_type {
|
||||
DbType::Sqlite => required(&db.path, &db.name, "path")?,
|
||||
_ => optional(&db.path),
|
||||
};
|
||||
|
||||
let max_packet_size = match db.db_type {
|
||||
DbType::Mysql | DbType::Mariadb => {
|
||||
db.max_packet_size.unwrap_or_else(|| "512M".to_string())
|
||||
}
|
||||
_ => String::new(),
|
||||
};
|
||||
|
||||
let volume_name = match db.db_type {
|
||||
DbType::DockerVolume => required(&db.volume_name, &db.name, "volume_name")?,
|
||||
_ => optional(&db.volume_name),
|
||||
};
|
||||
let container_name = db.container_name.clone();
|
||||
|
||||
databases.push(DatabaseConfig {
|
||||
name: db.name,
|
||||
database: database_name,
|
||||
db_type: db.db_type,
|
||||
username,
|
||||
password,
|
||||
host,
|
||||
port,
|
||||
generated_id: db.generated_id,
|
||||
path: path_val,
|
||||
max_packet_size,
|
||||
volume_name,
|
||||
container_name,
|
||||
options: db.options.unwrap_or_default(),
|
||||
});
|
||||
databases.push(build_config(db)?);
|
||||
}
|
||||
|
||||
info!("Databases: {} instances loaded", databases.len());
|
||||
Ok(DatabasesConfig { databases })
|
||||
}
|
||||
|
||||
pub fn load_optional(&self, file_path: Option<&str>) -> DatabasesConfig {
|
||||
let path = Self::resolve_path(file_path);
|
||||
|
||||
if !Path::new(&path).exists() {
|
||||
info!(
|
||||
"No local databases config at {}; using dashboard-defined databases only",
|
||||
path
|
||||
);
|
||||
return DatabasesConfig {
|
||||
databases: Vec::new(),
|
||||
};
|
||||
}
|
||||
|
||||
self.load(file_path).unwrap_or_else(|e| {
|
||||
tracing::warn!(
|
||||
"Local databases config unavailable ({e}); continuing with dashboard-defined databases only"
|
||||
);
|
||||
DatabasesConfig { databases: Vec::new() }
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use crate::services::api::models::agent::status::PingResult;
|
||||
use crate::services::config::{DatabaseConfig, DatabasesConfig};
|
||||
use std::path::Path;
|
||||
|
||||
pub fn merge(local: &[DatabaseConfig], dashboard: &[DatabaseConfig]) -> DatabasesConfig {
|
||||
let mut databases: Vec<DatabaseConfig> = local.to_vec();
|
||||
for d in dashboard {
|
||||
if let Some(slot) = databases
|
||||
.iter_mut()
|
||||
.find(|c| c.generated_id == d.generated_id)
|
||||
{
|
||||
*slot = d.clone();
|
||||
} else {
|
||||
databases.push(d.clone());
|
||||
}
|
||||
}
|
||||
DatabasesConfig { databases }
|
||||
}
|
||||
|
||||
pub fn collect_configs(ping: &PingResult) -> Vec<DatabaseConfig> {
|
||||
ping.databases
|
||||
.iter()
|
||||
.filter_map(|db| db.resolved_config.clone())
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn load_cache(path: &Path) -> Vec<DatabaseConfig> {
|
||||
let contents = match std::fs::read_to_string(path) {
|
||||
Ok(c) => c,
|
||||
Err(_) => return Vec::new(),
|
||||
};
|
||||
match serde_json::from_str::<DatabasesConfig>(&contents) {
|
||||
Ok(cfg) => cfg.databases,
|
||||
Err(e) => {
|
||||
tracing::warn!("Dashboard cache at {:?} is corrupt ({e}); ignoring", path);
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn persist_cache(path: &Path, databases: &[DatabaseConfig]) -> std::io::Result<()> {
|
||||
let wrapper = DatabasesConfig {
|
||||
databases: databases.to_vec(),
|
||||
};
|
||||
let json = serde_json::to_string_pretty(&wrapper)
|
||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
let tmp = path.with_extension("json.tmp");
|
||||
std::fs::write(&tmp, json)?;
|
||||
std::fs::rename(&tmp, path)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -2,6 +2,7 @@ pub mod api;
|
||||
pub mod backup;
|
||||
pub mod config;
|
||||
pub mod cron;
|
||||
pub mod dashboard_config;
|
||||
pub mod restore;
|
||||
pub mod status;
|
||||
pub mod storage;
|
||||
|
||||
@@ -8,6 +8,7 @@ use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::utils::retry::{RetryPolicy, retry};
|
||||
|
||||
fn human_size(bytes: u64) -> String {
|
||||
if bytes >= 1024 * 1024 {
|
||||
@@ -26,6 +27,34 @@ impl RestoreService {
|
||||
tmp_path: &Path,
|
||||
logger: Arc<JobLogger>,
|
||||
expected_size: Option<String>,
|
||||
) -> Result<PathBuf> {
|
||||
let policy = RetryPolicy::default();
|
||||
|
||||
let logger_ref = &logger;
|
||||
|
||||
let outcome = retry("Backup download", &logger, &policy, move |_| {
|
||||
let expected = expected_size.clone();
|
||||
|
||||
async move {
|
||||
self.download_once(file_url, tmp_path, Arc::clone(logger_ref), expected)
|
||||
.await
|
||||
}
|
||||
})
|
||||
.await;
|
||||
|
||||
if let Err(e) = &outcome {
|
||||
logger.log("error", format!("Download failed: {e}"));
|
||||
}
|
||||
|
||||
outcome
|
||||
}
|
||||
|
||||
pub async fn download_once(
|
||||
&self,
|
||||
file_url: &str,
|
||||
tmp_path: &Path,
|
||||
logger: Arc<JobLogger>,
|
||||
expected_size: Option<String>,
|
||||
) -> Result<PathBuf> {
|
||||
logger.log("info", "Start downloading backup archive".to_string());
|
||||
|
||||
|
||||
+26
-1
@@ -3,9 +3,10 @@
|
||||
use crate::core::context::Context;
|
||||
use crate::domain::factory::DatabaseFactory;
|
||||
use crate::services::api::endpoints::status::DatabasePayload;
|
||||
use crate::services::api::models::agent::status::DatabaseStatus;
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::services::api::models::agent::status::PingResult;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::services::config::{build_config, DatabaseConfig, InputDatabaseConfig};
|
||||
use crate::settings::CONFIG;
|
||||
use crate::utils::file::decrypt_json_gcm;
|
||||
use futures_util::future::try_join_all;
|
||||
@@ -14,6 +15,26 @@ use std::error::Error;
|
||||
use std::sync::Arc;
|
||||
use tracing::info;
|
||||
|
||||
pub fn resolve_dashboard_config(
|
||||
status: &mut DatabaseStatus,
|
||||
master_key_b64: &str,
|
||||
) -> Result<(), String> {
|
||||
if status.config_encrypted != Some(true) {
|
||||
return Ok(());
|
||||
}
|
||||
let ciphertext = status
|
||||
.config_ciphertext
|
||||
.as_deref()
|
||||
.ok_or("config_encrypted set but config_ciphertext missing")?;
|
||||
|
||||
let plaintext = decrypt_json_gcm(ciphertext, master_key_b64)
|
||||
.map_err(|e| format!("Failed to decrypt config: {e}"))?;
|
||||
let input: InputDatabaseConfig = serde_json::from_slice(&plaintext)
|
||||
.map_err(|e| format!("Failed to parse decrypted config: {e}"))?;
|
||||
status.resolved_config = Some(build_config(input)?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub struct StatusService {
|
||||
ctx: Arc<Context>,
|
||||
client: Client,
|
||||
@@ -67,6 +88,10 @@ impl StatusService {
|
||||
db.storages = serde_json::from_slice::<Vec<DatabaseStorage>>(&plaintext)
|
||||
.map_err(|e| format!("Failed to parse decrypted storages: {e}"))?;
|
||||
}
|
||||
|
||||
if let Err(e) = resolve_dashboard_config(db, &edge_key.master_key_b64) {
|
||||
tracing::warn!("Skipping dashboard config for {}: {e}", db.generated_id);
|
||||
}
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
@@ -2,9 +2,8 @@ use anyhow::{Context as _, Result, anyhow};
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose::STANDARD;
|
||||
use chrono::{Duration, Utc};
|
||||
use openssl::hash::MessageDigest;
|
||||
use openssl::pkey::PKey;
|
||||
use openssl::sign::Signer;
|
||||
use hmac::{Hmac, Mac};
|
||||
use sha2::Sha256;
|
||||
use url::Url;
|
||||
use azure_core::http::RequestContent;
|
||||
use azure_storage_blob::clients::{BlobClient, BlockBlobClient};
|
||||
@@ -36,12 +35,12 @@ impl SasResource {
|
||||
|
||||
pub(crate) const SAS_VERSION: &str = "2022-11-02";
|
||||
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
pub(crate) fn hmac_sha256_b64(key: &[u8], data: &str) -> Result<String> {
|
||||
let pkey = PKey::hmac(key).context("hmac key")?;
|
||||
let mut signer = Signer::new(MessageDigest::sha256(), &pkey).context("signer")?;
|
||||
signer.update(data.as_bytes()).context("signer update")?;
|
||||
let sig = signer.sign_to_vec().context("sign")?;
|
||||
Ok(STANDARD.encode(sig))
|
||||
let mut mac = HmacSha256::new_from_slice(key).context("hmac key")?;
|
||||
mac.update(data.as_bytes());
|
||||
Ok(STANDARD.encode(mac.finalize().into_bytes()))
|
||||
}
|
||||
|
||||
pub fn build_service_sas(
|
||||
|
||||
@@ -13,7 +13,9 @@ use aws_config::retry::RetryConfig;
|
||||
use aws_sdk_s3 as s3;
|
||||
use aws_sdk_s3::config::BehaviorVersion;
|
||||
use aws_sdk_s3::config::Region;
|
||||
use aws_sdk_s3::config::RequestChecksumCalculation;
|
||||
use aws_sdk_s3::config::retry::ReconnectMode;
|
||||
use aws_sdk_s3::error::DisplayErrorContext;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use futures::StreamExt;
|
||||
@@ -135,6 +137,7 @@ impl StorageProvider for S3Provider {
|
||||
.credentials_provider(credentials)
|
||||
.region(region)
|
||||
.force_path_style(true)
|
||||
.request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
|
||||
.endpoint_url(endpoint)
|
||||
.behavior_version(BehaviorVersion::latest())
|
||||
.build();
|
||||
@@ -164,11 +167,12 @@ impl StorageProvider for S3Provider {
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
error!("Failed to create multipart upload: {}", e);
|
||||
let detail = DisplayErrorContext(&e).to_string();
|
||||
error!("Failed to create multipart upload: {}", detail);
|
||||
return UploadResult {
|
||||
storage_id: storage.id.clone(),
|
||||
success: false,
|
||||
error: Some(e.to_string()),
|
||||
error: Some(detail),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
};
|
||||
@@ -251,7 +255,8 @@ impl StorageProvider for S3Provider {
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to upload part {}: {}", part_number, e);
|
||||
let detail = DisplayErrorContext(&e).to_string();
|
||||
error!("Failed to upload part {}: {}", part_number, detail);
|
||||
let _ = client
|
||||
.abort_multipart_upload()
|
||||
.bucket(bucket)
|
||||
@@ -262,7 +267,7 @@ impl StorageProvider for S3Provider {
|
||||
return UploadResult {
|
||||
storage_id: storage.id.clone(),
|
||||
success: false,
|
||||
error: Some(e.to_string()),
|
||||
error: Some(detail),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
};
|
||||
@@ -317,7 +322,8 @@ impl StorageProvider for S3Provider {
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to complete multipart upload: {}", e);
|
||||
let detail = DisplayErrorContext(&e).to_string();
|
||||
error!("Failed to complete multipart upload: {}", detail);
|
||||
let _ = client
|
||||
.abort_multipart_upload()
|
||||
.bucket(bucket)
|
||||
@@ -328,7 +334,7 @@ impl StorageProvider for S3Provider {
|
||||
UploadResult {
|
||||
storage_id: storage.id.clone(),
|
||||
success: false,
|
||||
error: Some(e.to_string()),
|
||||
error: Some(detail),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
}
|
||||
|
||||
+23
-1
@@ -16,6 +16,8 @@ pub struct Settings {
|
||||
pub timezone: String,
|
||||
pub log: String,
|
||||
pub chunk_size: usize, // bytes
|
||||
pub retry_attempts: u32,
|
||||
pub retry_backoff_ms: u64,
|
||||
}
|
||||
|
||||
impl Settings {
|
||||
@@ -49,6 +51,24 @@ impl Settings {
|
||||
|
||||
let chunk_size = chunk_size_mb * 1024 * 1024;
|
||||
|
||||
let retry_attempts = env::var("RETRY_ATTEMPTS")
|
||||
.unwrap_or_else(|_| "3".to_string())
|
||||
.parse::<u32>()
|
||||
.expect("RETRY_ATTEMPTS must be a valid positive integer");
|
||||
|
||||
if retry_attempts < 3 || retry_attempts > 5 {
|
||||
panic!("RETRY_ATTEMPTS must be between 3 and 5");
|
||||
}
|
||||
|
||||
let retry_backoff_ms = env::var("RETRY_BACKOFF_MS")
|
||||
.unwrap_or_else(|_| "1000".to_string())
|
||||
.parse::<u64>()
|
||||
.expect("RETRY_BACKOFF_MS must be a valid positive integer");
|
||||
|
||||
if retry_backoff_ms < 100 || retry_backoff_ms > 30_000 {
|
||||
panic!("RETRY_BACKOFF_MS must be between 100 and 30000 milliseconds");
|
||||
}
|
||||
|
||||
let tz = env::var("TZ").unwrap_or_else(|_| "UTC".to_string());
|
||||
|
||||
Self {
|
||||
@@ -64,7 +84,9 @@ impl Settings {
|
||||
pooling: pooling_seconds,
|
||||
timezone: tz,
|
||||
log: env::var("LOG").unwrap_or_else(|_| "info".into()),
|
||||
chunk_size
|
||||
chunk_size,
|
||||
retry_attempts,
|
||||
retry_backoff_ms,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+700
-14
@@ -1,9 +1,14 @@
|
||||
use crate::domain::factory::DatabaseFactory;
|
||||
use crate::domain::postgres::connection::{pg_restore_binary_name, select_pg_path, server_version};
|
||||
use crate::domain::postgres::format::PostgresDumpFormat;
|
||||
use crate::domain::postgres::restore::prepare_archive;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::{DatabaseConfig, DbType};
|
||||
use crate::tests::init_tracing_for_test;
|
||||
use crate::utils::compress::{compress_to_tar_gz_large, decompress_large_tar_gz};
|
||||
use oauth2::url;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use tempfile::TempDir;
|
||||
use testcontainers::runners::AsyncRunner;
|
||||
use testcontainers::{ContainerAsync, ImageExt};
|
||||
@@ -63,7 +68,6 @@ async fn postgres_ping_test() {
|
||||
async fn is_superuser_detects_superuser_role() {
|
||||
init_tracing_for_test();
|
||||
|
||||
// The testcontainer's POSTGRES_USER ("testuser") is the bootstrap superuser.
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let is_super = crate::domain::postgres::connection::is_superuser(&config)
|
||||
@@ -73,6 +77,31 @@ async fn is_superuser_detects_superuser_role() {
|
||||
assert!(is_super);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn can_drop_database_false_for_unprivileged_role() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, admin) = create_config().await;
|
||||
|
||||
let a = crate::domain::postgres::connection::connect(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
a.batch_execute("DROP ROLE IF EXISTS lowpriv; CREATE ROLE lowpriv LOGIN PASSWORD 'x';")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut low = admin.clone();
|
||||
low.username = "lowpriv".into();
|
||||
low.password = "x".into();
|
||||
|
||||
assert_eq!(
|
||||
crate::domain::postgres::connection::can_drop_database(&low)
|
||||
.await
|
||||
.unwrap(),
|
||||
false
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn postgres_backup_restore_test() {
|
||||
init_tracing_for_test();
|
||||
@@ -170,17 +199,585 @@ async fn postgres_password_with_slash_test() {
|
||||
assert_eq!(reachable, true);
|
||||
}
|
||||
|
||||
fn pg_dump_env(config: &DatabaseConfig) -> std::collections::HashMap<String, String> {
|
||||
let mut env = std::env::vars().collect::<std::collections::HashMap<_, _>>();
|
||||
env.insert("PGPASSWORD".to_string(), config.password.clone());
|
||||
env
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepare_archive_fd_locates_toc_dir() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
let backup_path = crate::domain::postgres::backup::run(
|
||||
config.clone(),
|
||||
PostgresDumpFormat::Fd,
|
||||
temp_dir.path().to_path_buf(),
|
||||
pg_dump_env(&config),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let compression = compress_to_tar_gz_large(&backup_path, Arc::new(JobLogger::new()))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(compression.compressed_path.is_file());
|
||||
|
||||
let version = server_version(&config).await.unwrap();
|
||||
let pg_restore = select_pg_path(&version).join(pg_restore_binary_name());
|
||||
|
||||
let logger = JobLogger::new();
|
||||
|
||||
let prepared = prepare_archive(
|
||||
PostgresDumpFormat::Fd,
|
||||
&compression.compressed_path,
|
||||
&pg_restore,
|
||||
&logger,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert!(prepared.path().join("toc.dat").exists());
|
||||
assert!(!prepared.toc().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepare_archive_fc_returns_file_path_unchanged() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
let backup_path = crate::domain::postgres::backup::run(
|
||||
config.clone(),
|
||||
PostgresDumpFormat::Fc,
|
||||
temp_dir.path().to_path_buf(),
|
||||
pg_dump_env(&config),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(backup_path.is_file());
|
||||
|
||||
let version = server_version(&config).await.unwrap();
|
||||
let pg_restore = select_pg_path(&version).join(pg_restore_binary_name());
|
||||
|
||||
let logger = JobLogger::new();
|
||||
|
||||
let prepared = prepare_archive(PostgresDumpFormat::Fc, &backup_path, &pg_restore, &logger).unwrap();
|
||||
|
||||
assert_eq!(prepared.path(), backup_path.as_path());
|
||||
assert!(!prepared.toc().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restore_run_unified_fc_roundtrip() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
client.execute("CREATE TABLE t(id int);", &[]).await.unwrap();
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
let dump_file = crate::domain::postgres::backup::run(
|
||||
config.clone(),
|
||||
PostgresDumpFormat::Fc,
|
||||
temp_dir.path().to_path_buf(),
|
||||
pg_dump_env(&config),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(dump_file.is_file());
|
||||
|
||||
let format = crate::domain::postgres::connection::detect_format_from_file(&dump_file);
|
||||
|
||||
let result = crate::domain::postgres::restore::run(
|
||||
config.clone(),
|
||||
format,
|
||||
dump_file,
|
||||
pg_dump_env(&config),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(result.is_ok(), "restore::run failed: {:?}", result);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn drop_all_schemas_removes_user_schema() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
client
|
||||
.batch_execute("CREATE SCHEMA IF NOT EXISTS extra_ns; CREATE TABLE IF NOT EXISTS extra_ns.t(id int);")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let dropped = crate::domain::postgres::connection::drop_all_schemas(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(dropped.iter().any(|s| s == "extra_ns"));
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
let row = client
|
||||
.query_one(
|
||||
"SELECT count(*) FROM pg_namespace WHERE nspname = 'extra_ns'",
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let n: i64 = row.get(0);
|
||||
assert_eq!(n, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restore_drop_schemas_removes_extra_objects() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
client
|
||||
.batch_execute("CREATE TABLE base_t(id int);")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
let dump_file = crate::domain::postgres::backup::run(
|
||||
config.clone(),
|
||||
PostgresDumpFormat::Fc,
|
||||
temp_dir.path().to_path_buf(),
|
||||
pg_dump_env(&config),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
client
|
||||
.batch_execute("CREATE TABLE orphan_only_here(id int);")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut cfg = config.clone();
|
||||
cfg.options
|
||||
.insert("clean_mode".into(), serde_json::json!("drop_schemas"));
|
||||
|
||||
let format = crate::domain::postgres::connection::detect_format_from_file(&dump_file);
|
||||
|
||||
let result = crate::domain::postgres::restore::run(
|
||||
cfg.clone(),
|
||||
format,
|
||||
dump_file,
|
||||
pg_dump_env(&cfg),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(result.is_ok(), "restore::run failed: {:?}", result);
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let n: i64 = client
|
||||
.query_one(
|
||||
"SELECT count(*) FROM information_schema.tables WHERE table_name = 'orphan_only_here'",
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.get(0);
|
||||
assert_eq!(n, 0);
|
||||
|
||||
let n: i64 = client
|
||||
.query_one(
|
||||
"SELECT count(*) FROM information_schema.tables WHERE table_name = 'base_t'",
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.get(0);
|
||||
assert_eq!(n, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restore_clean_leaves_divergent_object() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
client
|
||||
.batch_execute("CREATE TABLE base_t(id int);")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
let dump_file = crate::domain::postgres::backup::run(
|
||||
config.clone(),
|
||||
PostgresDumpFormat::Fc,
|
||||
temp_dir.path().to_path_buf(),
|
||||
pg_dump_env(&config),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
client
|
||||
.batch_execute("CREATE TABLE survives_clean(id int);")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut cfg = config.clone();
|
||||
cfg.options
|
||||
.insert("clean_mode".into(), serde_json::json!("clean"));
|
||||
|
||||
let format = crate::domain::postgres::connection::detect_format_from_file(&dump_file);
|
||||
|
||||
let result = crate::domain::postgres::restore::run(
|
||||
cfg.clone(),
|
||||
format,
|
||||
dump_file,
|
||||
pg_dump_env(&cfg),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(result.is_ok(), "restore::run failed: {:?}", result);
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let n: i64 = client
|
||||
.query_one(
|
||||
"SELECT count(*) FROM information_schema.tables WHERE table_name = 'survives_clean'",
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.get(0);
|
||||
assert_eq!(n, 1, "clean mode is not a reset; divergent object survives");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restore_unknown_clean_mode_falls_back() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
let dump_file = crate::domain::postgres::backup::run(
|
||||
config.clone(),
|
||||
PostgresDumpFormat::Fc,
|
||||
temp_dir.path().to_path_buf(),
|
||||
pg_dump_env(&config),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut cfg = config.clone();
|
||||
cfg.options
|
||||
.insert("clean_mode".into(), serde_json::json!("wat"));
|
||||
|
||||
let format = crate::domain::postgres::connection::detect_format_from_file(&dump_file);
|
||||
|
||||
let logger = Arc::new(JobLogger::new());
|
||||
|
||||
let result = crate::domain::postgres::restore::run(
|
||||
cfg.clone(),
|
||||
format,
|
||||
dump_file,
|
||||
pg_dump_env(&cfg),
|
||||
logger.clone(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(result.is_ok(), "restore::run failed: {:?}", result);
|
||||
|
||||
let entries = Arc::try_unwrap(logger)
|
||||
.expect("logger should have a single owner after run() completes")
|
||||
.into_entries();
|
||||
assert!(entries
|
||||
.iter()
|
||||
.any(|e| e.message.contains("Unknown clean_mode 'wat'")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn drop_database_preflight_preserves_data_when_unprivileged() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, admin) = create_config().await;
|
||||
|
||||
let a = crate::domain::postgres::connection::connect(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
a.batch_execute("DROP ROLE IF EXISTS lowpriv2; CREATE ROLE lowpriv2 LOGIN PASSWORD 'x';")
|
||||
.await
|
||||
.unwrap();
|
||||
a.batch_execute("CREATE TABLE IF NOT EXISTS keep_me(id int);")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
let dump_file = crate::domain::postgres::backup::run(
|
||||
admin.clone(),
|
||||
PostgresDumpFormat::Fc,
|
||||
temp_dir.path().to_path_buf(),
|
||||
pg_dump_env(&admin),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut low = admin.clone();
|
||||
low.username = "lowpriv2".into();
|
||||
low.password = "x".into();
|
||||
low.options
|
||||
.insert("clean_mode".into(), serde_json::json!("drop_database"));
|
||||
|
||||
let format = crate::domain::postgres::connection::detect_format_from_file(&dump_file);
|
||||
|
||||
let res = crate::domain::postgres::restore::run(
|
||||
low.clone(),
|
||||
format,
|
||||
dump_file,
|
||||
pg_dump_env(&low),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await;
|
||||
assert!(res.is_err(), "preflight must reject an unprivileged role");
|
||||
|
||||
let a = crate::domain::postgres::connection::connect(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
let n: i64 = a
|
||||
.query_one(
|
||||
"SELECT count(*) FROM information_schema.tables WHERE table_name = 'keep_me'",
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.get(0);
|
||||
assert_eq!(n, 1, "preflight must fail before dropping anything");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn drop_database_preserves_encoding_and_owner() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
client
|
||||
.batch_execute("CREATE TABLE base_t(id int);")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
let dump_file = crate::domain::postgres::backup::run(
|
||||
config.clone(),
|
||||
PostgresDumpFormat::Fc,
|
||||
temp_dir.path().to_path_buf(),
|
||||
pg_dump_env(&config),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let before = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap()
|
||||
.query_one(
|
||||
"SELECT pg_encoding_to_char(encoding), datcollate FROM pg_database WHERE datname = current_database()",
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let enc0: String = before.get(0);
|
||||
let coll0: String = before.get(1);
|
||||
|
||||
let mut cfg = config.clone();
|
||||
cfg.options
|
||||
.insert("clean_mode".into(), serde_json::json!("drop_database"));
|
||||
|
||||
let format = crate::domain::postgres::connection::detect_format_from_file(&dump_file);
|
||||
|
||||
let result = crate::domain::postgres::restore::run(
|
||||
cfg.clone(),
|
||||
format,
|
||||
dump_file,
|
||||
pg_dump_env(&cfg),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_ok(), "restore::run failed: {:?}", result);
|
||||
|
||||
let after = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap()
|
||||
.query_one(
|
||||
"SELECT pg_encoding_to_char(encoding), datcollate FROM pg_database WHERE datname = current_database()",
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let enc1: String = after.get(0);
|
||||
let coll1: String = after.get(1);
|
||||
assert_eq!(enc0, enc1);
|
||||
assert_eq!(coll0, coll1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn drop_database_force_wins_over_open_connection() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
let dump_file = crate::domain::postgres::backup::run(
|
||||
config.clone(),
|
||||
PostgresDumpFormat::Fc,
|
||||
temp_dir.path().to_path_buf(),
|
||||
pg_dump_env(&config),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut cfg = config.clone();
|
||||
cfg.options
|
||||
.insert("clean_mode".into(), serde_json::json!("drop_database"));
|
||||
|
||||
let squatter = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
let _keep = tokio::spawn(async move {
|
||||
let _ = squatter.query_one("SELECT pg_sleep(5)", &[]).await;
|
||||
});
|
||||
|
||||
let format = crate::domain::postgres::connection::detect_format_from_file(&dump_file);
|
||||
|
||||
let result = crate::domain::postgres::restore::run(
|
||||
cfg.clone(),
|
||||
format,
|
||||
dump_file,
|
||||
pg_dump_env(&cfg),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_ok(), "restore::run failed: {:?}", result);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sniff_format_detects_custom_and_gzip() {
|
||||
use crate::domain::postgres::connection::sniff_format;
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let fc = dir.path().join("a.dump");
|
||||
std::fs::write(&fc, b"PGDMP\x01\x0e\x00").unwrap();
|
||||
assert_eq!(sniff_format(&fc).unwrap(), PostgresDumpFormat::Fc);
|
||||
|
||||
let fd = dir.path().join("b.gz");
|
||||
std::fs::write(&fd, [0x1f, 0x8b, 0x08, 0x00]).unwrap();
|
||||
assert_eq!(sniff_format(&fd).unwrap(), PostgresDumpFormat::Fd);
|
||||
|
||||
let bad = dir.path().join("c.bin");
|
||||
std::fs::write(&bad, b"not a dump").unwrap();
|
||||
assert!(sniff_format(&bad).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn corrupt_archive_leaves_database_untouched() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
client
|
||||
.batch_execute("CREATE TABLE must_survive(id int);")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut cfg = config.clone();
|
||||
cfg.options
|
||||
.insert("clean_mode".into(), serde_json::json!("drop_schemas"));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let broken_file = dir.path().join("broken.tar.gz");
|
||||
std::fs::write(&broken_file, [0x1f, 0x8b, 0x08, 0x00, 0xde, 0xad, 0xbe, 0xef]).unwrap();
|
||||
|
||||
let result = crate::domain::postgres::restore::run(
|
||||
cfg.clone(),
|
||||
PostgresDumpFormat::Fd,
|
||||
broken_file,
|
||||
pg_dump_env(&cfg),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(result.is_err(), "corrupt archive must be rejected before any destructive step");
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
let n: i64 = client
|
||||
.query_one(
|
||||
"SELECT count(*) FROM information_schema.tables WHERE table_name = 'must_survive'",
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.get(0);
|
||||
assert_eq!(n, 1, "corrupt archive must never trigger the schema drop");
|
||||
}
|
||||
|
||||
mod select_pg_path_tests {
|
||||
use crate::domain::postgres::connection::{
|
||||
pg_dump_binary_name, pg_dump_exists_in, pg_dumpall_binary_name, psql_binary_name,
|
||||
select_pg_path_with,
|
||||
pg_dump_binary_name, pg_dump_exists_in, pg_dumpall_binary_name, pg_restore_binary_name,
|
||||
psql_binary_name, select_pg_path_with,
|
||||
};
|
||||
|
||||
// `select_pg_path_with` takes the `PG_BIN_DIR` override as a plain
|
||||
// argument, so these tests never touch process-global env state or the
|
||||
// cached `CONFIG`. They stay deterministic regardless of whether — or at
|
||||
// which version — a real PostgreSQL install exists on the host.
|
||||
|
||||
#[test]
|
||||
fn respects_pg_bin_dir_override() {
|
||||
let custom = if cfg!(target_os = "windows") {
|
||||
@@ -194,8 +791,6 @@ mod select_pg_path_tests {
|
||||
|
||||
#[test]
|
||||
fn pg_bin_dir_override_ignores_requested_version() {
|
||||
// The override is taken as-is, regardless of which version was
|
||||
// requested — this documents/locks in that behavior.
|
||||
let custom = if cfg!(target_os = "windows") {
|
||||
r"C:\custom\pg\bin"
|
||||
} else {
|
||||
@@ -207,10 +802,6 @@ mod select_pg_path_tests {
|
||||
|
||||
#[test]
|
||||
fn empty_pg_bin_dir_falls_through_to_detection() {
|
||||
// An empty override means "unset" (matches `CONFIG.pg_bin_dir` when
|
||||
// `PG_BIN_DIR` is absent). It must not be returned as a literal empty
|
||||
// path — resolution falls through to platform defaults / PATH lookup
|
||||
// and yields a non-empty path.
|
||||
let path = select_pg_path_with("17", "");
|
||||
assert_ne!(path, std::path::PathBuf::from(""));
|
||||
}
|
||||
@@ -250,4 +841,99 @@ mod select_pg_path_tests {
|
||||
assert_eq!(name, "psql");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pg_restore_binary_name_is_platform_correct() {
|
||||
let name = pg_restore_binary_name();
|
||||
if cfg!(target_os = "windows") {
|
||||
assert_eq!(name, "pg_restore.exe");
|
||||
} else {
|
||||
assert_eq!(name, "pg_restore");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mod quoting_tests {
|
||||
use crate::domain::postgres::connection::{quote_ident, quote_literal};
|
||||
|
||||
#[test]
|
||||
fn quote_ident_escapes_double_quotes() {
|
||||
assert_eq!(quote_ident("devdb"), "\"devdb\"");
|
||||
assert_eq!(quote_ident("a\"b"), "\"a\"\"b\"");
|
||||
assert_eq!(quote_ident("drop\"; --"), "\"drop\"\"; --\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quote_literal_escapes_single_quotes() {
|
||||
assert_eq!(quote_literal("UTF8"), "'UTF8'");
|
||||
assert_eq!(quote_literal("O'Brien"), "'O''Brien'");
|
||||
}
|
||||
}
|
||||
|
||||
mod clean_mode_tests {
|
||||
use crate::domain::postgres::clean_mode::RestoreCleanMode as M;
|
||||
use crate::services::config::{DatabaseConfig, DbType};
|
||||
|
||||
fn cfg_with(clean_mode: Option<&str>) -> DatabaseConfig {
|
||||
let mut options = std::collections::HashMap::new();
|
||||
if let Some(v) = clean_mode {
|
||||
options.insert("clean_mode".to_string(), serde_json::json!(v));
|
||||
}
|
||||
DatabaseConfig {
|
||||
name: "t".into(),
|
||||
database: "testdb".into(),
|
||||
db_type: DbType::Postgresql,
|
||||
username: "testuser".into(),
|
||||
password: "changeme".into(),
|
||||
port: 5432,
|
||||
host: "localhost".into(),
|
||||
generated_id: "00000000-0000-0000-0000-000000000000".into(),
|
||||
path: "".into(),
|
||||
max_packet_size: "".into(),
|
||||
volume_name: "".into(),
|
||||
container_name: None,
|
||||
options,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clean_mode_parsing() {
|
||||
assert_eq!(M::from_config(&cfg_with(None)), (M::Clean, None));
|
||||
assert_eq!(M::from_config(&cfg_with(Some("clean"))), (M::Clean, None));
|
||||
assert_eq!(M::from_config(&cfg_with(Some("none"))), (M::None, None));
|
||||
assert_eq!(
|
||||
M::from_config(&cfg_with(Some("drop_schemas"))),
|
||||
(M::DropSchemas, None)
|
||||
);
|
||||
assert_eq!(
|
||||
M::from_config(&cfg_with(Some("drop_database"))),
|
||||
(M::DropDatabase, None)
|
||||
);
|
||||
assert_eq!(
|
||||
M::from_config(&cfg_with(Some("bogus"))),
|
||||
(M::Clean, Some("bogus".to_string()))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uses_pg_restore_clean_behavior() {
|
||||
assert!(M::Clean.uses_pg_restore_clean());
|
||||
assert!(!M::DropSchemas.uses_pg_restore_clean());
|
||||
assert!(!M::None.uses_pg_restore_clean());
|
||||
assert!(!M::DropDatabase.uses_pg_restore_clean());
|
||||
}
|
||||
}
|
||||
|
||||
mod toc_tests {
|
||||
use crate::domain::postgres::restore::toc_creates_public_schema;
|
||||
|
||||
#[test]
|
||||
fn toc_public_schema_detection() {
|
||||
let with = "215; 2615 2200 SCHEMA - public pg_database_owner";
|
||||
let without_table = "200; 1259 12346 TABLE devschema users devuser";
|
||||
let without_similar_schema = "216; 2615 2201 SCHEMA - publicish someowner";
|
||||
assert!(toc_creates_public_schema(with));
|
||||
assert!(!toc_creates_public_schema(without_table));
|
||||
assert!(!toc_creates_public_schema(without_similar_schema));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,3 +148,98 @@ fn database_status_encrypted_envelope() {
|
||||
assert_eq!(status.storages_encrypted, Some(true));
|
||||
assert_eq!(status.storages_ciphertext.as_deref(), Some("AQIDBA=="));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn database_status_defaults_config_fields_absent() {
|
||||
let json = r#"{
|
||||
"dbms": "postgresql",
|
||||
"generatedId": "16678159-ff7e-4c97-8c83-0adeff214681",
|
||||
"encrypt": false,
|
||||
"data": { "backup": { "action": false, "cron": null },
|
||||
"restore": { "action": false, "file": null, "metaFile": null, "size": null } }
|
||||
}"#;
|
||||
let status: crate::services::api::models::agent::status::DatabaseStatus =
|
||||
serde_json::from_str(json).unwrap();
|
||||
assert_eq!(status.config_encrypted, None);
|
||||
assert!(status.config_ciphertext.is_none());
|
||||
assert!(status.resolved_config.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_dashboard_config_decrypts_full_entry() {
|
||||
use crate::services::status::resolve_dashboard_config;
|
||||
use base64::{engine::general_purpose, Engine};
|
||||
|
||||
// 32-byte master key, base64 STANDARD (matches decrypt_json_gcm).
|
||||
let master_key_b64 = general_purpose::STANDARD.encode([7u8; 32]);
|
||||
|
||||
// Full agent-entry shape the dashboard encrypts.
|
||||
let entry = r#"{
|
||||
"name": "Dashboard PG",
|
||||
"type": "postgresql",
|
||||
"database": "app",
|
||||
"username": "postgres",
|
||||
"password": "s3cret",
|
||||
"port": 5432,
|
||||
"host": "10.0.0.10",
|
||||
"generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"
|
||||
}"#;
|
||||
let ciphertext = encrypt_json_gcm(entry.as_bytes(), &master_key_b64);
|
||||
|
||||
let mut status: crate::services::api::models::agent::status::DatabaseStatus =
|
||||
serde_json::from_str(
|
||||
r#"{
|
||||
"dbms": "postgresql",
|
||||
"generatedId": "16678159-ff7e-4c97-8c83-0adeff214681",
|
||||
"encrypt": false,
|
||||
"config_encrypted": true,
|
||||
"config_ciphertext": "PLACEHOLDER",
|
||||
"data": { "backup": { "action": false, "cron": null },
|
||||
"restore": { "action": false, "file": null, "metaFile": null, "size": null } }
|
||||
}"#,
|
||||
)
|
||||
.unwrap();
|
||||
status.config_ciphertext = Some(ciphertext);
|
||||
|
||||
resolve_dashboard_config(&mut status, &master_key_b64).unwrap();
|
||||
|
||||
let cfg = status.resolved_config.expect("resolved");
|
||||
assert_eq!(cfg.name, "Dashboard PG");
|
||||
assert_eq!(cfg.password, "s3cret");
|
||||
assert_eq!(cfg.host, "10.0.0.10");
|
||||
assert_eq!(cfg.db_type.as_str(), "postgresql");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_dashboard_config_noop_when_not_encrypted() {
|
||||
use crate::services::status::resolve_dashboard_config;
|
||||
let mut status: crate::services::api::models::agent::status::DatabaseStatus =
|
||||
serde_json::from_str(
|
||||
r#"{
|
||||
"dbms": "postgresql",
|
||||
"generatedId": "16678159-ff7e-4c97-8c83-0adeff214681",
|
||||
"encrypt": false,
|
||||
"data": { "backup": { "action": false, "cron": null },
|
||||
"restore": { "action": false, "file": null, "metaFile": null, "size": null } }
|
||||
}"#,
|
||||
)
|
||||
.unwrap();
|
||||
resolve_dashboard_config(&mut status, "unused").unwrap();
|
||||
assert!(status.resolved_config.is_none());
|
||||
}
|
||||
|
||||
fn encrypt_json_gcm(plaintext: &[u8], master_key_b64: &str) -> String {
|
||||
use aes_gcm::aead::{Aead, KeyInit};
|
||||
use aes_gcm::{Aes256Gcm, Key, Nonce};
|
||||
use base64::{engine::general_purpose, Engine};
|
||||
|
||||
let key_bytes = general_purpose::STANDARD.decode(master_key_b64).unwrap();
|
||||
let key = Key::<Aes256Gcm>::try_from(key_bytes.as_slice()).unwrap();
|
||||
let cipher = Aes256Gcm::new(&key);
|
||||
let nonce_bytes = [0u8; 12];
|
||||
let nonce = Nonce::try_from(&nonce_bytes[..]).unwrap();
|
||||
let ct = cipher.encrypt(&nonce, plaintext).unwrap();
|
||||
let mut data = nonce_bytes.to_vec();
|
||||
data.extend_from_slice(&ct);
|
||||
general_purpose::STANDARD.encode(data)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
use crate::services::backup::BackupService;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::{DatabaseConfig, DbType};
|
||||
use crate::tests::init_tracing_for_test;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn sqlite_config(path: &str) -> DatabaseConfig {
|
||||
DatabaseConfig {
|
||||
name: "retry-test".to_string(),
|
||||
database: String::new(),
|
||||
db_type: DbType::Sqlite,
|
||||
username: String::new(),
|
||||
password: String::new(),
|
||||
port: 0,
|
||||
host: String::new(),
|
||||
generated_id: "retry-test-gen".to_string(),
|
||||
path: path.to_string(),
|
||||
max_packet_size: String::new(),
|
||||
volume_name: String::new(),
|
||||
container_name: None,
|
||||
options: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failing_backup_is_retried_and_leaves_no_attempt_directory() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let tmp_path = temp_dir.path();
|
||||
let logger = Arc::new(JobLogger::new());
|
||||
|
||||
let cfg = sqlite_config("/nonexistent/definitely-not-here.sqlite");
|
||||
|
||||
let result = BackupService::run(cfg, tmp_path, Arc::clone(&logger))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result.status, "failed");
|
||||
assert!(result.backup_file.is_none());
|
||||
|
||||
let entries = Arc::try_unwrap(logger).unwrap().into_entries();
|
||||
assert_eq!(
|
||||
entries.iter().filter(|e| e.level == "warn").count(),
|
||||
2,
|
||||
"expected one warn per non-final failed attempt"
|
||||
);
|
||||
assert!(
|
||||
entries
|
||||
.iter()
|
||||
.any(|e| e.level == "error" && e.message.starts_with("Backup failed:")),
|
||||
"expected a single terminal error from the runner"
|
||||
);
|
||||
|
||||
let leftovers: Vec<_> = std::fs::read_dir(tmp_path)
|
||||
.unwrap()
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| e.file_name().to_string_lossy().starts_with("attempt-"))
|
||||
.collect();
|
||||
assert!(
|
||||
leftovers.is_empty(),
|
||||
"failed attempt directories must be cleaned up, found {:?}",
|
||||
leftovers.iter().map(|e| e.file_name()).collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
@@ -14,7 +14,9 @@ use crate::utils::common::BackupMethod;
|
||||
use crate::utils::edge_key::EdgeKey;
|
||||
|
||||
use serde_json::json;
|
||||
use std::io::Write;
|
||||
use std::sync::Arc;
|
||||
use tempfile::NamedTempFile;
|
||||
use wiremock::matchers::{body_partial_json, method, path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
@@ -93,3 +95,111 @@ async fn failed_upload_reports_failed_status_to_server() {
|
||||
|
||||
// MockServer drop verifies both `.expect(1)` mounts were hit — including the "failed" PATCH.
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failing_upload_is_retried_until_it_succeeds() {
|
||||
init_tracing_for_test();
|
||||
let server = MockServer::start().await;
|
||||
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/agent/agent-1/backup/upload/init"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"message": "ok",
|
||||
"backupStorage": { "id": "bs-1" }
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/tus/files"))
|
||||
.respond_with(ResponseTemplate::new(500))
|
||||
.up_to_n_times(2)
|
||||
.with_priority(1)
|
||||
.expect(2)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/tus/files"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(201)
|
||||
.insert_header("Location", format!("{}/tus/files/upload-1", server.uri()).as_str()),
|
||||
)
|
||||
.with_priority(2)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
Mock::given(method("PATCH"))
|
||||
.and(path("/tus/files/upload-1"))
|
||||
.respond_with(ResponseTemplate::new(204))
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
Mock::given(method("PATCH"))
|
||||
.and(path("/agent/agent-1/backup/upload/status"))
|
||||
.and(body_partial_json(json!({ "status": "success" })))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"message": "ok",
|
||||
"backupStorage": { "id": "bs-1" }
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let mut backup_file = NamedTempFile::new().unwrap();
|
||||
backup_file.write_all(b"portabase-retry-test-payload").unwrap();
|
||||
backup_file.flush().unwrap();
|
||||
|
||||
let ctx = Context {
|
||||
edge_key: EdgeKey {
|
||||
server_url: server.uri(),
|
||||
agent_id: "agent-1".to_string(),
|
||||
master_key_b64: String::new(),
|
||||
},
|
||||
api: ApiClient::new(server.uri()),
|
||||
};
|
||||
|
||||
let service = BackupService::new(Arc::new(ctx));
|
||||
|
||||
let result = BackupResult {
|
||||
generated_id: "gen-1".to_string(),
|
||||
db_type: DbType::Postgresql,
|
||||
status: "success".to_string(),
|
||||
backup_file: Some(backup_file.path().to_path_buf()),
|
||||
code: None,
|
||||
};
|
||||
|
||||
let storage: DatabaseStorage = serde_json::from_value(json!({
|
||||
"id": "storage-1",
|
||||
"provider": "local",
|
||||
"config": {}
|
||||
}))
|
||||
.unwrap();
|
||||
|
||||
let backup_id = "backup-1".to_string();
|
||||
let logger = Arc::new(JobLogger::new());
|
||||
|
||||
let results = service
|
||||
.upload(
|
||||
result,
|
||||
BackupMethod::Manual,
|
||||
vec![storage],
|
||||
false,
|
||||
&backup_id,
|
||||
Arc::clone(&logger),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(results.len(), 1);
|
||||
assert!(results[0].success);
|
||||
|
||||
let entries = Arc::try_unwrap(logger).unwrap().into_entries();
|
||||
assert_eq!(entries.iter().filter(|e| e.level == "warn").count(), 2);
|
||||
assert!(
|
||||
entries.iter().any(|e| e.message
|
||||
== "Upload to storage storage-1 succeeded on attempt 3/3")
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,15 +1,12 @@
|
||||
use crate::core::context::Context;
|
||||
use crate::services::api::ApiClient;
|
||||
use crate::services::config::ConfigService;
|
||||
use crate::services::config::{build_config, DatabasesConfig, InputDatabaseConfig};
|
||||
use crate::utils::edge_key::EdgeKey;
|
||||
use std::io::Write;
|
||||
use std::sync::Arc;
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
// `ConfigService::load` never touches `self.ctx` on the `Some(file_path)` path,
|
||||
// so the values here don't matter — but `Context::new()` panics without an
|
||||
// `EDGE_KEY` env var, so build the struct directly (mirrors
|
||||
// backup_uploader_tests.rs's `ctx_pointing_at`).
|
||||
fn test_context() -> Arc<Context> {
|
||||
Arc::new(Context {
|
||||
edge_key: EdgeKey {
|
||||
@@ -264,3 +261,73 @@ fn docker_volume_requires_volume_name() {
|
||||
let err = service.load(Some(file.path().to_str().unwrap())).unwrap_err();
|
||||
assert!(err.contains("volume_name"), "error was: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_config_applies_type_defaults() {
|
||||
let input: InputDatabaseConfig = serde_json::from_str(
|
||||
r#"{
|
||||
"name": "cluster1",
|
||||
"type": "postgresql-cluster",
|
||||
"username": "postgres",
|
||||
"password": "p",
|
||||
"port": 5432,
|
||||
"host": "localhost",
|
||||
"generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"
|
||||
}"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let cfg = build_config(input).unwrap();
|
||||
assert_eq!(cfg.db_type.as_str(), "postgresql-cluster");
|
||||
assert_eq!(cfg.database, "postgres"); // cluster default
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_config_rejects_missing_required_field() {
|
||||
let input: InputDatabaseConfig = serde_json::from_str(
|
||||
r#"{
|
||||
"name": "pg",
|
||||
"type": "postgresql",
|
||||
"username": "postgres",
|
||||
"port": 5432,
|
||||
"host": "localhost",
|
||||
"generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"
|
||||
}"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let err = build_config(input).unwrap_err();
|
||||
assert!(err.contains("password"), "unexpected error: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_optional_returns_empty_when_file_missing() {
|
||||
let service = ConfigService::new(test_context());
|
||||
let cfg = service.load_optional(Some("/nonexistent/path/does-not-exist.json"));
|
||||
assert!(cfg.databases.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn databases_config_roundtrips_through_serde() {
|
||||
let input: InputDatabaseConfig = serde_json::from_str(
|
||||
r#"{
|
||||
"name": "pg",
|
||||
"type": "postgresql",
|
||||
"database": "app",
|
||||
"username": "postgres",
|
||||
"password": "secret",
|
||||
"port": 5432,
|
||||
"host": "localhost",
|
||||
"generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"
|
||||
}"#,
|
||||
)
|
||||
.unwrap();
|
||||
let cfg = build_config(input).unwrap();
|
||||
let wrapped = DatabasesConfig { databases: vec![cfg] };
|
||||
|
||||
let json = serde_json::to_string(&wrapped).unwrap();
|
||||
let back: DatabasesConfig = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(back.databases[0].name, "pg");
|
||||
assert_eq!(back.databases[0].db_type.as_str(), "postgresql");
|
||||
assert_eq!(back.databases[0].password, "secret");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
use crate::services::config::{build_config, DatabaseConfig, InputDatabaseConfig};
|
||||
use crate::services::dashboard_config::merge;
|
||||
use crate::services::dashboard_config::{load_cache, persist_cache};
|
||||
|
||||
fn cfg(name: &str, gen_id: &str, host: &str) -> DatabaseConfig {
|
||||
let json = format!(
|
||||
r#"{{ "name": "{name}", "type": "postgresql", "database": "app",
|
||||
"username": "u", "password": "p", "port": 5432,
|
||||
"host": "{host}", "generated_id": "{gen_id}" }}"#
|
||||
);
|
||||
let input: InputDatabaseConfig = serde_json::from_str(&json).unwrap();
|
||||
build_config(input).unwrap()
|
||||
}
|
||||
|
||||
const ID_A: &str = "16678159-ff7e-4c97-8c83-0adeff214681";
|
||||
const ID_B: &str = "16678124-ff7e-4c97-8c83-0adeff214681";
|
||||
|
||||
#[test]
|
||||
fn merge_keeps_local_only_databases() {
|
||||
let local = vec![cfg("local-a", ID_A, "local-host")];
|
||||
let merged = merge(&local, &[]);
|
||||
assert_eq!(merged.databases.len(), 1);
|
||||
assert_eq!(merged.databases[0].host, "local-host");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_appends_dashboard_only_databases() {
|
||||
let local = vec![cfg("local-a", ID_A, "local-host")];
|
||||
let dashboard = vec![cfg("dash-b", ID_B, "dash-host")];
|
||||
let merged = merge(&local, &dashboard);
|
||||
assert_eq!(merged.databases.len(), 2);
|
||||
assert!(merged.databases.iter().any(|d| d.generated_id == ID_B));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_dashboard_wins_on_id_collision() {
|
||||
let local = vec![cfg("local-a", ID_A, "local-host")];
|
||||
let dashboard = vec![cfg("dash-a", ID_A, "dash-host")];
|
||||
let merged = merge(&local, &dashboard);
|
||||
assert_eq!(merged.databases.len(), 1);
|
||||
assert_eq!(merged.databases[0].host, "dash-host"); // dashboard wins
|
||||
assert_eq!(merged.databases[0].name, "dash-a");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_roundtrips() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("dashboard_databases.json");
|
||||
|
||||
let dbs = vec![cfg("dash-a", ID_A, "dash-host")];
|
||||
persist_cache(&path, &dbs).unwrap();
|
||||
|
||||
let loaded = load_cache(&path);
|
||||
assert_eq!(loaded.len(), 1);
|
||||
assert_eq!(loaded[0].generated_id, ID_A);
|
||||
assert_eq!(loaded[0].host, "dash-host");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_cache_missing_file_is_empty() {
|
||||
let loaded = load_cache(std::path::Path::new("/nonexistent/dashboard_databases.json"));
|
||||
assert!(loaded.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_cache_corrupt_file_is_empty() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("dashboard_databases.json");
|
||||
std::fs::write(&path, b"{ this is not valid json").unwrap();
|
||||
|
||||
let loaded = load_cache(&path);
|
||||
assert!(loaded.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persist_cache_leaves_no_tmp_file() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("dashboard_databases.json");
|
||||
persist_cache(&path, &[cfg("dash-a", ID_A, "h")]).unwrap();
|
||||
|
||||
let tmp = path.with_extension("json.tmp");
|
||||
assert!(!tmp.exists(), "temp file should have been renamed away");
|
||||
assert!(path.exists());
|
||||
}
|
||||
@@ -1,3 +1,6 @@
|
||||
mod api_models_tests;
|
||||
mod backup_runner_tests;
|
||||
mod backup_uploader_tests;
|
||||
mod config_tests;
|
||||
mod dashboard_config_tests;
|
||||
mod restore_downloader_tests;
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
use crate::core::context::Context;
|
||||
use crate::services::api::ApiClient;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::restore::RestoreService;
|
||||
use crate::tests::init_tracing_for_test;
|
||||
use crate::utils::edge_key::EdgeKey;
|
||||
|
||||
use std::sync::Arc;
|
||||
use tempfile::TempDir;
|
||||
use wiremock::matchers::{method, path};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failing_download_is_retried_until_it_succeeds() {
|
||||
init_tracing_for_test();
|
||||
let server = MockServer::start().await;
|
||||
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/backups/archive.tar.gz"))
|
||||
.respond_with(ResponseTemplate::new(503))
|
||||
.up_to_n_times(2)
|
||||
.with_priority(1)
|
||||
.expect(2)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/backups/archive.tar.gz"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_bytes(b"portabase-archive".to_vec()))
|
||||
.with_priority(2)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let ctx = Context {
|
||||
edge_key: EdgeKey {
|
||||
server_url: server.uri(),
|
||||
agent_id: "agent-1".to_string(),
|
||||
master_key_b64: String::new(),
|
||||
},
|
||||
api: ApiClient::new(server.uri()),
|
||||
};
|
||||
|
||||
let service = RestoreService::new(Arc::new(ctx));
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let logger = Arc::new(JobLogger::new());
|
||||
let url = format!("{}/backups/archive.tar.gz", server.uri());
|
||||
|
||||
let downloaded = service
|
||||
.download_backup(&url, temp_dir.path(), Arc::clone(&logger), None)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(std::fs::read(&downloaded).unwrap(), b"portabase-archive");
|
||||
|
||||
let entries = Arc::try_unwrap(logger).unwrap().into_entries();
|
||||
assert_eq!(entries.iter().filter(|e| e.level == "warn").count(), 2);
|
||||
assert!(
|
||||
entries
|
||||
.iter()
|
||||
.any(|e| e.message == "Backup download succeeded on attempt 3/3")
|
||||
);
|
||||
}
|
||||
@@ -4,4 +4,5 @@ mod deserializer;
|
||||
mod edge_key_tests;
|
||||
mod file_tests;
|
||||
mod normalize_cron_tests;
|
||||
mod retry_tests;
|
||||
mod stream_tests;
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::tests::init_tracing_for_test;
|
||||
use crate::utils::retry::{RetryPolicy, retry};
|
||||
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
fn fast_policy(attempts: u32) -> RetryPolicy {
|
||||
RetryPolicy {
|
||||
attempts,
|
||||
base_backoff: Duration::from_millis(1),
|
||||
max_backoff: Duration::from_millis(4),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delay_grows_with_the_attempt_number() {
|
||||
let policy = RetryPolicy {
|
||||
attempts: 5,
|
||||
base_backoff: Duration::from_millis(100),
|
||||
max_backoff: Duration::from_secs(30),
|
||||
};
|
||||
|
||||
assert!(policy.delay(1) >= Duration::from_millis(50));
|
||||
assert!(policy.delay(1) <= Duration::from_millis(100));
|
||||
assert!(policy.delay(2) >= Duration::from_millis(100));
|
||||
assert!(policy.delay(2) <= Duration::from_millis(200));
|
||||
assert!(policy.delay(3) >= Duration::from_millis(200));
|
||||
assert!(policy.delay(3) <= Duration::from_millis(400));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delay_never_exceeds_max_backoff() {
|
||||
let policy = RetryPolicy {
|
||||
attempts: 5,
|
||||
base_backoff: Duration::from_millis(1000),
|
||||
max_backoff: Duration::from_millis(2000),
|
||||
};
|
||||
|
||||
for attempt in 1..=5 {
|
||||
assert!(policy.delay(attempt) <= Duration::from_millis(2000));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn first_attempt_success_logs_nothing() {
|
||||
init_tracing_for_test();
|
||||
let logger = JobLogger::new();
|
||||
|
||||
let result: Result<u32, anyhow::Error> =
|
||||
retry("Test op", &logger, &fast_policy(3), |_| async { Ok(7) }).await;
|
||||
|
||||
assert_eq!(result.unwrap(), 7);
|
||||
assert!(logger.into_entries().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retries_until_success_and_logs_each_attempt() {
|
||||
init_tracing_for_test();
|
||||
let logger = JobLogger::new();
|
||||
let calls = AtomicU32::new(0);
|
||||
|
||||
let result: Result<u32, anyhow::Error> =
|
||||
retry("Test op", &logger, &fast_policy(3), |_| async {
|
||||
let n = calls.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
if n < 3 {
|
||||
Err(anyhow::anyhow!("boom {n}"))
|
||||
} else {
|
||||
Ok(n)
|
||||
}
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(result.unwrap(), 3);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 3);
|
||||
|
||||
let entries = logger.into_entries();
|
||||
|
||||
let warns: Vec<_> = entries.iter().filter(|e| e.level == "warn").collect();
|
||||
assert_eq!(warns.len(), 2);
|
||||
assert!(warns[0].message.starts_with("Test op attempt 1/3 failed: boom 1"));
|
||||
assert!(warns[1].message.starts_with("Test op attempt 2/3 failed: boom 2"));
|
||||
|
||||
let infos: Vec<_> = entries.iter().filter(|e| e.level == "info").collect();
|
||||
assert_eq!(infos.len(), 1);
|
||||
assert_eq!(infos[0].message, "Test op succeeded on attempt 3/3");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exhausts_attempts_and_logs_no_terminal_error() {
|
||||
init_tracing_for_test();
|
||||
let logger = JobLogger::new();
|
||||
let calls = AtomicU32::new(0);
|
||||
|
||||
let result: Result<(), anyhow::Error> =
|
||||
retry("Test op", &logger, &fast_policy(3), |_| async {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
Err(anyhow::anyhow!("always"))
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(result.is_err());
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 3);
|
||||
|
||||
let entries = logger.into_entries();
|
||||
assert_eq!(entries.iter().filter(|e| e.level == "warn").count(), 2);
|
||||
assert_eq!(entries.iter().filter(|e| e.level == "error").count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn closure_receives_the_attempt_number() {
|
||||
init_tracing_for_test();
|
||||
let logger = JobLogger::new();
|
||||
let seen = Mutex::new(Vec::new());
|
||||
let seen_ref = &seen;
|
||||
|
||||
let result: Result<(), anyhow::Error> =
|
||||
retry("Test op", &logger, &fast_policy(3), move |attempt| async move {
|
||||
seen_ref.lock().unwrap().push(attempt);
|
||||
Err(anyhow::anyhow!("nope"))
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(result.is_err());
|
||||
assert_eq!(*seen.lock().unwrap(), vec![1, 2, 3]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_defaults_are_within_the_documented_range() {
|
||||
let policy = RetryPolicy::default();
|
||||
assert!(policy.attempts >= 3 && policy.attempts <= 5);
|
||||
assert!(policy.base_backoff >= Duration::from_millis(100));
|
||||
assert!(policy.base_backoff <= Duration::from_millis(30_000));
|
||||
}
|
||||
@@ -6,6 +6,7 @@ pub mod file;
|
||||
pub mod locks;
|
||||
pub mod logging;
|
||||
pub mod redis_client;
|
||||
pub mod retry;
|
||||
pub mod stream;
|
||||
pub mod task_manager;
|
||||
pub mod text;
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::settings::CONFIG;
|
||||
use rand::Rng;
|
||||
use std::fmt::Display;
|
||||
use std::future::Future;
|
||||
use std::time::Duration;
|
||||
|
||||
pub struct RetryPolicy {
|
||||
pub attempts: u32,
|
||||
pub base_backoff: Duration,
|
||||
pub max_backoff: Duration,
|
||||
}
|
||||
|
||||
impl Default for RetryPolicy {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
attempts: CONFIG.retry_attempts,
|
||||
base_backoff: Duration::from_millis(CONFIG.retry_backoff_ms),
|
||||
max_backoff: Duration::from_secs(30),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RetryPolicy {
|
||||
pub(crate) fn delay(&self, attempt: u32) -> Duration {
|
||||
let exp = self.base_backoff.saturating_mul(1u32 << (attempt - 1).min(16));
|
||||
let capped = exp.min(self.max_backoff);
|
||||
let half = capped / 2;
|
||||
let jitter = rand::rng().random_range(0..=half.as_millis() as u64);
|
||||
|
||||
half + Duration::from_millis(jitter)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn retry<T, E, F, Fut>(
|
||||
op: &str,
|
||||
logger: &JobLogger,
|
||||
policy: &RetryPolicy,
|
||||
mut f: F,
|
||||
) -> Result<T, E>
|
||||
where
|
||||
F: FnMut(u32) -> Fut,
|
||||
Fut: Future<Output = Result<T, E>> + Send,
|
||||
T: Send,
|
||||
E: Display + Send,
|
||||
{
|
||||
let total = policy.attempts;
|
||||
let mut attempt = 1;
|
||||
|
||||
loop {
|
||||
match f(attempt).await {
|
||||
Ok(v) => {
|
||||
if attempt > 1 {
|
||||
logger.log("info", format!("{op} succeeded on attempt {attempt}/{total}"));
|
||||
}
|
||||
return Ok(v);
|
||||
}
|
||||
Err(e) if attempt < total => {
|
||||
let delay = policy.delay(attempt);
|
||||
logger.log(
|
||||
"warn",
|
||||
format!(
|
||||
"{op} attempt {attempt}/{total} failed: {e} - retrying in {}ms",
|
||||
delay.as_millis()
|
||||
),
|
||||
);
|
||||
tokio::time::sleep(delay).await;
|
||||
attempt += 1;
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -79,7 +79,12 @@ pub async fn execute_task(
|
||||
let ctx = Arc::new(Context::new());
|
||||
let config_service = ConfigService::new(ctx.clone());
|
||||
let backup_service = BackupService::new(ctx.clone());
|
||||
let config = config_service.load(None).unwrap();
|
||||
|
||||
let local = config_service.load_optional(None);
|
||||
let cache_path = std::path::PathBuf::from(&crate::settings::CONFIG.data_path)
|
||||
.join("dashboard_databases.json");
|
||||
let dashboard = crate::services::dashboard_config::load_cache(&cache_path);
|
||||
let config = crate::services::dashboard_config::merge(&local.databases, &dashboard);
|
||||
|
||||
let metadata_obj = metadata
|
||||
.into_iter()
|
||||
|
||||
Reference in New Issue
Block a user