Compare commits

...

69 Commits

Author SHA1 Message Date
github-actions[bot] 6537e9df53 chore: release 1.19.0 2026-08-20 11:56:59 +00:00
Charles GTE b8d869d5a6 Merge pull request #97 from Portabase/feat/dashboard-databases
feat: dashboard-databases
2026-08-20 13:54:46 +02:00
Charles GTE 90941ea67d fix 2026-08-20 13:41:14 +02:00
charles-gauthereau 046d593e2b fix 2026-08-18 18:20:21 +02:00
charles-gauthereau cf9a59a138 fix: config.rs 2026-08-18 13:07:22 +02:00
charles-gauthereau 2464f6dfb0 Merge branch 'main' into feat/dashboard-databases
# Conflicts:
#	docker-compose.yml
#	src/services/config.rs
2026-08-18 13:05:56 +02:00
github-actions[bot] 069067ca55 chore: release 1.18.6 2026-08-17 09:10:08 +00:00
Charles GTE 04b654d219 Merge pull request #96 from Portabase/fix/mongo-cloud
fix: mongodb cloud cluster issue
2026-08-17 11:07:35 +02:00
charles-gauthereau d83511cb64 fix: mongodb cloud cluster issue 2026-08-17 10:52:21 +02:00
charles-gauthereau ca294e968c fix 2026-08-13 22:02:17 +02:00
charles-gauthereau 1be88ffdb9 Merge branch 'main' into feat/dashboard-databases
# Conflicts:
#	docker-compose.yml
2026-08-13 21:56:38 +02:00
charles-gauthereau 9d393a96a4 fix 2026-08-13 21:55:59 +02:00
charles-gauthereau 044bf80633 feat(agent): ingest dashboard databases via merged cycle + cache 2026-08-13 18:47:18 +02:00
charles-gauthereau 0a6eb6db22 feat(dashboard_config): atomic cache load/persist 2026-08-13 18:40:56 +02:00
charles-gauthereau b26ff81889 feat(dashboard_config): merge (dashboard-wins) + collect_configs 2026-08-13 18:39:54 +02:00
charles-gauthereau 4e2f29f4ca feat(status): decrypt dashboard config_ciphertext into resolved_config 2026-08-13 18:38:15 +02:00
charles-gauthereau d1c8df4cac refactor(config): extract build_config, add load_optional + Serialize 2026-08-13 18:35:52 +02:00
github-actions[bot] 30f83bafcf chore: release 1.18.5 2026-07-26 09:06:42 +00:00
Charles GTE de106c835e Merge pull request #93 from Portabase/fix/s3-storage
fix: s3 logs
2026-07-26 11:04:35 +02:00
Charles GTE 23d6822ddc fix: s3 logs 2026-07-26 11:04:07 +02:00
github-actions[bot] fe1d74945f chore: release 1.18.4 2026-07-25 14:03:12 +00:00
Charles GTE 424a646385 Merge pull request #92 from Portabase/fix/windows-build
fix: windows-build
2026-07-25 16:00:57 +02:00
Charles GTE 0ef4bba5d7 fix: docker-compose.yml 2026-07-25 15:45:56 +02:00
Charles GTE 6548140eaf fix: windows build 2026-07-25 15:19:46 +02:00
github-actions[bot] c9725c381e chore: release 1.18.3 2026-07-24 16:46:51 +00:00
Charles GTE 03695b1897 Merge pull request #91 from Portabase/fix/firebird
fix: firebird
2026-07-24 18:44:20 +02:00
charles-gauthereau 80b1c0dac0 fix: firebird with -g for gbak 2026-07-24 18:29:04 +02:00
charles-gauthereau 5ededb3764 Merge branch 'main' into dev 2026-07-24 17:45:46 +02:00
github-actions[bot] 54cfd673e4 chore: release 1.18.2 2026-07-23 18:04:50 +00:00
Charles GTE 39dbeb5d30 Merge pull request #90 from Portabase/fix/ghcr-ci-cd
fix: ghcr.yml
2026-07-23 20:02:28 +02:00
charles-gauthereau 380cea79fb fix: ghcr.yml 2026-07-23 20:02:01 +02:00
github-actions[bot] 65e4538c2c chore: release 1.18.1 2026-07-23 17:47:38 +00:00
Charles GTE 86a706bca5 Merge pull request #89 from Portabase/fix/ghcr-ci-cd
fix: ghcr.yml
2026-07-23 19:45:23 +02:00
charles-gauthereau e8bda60f16 fix: ghcr.yml 2026-07-23 18:47:41 +02:00
charles-gauthereau 034912b176 Merge branch 'main' into dev 2026-07-23 18:24:18 +02:00
github-actions[bot] 4fd50ccda4 chore: release 1.18.0 2026-07-23 15:33:29 +00:00
Charles GTE 84c6b765c8 Merge pull request #88 from Portabase/fix/postgres-provider
fix: postgres-provider
2026-07-23 17:31:02 +02:00
charles-gauthereau 9790837200 fix: postgres 2026-07-23 17:14:27 +02:00
charles-gauthereau 8827234344 refactor(postgres): split restore into restore/ module (run, prepare, command, toc) 2026-07-23 16:40:56 +02:00
charles-gauthereau b3417304bb fix(postgres): match real pg_restore -l schema line in toc_creates_public_schema 2026-07-23 10:06:01 +02:00
charles-gauthereau 5f4006b1ca feat(postgres): sniff dump format by magic bytes 2026-07-23 09:56:46 +02:00
charles-gauthereau f24b1ec3e4 feat(postgres): wire clean_mode=drop_database with preflight gate 2026-07-23 09:51:49 +02:00
charles-gauthereau bb1b16d0f2 feat(postgres): add drop_and_recreate_database + preflight + FORCE/retry 2026-07-23 09:43:24 +02:00
charles-gauthereau 2aca036ee4 feat(postgres): wire clean_mode none/clean/drop_schemas into restore 2026-07-23 09:36:25 +02:00
charles-gauthereau 2a72a55e57 feat(postgres): detect CREATE SCHEMA public in archive TOC 2026-07-23 09:31:50 +02:00
charles-gauthereau ae63a0df50 feat(postgres): add drop_all_schemas + recreate_public_schema 2026-07-23 09:27:36 +02:00
charles-gauthereau 560c5e02f2 feat(postgres): add RestoreCleanMode option parsing 2026-07-23 09:24:21 +02:00
charles-gauthereau ca17bf9cb9 feat(postgres): add quote_ident/quote_literal for DDL 2026-07-23 09:20:15 +02:00
charles-gauthereau 9fbaab6feb refactor(postgres): unify FC/FD restore via run_pg_restore, validate-before-destroy 2026-07-23 09:15:32 +02:00
charles-gauthereau 1289497d98 refactor(postgres): extract prepare_archive, validate before restore 2026-07-23 09:09:19 +02:00
charles-gauthereau 27c55dbfd8 refactor(postgres): drive async restore work via captured Tokio Handle 2026-07-23 09:01:50 +02:00
charles-gauthereau 45e3acd709 fix(postgres): add pg_restore_binary_name() for Windows restore 2026-07-23 08:58:49 +02:00
charles-gauthereau a93054518a Merge branch 'main' into dev 2026-07-22 00:06:17 +02:00
Charles GTE baf2edc829 fix (#87)
Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-07-21 23:58:11 +02:00
github-actions[bot] f050fa35e0 chore: release 1.17.1 2026-07-21 21:29:30 +00:00
Charles GTE 90d55f7630 fix: backup folder name (#86)
* fix: support configurable backup file prefix (#83)

* fix: backup folder name

* fix: ghcr publish agent image

---------

Co-authored-by: Antonin Jousson <18756890+Antoninj@users.noreply.github.com>
Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-07-21 23:27:10 +02:00
charles-gauthereau c186d57105 fix: ghcr publish agent image 2026-07-21 23:10:40 +02:00
charles-gauthereau 37b4c1fe6d fix: backup folder name 2026-07-21 23:05:59 +02:00
Antonin Jousson 92a3324ca2 fix: support configurable backup file prefix (#83) 2026-07-21 21:50:11 +02:00
github-actions[bot] 82ffb71496 chore: release 1.17.0 2026-07-11 10:08:21 +00:00
Charles GTE 20b3b58024 Merge pull request #80 from Portabase/fix/storage-encryption
fix/storage-encryption
2026-07-11 12:05:59 +02:00
charles-gauthereau 6cddc73565 Merge branch 'main' into fix/storage-encryption
# Conflicts:
#	docker-compose.yml
2026-07-10 18:49:01 +02:00
github-actions[bot] fd3b0076c0 chore: release 1.16.3 2026-07-09 09:11:23 +00:00
Charles GTE 334ab51125 fix: cron (#79)
* fix: cron log

---------

Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-07-09 11:03:55 +02:00
charles-gauthereau 32d4e22196 fix: status.rs 2026-07-08 22:24:50 +02:00
charles-gauthereau f277c5485f feat: decrypt encrypted storages in status ping 2026-07-08 20:52:17 +02:00
charles-gauthereau 585e0bea72 feat: accept encrypted storages markers in DatabaseStatus 2026-07-08 20:47:20 +02:00
charles-gauthereau ee97c6056a feat: add decrypt_json_gcm for encrypted status storages 2026-07-08 20:43:01 +02:00
charles-gauthereau b37a4f2180 fix: docker-compose.yml 2026-07-07 18:43:30 +02:00
47 changed files with 2406 additions and 517 deletions
+134
View File
@@ -0,0 +1,134 @@
name: GHCR Publish
on:
workflow_call:
inputs:
version:
required: true
type: string
ref:
required: true
type: string
add_latest:
required: false
type: boolean
default: false
dockerfile:
required: false
type: string
default: "./docker/Dockerfile"
target:
required: false
type: string
default: "prod"
permissions:
contents: read
packages: write
jobs:
build:
name: Build ${{ matrix.platform }}
runs-on: ${{ matrix.platform == 'linux/amd64' && 'ubuntu-latest' || 'ubuntu-24.04-arm' }}
strategy:
fail-fast: false
matrix:
platform: [ linux/amd64, linux/arm64 ]
steps:
- name: Prepare vars
id: prep
run: |
ARCH="${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}"
echo "arch=$ARCH" >> "$GITHUB_OUTPUT"
echo "image=ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/agent" >> "$GITHUB_OUTPUT"
- name: Checkout
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref }}
fetch-depth: 1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push by digest
id: build
uses: docker/build-push-action@v6
with:
context: .
file: ${{ inputs.dockerfile }}
platforms: ${{ matrix.platform }}
target: ${{ inputs.target }}
provenance: false
outputs: type=image,name=${{ steps.prep.outputs.image }},push-by-digest=true,name-canonical=true,push=true
cache-from: type=gha,scope=ghcr-${{ steps.prep.outputs.arch }}
cache-to: type=gha,mode=max,scope=ghcr-${{ steps.prep.outputs.arch }},ignore-error=true
- name: Export digest
env:
DIGEST: ${{ steps.build.outputs.digest }}
run: |
mkdir -p /tmp/digests
touch "/tmp/digests/${DIGEST#sha256:}"
- name: Upload digest
uses: actions/upload-artifact@v4
with:
name: digest-${{ steps.prep.outputs.arch }}
path: /tmp/digests/*
if-no-files-found: error
retention-days: 1
merge:
name: Create multi-arch manifest
runs-on: ubuntu-latest
needs: build
steps:
- name: Prepare vars
id: prep
run: echo "image=ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/agent" >> "$GITHUB_OUTPUT"
- name: Download digests
uses: actions/download-artifact@v4
with:
path: /tmp/digests
pattern: digest-*
merge-multiple: true
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ steps.prep.outputs.image }}
tags: |
type=semver,pattern={{version}},value=${{ inputs.version }}
type=semver,pattern={{major}}.{{minor}},value=${{ inputs.version }}
type=semver,pattern={{major}},value=${{ inputs.version }}
type=raw,value=latest,enable=${{ inputs.add_latest }}
- name: Create and push manifest list
working-directory: /tmp/digests
run: |
docker buildx imagetools create \
$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
$(printf '${{ steps.prep.outputs.image }}@sha256:%s ' *)
- name: Inspect
run: docker buildx imagetools inspect ${{ steps.prep.outputs.image }}:${{ inputs.version }}
+22
View File
@@ -100,6 +100,15 @@ jobs:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
publish-docker-ghcr:
needs: create-release
if: ${{ needs.create-release.result == 'success' }}
uses: ./.github/workflows/ghcr.yml
with:
version: ${{ needs.create-release.outputs.version }}
ref: ${{ needs.create-release.outputs.version }}
add_latest: true
publish-helm:
needs: create-release
if: ${{ needs.create-release.result == 'success' }}
@@ -109,11 +118,24 @@ jobs:
secrets:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
build-windows:
needs: create-release
if: ${{ needs.create-release.result == 'success' }}
uses: ./.github/workflows/windows-release.yml
with:
version: ${{ needs.create-release.outputs.version }}
ref: ${{ needs.create-release.outputs.version }}
draft_tag: ${{ needs.create-release.outputs.draft_tag }}
secrets:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
finalize-release:
needs:
- create-release
- publish-docker
- publish-docker-ghcr
- publish-helm
- build-windows
runs-on: ubuntu-latest
outputs:
release_tag: ${{ steps.publish_release_step.outputs.release_tag }}
+52 -48
View File
@@ -1,14 +1,30 @@
name: Build Windows release
on:
workflow_call:
inputs:
version:
description: 'Release version (git tag), e.g. 1.18.4'
type: string
required: false
ref:
description: 'Git ref to check out and build'
type: string
required: false
draft_tag:
description: 'Draft GitHub release tag to attach the asset to (e.g. untagged-xxxx). Empty = skip upload.'
type: string
required: false
secrets:
GH_TOKEN:
required: false
workflow_dispatch:
push:
tags:
- '[0-9]+.[0-9]+.[0-9]+'
branches:
- main
- master
inputs:
ref:
description: 'Git ref to check out and build'
type: string
required: false
jobs:
build-windows:
@@ -17,80 +33,68 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.ref }}
- name: Set up Rust toolchain (MSVC)
uses: actions-rs/toolchain@v1
uses: dtolnay/rust-toolchain@stable
with:
toolchain: stable-x86_64-pc-windows-msvc
profile: minimal
override: true
targets: x86_64-pc-windows-msvc
- name: Install vcpkg and OpenSSL (x64)
- name: Cache cargo build
uses: Swatinem/rust-cache@v2
- name: Cache vcpkg installed packages
uses: actions/cache@v4
with:
path: C:\vcpkg\installed
key: vcpkg-openssl-x64-windows-v1
- name: Install OpenSSL (x64) via vcpkg
shell: pwsh
run: |
# Install vcpkg and the prebuilt OpenSSL package
git clone https://github.com/microsoft/vcpkg C:\vcpkg
C:\vcpkg\bootstrap-vcpkg.bat
C:\vcpkg\vcpkg install openssl:x64-windows
# Export variables for subsequent steps
# windows-latest ships vcpkg preinstalled; the install is a no-op when the
# package is restored from cache.
& "$env:VCPKG_INSTALLATION_ROOT\vcpkg.exe" install openssl:x64-windows
'VCPKG_ROOT=C:\vcpkg' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
'OPENSSL_DIR=C:\vcpkg\installed\x64-windows' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
- name: Build (cargo release)
shell: pwsh
env:
# Cargo / openssl-sys will pick up OPENSSL_DIR from the environment
OPENSSL_DIR: ${{ env.OPENSSL_DIR }}
run: |
# Ensure the environment variable is present for this step
if (-Not $env:OPENSSL_DIR) { Write-Host "OPENSSL_DIR not set, printing env for debugging"; Get-ChildItem Env: | ForEach-Object { Write-Host $_ } }
# Build the declared bin target explicitly (Cargo.toml [[bin]] name = "app")
cargo build --release --bin app
run: cargo build --release --bin app
- name: Prepare artifact zip
id: prepare_artifact
shell: pwsh
env:
RELEASE_TAG: ${{ github.ref_name }}
RELEASE_VERSION: ${{ inputs.version }}
run: |
$tag = $env:RELEASE_TAG
$tag = $env:RELEASE_VERSION
if (-not $tag) { $tag = $env:GITHUB_SHA }
# Package the declared bin target deterministically (Cargo.toml [[bin]] name = "app")
$exe = "target\release\app.exe"
if (-not (Test-Path $exe)) { Write-Error "Built binary $exe not found in target/release"; exit 1 }
$outDir = "artifact"
New-Item -ItemType Directory -Path $outDir -Force | Out-Null
# Ship under the package name, not the internal bin name "app"
# Ship under the package name, not the internal bin name "app".
Copy-Item -Path $exe -Destination "$outDir\portabase-agent.exe"
$zipName = "windows-release-$tag.zip"
if (Test-Path $zipName) { Remove-Item $zipName }
Compress-Archive -Path "$outDir\*" -DestinationPath $zipName -Force
Write-Host "ZIP=$zipName"
Write-Output "zip=$zipName" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
"zip=$zipName" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
- name: Upload build artifact
uses: actions/upload-artifact@v4
with:
name: windows-release
path: windows-release-*.zip
path: ${{ steps.prepare_artifact.outputs.zip }}
- name: Create GitHub Release
if: startsWith(github.ref, 'refs/tags/')
id: create_release
uses: softprops/action-gh-release@v1
with:
tag_name: ${{ github.ref_name }}
- name: Attach asset to draft release
if: ${{ inputs.draft_tag != '' }}
shell: pwsh
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Upload release asset
if: startsWith(github.ref, 'refs/tags/')
uses: actions/upload-release-asset@v1
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: windows-release-${{ github.ref_name }}.zip
asset_name: windows-release-${{ github.ref_name }}.zip
asset_content_type: application/zip
GH_TOKEN: ${{ secrets.GH_TOKEN }}
run: |
gh release upload "${{ inputs.draft_tag }}" "${{ steps.prepare_artifact.outputs.zip }}" --clobber
+1 -1
View File
@@ -27,5 +27,5 @@ keywords:
- self-hosted
- portabase
license: Apache-2.0
version: 1.16.2
version: 1.19.0
date-released: '2026-02-24'
Generated
+2 -1
View File
@@ -3503,7 +3503,7 @@ dependencies = [
[[package]]
name = "portabase-agent"
version = "1.16.2"
version = "1.19.0"
dependencies = [
"aes",
"aes-gcm",
@@ -3535,6 +3535,7 @@ dependencies = [
"oauth2",
"once_cell",
"openssl",
"percent-encoding",
"postgres",
"rand 0.9.2",
"redis",
+2 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "portabase-agent"
version = "1.16.2"
version = "1.19.0"
edition = "2024"
[dependencies]
@@ -57,6 +57,7 @@ testcontainers = "0.27.1"
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "valkey", "mysql", "mariadb", "mongo"] }
postgres = "0.19.12"
url = "2.5.8"
percent-encoding = "2.3.2"
bollard = "0.20.0"
[dev-dependencies]
+6 -47
View File
@@ -1,13 +1,14 @@
services:
rust-app:
# build:
# context: .
# dockerfile: docker/Dockerfile
# target: prod
image: portabase/agent:latest
build:
context: .
dockerfile: docker/Dockerfile
target: prod
# image: portabase/agent:latest
container_name: rust-prod
volumes:
- ./databases.json:/config/config.json
- /var/run/docker.sock:/var/run/docker.sock
environment:
LOG: info
TZ: "Europe/Paris"
@@ -18,48 +19,6 @@ services:
networks:
- portabase
db-mongodb-auth:
container_name: db-mongodb-auth
image: mongo:latest
ports:
- "27082:27017"
environment:
MONGO_INITDB_ROOT_USERNAME: root
MONGO_INITDB_ROOT_PASSWORD: rootpassword
MONGO_INITDB_DATABASE: testdbauth
command: mongod --auth
networks:
- portabase
volumes:
- mongodb-data-auth:/data/db
healthcheck:
test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ]
interval: 5s
timeout: 5s
retries: 10
db-mongodb:
container_name: db-mongodb
image: mongo:latest
ports:
- "27083:27017"
volumes:
- mongodb-data:/data/db
healthcheck:
test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ]
interval: 5s
timeout: 5s
retries: 10
environment:
MONGO_INITDB_DATABASE: testdb
networks:
- portabase
volumes:
mongodb-data:
mongodb-data-auth:
networks:
portabase:
name: portabase_network
+3 -1
View File
@@ -15,11 +15,13 @@ services:
# - cargo-target:/app/target
- databases_sqlite-data:/sqlite-data/workspace/data
- ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data
# - /bigdisk:/scratch
environment:
APP_ENV: development
LOG: debug
TZ: "Europe/Paris"
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiMWM4NmQ5NGEtMGVjOC00NzkxLTk0ZTEtOWRlYmJmMWY0M2I5IiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ=="
# TMPDIR: /scratch
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiZDY4MzU2MTQtNzE2NC00OTQ4LWJlZjMtMTlkZDc5NGQzYmRhIiwibWFzdGVyS2V5QjY0IjoiV2NiM0pQQkVTaFBjRjg5UXZwRVJuamU4NGZmak1kNm4vS2dJOUpjMCtmVT0ifQ=="
#CHUNK_SIZE_MB: "1"
#POOLING: 1
#DATABASES_CONFIG_FILE: "config.toml"
+8
View File
@@ -45,6 +45,14 @@ seed-firebird:
echo "SELECT RDB\$RELATION_NAME FROM RDB\$RELATIONS WHERE RDB\$SYSTEM_FLAG = 0 AND RDB\$VIEW_BLR IS NULL;" \
| docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb
seed-firebird-large:
echo "Seeding Firebird..."
docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb < ./scripts/firebird/seed-large.sql
echo "Verifying Firebird tables..."
echo "SELECT RDB\$RELATION_NAME FROM RDB\$RELATIONS WHERE RDB\$SYSTEM_FLAG = 0 AND RDB\$VIEW_BLR IS NULL;" \
| docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb
seed-mssql:
echo "Seeding MSSQL..."
docker exec -i rust-dev sqlcmd -S "db-mssql,1433" -U sa -P "$MSSQL_SA_PASSWORD" -N disable -i /app/scripts/mssql/seed.sql
+259
View File
@@ -0,0 +1,259 @@
SET SQL DIALECT 3;
SET BAIL ON;
SET AUTODDL OFF;
CREATE TABLE users (
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
email VARCHAR(255) NOT NULL UNIQUE,
name VARCHAR(255),
payload BLOB SUB_TYPE TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
COMMIT;
INSERT INTO users (email, name, payload)
VALUES ('alice@example.com', 'Alice', 'Alice seed data');
INSERT INTO users (email, name, payload)
VALUES ('bob@example.com', 'Bob', 'Bob seed data');
COMMIT;
/*
* Each procedure call generates approximately 128 MiB:
*
* 128 rows
* × 128 chunks per row
* × 8191 bytes per chunk
* = approximately 128 MiB
*
* 40 calls = approximately 5 GiB.
*/
SET TERM ^;
CREATE PROCEDURE seed_users_batch (
p_rows INTEGER,
p_chunks_per_row INTEGER
)
AS
DECLARE VARIABLE v_row_index INTEGER;
DECLARE VARIABLE v_chunk_index INTEGER;
DECLARE VARIABLE v_uuid VARCHAR(36);
DECLARE VARIABLE v_chunk VARCHAR(8191);
DECLARE VARIABLE v_payload BLOB SUB_TYPE TEXT;
BEGIN
v_row_index = 0;
WHILE (v_row_index < p_rows) DO
BEGIN
v_payload = NULL;
v_chunk_index = 0;
WHILE (v_chunk_index < p_chunks_per_row) DO
BEGIN
/*
* Generate a different chunk to avoid producing a completely
* uniform BLOB.
*/
v_chunk = RPAD(
UUID_TO_CHAR(GEN_UUID()),
8191,
UUID_TO_CHAR(GEN_UUID())
);
v_payload = BLOB_APPEND(v_payload, v_chunk);
v_chunk_index = v_chunk_index + 1;
END
v_uuid = UUID_TO_CHAR(GEN_UUID());
INSERT INTO users (
email,
name,
payload
)
VALUES (
:v_uuid || '@example.test',
'Seed User ' || :v_uuid,
:v_payload
);
v_row_index = v_row_index + 1;
END
END^
SET TERM ;^
/* Batch 01 — approximately 128 MiB */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 02 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 03 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 04 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 05 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 06 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 07 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 08 — approximately 1 GiB total */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 09 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 10 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 11 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 12 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 13 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 14 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 15 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 16 — approximately 2 GiB total */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 17 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 18 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 19 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 20 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 21 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 22 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 23 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 24 — approximately 3 GiB total */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 25 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 26 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 27 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 28 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 29 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 30 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 31 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 32 — approximately 4 GiB total */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 33 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 34 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 35 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 36 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 37 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 38 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 39 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 40 — approximately 5 GiB total */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
DROP PROCEDURE seed_users_batch;
COMMIT;
SELECT
COUNT(*) AS user_count,
CAST(SUM(OCTET_LENGTH(payload)) / 1073741824.0 AS DECIMAL(18, 2))
AS payload_size_gib
FROM users;
COMMIT;
+30 -8
View File
@@ -2,13 +2,16 @@
use crate::core::context::Context;
use crate::services::backup::BackupService;
use crate::services::config::ConfigService;
use crate::services::config::{ConfigService, DatabaseConfig};
use crate::services::cron::CronService;
use crate::services::dashboard_config::{collect_configs, load_cache, merge, persist_cache};
use crate::services::restore::RestoreService;
use crate::services::status::StatusService;
use crate::settings::CONFIG;
use crate::utils::common::BackupMethod;
use std::path::PathBuf;
use std::sync::Arc;
use tracing::info;
use tracing::{error, info, warn};
pub struct Agent {
ctx: Arc<Context>,
@@ -17,6 +20,8 @@ pub struct Agent {
cron_service: CronService,
backup_service: BackupService,
restore_service: RestoreService,
dashboard_cache: Vec<DatabaseConfig>,
cache_path: PathBuf,
}
impl Agent {
@@ -28,6 +33,9 @@ impl Agent {
let backup_service = BackupService::new(ctx.clone());
let restore_service = RestoreService::new(ctx.clone());
let cache_path = PathBuf::from(&CONFIG.data_path).join("dashboard_databases.json");
let dashboard_cache = load_cache(&cache_path);
Agent {
ctx,
config_service,
@@ -35,19 +43,33 @@ impl Agent {
cron_service,
backup_service,
restore_service,
dashboard_cache,
cache_path,
}
}
pub async fn run(&mut self, method: BackupMethod) -> Result<(), Box<dyn std::error::Error>> {
let config = self.config_service.load(None)?;
let ping_result = self.status_service.ping(&config.databases).await?;
let local = self.config_service.load_optional(None);
let merged_in = merge(&local.databases, &self.dashboard_cache);
let ping_result = self.status_service.ping(&merged_in.databases).await?;
self.dashboard_cache = collect_configs(&ping_result);
if let Err(e) = persist_cache(&self.cache_path, &self.dashboard_cache) {
error!("Failed to persist dashboard cache: {e}");
}
let merged = merge(&local.databases, &self.dashboard_cache);
for db in ping_result.databases.iter() {
let database = config
let Some(database) = merged
.databases
.iter()
.find(|cfg_db| cfg_db.generated_id == db.generated_id)
.unwrap();
else {
warn!("No config for returned database {}; skipping", db.generated_id);
continue;
};
info!(
"Generated Id: {} | backup action: {} | restore action: {} | Database Name: {}",
db.generated_id, db.data.backup.action, db.data.restore.action, database.name,
@@ -59,14 +81,14 @@ impl Agent {
.backup_service
.dispatch(
&db.generated_id,
&config,
&merged,
method.clone(),
&db.storages,
db.encrypt,
)
.await;
} else if db.data.restore.action {
let _ = self.restore_service.dispatch(db, &config).await;
let _ = self.restore_service.dispatch(db, &merged).await;
}
}
+1 -1
View File
@@ -15,7 +15,7 @@ pub const EPHEMERAL_LABEL: &str = "io.portabase.ephemeral";
const HELPER_MOUNT: &str = "/vol";
pub fn client() -> Result<Docker> {
Docker::connect_with_unix_defaults().context("Failed to connect to Docker daemon socket")
Docker::connect_with_defaults().context("Failed to connect to Docker daemon socket")
}
pub fn parse_container_id(mountinfo: &str, cgroup: &str) -> Option<String> {
+1
View File
@@ -23,6 +23,7 @@ pub async fn run(
let start = Instant::now();
let output = Command::new("gbak")
.arg("-b")
.arg("-g")
.arg("-v")
.arg("-user").arg(&cfg.username)
.arg("-password").arg(&cfg.password)
+106 -11
View File
@@ -1,6 +1,13 @@
use crate::services::config::DatabaseConfig;
use anyhow::Result;
use mongodb::Client;
use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
const USERINFO_ENCODE: &AsciiSet = &NON_ALPHANUMERIC
.remove(b'-')
.remove(b'_')
.remove(b'.')
.remove(b'~');
pub async fn connect(cfg: DatabaseConfig) -> Result<Client> {
let uri = get_mongo_uri(cfg)?;
@@ -16,19 +23,40 @@ pub fn select_mongo_path() -> std::path::PathBuf {
}
pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result<String> {
if cfg.username.is_empty() || cfg.password.is_empty() {
Ok(format!(
"mongodb://{}:{}/{}",
cfg.host, cfg.port, cfg.database
))
} else {
Ok(format!(
"mongodb://{}:{}@{}:{}/{}?authSource=admin",
cfg.username, cfg.password, cfg.host, cfg.port, cfg.database
))
}
Ok(build_mongo_uri(&cfg, true))
}
pub fn build_mongo_uri(cfg: &DatabaseConfig, include_db: bool) -> String {
let is_srv = cfg.port == 0;
let scheme = if is_srv { "mongodb+srv" } else { "mongodb" };
let has_auth = !cfg.username.is_empty() && !cfg.password.is_empty();
let credentials = if has_auth {
format!(
"{}:{}@",
utf8_percent_encode(&cfg.username, USERINFO_ENCODE),
utf8_percent_encode(&cfg.password, USERINFO_ENCODE)
)
} else {
String::new()
};
let authority = if is_srv {
cfg.host.clone()
} else {
format!("{}:{}", cfg.host, cfg.port)
};
let path = if include_db {
format!("/{}", cfg.database)
} else {
"/".to_string()
};
let query = if has_auth { "?authSource=admin" } else { "" };
format!("{}://{}{}{}{}", scheme, credentials, authority, path, query)
}
pub fn extract_db_name(dry_output: &str) -> Option<String> {
let mut dbs = std::collections::HashSet::new();
@@ -43,3 +71,70 @@ pub fn extract_db_name(dry_output: &str) -> Option<String> {
}
dbs.into_iter().next()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::services::config::{DatabaseConfig, DbType};
use std::collections::HashMap;
fn cfg(host: &str, port: u16, user: &str, pass: &str) -> DatabaseConfig {
DatabaseConfig {
name: "t".into(),
database: "mydb".into(),
db_type: DbType::MongoDB,
username: user.into(),
password: pass.into(),
port,
host: host.into(),
generated_id: "id".into(),
path: String::new(),
max_packet_size: String::new(),
volume_name: String::new(),
container_name: None,
options: HashMap::new(),
}
}
#[test]
fn standard_with_auth() {
let c = cfg("localhost", 27017, "user", "pass");
assert_eq!(
build_mongo_uri(&c, true),
"mongodb://user:pass@localhost:27017/mydb?authSource=admin"
);
}
#[test]
fn standard_no_auth() {
let c = cfg("localhost", 27017, "", "");
assert_eq!(build_mongo_uri(&c, true), "mongodb://localhost:27017/mydb");
}
#[test]
fn srv_with_auth() {
let c = cfg("cluster.example.mongodb.net", 0, "user", "pass");
assert_eq!(
build_mongo_uri(&c, true),
"mongodb+srv://user:pass@cluster.example.mongodb.net/mydb?authSource=admin"
);
}
#[test]
fn srv_no_db_for_dryrun() {
let c = cfg("cluster.example.mongodb.net", 0, "user", "pass");
assert_eq!(
build_mongo_uri(&c, false),
"mongodb+srv://user:pass@cluster.example.mongodb.net/?authSource=admin"
);
}
#[test]
fn encodes_special_chars_in_credentials() {
let c = cfg("cluster.example.mongodb.net", 0, "user", "p@ss:w/rd?");
assert_eq!(
build_mongo_uri(&c, true),
"mongodb+srv://user:p%40ss%3Aw%2Frd%3F@cluster.example.mongodb.net/mydb?authSource=admin"
);
}
}
+5 -1
View File
@@ -19,7 +19,11 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
Ok(_) => Ok(true),
Err(e) => {
error!("--- MongoDB Connection Error Details ---");
error!("Target Host: {}:{}", cfg.host, cfg.port);
if cfg.port == 0 {
error!("Target Host: {} (srv)", cfg.host);
} else {
error!("Target Host: {}:{}", cfg.host, cfg.port);
}
error!("Error Kind: {:?}", e.kind);
error!("Full Error: {}", e);
error!("Check you database network connectivity");
+4 -8
View File
@@ -1,4 +1,6 @@
use crate::domain::mongodb::connection::{extract_db_name, get_mongo_uri, select_mongo_path};
use crate::domain::mongodb::connection::{
build_mongo_uri, extract_db_name, get_mongo_uri, select_mongo_path,
};
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
@@ -16,13 +18,7 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogg
let dry_start = Instant::now();
let dry_run = Command::new(&mongorestore)
.arg(format!(
"--uri={}",
format!(
"mongodb://{}:{}@{}:{}/?authSource=admin",
cfg.username, cfg.password, cfg.host, cfg.port
)
))
.arg(format!("--uri={}", build_mongo_uri(&cfg, false)))
.arg(format!("--archive={}", restore_file.display()))
.arg("--gzip")
.arg("--dryRun")
+25
View File
@@ -0,0 +1,25 @@
use crate::services::config::DatabaseConfig;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RestoreCleanMode {
None,
Clean,
DropSchemas,
DropDatabase,
}
impl RestoreCleanMode {
pub fn from_config(cfg: &DatabaseConfig) -> (Self, Option<String>) {
match cfg.options.get("clean_mode").and_then(|v| v.as_str()) {
None | Some("clean") => (Self::Clean, None),
Some("none") => (Self::None, None),
Some("drop_schemas") => (Self::DropSchemas, None),
Some("drop_database") => (Self::DropDatabase, None),
Some(other) => (Self::Clean, Some(other.to_string())),
}
}
pub fn uses_pg_restore_clean(self) -> bool {
matches!(self, Self::Clean)
}
}
+4 -3
View File
@@ -15,10 +15,11 @@ pub async fn run(
env: HashMap<String, String>,
logger: Arc<JobLogger>,
) -> Result<()> {
let handle = tokio::runtime::Handle::current();
tokio::task::spawn_blocking(move || -> Result<()> {
logger.log("info", format!("Starting cluster restore for {}", cfg.name));
let version = match futures::executor::block_on(server_version(&cfg)) {
let version = match handle.block_on(server_version(&cfg)) {
Ok(v) => v,
Err(e) => {
logger.log("error", format!("Failed to get server version for {}: {:?}", cfg.name, e));
@@ -26,7 +27,7 @@ pub async fn run(
}
};
match futures::executor::block_on(is_superuser(&cfg)) {
match handle.block_on(is_superuser(&cfg)) {
Ok(true) => {}
Ok(false) => {
logger.log("error", format!("postgresql-cluster restore requires a superuser role for {}", cfg.name));
@@ -40,7 +41,7 @@ pub async fn run(
let psql = select_pg_path(&version).join(psql_binary_name());
if let Err(e) = futures::executor::block_on(terminate_all_connections(&cfg)) {
if let Err(e) = handle.block_on(terminate_all_connections(&cfg)) {
logger.log("error", format!("Failed to terminate connections for cluster {}: {:?}", cfg.name, e));
return Err(e.into());
}
+176
View File
@@ -33,6 +33,14 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
Ok(version)
}
pub async fn server_version_major(cfg: &DatabaseConfig) -> Result<u32> {
let v = server_version(cfg).await?;
Ok(v.split(['.', ' '])
.next()
.and_then(|s| s.parse().ok())
.unwrap_or(17))
}
pub async fn is_superuser(cfg: &DatabaseConfig) -> Result<bool> {
let client = connect(cfg).await?;
let is_super: bool = client
@@ -43,6 +51,19 @@ pub async fn is_superuser(cfg: &DatabaseConfig) -> Result<bool> {
Ok(is_super)
}
pub async fn can_drop_database(cfg: &DatabaseConfig) -> Result<bool> {
let client = connect(cfg).await?;
let row = client
.query_one(
"SELECT r.rolsuper OR (r.rolcreatedb AND pg_catalog.pg_has_role(current_user, d.datdba, 'USAGE')) \
FROM pg_roles r, pg_database d \
WHERE r.rolname = current_user AND d.datname = current_database()",
&[],
)
.await?;
Ok(row.get(0))
}
pub fn select_pg_path(version: &str) -> std::path::PathBuf {
select_pg_path_with(version, &CONFIG.pg_bin_dir)
@@ -114,6 +135,22 @@ pub(crate) fn psql_binary_name() -> &'static str {
}
}
pub(crate) fn pg_restore_binary_name() -> &'static str {
if cfg!(target_os = "windows") {
"pg_restore.exe"
} else {
"pg_restore"
}
}
pub(crate) fn quote_ident(s: &str) -> String {
format!("\"{}\"", s.replace('"', "\"\""))
}
pub(crate) fn quote_literal(s: &str) -> String {
format!("'{}'", s.replace('\'', "''"))
}
pub(crate) fn pg_dump_exists_in(dir: &std::path::Path) -> bool {
dir.join(pg_dump_binary_name()).is_file()
}
@@ -165,6 +202,107 @@ pub async fn terminate_all_connections(cfg: &DatabaseConfig) -> Result<()> {
Ok(())
}
pub async fn drop_and_recreate_database(cfg: &DatabaseConfig) -> Result<()> {
let mut admin_cfg = cfg.clone();
admin_cfg.database = "postgres".to_string();
let admin = connect(&admin_cfg).await?;
let row = admin
.query_opt(
r#"
SELECT pg_encoding_to_char(encoding), datcollate, datctype,
pg_get_userbyid(datdba), datistemplate
FROM pg_database WHERE datname = $1
"#,
&[&cfg.database],
)
.await?;
let (encoding, collate, ctype, owner) = match &row {
Some(r) => (
r.get::<_, String>(0),
r.get::<_, String>(1),
r.get::<_, String>(2),
r.get::<_, String>(3),
),
None => ("UTF8".into(), "C".into(), "C".into(), cfg.username.clone()),
};
if let Some(r) = &row {
if r.get::<_, bool>(4) {
anyhow::bail!("Refusing to drop template database {}", cfg.database);
}
}
let db = quote_ident(&cfg.database);
if let Err(e) = admin
.batch_execute(&format!("ALTER DATABASE {db} WITH ALLOW_CONNECTIONS false"))
.await
{
tracing::warn!("ALLOW_CONNECTIONS false failed for {}: {e}", cfg.database);
}
let major = server_version_major(&admin_cfg).await?;
let drop_stmt = if major >= 13 {
format!("DROP DATABASE IF EXISTS {db} WITH (FORCE)")
} else {
format!("DROP DATABASE IF EXISTS {db}")
};
let mut last_err = None;
let mut dropped = false;
for _ in 0..3 {
let _ = terminate_connections(cfg).await;
match admin.batch_execute(&drop_stmt).await {
Ok(()) => {
dropped = true;
break;
}
Err(e) => {
last_err = Some(e);
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
}
}
}
if !dropped {
let _ = admin
.batch_execute(&format!("ALTER DATABASE {db} WITH ALLOW_CONNECTIONS true"))
.await;
return Err(last_err
.map(anyhow::Error::from)
.unwrap_or_else(|| anyhow::anyhow!("DROP DATABASE {} failed", cfg.database)));
}
admin
.batch_execute(&format!(
"CREATE DATABASE {db} OWNER {} TEMPLATE template0 ENCODING {} LC_COLLATE {} LC_CTYPE {}",
quote_ident(&owner),
quote_literal(&encoding),
quote_literal(&collate),
quote_literal(&ctype),
))
.await?;
Ok(())
}
pub fn sniff_format(restore_file: &Path) -> Result<PostgresDumpFormat> {
use std::io::Read;
let mut f = std::fs::File::open(restore_file)?;
let mut magic = [0u8; 5];
let n = f.read(&mut magic)?;
let head = &magic[..n];
if head.starts_with(b"PGDMP") {
Ok(PostgresDumpFormat::Fc)
} else if head.starts_with(&[0x1f, 0x8b]) {
Ok(PostgresDumpFormat::Fd)
} else {
anyhow::bail!("Unrecognized dump format for {:?}", restore_file)
}
}
pub fn detect_format_from_file(restore_file: &Path) -> PostgresDumpFormat {
match restore_file.extension().and_then(|e| e.to_str()) {
Some("dump") => PostgresDumpFormat::Fc,
@@ -174,6 +312,44 @@ pub fn detect_format_from_file(restore_file: &Path) -> PostgresDumpFormat {
}
}
pub async fn drop_all_schemas(cfg: &DatabaseConfig) -> Result<Vec<String>> {
let client = connect(cfg).await?;
let rows = client
.query(
r#"
SELECT nspname FROM pg_namespace
WHERE nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast')
AND nspname NOT LIKE 'pg\_temp\_%'
AND nspname NOT LIKE 'pg\_toast\_temp\_%'
ORDER BY nspname
"#,
&[],
)
.await?;
let schemas: Vec<String> = rows.iter().map(|r| r.get::<_, String>(0)).collect();
for s in &schemas {
client
.batch_execute(&format!("DROP SCHEMA IF EXISTS {} CASCADE", quote_ident(s)))
.await?;
}
client
.batch_execute("SELECT lo_unlink(oid) FROM pg_largeobject_metadata")
.await
.ok();
Ok(schemas)
}
pub async fn recreate_public_schema(cfg: &DatabaseConfig, owner: &str) -> Result<()> {
let client = connect(cfg).await?;
client
.batch_execute(&format!(
"CREATE SCHEMA IF NOT EXISTS public AUTHORIZATION {}; GRANT USAGE ON SCHEMA public TO PUBLIC;",
quote_ident(owner)
))
.await?;
Ok(())
}
pub async fn detect_format_from_size(cfg: &DatabaseConfig) -> PostgresDumpFormat {
info!(
"Detecting database format {:?} - {:?}",
+1 -1
View File
@@ -1,4 +1,4 @@
#[derive(Clone, Copy)]
#[derive(Clone, Copy, PartialEq, Debug)]
pub enum PostgresDumpFormat {
Fc,
Fd,
+3 -2
View File
@@ -1,9 +1,10 @@
pub mod backup;
pub(crate) mod cluster;
pub(crate) mod clean_mode;
pub(crate) mod connection;
pub mod database;
mod format;
pub(crate) mod format;
mod ping;
mod restore;
pub(crate) mod restore;
pub use connection::{detect_format_from_file, detect_format_from_size};
-214
View File
@@ -1,214 +0,0 @@
use anyhow::Result;
use std::collections::HashMap;
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use std::time::Instant;
use super::connection::{select_pg_path, server_version, terminate_connections};
use super::format::PostgresDumpFormat;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
pub async fn run(
cfg: DatabaseConfig,
format: PostgresDumpFormat,
restore_file: PathBuf,
env: HashMap<String, String>,
logger: Arc<JobLogger>,
) -> Result<()> {
tokio::task::spawn_blocking(move || -> Result<()> {
logger.log("info", format!("Starting restore for database {}", cfg.name));
let version = match futures::executor::block_on(server_version(&cfg)) {
Ok(v) => {
logger.log("debug", format!("Postgres version detected: {}", v));
v
}
Err(e) => {
logger.log("error", format!("Failed to get server version for {}: {:?}", cfg.name, e));
return Err(e.into());
}
};
let pg_restore = select_pg_path(&version).join("pg_restore");
logger.log("debug", format!("Using pg_restore at {:?}", pg_restore));
if let Err(e) = futures::executor::block_on(terminate_connections(&cfg)) {
logger.log("error", format!("Failed to terminate connections for {}: {:?}", cfg.name, e));
return Err(e.into());
}
logger.log("info", format!("Connections terminated for database {}", cfg.name));
let keep_ownership = cfg.options
.get("keep_ownership")
.and_then(|v| v.as_bool())
.unwrap_or(false);
if keep_ownership {
logger.log("info", format!("Restoring ownership and privileges for {}", cfg.name));
} else {
logger.log("info", format!("Stripping ownership and privileges for {} (--no-owner --no-privileges)", cfg.name));
}
match format {
PostgresDumpFormat::Fc => {
logger.log("info", format!("Running FC restore for {}", cfg.name));
let start = Instant::now();
let mut cmd = Command::new(&pg_restore);
if !keep_ownership {
cmd.arg("--no-owner").arg("--no-privileges");
}
let output = cmd
.arg("--clean")
.arg("--if-exists")
// .arg("--create")
.arg("--host").arg(&cfg.host)
.arg("--port").arg(cfg.port.to_string())
.arg("--username").arg(&cfg.username)
.arg("--dbname").arg(&cfg.database)
.arg("-v")
.arg(&restore_file)
.envs(env)
.output();
let duration_ms = start.elapsed().as_millis() as f64;
match output {
Ok(o) => {
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
let stdout = String::from_utf8_lossy(&o.stdout).to_string();
let combined = format!("{}{}", stdout, stderr);
let exit_code = o.status.code().unwrap_or(-1);
if o.status.success() {
logger.log_command("pg_restore", if combined.is_empty() { None } else { Some(combined) }, Some(0), Some(duration_ms));
logger.log("info", format!("FC restore completed successfully for {}", cfg.name))
} else {
logger.log_command("pg_restore", if combined.is_empty() { None } else { Some(combined) }, Some(exit_code), Some(duration_ms));
logger.log("error", format!("FC restore failed with status {:?} for {}", o.status, cfg.name));
anyhow::bail!("Postgres restore failed for {}", cfg.name);
}
}
Err(e) => {
logger.log_command("pg_restore", Some(e.to_string()), Some(-1), Some(duration_ms));
logger.log("error", format!("Error executing pg_restore for {}: {:?}", cfg.name, e));
return Err(e.into());
}
}
}
PostgresDumpFormat::Fd => {
logger.log("info", format!("Running FD restore for {}", cfg.name));
let tar_gz = match std::fs::File::open(&restore_file) {
Ok(f) => f,
Err(e) => {
logger.log("error", format!(
"Failed to open restore file {:?} for {}: {:?}",
restore_file, cfg.name, e
));
return Err(e.into());
}
};
logger.log("info", format!("tar_gz {:?}", tar_gz));
let dec = flate2::read::GzDecoder::new(tar_gz);
let mut archive = tar::Archive::new(dec);
let tmp_dir = match tempfile::TempDir::new() {
Ok(d) => d,
Err(e) => {
logger.log("error", format!(
"Failed to create temporary directory for FD restore of {}: {:?}",
cfg.name, e
));
return Err(e.into());
}
};
if let Err(e) = archive.unpack(tmp_dir.path()) {
logger.log("error", format!("Failed to unpack FD archive for {}: {:?}", cfg.name, e));
return Err(e.into());
}
logger.log("debug", format!("Listing contents of temp dir: {}", tmp_dir.path().display()));
for entry in std::fs::read_dir(tmp_dir.path())? {
if let Ok(entry) = entry {
let path = entry.path();
let file_type = entry.file_type()?;
logger.log("debug", format!(
" - {} | is_dir: {} | is_file: {}",
path.display(),
file_type.is_dir(),
file_type.is_file()
));
}
}
let dump_dir = if tmp_dir.path().join("toc.dat").exists() {
tmp_dir.path().to_path_buf()
} else {
std::fs::read_dir(tmp_dir.path())?
.filter_map(|e| e.ok())
.find(|entry| entry.path().join("toc.dat").exists())
.map(|e| e.path())
.ok_or_else(|| anyhow::anyhow!("Invalid FD archive: toc.dat not found"))?
};
let start = Instant::now();
let mut cmd = Command::new(&pg_restore);
if !keep_ownership {
cmd.arg("--no-owner").arg("--no-privileges");
}
let output = cmd
.arg("--clean")
.arg("--if-exists")
// .arg("--create")
.arg("--host").arg(&cfg.host)
.arg("--port").arg(cfg.port.to_string())
.arg("--username").arg(&cfg.username)
.arg("--dbname").arg(&cfg.database)
.arg("-v")
.arg("-j")
.arg("4")
.arg(dump_dir)
.envs(env)
.output();
let duration_ms = start.elapsed().as_millis() as f64;
match output {
Ok(o) => {
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
let stdout = String::from_utf8_lossy(&o.stdout).to_string();
let combined = format!("{}{}", stdout, stderr);
let exit_code = o.status.code().unwrap_or(-1);
if o.status.success() {
logger.log_command("pg_restore", if combined.is_empty() { None } else { Some(combined) }, Some(0), Some(duration_ms));
logger.log("info", format!("FD restore completed successfully for {}", cfg.name))
} else {
logger.log_command("pg_restore", if combined.is_empty() { None } else { Some(combined) }, Some(exit_code), Some(duration_ms));
logger.log("error", format!("FD restore failed with status {:?} for {}", o.status, cfg.name));
anyhow::bail!("Postgres FD restore failed for {}", cfg.name);
}
}
Err(e) => {
logger.log_command("pg_restore", Some(e.to_string()), Some(-1), Some(duration_ms));
logger.log("error", format!("Error executing pg_restore for {}: {:?}", cfg.name, e));
return Err(e.into());
}
}
}
}
logger.log("info", format!("Restore finished for database {}", cfg.name));
Ok(())
})
.await?
}
+41
View File
@@ -0,0 +1,41 @@
use anyhow::Result;
use std::process::Command;
use std::time::Instant;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
pub(crate) fn run_pg_restore(
mut cmd: Command,
logger: &JobLogger,
cfg: &DatabaseConfig,
) -> Result<()> {
let start = Instant::now();
let output = cmd.output();
let duration_ms = start.elapsed().as_millis() as f64;
match output {
Ok(o) => {
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
let stdout = String::from_utf8_lossy(&o.stdout).to_string();
let combined = format!("{}{}", stdout, stderr);
let exit_code = o.status.code().unwrap_or(-1);
let payload = if combined.is_empty() { None } else { Some(combined) };
if o.status.success() {
logger.log_command("pg_restore", payload, Some(0), Some(duration_ms));
logger.log("info", format!("Restore completed successfully for {}", cfg.name));
Ok(())
} else {
logger.log_command("pg_restore", payload, Some(exit_code), Some(duration_ms));
logger.log("error", format!("Restore failed with status {:?} for {}", o.status, cfg.name));
anyhow::bail!("Postgres restore failed for {}", cfg.name);
}
}
Err(e) => {
logger.log_command("pg_restore", Some(e.to_string()), Some(-1), Some(duration_ms));
logger.log("error", format!("Error executing pg_restore for {}: {:?}", cfg.name, e));
Err(e.into())
}
}
}
+9
View File
@@ -0,0 +1,9 @@
mod command;
mod prepare;
mod run;
mod toc;
pub use run::run;
pub(crate) use command::run_pg_restore;
pub(crate) use prepare::prepare_archive;
pub(crate) use toc::toc_creates_public_schema;
+67
View File
@@ -0,0 +1,67 @@
use anyhow::Result;
use std::path::{Path, PathBuf};
use std::process::Command;
use crate::domain::postgres::connection::sniff_format;
use crate::domain::postgres::format::PostgresDumpFormat;
use crate::services::backup::logger::JobLogger;
pub(crate) struct PreparedArchive {
path: PathBuf,
_tmp: Option<tempfile::TempDir>,
toc: String,
}
impl PreparedArchive {
pub(crate) fn path(&self) -> &Path {
&self.path
}
pub(crate) fn toc(&self) -> &str {
&self.toc
}
}
pub(crate) fn prepare_archive(
format: PostgresDumpFormat,
restore_file: &Path,
pg_restore: &Path,
logger: &JobLogger,
) -> Result<PreparedArchive> {
let sniffed = sniff_format(restore_file)?;
if sniffed != format {
logger.log("warn", format!("Declared format {:?} != sniffed {:?}; using sniffed", format, sniffed));
}
let format = sniffed;
let (path, tmp) = match format {
PostgresDumpFormat::Fc => (restore_file.to_path_buf(), None),
PostgresDumpFormat::Fd => {
let tar_gz = std::fs::File::open(restore_file)?;
let dec = flate2::read::GzDecoder::new(tar_gz);
let mut archive = tar::Archive::new(dec);
let tmp_dir = tempfile::TempDir::new()?;
archive.unpack(tmp_dir.path())?;
let dump_dir = if tmp_dir.path().join("toc.dat").exists() {
tmp_dir.path().to_path_buf()
} else {
std::fs::read_dir(tmp_dir.path())?
.filter_map(|e| e.ok())
.find(|entry| entry.path().join("toc.dat").exists())
.map(|e| e.path())
.ok_or_else(|| anyhow::anyhow!("Invalid FD archive: toc.dat not found"))?
};
(dump_dir, Some(tmp_dir))
}
};
let toc_out = Command::new(pg_restore).arg("-l").arg(&path).output()?;
if !toc_out.status.success() {
let stderr = String::from_utf8_lossy(&toc_out.stderr).to_string();
logger.log("error", format!("pg_restore -l failed: {}", stderr));
anyhow::bail!("Archive validation failed (pg_restore -l): {}", stderr);
}
let toc = String::from_utf8_lossy(&toc_out.stdout).to_string();
Ok(PreparedArchive { path, _tmp: tmp, toc })
}
+108
View File
@@ -0,0 +1,108 @@
use anyhow::Result;
use std::collections::HashMap;
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use super::{prepare_archive, run_pg_restore, toc_creates_public_schema};
use crate::domain::postgres::clean_mode::RestoreCleanMode;
use crate::domain::postgres::connection::{
can_drop_database, drop_all_schemas, drop_and_recreate_database, pg_restore_binary_name,
recreate_public_schema, select_pg_path, server_version, terminate_connections,
};
use crate::domain::postgres::format::PostgresDumpFormat;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
pub async fn run(
cfg: DatabaseConfig,
format: PostgresDumpFormat,
restore_file: PathBuf,
env: HashMap<String, String>,
logger: Arc<JobLogger>,
) -> Result<()> {
let handle = tokio::runtime::Handle::current();
tokio::task::spawn_blocking(move || -> Result<()> {
logger.log("info", format!("Starting restore for database {}", cfg.name));
let version = match handle.block_on(server_version(&cfg)) {
Ok(v) => {
logger.log("debug", format!("Postgres version detected: {}", v));
v
}
Err(e) => {
logger.log("error", format!("Failed to get server version for {}: {:?}", cfg.name, e));
return Err(e.into());
}
};
let pg_restore = select_pg_path(&version).join(pg_restore_binary_name());
logger.log("debug", format!("Using pg_restore at {:?}", pg_restore));
let keep_ownership = cfg.options
.get("keep_ownership")
.and_then(|v| v.as_bool())
.unwrap_or(false);
if keep_ownership {
logger.log("info", format!("Restoring ownership and privileges for {}", cfg.name));
} else {
logger.log("info", format!("Stripping ownership and privileges for {} (--no-owner --no-privileges)", cfg.name));
}
let (mode, bad_value) = RestoreCleanMode::from_config(&cfg);
if let Some(v) = bad_value {
logger.log("warn", format!("Unknown clean_mode '{}' for {}, falling back to 'clean'", v, cfg.name));
}
let prepared = prepare_archive(format, &restore_file, &pg_restore, &logger)?;
match mode {
RestoreCleanMode::DropSchemas => {
handle.block_on(terminate_connections(&cfg))?;
let owner = cfg.username.clone();
let dropped = handle.block_on(drop_all_schemas(&cfg))?;
logger.log("warn", format!("clean_mode=drop_schemas dropped schemas {:?} in {}", dropped, cfg.database));
if !toc_creates_public_schema(prepared.toc()) {
handle.block_on(recreate_public_schema(&cfg, &owner))?;
}
}
RestoreCleanMode::DropDatabase => {
if !handle.block_on(can_drop_database(&cfg))? {
anyhow::bail!(
"clean_mode=drop_database requires CREATEDB + ownership on {}; use clean_mode=drop_schemas instead",
cfg.database
);
}
logger.log("warn", format!("clean_mode=drop_database DROPPING database {} before restore", cfg.database));
handle.block_on(drop_and_recreate_database(&cfg))?;
}
RestoreCleanMode::Clean | RestoreCleanMode::None => {
handle.block_on(terminate_connections(&cfg))?;
}
}
let mut cmd = Command::new(&pg_restore);
if !keep_ownership {
cmd.args(["--no-owner", "--no-privileges"]);
}
if mode.uses_pg_restore_clean() {
cmd.args(["--clean", "--if-exists"]);
}
cmd.arg("--host").arg(&cfg.host)
.arg("--port").arg(cfg.port.to_string())
.arg("--username").arg(&cfg.username)
.arg("--dbname").arg(&cfg.database)
.arg("-v");
if matches!(format, PostgresDumpFormat::Fd) {
cmd.arg("-j").arg("4");
}
cmd.arg(prepared.path()).envs(env);
run_pg_restore(cmd, &logger, &cfg)?;
logger.log("info", format!("Restore finished for database {}", cfg.name));
Ok(())
})
.await?
}
+8
View File
@@ -0,0 +1,8 @@
pub(crate) fn toc_creates_public_schema(toc: &str) -> bool {
toc.lines().any(|l| {
l.split(" SCHEMA - ")
.nth(1)
.and_then(|rest| rest.split_whitespace().next())
== Some("public")
})
}
-1
View File
@@ -22,7 +22,6 @@ async fn main() {
eprintln!("Failed to clean locks on startup: {:?}", e);
}
// Best-effort cleanup of ephemeral helper containers orphaned by a crash.
match crate::domain::docker_volume::docker::client() {
Ok(docker) => match crate::domain::docker_volume::docker::sweep_ephemeral(&docker).await {
Ok(n) if n > 0 => tracing::info!("Removed {n} orphaned ephemeral helper container(s)"),
+15
View File
@@ -1,5 +1,6 @@
#![allow(dead_code)]
use crate::services::config::DatabaseConfig;
use crate::utils::deserializer::{deserialize_snake_case, string_or_number_to_string};
use serde::{Deserialize, Serialize};
use toml::Value;
@@ -24,6 +25,8 @@ pub struct DatabaseStorage {
#[serde(deserialize_with = "deserialize_snake_case")]
pub config: Value,
pub provider: String,
#[serde(default, rename = "folderName")]
pub folder_name: Option<String>,
}
#[derive(Debug, Deserialize)]
@@ -31,7 +34,19 @@ pub struct DatabaseStatus {
pub dbms: String,
#[serde(rename = "generatedId")]
pub generated_id: String,
#[serde(default)]
pub storages: Vec<DatabaseStorage>,
#[serde(default)]
pub storages_encrypted: Option<bool>,
#[serde(default)]
pub storages_ciphertext: Option<String>,
#[serde(default)]
pub config_encrypted: Option<bool>,
#[serde(default)]
pub config_ciphertext: Option<String>,
/// Filled in memory after decrypting `config_ciphertext`; never on the wire.
#[serde(skip)]
pub resolved_config: Option<DatabaseConfig>,
pub encrypt: bool,
pub data: DatabaseData,
}
+114 -121
View File
@@ -1,7 +1,7 @@
#![allow(dead_code)]
use crate::core::context::Context;
use serde::Deserialize;
use serde::{Deserialize, Serialize};
use serde_json;
use std::collections::HashMap;
use std::fs::File;
@@ -12,7 +12,7 @@ use toml;
use tracing::info;
use uuid::Uuid;
#[derive(Debug, Deserialize, Clone)]
#[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(rename_all = "lowercase")]
pub enum DbType {
Mysql,
@@ -49,7 +49,7 @@ impl DbType {
}
#[allow(dead_code)]
#[derive(Debug, Deserialize, Clone)]
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct DatabaseConfig {
pub name: String,
pub database: String,
@@ -68,7 +68,7 @@ pub struct DatabaseConfig {
}
#[allow(dead_code)]
#[derive(Debug, Deserialize, Clone)]
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct DatabasesConfig {
pub databases: Vec<DatabaseConfig>,
}
@@ -98,6 +98,106 @@ pub struct InputDatabasesConfig {
pub databases: Vec<InputDatabaseConfig>,
}
fn required<T: Clone>(opt: &Option<T>, db_name: &str, field_name: &str) -> Result<T, String> {
match opt {
Some(v) => Ok(v.clone()),
None => Err(format!(
"Missing required field '{}' for database '{}'",
field_name, db_name
)),
}
}
fn optional<T: Clone + Default>(opt: &Option<T>) -> T {
opt.clone().unwrap_or_default()
}
pub fn build_config(db: InputDatabaseConfig) -> Result<DatabaseConfig, String> {
if Uuid::parse_str(&db.generated_id).is_err() {
return Err(format!("Invalid UUID for database '{}'", db.name));
}
let username = match db.db_type {
DbType::Postgresql
| DbType::PostgresqlCluster
| DbType::Mysql
| DbType::Mariadb
| DbType::Mssql => required(&db.username, &db.name, "username")?,
_ => optional(&db.username),
};
let password = match db.db_type {
DbType::Postgresql
| DbType::PostgresqlCluster
| DbType::Mysql
| DbType::Mariadb
| DbType::Mssql => required(&db.password, &db.name, "password")?,
_ => optional(&db.password),
};
let host = match db.db_type {
DbType::Postgresql
| DbType::PostgresqlCluster
| DbType::Mysql
| DbType::Mariadb
| DbType::MongoDB
| DbType::Redis
| DbType::Firebird
| DbType::Valkey
| DbType::Mssql => required(&db.host, &db.name, "host")?,
DbType::Sqlite | DbType::DockerVolume => optional(&db.host),
};
let port = match db.db_type {
DbType::Postgresql
| DbType::PostgresqlCluster
| DbType::Mysql
| DbType::Mariadb
| DbType::Redis
| DbType::Firebird
| DbType::Valkey
| DbType::Mssql => required(&db.port, &db.name, "port")?,
DbType::MongoDB | DbType::Sqlite | DbType::DockerVolume => db.port.unwrap_or(0),
};
let database_name = match db.db_type {
DbType::Sqlite | DbType::Redis | DbType::Valkey | DbType::DockerVolume => {
optional(&db.database)
}
DbType::PostgresqlCluster => db
.database
.clone()
.unwrap_or_else(|| "postgres".to_string()),
_ => required(&db.database, &db.name, "database")?,
};
let path_val = match db.db_type {
DbType::Sqlite => required(&db.path, &db.name, "path")?,
_ => optional(&db.path),
};
let max_packet_size = match db.db_type {
DbType::Mysql | DbType::Mariadb => db.max_packet_size.unwrap_or_else(|| "512M".to_string()),
_ => String::new(),
};
let volume_name = match db.db_type {
DbType::DockerVolume => required(&db.volume_name, &db.name, "volume_name")?,
_ => optional(&db.volume_name),
};
Ok(DatabaseConfig {
name: db.name,
database: database_name,
db_type: db.db_type,
username,
password,
host,
port,
generated_id: db.generated_id,
path: path_val,
max_packet_size,
volume_name,
container_name: db.container_name.clone(),
options: db.options.unwrap_or_default(),
})
}
pub struct ConfigService {
ctx: Arc<Context>,
}
@@ -150,128 +250,21 @@ impl ConfigService {
_ => return Err("Unsupported config file format. Use .json or .toml".to_string()),
};
fn required<T: Clone>(
opt: &Option<T>,
db_name: &str,
field_name: &str,
) -> Result<T, String> {
match opt {
Some(v) => Ok(v.clone()),
None => {
let msg = format!(
"Missing required field '{}' for database '{}'",
field_name, db_name
);
Err(msg)
}
}
}
fn optional<T: Clone>(opt: &Option<T>) -> T
where
T: Default,
{
opt.clone().unwrap_or_default()
}
let mut databases = Vec::with_capacity(input_config.databases.len());
for db in input_config.databases {
if Uuid::parse_str(&db.generated_id).is_err() {
return Err(format!("Invalid UUID for database '{}'", db.name));
}
let username = match db.db_type {
DbType::Postgresql
| DbType::PostgresqlCluster
| DbType::Mysql
| DbType::Mariadb
| DbType::Mssql => required(&db.username, &db.name, "username")?,
_ => optional(&db.username),
};
let password = match db.db_type {
DbType::Postgresql
| DbType::PostgresqlCluster
| DbType::Mysql
| DbType::Mariadb
| DbType::Mssql => required(&db.password, &db.name, "password")?,
_ => optional(&db.password),
};
let host = match db.db_type {
DbType::Postgresql
| DbType::PostgresqlCluster
| DbType::Mysql
| DbType::Mariadb
| DbType::MongoDB
| DbType::Redis
| DbType::Firebird
| DbType::Valkey
| DbType::Mssql => required(&db.host, &db.name, "host")?,
DbType::Sqlite | DbType::DockerVolume => optional(&db.host),
};
let port = match db.db_type {
DbType::Postgresql
| DbType::PostgresqlCluster
| DbType::Mysql
| DbType::Mariadb
| DbType::MongoDB
| DbType::Redis
| DbType::Firebird
| DbType::Valkey
| DbType::Mssql => required(&db.port, &db.name, "port")?,
DbType::Sqlite | DbType::DockerVolume => db.port.unwrap_or(0),
};
let database_name = match db.db_type {
DbType::Sqlite | DbType::Redis | DbType::Valkey | DbType::DockerVolume => {
optional(&db.database)
}
DbType::PostgresqlCluster => db
.database
.clone()
.unwrap_or_else(|| "postgres".to_string()),
_ => required(&db.database, &db.name, "database")?,
};
let path_val = match db.db_type {
DbType::Sqlite => required(&db.path, &db.name, "path")?,
_ => optional(&db.path),
};
let max_packet_size = match db.db_type {
DbType::Mysql | DbType::Mariadb => {
db.max_packet_size.unwrap_or_else(|| "512M".to_string())
}
_ => String::new(),
};
let volume_name = match db.db_type {
DbType::DockerVolume => required(&db.volume_name, &db.name, "volume_name")?,
_ => optional(&db.volume_name),
};
let container_name = db.container_name.clone();
databases.push(DatabaseConfig {
name: db.name,
database: database_name,
db_type: db.db_type,
username,
password,
host,
port,
generated_id: db.generated_id,
path: path_val,
max_packet_size,
volume_name,
container_name,
options: db.options.unwrap_or_default(),
});
databases.push(build_config(db)?);
}
info!("Databases: {} instances loaded", databases.len());
Ok(DatabasesConfig { databases })
}
pub fn load_optional(&self, file_path: Option<&str>) -> DatabasesConfig {
self.load(file_path).unwrap_or_else(|e| {
tracing::warn!(
"Local databases config unavailable ({e}); continuing with dashboard-defined databases only"
);
DatabasesConfig { databases: Vec::new() }
})
}
}
+53
View File
@@ -0,0 +1,53 @@
#![allow(dead_code)]
use crate::services::api::models::agent::status::PingResult;
use crate::services::config::{DatabaseConfig, DatabasesConfig};
use std::path::Path;
pub fn merge(local: &[DatabaseConfig], dashboard: &[DatabaseConfig]) -> DatabasesConfig {
let mut databases: Vec<DatabaseConfig> = local.to_vec();
for d in dashboard {
if let Some(slot) = databases
.iter_mut()
.find(|c| c.generated_id == d.generated_id)
{
*slot = d.clone();
} else {
databases.push(d.clone());
}
}
DatabasesConfig { databases }
}
pub fn collect_configs(ping: &PingResult) -> Vec<DatabaseConfig> {
ping.databases
.iter()
.filter_map(|db| db.resolved_config.clone())
.collect()
}
pub fn load_cache(path: &Path) -> Vec<DatabaseConfig> {
let contents = match std::fs::read_to_string(path) {
Ok(c) => c,
Err(_) => return Vec::new(),
};
match serde_json::from_str::<DatabasesConfig>(&contents) {
Ok(cfg) => cfg.databases,
Err(e) => {
tracing::warn!("Dashboard cache at {:?} is corrupt ({e}); ignoring", path);
Vec::new()
}
}
}
pub fn persist_cache(path: &Path, databases: &[DatabaseConfig]) -> std::io::Result<()> {
let wrapper = DatabasesConfig {
databases: databases.to_vec(),
};
let json = serde_json::to_string_pretty(&wrapper)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
let tmp = path.with_extension("json.tmp");
std::fs::write(&tmp, json)?;
std::fs::rename(&tmp, path)?;
Ok(())
}
+1
View File
@@ -2,6 +2,7 @@ pub mod api;
pub mod backup;
pub mod config;
pub mod cron;
pub mod dashboard_config;
pub mod restore;
pub mod status;
pub mod storage;
+51 -9
View File
@@ -1,16 +1,39 @@
#![allow(dead_code)]
use crate::core::context::Context;
use crate::domain::factory::DatabaseFactory;
use crate::services::api::endpoints::status::DatabasePayload;
use crate::services::api::models::agent::status::DatabaseStatus;
use crate::services::api::models::agent::status::DatabaseStorage;
use crate::services::api::models::agent::status::PingResult;
use crate::services::config::DatabaseConfig;
use crate::services::config::{build_config, DatabaseConfig, InputDatabaseConfig};
use crate::settings::CONFIG;
use crate::utils::file::decrypt_json_gcm;
use futures_util::future::try_join_all;
use reqwest::Client;
use std::error::Error;
use std::sync::Arc;
use futures_util::future::try_join_all;
use tracing::info;
use crate::domain::factory::DatabaseFactory;
pub fn resolve_dashboard_config(
status: &mut DatabaseStatus,
master_key_b64: &str,
) -> Result<(), String> {
if status.config_encrypted != Some(true) {
return Ok(());
}
let ciphertext = status
.config_ciphertext
.as_deref()
.ok_or("config_encrypted set but config_ciphertext missing")?;
let plaintext = decrypt_json_gcm(ciphertext, master_key_b64)
.map_err(|e| format!("Failed to decrypt config: {e}"))?;
let input: InputDatabaseConfig = serde_json::from_slice(&plaintext)
.map_err(|e| format!("Failed to parse decrypted config: {e}"))?;
status.resolved_config = Some(build_config(input)?);
Ok(())
}
pub struct StatusService {
ctx: Arc<Context>,
@@ -28,12 +51,12 @@ impl StatusService {
pub async fn ping(&self, databases: &[DatabaseConfig]) -> Result<PingResult, Box<dyn Error>> {
let edge_key = &self.ctx.edge_key;
let databases_payload: Vec<DatabasePayload> = try_join_all(
databases.into_iter().map(|db| async move {
let databases_payload: Vec<DatabasePayload> =
try_join_all(databases.into_iter().map(|db| async move {
let db_engine = DatabaseFactory::create_for_backup(db.clone()).await;
let reachable = db_engine.ping().await?;
info!("Ping {} => {:?}",db.name, reachable);
info!("Ping {} => {:?}", db.name, reachable);
Ok::<DatabasePayload, anyhow::Error>(DatabasePayload {
name: &db.name,
@@ -41,16 +64,35 @@ impl StatusService {
generated_id: &db.generated_id,
ping_status: reachable,
})
})
).await?;
}))
.await?;
let version_str = CONFIG.app_version.as_str();
let result = self
let mut result = self
.ctx
.api
.agent_status(&edge_key.agent_id, &version_str, databases_payload)
.await?
.unwrap();
for db in result.databases.iter_mut() {
if db.storages_encrypted == Some(true) {
let ciphertext = db
.storages_ciphertext
.as_deref()
.ok_or("storages_encrypted set but storages_ciphertext missing")?;
let plaintext = decrypt_json_gcm(ciphertext, &edge_key.master_key_b64)
.map_err(|e| format!("Failed to decrypt storages: {e}"))?;
db.storages = serde_json::from_slice::<Vec<DatabaseStorage>>(&plaintext)
.map_err(|e| format!("Failed to parse decrypted storages: {e}"))?;
}
if let Err(e) = resolve_dashboard_config(db, &edge_key.master_key_b64) {
tracing::warn!("Skipping dashboard config for {}: {e}", db.generated_id);
}
}
Ok(result)
}
}
@@ -95,7 +95,7 @@ impl StorageProvider for AzureBlobProvider {
};
let file_name = full_file_name(encrypt);
let remote_file_path = full_file_path(&file_name);
let remote_file_path = full_file_path(&file_name, storage.folder_name.as_deref());
info!(
"Starting block upload to azure blob {}/{}",
config.container_name, remote_file_path
@@ -85,7 +85,7 @@ impl StorageProvider for GoogleCloudStorageProvider {
let file_name = full_file_name(encrypt);
info!("Uploading file {}", file_name);
let remote_file_path = full_file_path(&file_name);
let remote_file_path = full_file_path(&file_name, storage.folder_name.as_deref());
let client = match build_client(&config).await {
Ok(c) => c,
@@ -85,7 +85,7 @@ impl StorageProvider for GoogleDriveProvider {
info!("Uploading file {}", file_name);
let remote_file_path = full_file_path(&file_name);
let remote_file_path = full_file_path(&file_name, storage.folder_name.as_deref());
match upload_stream_to_google_drive(
&config,
+1 -1
View File
@@ -38,7 +38,7 @@ impl StorageProvider for LocalProvider {
let encrypt = encrypt.unwrap_or(false);
let file_name = full_file_name(encrypt);
let remote_file_path = full_file_path(&file_name);
let remote_file_path = full_file_path(&file_name, storage.folder_name.as_deref());
let total_size = match fs::metadata(&file_path).await {
Ok(meta) => meta.len(),
+13 -7
View File
@@ -13,7 +13,9 @@ use aws_config::retry::RetryConfig;
use aws_sdk_s3 as s3;
use aws_sdk_s3::config::BehaviorVersion;
use aws_sdk_s3::config::Region;
use aws_sdk_s3::config::RequestChecksumCalculation;
use aws_sdk_s3::config::retry::ReconnectMode;
use aws_sdk_s3::error::DisplayErrorContext;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use futures::StreamExt;
@@ -135,6 +137,7 @@ impl StorageProvider for S3Provider {
.credentials_provider(credentials)
.region(region)
.force_path_style(true)
.request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
.endpoint_url(endpoint)
.behavior_version(BehaviorVersion::latest())
.build();
@@ -148,7 +151,7 @@ impl StorageProvider for S3Provider {
info!("Uploading file {}", file_name);
let bucket = &config.bucket_name;
let remote_file_path = full_file_path(&file_name);
let remote_file_path = full_file_path(&file_name, storage.folder_name.as_deref());
info!("S3 key {:}", remote_file_path);
info!(
"Starting multipart upload to s3://{}/{}",
@@ -164,11 +167,12 @@ impl StorageProvider for S3Provider {
{
Ok(r) => r,
Err(e) => {
error!("Failed to create multipart upload: {}", e);
let detail = DisplayErrorContext(&e).to_string();
error!("Failed to create multipart upload: {}", detail);
return UploadResult {
storage_id: storage.id.clone(),
success: false,
error: Some(e.to_string()),
error: Some(detail),
remote_file_path: None,
total_size: None,
};
@@ -251,7 +255,8 @@ impl StorageProvider for S3Provider {
}
}
Err(e) => {
error!("Failed to upload part {}: {}", part_number, e);
let detail = DisplayErrorContext(&e).to_string();
error!("Failed to upload part {}: {}", part_number, detail);
let _ = client
.abort_multipart_upload()
.bucket(bucket)
@@ -262,7 +267,7 @@ impl StorageProvider for S3Provider {
return UploadResult {
storage_id: storage.id.clone(),
success: false,
error: Some(e.to_string()),
error: Some(detail),
remote_file_path: None,
total_size: None,
};
@@ -317,7 +322,8 @@ impl StorageProvider for S3Provider {
}
}
Err(e) => {
error!("Failed to complete multipart upload: {}", e);
let detail = DisplayErrorContext(&e).to_string();
error!("Failed to complete multipart upload: {}", detail);
let _ = client
.abort_multipart_upload()
.bucket(bucket)
@@ -328,7 +334,7 @@ impl StorageProvider for S3Provider {
UploadResult {
storage_id: storage.id.clone(),
success: false,
error: Some(e.to_string()),
error: Some(detail),
remote_file_path: None,
total_size: None,
}
+700 -14
View File
@@ -1,9 +1,14 @@
use crate::domain::factory::DatabaseFactory;
use crate::domain::postgres::connection::{pg_restore_binary_name, select_pg_path, server_version};
use crate::domain::postgres::format::PostgresDumpFormat;
use crate::domain::postgres::restore::prepare_archive;
use crate::services::backup::logger::JobLogger;
use crate::services::config::{DatabaseConfig, DbType};
use crate::tests::init_tracing_for_test;
use crate::utils::compress::{compress_to_tar_gz_large, decompress_large_tar_gz};
use oauth2::url;
use std::path::PathBuf;
use std::sync::Arc;
use tempfile::TempDir;
use testcontainers::runners::AsyncRunner;
use testcontainers::{ContainerAsync, ImageExt};
@@ -63,7 +68,6 @@ async fn postgres_ping_test() {
async fn is_superuser_detects_superuser_role() {
init_tracing_for_test();
// The testcontainer's POSTGRES_USER ("testuser") is the bootstrap superuser.
let (_container, config) = create_config().await;
let is_super = crate::domain::postgres::connection::is_superuser(&config)
@@ -73,6 +77,31 @@ async fn is_superuser_detects_superuser_role() {
assert!(is_super);
}
#[tokio::test]
async fn can_drop_database_false_for_unprivileged_role() {
init_tracing_for_test();
let (_container, admin) = create_config().await;
let a = crate::domain::postgres::connection::connect(&admin)
.await
.unwrap();
a.batch_execute("DROP ROLE IF EXISTS lowpriv; CREATE ROLE lowpriv LOGIN PASSWORD 'x';")
.await
.unwrap();
let mut low = admin.clone();
low.username = "lowpriv".into();
low.password = "x".into();
assert_eq!(
crate::domain::postgres::connection::can_drop_database(&low)
.await
.unwrap(),
false
);
}
#[tokio::test]
async fn postgres_backup_restore_test() {
init_tracing_for_test();
@@ -170,17 +199,585 @@ async fn postgres_password_with_slash_test() {
assert_eq!(reachable, true);
}
fn pg_dump_env(config: &DatabaseConfig) -> std::collections::HashMap<String, String> {
let mut env = std::env::vars().collect::<std::collections::HashMap<_, _>>();
env.insert("PGPASSWORD".to_string(), config.password.clone());
env
}
#[tokio::test]
async fn prepare_archive_fd_locates_toc_dir() {
init_tracing_for_test();
let (_container, config) = create_config().await;
let temp_dir = TempDir::new().unwrap();
let backup_path = crate::domain::postgres::backup::run(
config.clone(),
PostgresDumpFormat::Fd,
temp_dir.path().to_path_buf(),
pg_dump_env(&config),
Arc::new(JobLogger::new()),
)
.await
.unwrap();
let compression = compress_to_tar_gz_large(&backup_path, Arc::new(JobLogger::new()))
.await
.unwrap();
assert!(compression.compressed_path.is_file());
let version = server_version(&config).await.unwrap();
let pg_restore = select_pg_path(&version).join(pg_restore_binary_name());
let logger = JobLogger::new();
let prepared = prepare_archive(
PostgresDumpFormat::Fd,
&compression.compressed_path,
&pg_restore,
&logger,
)
.unwrap();
assert!(prepared.path().join("toc.dat").exists());
assert!(!prepared.toc().is_empty());
}
#[tokio::test]
async fn prepare_archive_fc_returns_file_path_unchanged() {
init_tracing_for_test();
let (_container, config) = create_config().await;
let temp_dir = TempDir::new().unwrap();
let backup_path = crate::domain::postgres::backup::run(
config.clone(),
PostgresDumpFormat::Fc,
temp_dir.path().to_path_buf(),
pg_dump_env(&config),
Arc::new(JobLogger::new()),
)
.await
.unwrap();
assert!(backup_path.is_file());
let version = server_version(&config).await.unwrap();
let pg_restore = select_pg_path(&version).join(pg_restore_binary_name());
let logger = JobLogger::new();
let prepared = prepare_archive(PostgresDumpFormat::Fc, &backup_path, &pg_restore, &logger).unwrap();
assert_eq!(prepared.path(), backup_path.as_path());
assert!(!prepared.toc().is_empty());
}
#[tokio::test]
async fn restore_run_unified_fc_roundtrip() {
init_tracing_for_test();
let (_container, config) = create_config().await;
let client = crate::domain::postgres::connection::connect(&config)
.await
.unwrap();
client.execute("CREATE TABLE t(id int);", &[]).await.unwrap();
let temp_dir = TempDir::new().unwrap();
let dump_file = crate::domain::postgres::backup::run(
config.clone(),
PostgresDumpFormat::Fc,
temp_dir.path().to_path_buf(),
pg_dump_env(&config),
Arc::new(JobLogger::new()),
)
.await
.unwrap();
assert!(dump_file.is_file());
let format = crate::domain::postgres::connection::detect_format_from_file(&dump_file);
let result = crate::domain::postgres::restore::run(
config.clone(),
format,
dump_file,
pg_dump_env(&config),
Arc::new(JobLogger::new()),
)
.await;
assert!(result.is_ok(), "restore::run failed: {:?}", result);
}
#[tokio::test]
async fn drop_all_schemas_removes_user_schema() {
init_tracing_for_test();
let (_container, config) = create_config().await;
let client = crate::domain::postgres::connection::connect(&config)
.await
.unwrap();
client
.batch_execute("CREATE SCHEMA IF NOT EXISTS extra_ns; CREATE TABLE IF NOT EXISTS extra_ns.t(id int);")
.await
.unwrap();
let dropped = crate::domain::postgres::connection::drop_all_schemas(&config)
.await
.unwrap();
assert!(dropped.iter().any(|s| s == "extra_ns"));
let client = crate::domain::postgres::connection::connect(&config)
.await
.unwrap();
let row = client
.query_one(
"SELECT count(*) FROM pg_namespace WHERE nspname = 'extra_ns'",
&[],
)
.await
.unwrap();
let n: i64 = row.get(0);
assert_eq!(n, 0);
}
#[tokio::test]
async fn restore_drop_schemas_removes_extra_objects() {
init_tracing_for_test();
let (_container, config) = create_config().await;
let client = crate::domain::postgres::connection::connect(&config)
.await
.unwrap();
client
.batch_execute("CREATE TABLE base_t(id int);")
.await
.unwrap();
let temp_dir = TempDir::new().unwrap();
let dump_file = crate::domain::postgres::backup::run(
config.clone(),
PostgresDumpFormat::Fc,
temp_dir.path().to_path_buf(),
pg_dump_env(&config),
Arc::new(JobLogger::new()),
)
.await
.unwrap();
let client = crate::domain::postgres::connection::connect(&config)
.await
.unwrap();
client
.batch_execute("CREATE TABLE orphan_only_here(id int);")
.await
.unwrap();
let mut cfg = config.clone();
cfg.options
.insert("clean_mode".into(), serde_json::json!("drop_schemas"));
let format = crate::domain::postgres::connection::detect_format_from_file(&dump_file);
let result = crate::domain::postgres::restore::run(
cfg.clone(),
format,
dump_file,
pg_dump_env(&cfg),
Arc::new(JobLogger::new()),
)
.await;
assert!(result.is_ok(), "restore::run failed: {:?}", result);
let client = crate::domain::postgres::connection::connect(&config)
.await
.unwrap();
let n: i64 = client
.query_one(
"SELECT count(*) FROM information_schema.tables WHERE table_name = 'orphan_only_here'",
&[],
)
.await
.unwrap()
.get(0);
assert_eq!(n, 0);
let n: i64 = client
.query_one(
"SELECT count(*) FROM information_schema.tables WHERE table_name = 'base_t'",
&[],
)
.await
.unwrap()
.get(0);
assert_eq!(n, 1);
}
#[tokio::test]
async fn restore_clean_leaves_divergent_object() {
init_tracing_for_test();
let (_container, config) = create_config().await;
let client = crate::domain::postgres::connection::connect(&config)
.await
.unwrap();
client
.batch_execute("CREATE TABLE base_t(id int);")
.await
.unwrap();
let temp_dir = TempDir::new().unwrap();
let dump_file = crate::domain::postgres::backup::run(
config.clone(),
PostgresDumpFormat::Fc,
temp_dir.path().to_path_buf(),
pg_dump_env(&config),
Arc::new(JobLogger::new()),
)
.await
.unwrap();
let client = crate::domain::postgres::connection::connect(&config)
.await
.unwrap();
client
.batch_execute("CREATE TABLE survives_clean(id int);")
.await
.unwrap();
let mut cfg = config.clone();
cfg.options
.insert("clean_mode".into(), serde_json::json!("clean"));
let format = crate::domain::postgres::connection::detect_format_from_file(&dump_file);
let result = crate::domain::postgres::restore::run(
cfg.clone(),
format,
dump_file,
pg_dump_env(&cfg),
Arc::new(JobLogger::new()),
)
.await;
assert!(result.is_ok(), "restore::run failed: {:?}", result);
let client = crate::domain::postgres::connection::connect(&config)
.await
.unwrap();
let n: i64 = client
.query_one(
"SELECT count(*) FROM information_schema.tables WHERE table_name = 'survives_clean'",
&[],
)
.await
.unwrap()
.get(0);
assert_eq!(n, 1, "clean mode is not a reset; divergent object survives");
}
#[tokio::test]
async fn restore_unknown_clean_mode_falls_back() {
init_tracing_for_test();
let (_container, config) = create_config().await;
let temp_dir = TempDir::new().unwrap();
let dump_file = crate::domain::postgres::backup::run(
config.clone(),
PostgresDumpFormat::Fc,
temp_dir.path().to_path_buf(),
pg_dump_env(&config),
Arc::new(JobLogger::new()),
)
.await
.unwrap();
let mut cfg = config.clone();
cfg.options
.insert("clean_mode".into(), serde_json::json!("wat"));
let format = crate::domain::postgres::connection::detect_format_from_file(&dump_file);
let logger = Arc::new(JobLogger::new());
let result = crate::domain::postgres::restore::run(
cfg.clone(),
format,
dump_file,
pg_dump_env(&cfg),
logger.clone(),
)
.await;
assert!(result.is_ok(), "restore::run failed: {:?}", result);
let entries = Arc::try_unwrap(logger)
.expect("logger should have a single owner after run() completes")
.into_entries();
assert!(entries
.iter()
.any(|e| e.message.contains("Unknown clean_mode 'wat'")));
}
#[tokio::test]
async fn drop_database_preflight_preserves_data_when_unprivileged() {
init_tracing_for_test();
let (_container, admin) = create_config().await;
let a = crate::domain::postgres::connection::connect(&admin)
.await
.unwrap();
a.batch_execute("DROP ROLE IF EXISTS lowpriv2; CREATE ROLE lowpriv2 LOGIN PASSWORD 'x';")
.await
.unwrap();
a.batch_execute("CREATE TABLE IF NOT EXISTS keep_me(id int);")
.await
.unwrap();
let temp_dir = TempDir::new().unwrap();
let dump_file = crate::domain::postgres::backup::run(
admin.clone(),
PostgresDumpFormat::Fc,
temp_dir.path().to_path_buf(),
pg_dump_env(&admin),
Arc::new(JobLogger::new()),
)
.await
.unwrap();
let mut low = admin.clone();
low.username = "lowpriv2".into();
low.password = "x".into();
low.options
.insert("clean_mode".into(), serde_json::json!("drop_database"));
let format = crate::domain::postgres::connection::detect_format_from_file(&dump_file);
let res = crate::domain::postgres::restore::run(
low.clone(),
format,
dump_file,
pg_dump_env(&low),
Arc::new(JobLogger::new()),
)
.await;
assert!(res.is_err(), "preflight must reject an unprivileged role");
let a = crate::domain::postgres::connection::connect(&admin)
.await
.unwrap();
let n: i64 = a
.query_one(
"SELECT count(*) FROM information_schema.tables WHERE table_name = 'keep_me'",
&[],
)
.await
.unwrap()
.get(0);
assert_eq!(n, 1, "preflight must fail before dropping anything");
}
#[tokio::test]
async fn drop_database_preserves_encoding_and_owner() {
init_tracing_for_test();
let (_container, config) = create_config().await;
let client = crate::domain::postgres::connection::connect(&config)
.await
.unwrap();
client
.batch_execute("CREATE TABLE base_t(id int);")
.await
.unwrap();
let temp_dir = TempDir::new().unwrap();
let dump_file = crate::domain::postgres::backup::run(
config.clone(),
PostgresDumpFormat::Fc,
temp_dir.path().to_path_buf(),
pg_dump_env(&config),
Arc::new(JobLogger::new()),
)
.await
.unwrap();
let before = crate::domain::postgres::connection::connect(&config)
.await
.unwrap()
.query_one(
"SELECT pg_encoding_to_char(encoding), datcollate FROM pg_database WHERE datname = current_database()",
&[],
)
.await
.unwrap();
let enc0: String = before.get(0);
let coll0: String = before.get(1);
let mut cfg = config.clone();
cfg.options
.insert("clean_mode".into(), serde_json::json!("drop_database"));
let format = crate::domain::postgres::connection::detect_format_from_file(&dump_file);
let result = crate::domain::postgres::restore::run(
cfg.clone(),
format,
dump_file,
pg_dump_env(&cfg),
Arc::new(JobLogger::new()),
)
.await;
assert!(result.is_ok(), "restore::run failed: {:?}", result);
let after = crate::domain::postgres::connection::connect(&config)
.await
.unwrap()
.query_one(
"SELECT pg_encoding_to_char(encoding), datcollate FROM pg_database WHERE datname = current_database()",
&[],
)
.await
.unwrap();
let enc1: String = after.get(0);
let coll1: String = after.get(1);
assert_eq!(enc0, enc1);
assert_eq!(coll0, coll1);
}
#[tokio::test]
async fn drop_database_force_wins_over_open_connection() {
init_tracing_for_test();
let (_container, config) = create_config().await;
let temp_dir = TempDir::new().unwrap();
let dump_file = crate::domain::postgres::backup::run(
config.clone(),
PostgresDumpFormat::Fc,
temp_dir.path().to_path_buf(),
pg_dump_env(&config),
Arc::new(JobLogger::new()),
)
.await
.unwrap();
let mut cfg = config.clone();
cfg.options
.insert("clean_mode".into(), serde_json::json!("drop_database"));
let squatter = crate::domain::postgres::connection::connect(&config)
.await
.unwrap();
let _keep = tokio::spawn(async move {
let _ = squatter.query_one("SELECT pg_sleep(5)", &[]).await;
});
let format = crate::domain::postgres::connection::detect_format_from_file(&dump_file);
let result = crate::domain::postgres::restore::run(
cfg.clone(),
format,
dump_file,
pg_dump_env(&cfg),
Arc::new(JobLogger::new()),
)
.await;
assert!(result.is_ok(), "restore::run failed: {:?}", result);
}
#[test]
fn sniff_format_detects_custom_and_gzip() {
use crate::domain::postgres::connection::sniff_format;
let dir = TempDir::new().unwrap();
let fc = dir.path().join("a.dump");
std::fs::write(&fc, b"PGDMP\x01\x0e\x00").unwrap();
assert_eq!(sniff_format(&fc).unwrap(), PostgresDumpFormat::Fc);
let fd = dir.path().join("b.gz");
std::fs::write(&fd, [0x1f, 0x8b, 0x08, 0x00]).unwrap();
assert_eq!(sniff_format(&fd).unwrap(), PostgresDumpFormat::Fd);
let bad = dir.path().join("c.bin");
std::fs::write(&bad, b"not a dump").unwrap();
assert!(sniff_format(&bad).is_err());
}
#[tokio::test]
async fn corrupt_archive_leaves_database_untouched() {
init_tracing_for_test();
let (_container, config) = create_config().await;
let client = crate::domain::postgres::connection::connect(&config)
.await
.unwrap();
client
.batch_execute("CREATE TABLE must_survive(id int);")
.await
.unwrap();
let mut cfg = config.clone();
cfg.options
.insert("clean_mode".into(), serde_json::json!("drop_schemas"));
let dir = TempDir::new().unwrap();
let broken_file = dir.path().join("broken.tar.gz");
std::fs::write(&broken_file, [0x1f, 0x8b, 0x08, 0x00, 0xde, 0xad, 0xbe, 0xef]).unwrap();
let result = crate::domain::postgres::restore::run(
cfg.clone(),
PostgresDumpFormat::Fd,
broken_file,
pg_dump_env(&cfg),
Arc::new(JobLogger::new()),
)
.await;
assert!(result.is_err(), "corrupt archive must be rejected before any destructive step");
let client = crate::domain::postgres::connection::connect(&config)
.await
.unwrap();
let n: i64 = client
.query_one(
"SELECT count(*) FROM information_schema.tables WHERE table_name = 'must_survive'",
&[],
)
.await
.unwrap()
.get(0);
assert_eq!(n, 1, "corrupt archive must never trigger the schema drop");
}
mod select_pg_path_tests {
use crate::domain::postgres::connection::{
pg_dump_binary_name, pg_dump_exists_in, pg_dumpall_binary_name, psql_binary_name,
select_pg_path_with,
pg_dump_binary_name, pg_dump_exists_in, pg_dumpall_binary_name, pg_restore_binary_name,
psql_binary_name, select_pg_path_with,
};
// `select_pg_path_with` takes the `PG_BIN_DIR` override as a plain
// argument, so these tests never touch process-global env state or the
// cached `CONFIG`. They stay deterministic regardless of whether — or at
// which version — a real PostgreSQL install exists on the host.
#[test]
fn respects_pg_bin_dir_override() {
let custom = if cfg!(target_os = "windows") {
@@ -194,8 +791,6 @@ mod select_pg_path_tests {
#[test]
fn pg_bin_dir_override_ignores_requested_version() {
// The override is taken as-is, regardless of which version was
// requested — this documents/locks in that behavior.
let custom = if cfg!(target_os = "windows") {
r"C:\custom\pg\bin"
} else {
@@ -207,10 +802,6 @@ mod select_pg_path_tests {
#[test]
fn empty_pg_bin_dir_falls_through_to_detection() {
// An empty override means "unset" (matches `CONFIG.pg_bin_dir` when
// `PG_BIN_DIR` is absent). It must not be returned as a literal empty
// path — resolution falls through to platform defaults / PATH lookup
// and yields a non-empty path.
let path = select_pg_path_with("17", "");
assert_ne!(path, std::path::PathBuf::from(""));
}
@@ -250,4 +841,99 @@ mod select_pg_path_tests {
assert_eq!(name, "psql");
}
}
#[test]
fn pg_restore_binary_name_is_platform_correct() {
let name = pg_restore_binary_name();
if cfg!(target_os = "windows") {
assert_eq!(name, "pg_restore.exe");
} else {
assert_eq!(name, "pg_restore");
}
}
}
mod quoting_tests {
use crate::domain::postgres::connection::{quote_ident, quote_literal};
#[test]
fn quote_ident_escapes_double_quotes() {
assert_eq!(quote_ident("devdb"), "\"devdb\"");
assert_eq!(quote_ident("a\"b"), "\"a\"\"b\"");
assert_eq!(quote_ident("drop\"; --"), "\"drop\"\"; --\"");
}
#[test]
fn quote_literal_escapes_single_quotes() {
assert_eq!(quote_literal("UTF8"), "'UTF8'");
assert_eq!(quote_literal("O'Brien"), "'O''Brien'");
}
}
mod clean_mode_tests {
use crate::domain::postgres::clean_mode::RestoreCleanMode as M;
use crate::services::config::{DatabaseConfig, DbType};
fn cfg_with(clean_mode: Option<&str>) -> DatabaseConfig {
let mut options = std::collections::HashMap::new();
if let Some(v) = clean_mode {
options.insert("clean_mode".to_string(), serde_json::json!(v));
}
DatabaseConfig {
name: "t".into(),
database: "testdb".into(),
db_type: DbType::Postgresql,
username: "testuser".into(),
password: "changeme".into(),
port: 5432,
host: "localhost".into(),
generated_id: "00000000-0000-0000-0000-000000000000".into(),
path: "".into(),
max_packet_size: "".into(),
volume_name: "".into(),
container_name: None,
options,
}
}
#[test]
fn clean_mode_parsing() {
assert_eq!(M::from_config(&cfg_with(None)), (M::Clean, None));
assert_eq!(M::from_config(&cfg_with(Some("clean"))), (M::Clean, None));
assert_eq!(M::from_config(&cfg_with(Some("none"))), (M::None, None));
assert_eq!(
M::from_config(&cfg_with(Some("drop_schemas"))),
(M::DropSchemas, None)
);
assert_eq!(
M::from_config(&cfg_with(Some("drop_database"))),
(M::DropDatabase, None)
);
assert_eq!(
M::from_config(&cfg_with(Some("bogus"))),
(M::Clean, Some("bogus".to_string()))
);
}
#[test]
fn uses_pg_restore_clean_behavior() {
assert!(M::Clean.uses_pg_restore_clean());
assert!(!M::DropSchemas.uses_pg_restore_clean());
assert!(!M::None.uses_pg_restore_clean());
assert!(!M::DropDatabase.uses_pg_restore_clean());
}
}
mod toc_tests {
use crate::domain::postgres::restore::toc_creates_public_schema;
#[test]
fn toc_public_schema_detection() {
let with = "215; 2615 2200 SCHEMA - public pg_database_owner";
let without_table = "200; 1259 12346 TABLE devschema users devuser";
let without_similar_schema = "216; 2615 2201 SCHEMA - publicish someowner";
assert!(toc_creates_public_schema(with));
assert!(!toc_creates_public_schema(without_table));
assert!(!toc_creates_public_schema(without_similar_schema));
}
}
+134 -1
View File
@@ -2,7 +2,7 @@ use serde_json::json;
use crate::services::api::models::agent::backup::{BackupResponse, BackupUploadResponse};
use crate::services::api::models::agent::restore::ResultRestoreResponse;
use crate::services::api::models::agent::status::PingResult;
use crate::services::api::models::agent::status::{DatabaseStatus, PingResult};
#[test]
fn backup_response_deserializes_nested_backup_id() {
@@ -110,3 +110,136 @@ fn ping_result_deserializes_and_normalizes_storage_config_keys() {
assert!(result.databases[0].data.restore.file.is_none());
assert!(result.databases[0].data.restore.meta_file.is_none());
}
#[test]
fn database_status_legacy_plaintext_storages() {
let status: DatabaseStatus = serde_json::from_value(json!({
"dbms": "postgres",
"generatedId": "gen-1",
"storages": [ { "id": "s1", "config": { "bucket": "b" }, "provider": "s3" } ],
"encrypt": true,
"data": {
"backup": { "action": false, "cron": null },
"restore": { "action": false, "file": null, "metaFile": null, "size": null }
}
})).unwrap();
assert_eq!(status.storages.len(), 1);
assert_eq!(status.storages_encrypted, None);
assert!(status.storages_ciphertext.is_none());
}
#[test]
fn database_status_encrypted_envelope() {
let status: DatabaseStatus = serde_json::from_value(json!({
"dbms": "postgres",
"generatedId": "gen-1",
"storages": [],
"storages_encrypted": true,
"storages_ciphertext": "AQIDBA==",
"encrypt": true,
"data": {
"backup": { "action": true, "cron": null },
"restore": { "action": false, "file": null, "metaFile": null, "size": null }
}
})).unwrap();
assert!(status.storages.is_empty());
assert_eq!(status.storages_encrypted, Some(true));
assert_eq!(status.storages_ciphertext.as_deref(), Some("AQIDBA=="));
}
#[test]
fn database_status_defaults_config_fields_absent() {
let json = r#"{
"dbms": "postgresql",
"generatedId": "16678159-ff7e-4c97-8c83-0adeff214681",
"encrypt": false,
"data": { "backup": { "action": false, "cron": null },
"restore": { "action": false, "file": null, "metaFile": null, "size": null } }
}"#;
let status: crate::services::api::models::agent::status::DatabaseStatus =
serde_json::from_str(json).unwrap();
assert_eq!(status.config_encrypted, None);
assert!(status.config_ciphertext.is_none());
assert!(status.resolved_config.is_none());
}
#[test]
fn resolve_dashboard_config_decrypts_full_entry() {
use crate::services::status::resolve_dashboard_config;
use base64::{engine::general_purpose, Engine};
// 32-byte master key, base64 STANDARD (matches decrypt_json_gcm).
let master_key_b64 = general_purpose::STANDARD.encode([7u8; 32]);
// Full agent-entry shape the dashboard encrypts.
let entry = r#"{
"name": "Dashboard PG",
"type": "postgresql",
"database": "app",
"username": "postgres",
"password": "s3cret",
"port": 5432,
"host": "10.0.0.10",
"generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"
}"#;
let ciphertext = encrypt_json_gcm(entry.as_bytes(), &master_key_b64);
let mut status: crate::services::api::models::agent::status::DatabaseStatus =
serde_json::from_str(
r#"{
"dbms": "postgresql",
"generatedId": "16678159-ff7e-4c97-8c83-0adeff214681",
"encrypt": false,
"config_encrypted": true,
"config_ciphertext": "PLACEHOLDER",
"data": { "backup": { "action": false, "cron": null },
"restore": { "action": false, "file": null, "metaFile": null, "size": null } }
}"#,
)
.unwrap();
status.config_ciphertext = Some(ciphertext);
resolve_dashboard_config(&mut status, &master_key_b64).unwrap();
let cfg = status.resolved_config.expect("resolved");
assert_eq!(cfg.name, "Dashboard PG");
assert_eq!(cfg.password, "s3cret");
assert_eq!(cfg.host, "10.0.0.10");
assert_eq!(cfg.db_type.as_str(), "postgresql");
}
#[test]
fn resolve_dashboard_config_noop_when_not_encrypted() {
use crate::services::status::resolve_dashboard_config;
let mut status: crate::services::api::models::agent::status::DatabaseStatus =
serde_json::from_str(
r#"{
"dbms": "postgresql",
"generatedId": "16678159-ff7e-4c97-8c83-0adeff214681",
"encrypt": false,
"data": { "backup": { "action": false, "cron": null },
"restore": { "action": false, "file": null, "metaFile": null, "size": null } }
}"#,
)
.unwrap();
resolve_dashboard_config(&mut status, "unused").unwrap();
assert!(status.resolved_config.is_none());
}
fn encrypt_json_gcm(plaintext: &[u8], master_key_b64: &str) -> String {
use aes_gcm::aead::{Aead, KeyInit};
use aes_gcm::{Aes256Gcm, Key, Nonce};
use base64::{engine::general_purpose, Engine};
let key_bytes = general_purpose::STANDARD.decode(master_key_b64).unwrap();
let key = Key::<Aes256Gcm>::try_from(key_bytes.as_slice()).unwrap();
let cipher = Aes256Gcm::new(&key);
let nonce_bytes = [0u8; 12];
let nonce = Nonce::try_from(&nonce_bytes[..]).unwrap();
let ct = cipher.encrypt(&nonce, plaintext).unwrap();
let mut data = nonce_bytes.to_vec();
data.extend_from_slice(&ct);
general_purpose::STANDARD.encode(data)
}
+71 -4
View File
@@ -1,15 +1,12 @@
use crate::core::context::Context;
use crate::services::api::ApiClient;
use crate::services::config::ConfigService;
use crate::services::config::{build_config, DatabasesConfig, InputDatabaseConfig};
use crate::utils::edge_key::EdgeKey;
use std::io::Write;
use std::sync::Arc;
use tempfile::NamedTempFile;
// `ConfigService::load` never touches `self.ctx` on the `Some(file_path)` path,
// so the values here don't matter — but `Context::new()` panics without an
// `EDGE_KEY` env var, so build the struct directly (mirrors
// backup_uploader_tests.rs's `ctx_pointing_at`).
fn test_context() -> Arc<Context> {
Arc::new(Context {
edge_key: EdgeKey {
@@ -264,3 +261,73 @@ fn docker_volume_requires_volume_name() {
let err = service.load(Some(file.path().to_str().unwrap())).unwrap_err();
assert!(err.contains("volume_name"), "error was: {err}");
}
#[test]
fn build_config_applies_type_defaults() {
let input: InputDatabaseConfig = serde_json::from_str(
r#"{
"name": "cluster1",
"type": "postgresql-cluster",
"username": "postgres",
"password": "p",
"port": 5432,
"host": "localhost",
"generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"
}"#,
)
.unwrap();
let cfg = build_config(input).unwrap();
assert_eq!(cfg.db_type.as_str(), "postgresql-cluster");
assert_eq!(cfg.database, "postgres"); // cluster default
}
#[test]
fn build_config_rejects_missing_required_field() {
let input: InputDatabaseConfig = serde_json::from_str(
r#"{
"name": "pg",
"type": "postgresql",
"username": "postgres",
"port": 5432,
"host": "localhost",
"generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"
}"#,
)
.unwrap();
let err = build_config(input).unwrap_err();
assert!(err.contains("password"), "unexpected error: {err}");
}
#[test]
fn load_optional_returns_empty_when_file_missing() {
let service = ConfigService::new(test_context());
let cfg = service.load_optional(Some("/nonexistent/path/does-not-exist.json"));
assert!(cfg.databases.is_empty());
}
#[test]
fn databases_config_roundtrips_through_serde() {
let input: InputDatabaseConfig = serde_json::from_str(
r#"{
"name": "pg",
"type": "postgresql",
"database": "app",
"username": "postgres",
"password": "secret",
"port": 5432,
"host": "localhost",
"generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"
}"#,
)
.unwrap();
let cfg = build_config(input).unwrap();
let wrapped = DatabasesConfig { databases: vec![cfg] };
let json = serde_json::to_string(&wrapped).unwrap();
let back: DatabasesConfig = serde_json::from_str(&json).unwrap();
assert_eq!(back.databases[0].name, "pg");
assert_eq!(back.databases[0].db_type.as_str(), "postgresql");
assert_eq!(back.databases[0].password, "secret");
}
@@ -0,0 +1,84 @@
use crate::services::config::{build_config, DatabaseConfig, InputDatabaseConfig};
use crate::services::dashboard_config::merge;
use crate::services::dashboard_config::{load_cache, persist_cache};
fn cfg(name: &str, gen_id: &str, host: &str) -> DatabaseConfig {
let json = format!(
r#"{{ "name": "{name}", "type": "postgresql", "database": "app",
"username": "u", "password": "p", "port": 5432,
"host": "{host}", "generated_id": "{gen_id}" }}"#
);
let input: InputDatabaseConfig = serde_json::from_str(&json).unwrap();
build_config(input).unwrap()
}
const ID_A: &str = "16678159-ff7e-4c97-8c83-0adeff214681";
const ID_B: &str = "16678124-ff7e-4c97-8c83-0adeff214681";
#[test]
fn merge_keeps_local_only_databases() {
let local = vec![cfg("local-a", ID_A, "local-host")];
let merged = merge(&local, &[]);
assert_eq!(merged.databases.len(), 1);
assert_eq!(merged.databases[0].host, "local-host");
}
#[test]
fn merge_appends_dashboard_only_databases() {
let local = vec![cfg("local-a", ID_A, "local-host")];
let dashboard = vec![cfg("dash-b", ID_B, "dash-host")];
let merged = merge(&local, &dashboard);
assert_eq!(merged.databases.len(), 2);
assert!(merged.databases.iter().any(|d| d.generated_id == ID_B));
}
#[test]
fn merge_dashboard_wins_on_id_collision() {
let local = vec![cfg("local-a", ID_A, "local-host")];
let dashboard = vec![cfg("dash-a", ID_A, "dash-host")];
let merged = merge(&local, &dashboard);
assert_eq!(merged.databases.len(), 1);
assert_eq!(merged.databases[0].host, "dash-host"); // dashboard wins
assert_eq!(merged.databases[0].name, "dash-a");
}
#[test]
fn cache_roundtrips() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("dashboard_databases.json");
let dbs = vec![cfg("dash-a", ID_A, "dash-host")];
persist_cache(&path, &dbs).unwrap();
let loaded = load_cache(&path);
assert_eq!(loaded.len(), 1);
assert_eq!(loaded[0].generated_id, ID_A);
assert_eq!(loaded[0].host, "dash-host");
}
#[test]
fn load_cache_missing_file_is_empty() {
let loaded = load_cache(std::path::Path::new("/nonexistent/dashboard_databases.json"));
assert!(loaded.is_empty());
}
#[test]
fn load_cache_corrupt_file_is_empty() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("dashboard_databases.json");
std::fs::write(&path, b"{ this is not valid json").unwrap();
let loaded = load_cache(&path);
assert!(loaded.is_empty());
}
#[test]
fn persist_cache_leaves_no_tmp_file() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("dashboard_databases.json");
persist_cache(&path, &[cfg("dash-a", ID_A, "h")]).unwrap();
let tmp = path.with_extension("json.tmp");
assert!(!tmp.exists(), "temp file should have been renamed away");
assert!(path.exists());
}
+1
View File
@@ -1,3 +1,4 @@
mod api_models_tests;
mod backup_uploader_tests;
mod config_tests;
mod dashboard_config_tests;
+32 -4
View File
@@ -30,12 +30,16 @@ fn full_file_name_matches_expected_suffix() {
}
#[test]
fn full_file_path_prefixes_backups_directory_and_date() {
fn full_file_path_uses_default_or_configured_folder_name() {
let file_name = "backup.tar.gz".to_string();
let full_path = full_file_path(&file_name);
assert!(full_path.starts_with("backups/"));
assert!(full_path.ends_with("/backup.tar.gz"));
let default_path = full_file_path(&file_name, None);
assert!(default_path.starts_with("backups/"));
assert!(default_path.ends_with("/backup.tar.gz"));
let configured_path = full_file_path(&file_name, Some("/portabase/"));
assert!(configured_path.starts_with("portabase/"));
assert!(configured_path.ends_with("/backup.tar.gz"));
}
#[tokio::test]
@@ -86,3 +90,27 @@ async fn encrypt_stream_starts_with_json_header_line() -> Result<()> {
Ok(())
}
use crate::utils::file::decrypt_json_gcm;
const VECTOR_KEY_B64: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=";
const VECTOR_ENVELOPE_B64: &str = "AQIDBAUGBwgJCgsM4AfK9w7I2A7UDzMvpJaScfnUYAGDZgPWT5Chrp1pdzMPPQVpNjb6ZEiFea9YdWVFv1UEo9RGmmf+zYUv4I3gE4SU/SBrMwkCHEpJGJOzJtK3tSpJmzLVX3+7EeUNwp4qjZheL8p0pe1x6dRUtx3JmLjz1W/RhWd6zuReDItv6+0jg4CaPOHvFXBreaGNCTRslxbImD+lFBoEOvw8lsbH";
const VECTOR_PLAINTEXT: &str = "[{\"id\":\"11111111-1111-1111-1111-111111111111\",\"config\":{\"bucket\":\"my-bucket\",\"accessKeyId\":\"AKIA\",\"secretAccessKey\":\"s3cr3t\"},\"provider\":\"s3\"}]";
#[test]
fn decrypt_json_gcm_decrypts_node_vector() {
let plaintext = decrypt_json_gcm(VECTOR_ENVELOPE_B64, VECTOR_KEY_B64).unwrap();
assert_eq!(String::from_utf8(plaintext).unwrap(), VECTOR_PLAINTEXT);
}
#[test]
fn decrypt_json_gcm_rejects_wrong_key() {
let wrong_key = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
assert!(decrypt_json_gcm(VECTOR_ENVELOPE_B64, wrong_key).is_err());
}
#[test]
fn decrypt_json_gcm_rejects_short_input() {
// 8 bytes base64 -> shorter than nonce(12)+tag(16)
assert!(decrypt_json_gcm("AAAAAAAAAAA=", VECTOR_KEY_B64).is_err());
}
+15
View File
@@ -53,6 +53,21 @@ fn normalize_converts_unix_saturday_to_crate_dow() {
assert_eq!(normalize_cron("00 06 * * 6"), "0 00 06 * * 7");
}
#[test]
fn normalized_sunday_actually_fires_on_sunday() {
use chrono::{Datelike, Timelike, Weekday};
let normalized = normalize_cron("00 03 * * 0");
assert_eq!(normalized, "0 00 03 * * 1");
let schedule = Schedule::from_str(&normalized).unwrap();
let next = schedule.upcoming(chrono::Utc).next().unwrap();
assert_eq!(next.weekday(), Weekday::Sun);
assert_eq!(next.hour(), 3);
assert_eq!(next.minute(), 0);
}
#[test]
fn normalize_converts_unix_dow_range() {
assert_eq!(normalize_cron("00 06 * * 0-4"), "0 00 06 * * 1-5");
+37 -2
View File
@@ -47,8 +47,14 @@ pub fn full_file_name(encrypt: bool) -> String {
}
}
pub fn full_file_path(file_name: &String) -> String {
format!("backups/{}/{}", Utc::now().format("%Y-%m-%d"), file_name)
pub fn full_file_path(file_name: &String, folder_name: Option<&str>) -> String {
let folder_name = folder_name
.map(str::trim)
.map(|folder_name| folder_name.trim_matches(char::from(47)))
.filter(|folder_name| !folder_name.is_empty())
.unwrap_or("backups");
format!("{}/{}/{}", folder_name, Utc::now().format("%Y-%m-%d"), file_name)
}
const CHUNK_SIZE: usize = 16 * 1024 * 1024;
@@ -175,3 +181,32 @@ pub async fn decrypt_file_stream_gcm(
writer.flush().await?;
Ok(())
}
/// Decrypt a base64 `nonce(12) ‖ ciphertext ‖ tag(16)` AES-256-GCM envelope
/// using the raw master key (STANDARD base64). Returns the plaintext bytes.
pub fn decrypt_json_gcm(ciphertext_b64: &str, master_key_b64: &str) -> Result<Vec<u8>> {
let master_key_bytes = general_purpose::STANDARD
.decode(master_key_b64)
.map_err(|_| anyhow::anyhow!("Invalid base64 master key"))?;
let data = general_purpose::STANDARD
.decode(ciphertext_b64)
.map_err(|_| anyhow::anyhow!("Invalid base64 ciphertext"))?;
if data.len() < 12 + 16 {
return Err(anyhow::anyhow!("Ciphertext too short"));
}
let key = Key::<Aes256Gcm>::try_from(master_key_bytes.as_slice())
.map_err(|_| anyhow::anyhow!("Invalid AES-256 key length"))?;
let cipher = Aes256Gcm::new(&key);
let nonce = Nonce::try_from(&data[..12])
.map_err(|_| anyhow::anyhow!("Invalid nonce length"))?;
let plaintext = cipher
.decrypt(&nonce, &data[12..])
.map_err(|e| anyhow::anyhow!("AES-GCM decryption failed: {:?}", e))?;
Ok(plaintext)
}
+3 -2
View File
@@ -38,8 +38,9 @@ pub async fn check_and_update_cron(
}
Some(cron) => {
let cron = normalize_cron(&cron);
debug!("Task cron (normalized): {:?}", cron);
let raw_cron = cron;
let cron = normalize_cron(&raw_cron);
debug!("Task cron normalized: unix \"{}\" -> crate \"{}\"", raw_cron, cron);
if exists {
let raw: String = conn.hget(&redis_key, "data").await.unwrap();