mirror of
https://github.com/Portabase/agent.git
synced 2026-09-11 02:27:10 +00:00
Compare commits
27 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 069067ca55 | |||
| 04b654d219 | |||
| d83511cb64 | |||
| 30f83bafcf | |||
| de106c835e | |||
| 23d6822ddc | |||
| fe1d74945f | |||
| 424a646385 | |||
| 0ef4bba5d7 | |||
| 6548140eaf | |||
| c9725c381e | |||
| 03695b1897 | |||
| 80b1c0dac0 | |||
| 5ededb3764 | |||
| 54cfd673e4 | |||
| 39dbeb5d30 | |||
| 380cea79fb | |||
| 65e4538c2c | |||
| 86a706bca5 | |||
| e8bda60f16 | |||
| 034912b176 | |||
| 4fd50ccda4 | |||
| 84c6b765c8 | |||
| a93054518a | |||
| c186d57105 | |||
| 37b4c1fe6d | |||
| 92a3324ca2 |
+79
-17
@@ -27,15 +27,80 @@ permissions:
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Build and push to GHCR
|
||||
runs-on: ubuntu-latest
|
||||
build:
|
||||
name: Build ${{ matrix.platform }}
|
||||
runs-on: ${{ matrix.platform == 'linux/amd64' && 'ubuntu-latest' || 'ubuntu-24.04-arm' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
platform: [ linux/amd64, linux/arm64 ]
|
||||
steps:
|
||||
- name: Prepare vars
|
||||
id: prep
|
||||
run: |
|
||||
ARCH="${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}"
|
||||
echo "arch=$ARCH" >> "$GITHUB_OUTPUT"
|
||||
echo "image=ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/agent" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
fetch-depth: 0
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and push by digest
|
||||
id: build
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: ${{ inputs.dockerfile }}
|
||||
platforms: ${{ matrix.platform }}
|
||||
target: ${{ inputs.target }}
|
||||
provenance: false
|
||||
outputs: type=image,name=${{ steps.prep.outputs.image }},push-by-digest=true,name-canonical=true,push=true
|
||||
cache-from: type=gha,scope=ghcr-${{ steps.prep.outputs.arch }}
|
||||
cache-to: type=gha,mode=max,scope=ghcr-${{ steps.prep.outputs.arch }},ignore-error=true
|
||||
|
||||
- name: Export digest
|
||||
env:
|
||||
DIGEST: ${{ steps.build.outputs.digest }}
|
||||
run: |
|
||||
mkdir -p /tmp/digests
|
||||
touch "/tmp/digests/${DIGEST#sha256:}"
|
||||
|
||||
- name: Upload digest
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: digest-${{ steps.prep.outputs.arch }}
|
||||
path: /tmp/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
merge:
|
||||
name: Create multi-arch manifest
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
steps:
|
||||
- name: Prepare vars
|
||||
id: prep
|
||||
run: echo "image=ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/agent" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Download digests
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: /tmp/digests
|
||||
pattern: digest-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
@@ -51,22 +116,19 @@ jobs:
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ghcr.io/${{ github.repository_owner }}/agent
|
||||
images: ${{ steps.prep.outputs.image }}
|
||||
tags: |
|
||||
type=semver,pattern={{version}},value=${{ inputs.version }}
|
||||
type=semver,pattern={{major}}.{{minor}},value=${{ inputs.version }}
|
||||
type=semver,pattern={{major}},value=${{ inputs.version }}
|
||||
type=raw,value=latest,enable=${{ inputs.add_latest }}
|
||||
|
||||
- name: Build and push image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: ${{ inputs.dockerfile }}
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
target: ${{ inputs.target }}
|
||||
provenance: false
|
||||
cache-from: type=gha,scope=ghcr-build
|
||||
cache-to: type=gha,mode=max,scope=ghcr-build,ignore-error=true
|
||||
- name: Create and push manifest list
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
docker buildx imagetools create \
|
||||
$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
|
||||
$(printf '${{ steps.prep.outputs.image }}@sha256:%s ' *)
|
||||
|
||||
- name: Inspect
|
||||
run: docker buildx imagetools inspect ${{ steps.prep.outputs.image }}:${{ inputs.version }}
|
||||
|
||||
@@ -118,11 +118,24 @@ jobs:
|
||||
secrets:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build-windows:
|
||||
needs: create-release
|
||||
if: ${{ needs.create-release.result == 'success' }}
|
||||
uses: ./.github/workflows/windows-release.yml
|
||||
with:
|
||||
version: ${{ needs.create-release.outputs.version }}
|
||||
ref: ${{ needs.create-release.outputs.version }}
|
||||
draft_tag: ${{ needs.create-release.outputs.draft_tag }}
|
||||
secrets:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
finalize-release:
|
||||
needs:
|
||||
- create-release
|
||||
- publish-docker
|
||||
- publish-docker-ghcr
|
||||
- publish-helm
|
||||
- build-windows
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
release_tag: ${{ steps.publish_release_step.outputs.release_tag }}
|
||||
|
||||
@@ -1,14 +1,30 @@
|
||||
name: Build Windows release
|
||||
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
version:
|
||||
description: 'Release version (git tag), e.g. 1.18.4'
|
||||
type: string
|
||||
required: false
|
||||
ref:
|
||||
description: 'Git ref to check out and build'
|
||||
type: string
|
||||
required: false
|
||||
draft_tag:
|
||||
description: 'Draft GitHub release tag to attach the asset to (e.g. untagged-xxxx). Empty = skip upload.'
|
||||
type: string
|
||||
required: false
|
||||
secrets:
|
||||
GH_TOKEN:
|
||||
required: false
|
||||
|
||||
workflow_dispatch:
|
||||
push:
|
||||
tags:
|
||||
- '[0-9]+.[0-9]+.[0-9]+'
|
||||
branches:
|
||||
- main
|
||||
- master
|
||||
inputs:
|
||||
ref:
|
||||
description: 'Git ref to check out and build'
|
||||
type: string
|
||||
required: false
|
||||
|
||||
jobs:
|
||||
build-windows:
|
||||
@@ -17,80 +33,68 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Set up Rust toolchain (MSVC)
|
||||
uses: actions-rs/toolchain@v1
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: stable-x86_64-pc-windows-msvc
|
||||
profile: minimal
|
||||
override: true
|
||||
targets: x86_64-pc-windows-msvc
|
||||
|
||||
- name: Install vcpkg and OpenSSL (x64)
|
||||
- name: Cache cargo build
|
||||
uses: Swatinem/rust-cache@v2
|
||||
|
||||
- name: Cache vcpkg installed packages
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: C:\vcpkg\installed
|
||||
key: vcpkg-openssl-x64-windows-v1
|
||||
|
||||
- name: Install OpenSSL (x64) via vcpkg
|
||||
shell: pwsh
|
||||
run: |
|
||||
# Install vcpkg and the prebuilt OpenSSL package
|
||||
git clone https://github.com/microsoft/vcpkg C:\vcpkg
|
||||
C:\vcpkg\bootstrap-vcpkg.bat
|
||||
C:\vcpkg\vcpkg install openssl:x64-windows
|
||||
# Export variables for subsequent steps
|
||||
# windows-latest ships vcpkg preinstalled; the install is a no-op when the
|
||||
# package is restored from cache.
|
||||
& "$env:VCPKG_INSTALLATION_ROOT\vcpkg.exe" install openssl:x64-windows
|
||||
'VCPKG_ROOT=C:\vcpkg' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
'OPENSSL_DIR=C:\vcpkg\installed\x64-windows' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
|
||||
|
||||
- name: Build (cargo release)
|
||||
shell: pwsh
|
||||
env:
|
||||
# Cargo / openssl-sys will pick up OPENSSL_DIR from the environment
|
||||
OPENSSL_DIR: ${{ env.OPENSSL_DIR }}
|
||||
run: |
|
||||
# Ensure the environment variable is present for this step
|
||||
if (-Not $env:OPENSSL_DIR) { Write-Host "OPENSSL_DIR not set, printing env for debugging"; Get-ChildItem Env: | ForEach-Object { Write-Host $_ } }
|
||||
# Build the declared bin target explicitly (Cargo.toml [[bin]] name = "app")
|
||||
cargo build --release --bin app
|
||||
run: cargo build --release --bin app
|
||||
|
||||
- name: Prepare artifact zip
|
||||
id: prepare_artifact
|
||||
shell: pwsh
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.ref_name }}
|
||||
RELEASE_VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
$tag = $env:RELEASE_TAG
|
||||
$tag = $env:RELEASE_VERSION
|
||||
if (-not $tag) { $tag = $env:GITHUB_SHA }
|
||||
|
||||
# Package the declared bin target deterministically (Cargo.toml [[bin]] name = "app")
|
||||
$exe = "target\release\app.exe"
|
||||
if (-not (Test-Path $exe)) { Write-Error "Built binary $exe not found in target/release"; exit 1 }
|
||||
|
||||
$outDir = "artifact"
|
||||
New-Item -ItemType Directory -Path $outDir -Force | Out-Null
|
||||
# Ship under the package name, not the internal bin name "app"
|
||||
# Ship under the package name, not the internal bin name "app".
|
||||
Copy-Item -Path $exe -Destination "$outDir\portabase-agent.exe"
|
||||
|
||||
$zipName = "windows-release-$tag.zip"
|
||||
if (Test-Path $zipName) { Remove-Item $zipName }
|
||||
Compress-Archive -Path "$outDir\*" -DestinationPath $zipName -Force
|
||||
Write-Host "ZIP=$zipName"
|
||||
Write-Output "zip=$zipName" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
|
||||
"zip=$zipName" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
|
||||
|
||||
- name: Upload build artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: windows-release
|
||||
path: windows-release-*.zip
|
||||
path: ${{ steps.prepare_artifact.outputs.zip }}
|
||||
|
||||
- name: Create GitHub Release
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
id: create_release
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
tag_name: ${{ github.ref_name }}
|
||||
- name: Attach asset to draft release
|
||||
if: ${{ inputs.draft_tag != '' }}
|
||||
shell: pwsh
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload release asset
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: actions/upload-release-asset@v1
|
||||
with:
|
||||
upload_url: ${{ steps.create_release.outputs.upload_url }}
|
||||
asset_path: windows-release-${{ github.ref_name }}.zip
|
||||
asset_name: windows-release-${{ github.ref_name }}.zip
|
||||
asset_content_type: application/zip
|
||||
GH_TOKEN: ${{ secrets.GH_TOKEN }}
|
||||
run: |
|
||||
gh release upload "${{ inputs.draft_tag }}" "${{ steps.prepare_artifact.outputs.zip }}" --clobber
|
||||
|
||||
+1
-1
@@ -27,5 +27,5 @@ keywords:
|
||||
- self-hosted
|
||||
- portabase
|
||||
license: Apache-2.0
|
||||
version: 1.17.1
|
||||
version: 1.18.6
|
||||
date-released: '2026-02-24'
|
||||
|
||||
Generated
+2
-1
@@ -3503,7 +3503,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "portabase-agent"
|
||||
version = "1.17.1"
|
||||
version = "1.18.6"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"aes-gcm",
|
||||
@@ -3535,6 +3535,7 @@ dependencies = [
|
||||
"oauth2",
|
||||
"once_cell",
|
||||
"openssl",
|
||||
"percent-encoding",
|
||||
"postgres",
|
||||
"rand 0.9.2",
|
||||
"redis",
|
||||
|
||||
+2
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "portabase-agent"
|
||||
version = "1.17.1"
|
||||
version = "1.18.6"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
@@ -57,6 +57,7 @@ testcontainers = "0.27.1"
|
||||
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "valkey", "mysql", "mariadb", "mongo"] }
|
||||
postgres = "0.19.12"
|
||||
url = "2.5.8"
|
||||
percent-encoding = "2.3.2"
|
||||
bollard = "0.20.0"
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
+6
-47
@@ -1,13 +1,14 @@
|
||||
services:
|
||||
rust-app:
|
||||
# build:
|
||||
# context: .
|
||||
# dockerfile: docker/Dockerfile
|
||||
# target: prod
|
||||
image: portabase/agent:latest
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/Dockerfile
|
||||
target: prod
|
||||
# image: portabase/agent:latest
|
||||
container_name: rust-prod
|
||||
volumes:
|
||||
- ./databases.json:/config/config.json
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
environment:
|
||||
LOG: info
|
||||
TZ: "Europe/Paris"
|
||||
@@ -18,48 +19,6 @@ services:
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
|
||||
db-mongodb-auth:
|
||||
container_name: db-mongodb-auth
|
||||
image: mongo:latest
|
||||
ports:
|
||||
- "27082:27017"
|
||||
environment:
|
||||
MONGO_INITDB_ROOT_USERNAME: root
|
||||
MONGO_INITDB_ROOT_PASSWORD: rootpassword
|
||||
MONGO_INITDB_DATABASE: testdbauth
|
||||
command: mongod --auth
|
||||
networks:
|
||||
- portabase
|
||||
volumes:
|
||||
- mongodb-data-auth:/data/db
|
||||
healthcheck:
|
||||
test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
db-mongodb:
|
||||
container_name: db-mongodb
|
||||
image: mongo:latest
|
||||
ports:
|
||||
- "27083:27017"
|
||||
volumes:
|
||||
- mongodb-data:/data/db
|
||||
healthcheck:
|
||||
test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
environment:
|
||||
MONGO_INITDB_DATABASE: testdb
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
volumes:
|
||||
mongodb-data:
|
||||
mongodb-data-auth:
|
||||
|
||||
networks:
|
||||
portabase:
|
||||
name: portabase_network
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@ services:
|
||||
LOG: debug
|
||||
TZ: "Europe/Paris"
|
||||
# TMPDIR: /scratch
|
||||
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiNmM4NWE3ODQtODRkMi00YzUyLTgzYmUtZTc2MDZkZjg2YjM5IiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ=="
|
||||
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiMGNlNjVjMDQtMDJiOS00YjMzLTk5MzYtMzIzYTFhOGU4MTk3IiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ=="
|
||||
#CHUNK_SIZE_MB: "1"
|
||||
#POOLING: 1
|
||||
#DATABASES_CONFIG_FILE: "config.toml"
|
||||
|
||||
@@ -45,6 +45,14 @@ seed-firebird:
|
||||
echo "SELECT RDB\$RELATION_NAME FROM RDB\$RELATIONS WHERE RDB\$SYSTEM_FLAG = 0 AND RDB\$VIEW_BLR IS NULL;" \
|
||||
| docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb
|
||||
|
||||
seed-firebird-large:
|
||||
echo "Seeding Firebird..."
|
||||
docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb < ./scripts/firebird/seed-large.sql
|
||||
|
||||
echo "Verifying Firebird tables..."
|
||||
echo "SELECT RDB\$RELATION_NAME FROM RDB\$RELATIONS WHERE RDB\$SYSTEM_FLAG = 0 AND RDB\$VIEW_BLR IS NULL;" \
|
||||
| docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb
|
||||
|
||||
seed-mssql:
|
||||
echo "Seeding MSSQL..."
|
||||
docker exec -i rust-dev sqlcmd -S "db-mssql,1433" -U sa -P "$MSSQL_SA_PASSWORD" -N disable -i /app/scripts/mssql/seed.sql
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
SET SQL DIALECT 3;
|
||||
SET BAIL ON;
|
||||
SET AUTODDL OFF;
|
||||
|
||||
CREATE TABLE users (
|
||||
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
email VARCHAR(255) NOT NULL UNIQUE,
|
||||
name VARCHAR(255),
|
||||
payload BLOB SUB_TYPE TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
COMMIT;
|
||||
|
||||
INSERT INTO users (email, name, payload)
|
||||
VALUES ('alice@example.com', 'Alice', 'Alice seed data');
|
||||
|
||||
INSERT INTO users (email, name, payload)
|
||||
VALUES ('bob@example.com', 'Bob', 'Bob seed data');
|
||||
|
||||
COMMIT;
|
||||
|
||||
/*
|
||||
* Each procedure call generates approximately 128 MiB:
|
||||
*
|
||||
* 128 rows
|
||||
* × 128 chunks per row
|
||||
* × 8191 bytes per chunk
|
||||
* = approximately 128 MiB
|
||||
*
|
||||
* 40 calls = approximately 5 GiB.
|
||||
*/
|
||||
|
||||
SET TERM ^;
|
||||
|
||||
CREATE PROCEDURE seed_users_batch (
|
||||
p_rows INTEGER,
|
||||
p_chunks_per_row INTEGER
|
||||
)
|
||||
AS
|
||||
DECLARE VARIABLE v_row_index INTEGER;
|
||||
DECLARE VARIABLE v_chunk_index INTEGER;
|
||||
DECLARE VARIABLE v_uuid VARCHAR(36);
|
||||
DECLARE VARIABLE v_chunk VARCHAR(8191);
|
||||
DECLARE VARIABLE v_payload BLOB SUB_TYPE TEXT;
|
||||
BEGIN
|
||||
v_row_index = 0;
|
||||
|
||||
WHILE (v_row_index < p_rows) DO
|
||||
BEGIN
|
||||
v_payload = NULL;
|
||||
v_chunk_index = 0;
|
||||
|
||||
WHILE (v_chunk_index < p_chunks_per_row) DO
|
||||
BEGIN
|
||||
/*
|
||||
* Generate a different chunk to avoid producing a completely
|
||||
* uniform BLOB.
|
||||
*/
|
||||
v_chunk = RPAD(
|
||||
UUID_TO_CHAR(GEN_UUID()),
|
||||
8191,
|
||||
UUID_TO_CHAR(GEN_UUID())
|
||||
);
|
||||
|
||||
v_payload = BLOB_APPEND(v_payload, v_chunk);
|
||||
v_chunk_index = v_chunk_index + 1;
|
||||
END
|
||||
|
||||
v_uuid = UUID_TO_CHAR(GEN_UUID());
|
||||
|
||||
INSERT INTO users (
|
||||
email,
|
||||
name,
|
||||
payload
|
||||
)
|
||||
VALUES (
|
||||
:v_uuid || '@example.test',
|
||||
'Seed User ' || :v_uuid,
|
||||
:v_payload
|
||||
);
|
||||
|
||||
v_row_index = v_row_index + 1;
|
||||
END
|
||||
END^
|
||||
|
||||
SET TERM ;^
|
||||
|
||||
/* Batch 01 — approximately 128 MiB */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 02 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 03 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 04 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 05 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 06 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 07 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 08 — approximately 1 GiB total */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 09 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 10 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 11 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 12 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 13 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 14 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 15 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 16 — approximately 2 GiB total */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 17 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 18 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 19 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 20 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 21 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 22 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 23 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 24 — approximately 3 GiB total */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 25 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 26 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 27 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 28 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 29 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 30 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 31 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 32 — approximately 4 GiB total */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 33 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 34 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 35 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 36 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 37 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 38 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 39 */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
/* Batch 40 — approximately 5 GiB total */
|
||||
EXECUTE PROCEDURE seed_users_batch(128, 128);
|
||||
COMMIT;
|
||||
|
||||
DROP PROCEDURE seed_users_batch;
|
||||
|
||||
COMMIT;
|
||||
|
||||
SELECT
|
||||
COUNT(*) AS user_count,
|
||||
CAST(SUM(OCTET_LENGTH(payload)) / 1073741824.0 AS DECIMAL(18, 2))
|
||||
AS payload_size_gib
|
||||
FROM users;
|
||||
|
||||
COMMIT;
|
||||
@@ -15,7 +15,7 @@ pub const EPHEMERAL_LABEL: &str = "io.portabase.ephemeral";
|
||||
const HELPER_MOUNT: &str = "/vol";
|
||||
|
||||
pub fn client() -> Result<Docker> {
|
||||
Docker::connect_with_unix_defaults().context("Failed to connect to Docker daemon socket")
|
||||
Docker::connect_with_defaults().context("Failed to connect to Docker daemon socket")
|
||||
}
|
||||
|
||||
pub fn parse_container_id(mountinfo: &str, cgroup: &str) -> Option<String> {
|
||||
|
||||
@@ -23,6 +23,7 @@ pub async fn run(
|
||||
let start = Instant::now();
|
||||
let output = Command::new("gbak")
|
||||
.arg("-b")
|
||||
.arg("-g")
|
||||
.arg("-v")
|
||||
.arg("-user").arg(&cfg.username)
|
||||
.arg("-password").arg(&cfg.password)
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::Result;
|
||||
use mongodb::Client;
|
||||
use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
|
||||
|
||||
const USERINFO_ENCODE: &AsciiSet = &NON_ALPHANUMERIC
|
||||
.remove(b'-')
|
||||
.remove(b'_')
|
||||
.remove(b'.')
|
||||
.remove(b'~');
|
||||
|
||||
pub async fn connect(cfg: DatabaseConfig) -> Result<Client> {
|
||||
let uri = get_mongo_uri(cfg)?;
|
||||
@@ -16,19 +23,40 @@ pub fn select_mongo_path() -> std::path::PathBuf {
|
||||
}
|
||||
|
||||
pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result<String> {
|
||||
if cfg.username.is_empty() || cfg.password.is_empty() {
|
||||
Ok(format!(
|
||||
"mongodb://{}:{}/{}",
|
||||
cfg.host, cfg.port, cfg.database
|
||||
))
|
||||
} else {
|
||||
Ok(format!(
|
||||
"mongodb://{}:{}@{}:{}/{}?authSource=admin",
|
||||
cfg.username, cfg.password, cfg.host, cfg.port, cfg.database
|
||||
))
|
||||
}
|
||||
Ok(build_mongo_uri(&cfg, true))
|
||||
}
|
||||
|
||||
pub fn build_mongo_uri(cfg: &DatabaseConfig, include_db: bool) -> String {
|
||||
let is_srv = cfg.port == 0;
|
||||
let scheme = if is_srv { "mongodb+srv" } else { "mongodb" };
|
||||
let has_auth = !cfg.username.is_empty() && !cfg.password.is_empty();
|
||||
|
||||
let credentials = if has_auth {
|
||||
format!(
|
||||
"{}:{}@",
|
||||
utf8_percent_encode(&cfg.username, USERINFO_ENCODE),
|
||||
utf8_percent_encode(&cfg.password, USERINFO_ENCODE)
|
||||
)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
|
||||
let authority = if is_srv {
|
||||
cfg.host.clone()
|
||||
} else {
|
||||
format!("{}:{}", cfg.host, cfg.port)
|
||||
};
|
||||
|
||||
let path = if include_db {
|
||||
format!("/{}", cfg.database)
|
||||
} else {
|
||||
"/".to_string()
|
||||
};
|
||||
|
||||
let query = if has_auth { "?authSource=admin" } else { "" };
|
||||
|
||||
format!("{}://{}{}{}{}", scheme, credentials, authority, path, query)
|
||||
}
|
||||
|
||||
pub fn extract_db_name(dry_output: &str) -> Option<String> {
|
||||
let mut dbs = std::collections::HashSet::new();
|
||||
@@ -43,3 +71,70 @@ pub fn extract_db_name(dry_output: &str) -> Option<String> {
|
||||
}
|
||||
dbs.into_iter().next()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::services::config::{DatabaseConfig, DbType};
|
||||
use std::collections::HashMap;
|
||||
|
||||
fn cfg(host: &str, port: u16, user: &str, pass: &str) -> DatabaseConfig {
|
||||
DatabaseConfig {
|
||||
name: "t".into(),
|
||||
database: "mydb".into(),
|
||||
db_type: DbType::MongoDB,
|
||||
username: user.into(),
|
||||
password: pass.into(),
|
||||
port,
|
||||
host: host.into(),
|
||||
generated_id: "id".into(),
|
||||
path: String::new(),
|
||||
max_packet_size: String::new(),
|
||||
volume_name: String::new(),
|
||||
container_name: None,
|
||||
options: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standard_with_auth() {
|
||||
let c = cfg("localhost", 27017, "user", "pass");
|
||||
assert_eq!(
|
||||
build_mongo_uri(&c, true),
|
||||
"mongodb://user:pass@localhost:27017/mydb?authSource=admin"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standard_no_auth() {
|
||||
let c = cfg("localhost", 27017, "", "");
|
||||
assert_eq!(build_mongo_uri(&c, true), "mongodb://localhost:27017/mydb");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn srv_with_auth() {
|
||||
let c = cfg("cluster.example.mongodb.net", 0, "user", "pass");
|
||||
assert_eq!(
|
||||
build_mongo_uri(&c, true),
|
||||
"mongodb+srv://user:pass@cluster.example.mongodb.net/mydb?authSource=admin"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn srv_no_db_for_dryrun() {
|
||||
let c = cfg("cluster.example.mongodb.net", 0, "user", "pass");
|
||||
assert_eq!(
|
||||
build_mongo_uri(&c, false),
|
||||
"mongodb+srv://user:pass@cluster.example.mongodb.net/?authSource=admin"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encodes_special_chars_in_credentials() {
|
||||
let c = cfg("cluster.example.mongodb.net", 0, "user", "p@ss:w/rd?");
|
||||
assert_eq!(
|
||||
build_mongo_uri(&c, true),
|
||||
"mongodb+srv://user:p%40ss%3Aw%2Frd%3F@cluster.example.mongodb.net/mydb?authSource=admin"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,11 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
|
||||
Ok(_) => Ok(true),
|
||||
Err(e) => {
|
||||
error!("--- MongoDB Connection Error Details ---");
|
||||
error!("Target Host: {}:{}", cfg.host, cfg.port);
|
||||
if cfg.port == 0 {
|
||||
error!("Target Host: {} (srv)", cfg.host);
|
||||
} else {
|
||||
error!("Target Host: {}:{}", cfg.host, cfg.port);
|
||||
}
|
||||
error!("Error Kind: {:?}", e.kind);
|
||||
error!("Full Error: {}", e);
|
||||
error!("Check you database network connectivity");
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
use crate::domain::mongodb::connection::{extract_db_name, get_mongo_uri, select_mongo_path};
|
||||
use crate::domain::mongodb::connection::{
|
||||
build_mongo_uri, extract_db_name, get_mongo_uri, select_mongo_path,
|
||||
};
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
@@ -16,13 +18,7 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogg
|
||||
|
||||
let dry_start = Instant::now();
|
||||
let dry_run = Command::new(&mongorestore)
|
||||
.arg(format!(
|
||||
"--uri={}",
|
||||
format!(
|
||||
"mongodb://{}:{}@{}:{}/?authSource=admin",
|
||||
cfg.username, cfg.password, cfg.host, cfg.port
|
||||
)
|
||||
))
|
||||
.arg(format!("--uri={}", build_mongo_uri(&cfg, false)))
|
||||
.arg(format!("--archive={}", restore_file.display()))
|
||||
.arg("--gzip")
|
||||
.arg("--dryRun")
|
||||
|
||||
@@ -217,12 +217,11 @@ impl ConfigService {
|
||||
| DbType::PostgresqlCluster
|
||||
| DbType::Mysql
|
||||
| DbType::Mariadb
|
||||
| DbType::MongoDB
|
||||
| DbType::Redis
|
||||
| DbType::Firebird
|
||||
| DbType::Valkey
|
||||
| DbType::Mssql => required(&db.port, &db.name, "port")?,
|
||||
DbType::Sqlite | DbType::DockerVolume => db.port.unwrap_or(0),
|
||||
DbType::MongoDB | DbType::Sqlite | DbType::DockerVolume => db.port.unwrap_or(0),
|
||||
};
|
||||
|
||||
let database_name = match db.db_type {
|
||||
|
||||
@@ -13,7 +13,9 @@ use aws_config::retry::RetryConfig;
|
||||
use aws_sdk_s3 as s3;
|
||||
use aws_sdk_s3::config::BehaviorVersion;
|
||||
use aws_sdk_s3::config::Region;
|
||||
use aws_sdk_s3::config::RequestChecksumCalculation;
|
||||
use aws_sdk_s3::config::retry::ReconnectMode;
|
||||
use aws_sdk_s3::error::DisplayErrorContext;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use futures::StreamExt;
|
||||
@@ -135,6 +137,10 @@ impl StorageProvider for S3Provider {
|
||||
.credentials_provider(credentials)
|
||||
.region(region)
|
||||
.force_path_style(true)
|
||||
// S3-compatible endpoints (MinIO, Garage, RustFS, Synology, ...) reject the
|
||||
// default CRC32 integrity checksums the SDK attaches to multipart uploads.
|
||||
// Only send checksums when the operation actually requires them.
|
||||
.request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
|
||||
.endpoint_url(endpoint)
|
||||
.behavior_version(BehaviorVersion::latest())
|
||||
.build();
|
||||
@@ -164,11 +170,12 @@ impl StorageProvider for S3Provider {
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
error!("Failed to create multipart upload: {}", e);
|
||||
let detail = DisplayErrorContext(&e).to_string();
|
||||
error!("Failed to create multipart upload: {}", detail);
|
||||
return UploadResult {
|
||||
storage_id: storage.id.clone(),
|
||||
success: false,
|
||||
error: Some(e.to_string()),
|
||||
error: Some(detail),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
};
|
||||
@@ -251,7 +258,8 @@ impl StorageProvider for S3Provider {
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to upload part {}: {}", part_number, e);
|
||||
let detail = DisplayErrorContext(&e).to_string();
|
||||
error!("Failed to upload part {}: {}", part_number, detail);
|
||||
let _ = client
|
||||
.abort_multipart_upload()
|
||||
.bucket(bucket)
|
||||
@@ -262,7 +270,7 @@ impl StorageProvider for S3Provider {
|
||||
return UploadResult {
|
||||
storage_id: storage.id.clone(),
|
||||
success: false,
|
||||
error: Some(e.to_string()),
|
||||
error: Some(detail),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
};
|
||||
@@ -317,7 +325,8 @@ impl StorageProvider for S3Provider {
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to complete multipart upload: {}", e);
|
||||
let detail = DisplayErrorContext(&e).to_string();
|
||||
error!("Failed to complete multipart upload: {}", detail);
|
||||
let _ = client
|
||||
.abort_multipart_upload()
|
||||
.bucket(bucket)
|
||||
@@ -328,7 +337,7 @@ impl StorageProvider for S3Provider {
|
||||
UploadResult {
|
||||
storage_id: storage.id.clone(),
|
||||
success: false,
|
||||
error: Some(e.to_string()),
|
||||
error: Some(detail),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user