mirror of
https://github.com/Portabase/agent.git
synced 2026-09-11 14:00:14 +00:00
Compare commits
87 Commits
1.1.0-rc.1
..
1.4.0
| Author | SHA1 | Date | |
|---|---|---|---|
| a2be03751f | |||
| 7df9df1605 | |||
| df5b16a153 | |||
| 824a5d52a3 | |||
| 1496208db4 | |||
| 454e1b442f | |||
| 54bca7452a | |||
| a96cef6cb5 | |||
| fd9f183166 | |||
| 94fb2535ec | |||
| 189ad866de | |||
| 55bbd4e724 | |||
| 7fabc9c98c | |||
| 4f67a57681 | |||
| 45a1118f64 | |||
| 063b2e1c2c | |||
| f2275d5aca | |||
| b136140b55 | |||
| 924177a4fe | |||
| 773077f66b | |||
| 46ff086cf9 | |||
| 678df8e2bc | |||
| f563c187db | |||
| df0dffa47b | |||
| 864396422b | |||
| e280c1d2f8 | |||
| 91a9d0654b | |||
| bc857a4c18 | |||
| 2454915866 | |||
| a4a7a9a2c4 | |||
| 997ba2afa2 | |||
| 7e0c7c0688 | |||
| f973bc1f52 | |||
| b9b9fcf6ee | |||
| 0224605c59 | |||
| 6ea9fdf8bc | |||
| ad68be8a5a | |||
| 434c131afb | |||
| 675ac5976d | |||
| 70543fe032 | |||
| 77f90308b4 | |||
| 63cb254abd | |||
| 7738745def | |||
| 82659f681c | |||
| eee3bebd50 | |||
| 58dd0259f9 | |||
| 5b7453208b | |||
| 35933aae52 | |||
| 178628cb1c | |||
| 3fc60e3f40 | |||
| e62de95827 | |||
| fc3612fa3f | |||
| 18fe211ad1 | |||
| 74ada01232 | |||
| 4fef360c06 | |||
| 2647941008 | |||
| b4bdc89888 | |||
| 829e5bade3 | |||
| fc9a4ef8fb | |||
| 3c5734ed4e | |||
| 4b77896ecb | |||
| 40c4a4e2bb | |||
| 8a05b93ff3 | |||
| c186932a79 | |||
| bf74173cac | |||
| 3dcebf4fcc | |||
| 4de2b81b33 | |||
| b06f9d8186 | |||
| f5a2182723 | |||
| e8a96980f1 | |||
| a64792bcc6 | |||
| 2d4542be5f | |||
| 4a00ed574d | |||
| fd220598fc | |||
| d8b239a123 | |||
| ebf0526366 | |||
| 6461c1523b | |||
| 2809373de1 | |||
| 5349e1fef9 | |||
| 2bcb393d60 | |||
| 406273796e | |||
| cb35347cb6 | |||
| b92138fd84 | |||
| f4c54836f1 | |||
| e678cf4b06 | |||
| 3c599e94de | |||
| 6bbc856f62 |
@@ -0,0 +1,23 @@
|
||||
reviews:
|
||||
auto_review:
|
||||
enabled: true
|
||||
ignore_usernames: ["dependabot[bot]"]
|
||||
labels: ["!wip", "!draft"]
|
||||
auto_apply_labels: true
|
||||
suggested_reviewers: true
|
||||
auto_assign_reviewers: true
|
||||
|
||||
issue_enrichment:
|
||||
auto_enrich:
|
||||
enabled: true
|
||||
labeling:
|
||||
auto_apply_labels: true
|
||||
labeling_instructions:
|
||||
- label: bug
|
||||
instructions: "Error, crash, or incorrect behavior reports"
|
||||
- label: enhancement
|
||||
instructions: "Feature requests or improvements"
|
||||
- label: documentation
|
||||
instructions: "Docs updates or missing documentation"
|
||||
- label: good first issue
|
||||
instructions: "Beginner-friendly tasks suitable for new contributors"
|
||||
@@ -0,0 +1,73 @@
|
||||
name: Discord Notification
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
release_tag:
|
||||
required: true
|
||||
type: string
|
||||
discord_title:
|
||||
required: true
|
||||
type: string
|
||||
discord_color:
|
||||
required: true
|
||||
type: number
|
||||
discord_footer:
|
||||
required: true
|
||||
type: string
|
||||
secrets:
|
||||
DISCORD_WEBHOOK:
|
||||
required: true
|
||||
GH_TOKEN:
|
||||
required: true
|
||||
|
||||
jobs:
|
||||
notify-discord:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Send Discord Notification
|
||||
env:
|
||||
DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
|
||||
GH_TOKEN: ${{ secrets.GH_TOKEN }}
|
||||
run: |
|
||||
RELEASE_INFO=$(gh release view "${{ inputs.release_tag }}" -R ${{ github.repository }} --json name,url,body,author)
|
||||
|
||||
RELEASE_TITLE=$(echo "$RELEASE_INFO" | jq -r .name)
|
||||
if [ -z "$RELEASE_TITLE" ] || [ "$RELEASE_TITLE" = "null" ]; then RELEASE_TITLE="${{ inputs.release_tag }}"; fi
|
||||
|
||||
RELEASE_URL=$(echo "$RELEASE_INFO" | jq -r .url)
|
||||
RELEASE_BODY=$(echo "$RELEASE_INFO" | jq -r .body)
|
||||
|
||||
AUTHOR_NAME="Portabase"
|
||||
AUTHOR_ICON="https://github.com/Portabase.png"
|
||||
|
||||
PAYLOAD=$(jq -n \
|
||||
--arg title "$RELEASE_TITLE" \
|
||||
--arg description "$RELEASE_BODY" \
|
||||
--arg url "$RELEASE_URL" \
|
||||
--arg author "$AUTHOR_NAME" \
|
||||
--arg icon "$AUTHOR_ICON" \
|
||||
--arg discord_title "${{ inputs.discord_title }}" \
|
||||
--arg discord_footer "${{ inputs.discord_footer }}" \
|
||||
--argjson discord_color ${{ inputs.discord_color }} \
|
||||
'{
|
||||
content: $discord_title,
|
||||
embeds: [{
|
||||
title: $title,
|
||||
url: $url,
|
||||
description: $description,
|
||||
color: $discord_color,
|
||||
author: {
|
||||
name: $author,
|
||||
icon_url: $icon
|
||||
},
|
||||
footer: {
|
||||
text: $discord_footer
|
||||
}
|
||||
}]
|
||||
}'
|
||||
)
|
||||
|
||||
curl -H "Content-Type: application/json" \
|
||||
-d "$PAYLOAD" \
|
||||
"$DISCORD_WEBHOOK"
|
||||
@@ -3,10 +3,16 @@ name: Docker Publish
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
version:
|
||||
required: true
|
||||
type: string
|
||||
ref:
|
||||
required: true
|
||||
type: string
|
||||
image_name:
|
||||
required: false
|
||||
type: string
|
||||
default: 'portabase/agent'
|
||||
default: "portabase/agent"
|
||||
add_latest:
|
||||
required: false
|
||||
type: boolean
|
||||
@@ -14,11 +20,11 @@ on:
|
||||
target:
|
||||
required: false
|
||||
type: string
|
||||
default: 'prod'
|
||||
default: "prod"
|
||||
dockerfile:
|
||||
required: false
|
||||
type: string
|
||||
default: './docker/Dockerfile'
|
||||
default: "./docker/Dockerfile"
|
||||
secrets:
|
||||
DOCKER_USERNAME:
|
||||
required: true
|
||||
@@ -27,32 +33,34 @@ on:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build and push Docker images
|
||||
runs-on: ${{ matrix.platform == 'linux/amd64' && 'ubuntu-latest' || matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' }}
|
||||
name: Build architectures
|
||||
runs-on: ${{ matrix.platform == 'linux/amd64' && 'ubuntu-latest' || 'ubuntu-24.04-arm' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
platform: [linux/amd64, linux/arm64]
|
||||
platform: [ linux/amd64, linux/arm64 ]
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker
|
||||
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Set image tag
|
||||
id: set-tags
|
||||
- name: Prepare Image Tags
|
||||
id: prep
|
||||
run: |
|
||||
REF_NAME=${GITHUB_REF#refs/tags/}
|
||||
if [ "${{ matrix.platform }}" = "linux/amd64" ]; then
|
||||
IMAGE="${{ inputs.image_name }}:$REF_NAME-amd64"
|
||||
else
|
||||
IMAGE="${{ inputs.image_name }}:$REF_NAME-arm64"
|
||||
fi
|
||||
echo "image=$IMAGE" >> $GITHUB_OUTPUT
|
||||
ARCH=${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
|
||||
echo "image=${{ inputs.image_name }}:${{inputs.version}}-$ARCH" >> $GITHUB_OUTPUT
|
||||
echo "safe_platform=${{ matrix.platform == 'linux/amd64' && 'linux-amd64' || 'linux-arm64' }}" >> $GITHUB_OUTPUT
|
||||
|
||||
|
||||
- name: Build and push image
|
||||
uses: docker/build-push-action@v6
|
||||
@@ -61,57 +69,71 @@ jobs:
|
||||
file: ${{ inputs.dockerfile }}
|
||||
platforms: ${{ matrix.platform }}
|
||||
push: true
|
||||
tags: ${{ steps.set-tags.outputs.image }}
|
||||
tags: ${{ steps.prep.outputs.image }}
|
||||
target: ${{ inputs.target }}
|
||||
|
||||
- name: Prepare artifact name
|
||||
id: artifact
|
||||
run: |
|
||||
platform=${{ matrix.platform }}
|
||||
echo "safe_platform=${platform//\//-}" >> $GITHUB_OUTPUT
|
||||
echo "${{ steps.set-tags.outputs.image }}" > image.txt
|
||||
- name: Save image name for manifest
|
||||
run: echo "${{ steps.prep.outputs.image }}" > image.txt
|
||||
|
||||
- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: image-${{ steps.artifact.outputs.safe_platform }}
|
||||
name: image-${{ steps.prep.outputs.safe_platform }}
|
||||
path: image.txt
|
||||
if-no-files-found: warn
|
||||
compression-level: 6
|
||||
overwrite: false
|
||||
include-hidden-files: false
|
||||
retention-days: 1
|
||||
|
||||
create-manifest:
|
||||
name: Create multi-arch Docker manifest
|
||||
name: Create multi-arch manifest
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
steps:
|
||||
- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: image-linux-amd64
|
||||
path: /tmp/digests/amd64
|
||||
|
||||
- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: image-linux-arm64
|
||||
path: /tmp/digests/arm64
|
||||
|
||||
- name: Login to Docker
|
||||
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Generate semantic tags
|
||||
id: tags
|
||||
run: |
|
||||
VERSION="${{ inputs.version }}"
|
||||
IFS='.' read -r MAJOR MINOR PATCH <<< "$VERSION"
|
||||
|
||||
echo "VERSION_TAG=$VERSION" >> $GITHUB_OUTPUT
|
||||
if [ -n "$MINOR" ]; then
|
||||
echo "MINOR_TAG=$MAJOR.$MINOR" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
if [ -n "$MAJOR" ]; then
|
||||
echo "MAJOR_TAG=$MAJOR" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
if [ "${{ inputs.add_latest }}" = "true" ]; then
|
||||
echo "LATEST_TAG=latest" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Extract Docker metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ inputs.image_name }}
|
||||
tags: |
|
||||
type=raw,value=${{ steps.tags.outputs.VERSION_TAG }}
|
||||
type=raw,value=${{ steps.tags.outputs.MINOR_TAG }}
|
||||
type=raw,value=${{ steps.tags.outputs.MAJOR_TAG }}
|
||||
type=raw,value=${{ steps.tags.outputs.LATEST_TAG }}
|
||||
|
||||
- name: Create and push manifest list
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
DOCKER_IMAGES="$(cat amd64/image.txt) $(cat arm64/image.txt)"
|
||||
REF_NAME=${GITHUB_REF#refs/tags/}
|
||||
MANIFEST_IMAGE="${{ inputs.image_name }}:$REF_NAME"
|
||||
|
||||
docker buildx imagetools create $DOCKER_IMAGES -t $MANIFEST_IMAGE
|
||||
docker buildx imagetools inspect $MANIFEST_IMAGE
|
||||
|
||||
if [ "${{ inputs.add_latest }}" = "true" ]; then
|
||||
docker buildx imagetools create $DOCKER_IMAGES -t ${{ inputs.image_name }}:latest
|
||||
docker buildx imagetools inspect ${{ inputs.image_name }}:latest
|
||||
fi
|
||||
TAG_ARGS=$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON")
|
||||
echo $TAG_ARGS
|
||||
docker buildx imagetools create $TAG_ARGS $DOCKER_IMAGES
|
||||
|
||||
@@ -1,128 +0,0 @@
|
||||
name: GitHub Release
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
prerelease:
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
make_latest:
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
discord_title:
|
||||
required: true
|
||||
type: string
|
||||
discord_color:
|
||||
required: true
|
||||
type: number
|
||||
discord_footer:
|
||||
required: true
|
||||
type: string
|
||||
secrets:
|
||||
DISCORD_WEBHOOK:
|
||||
required: true
|
||||
GH_TOKEN:
|
||||
required: true
|
||||
|
||||
jobs:
|
||||
create-release:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Check out the repo
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build Changelog
|
||||
id: build_changelog
|
||||
uses: mikepenz/release-changelog-builder-action@v5
|
||||
with:
|
||||
mode: "COMMIT"
|
||||
configurationJson: |
|
||||
{
|
||||
"template": "#{{CHANGELOG}}",
|
||||
"categories": [
|
||||
{
|
||||
"title": "## Feature",
|
||||
"labels": ["feat", "feature"]
|
||||
},
|
||||
{
|
||||
"title": "## Fix",
|
||||
"labels": ["fix", "bug"]
|
||||
},
|
||||
{
|
||||
"title": "## Other",
|
||||
"labels": []
|
||||
}
|
||||
],
|
||||
"label_extractor": [
|
||||
{
|
||||
"pattern": "^(build|chore|ci|docs|feat|fix|perf|refactor|revert|style|test){1}(\([\w\-\.]+\))?(!)?: ([\w ])+([\s\S]*)",
|
||||
"on_property": "title",
|
||||
"target": "$1"
|
||||
}
|
||||
]
|
||||
}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
|
||||
|
||||
- name: Create GitHub Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
generate_release_notes: false
|
||||
body: ${{ steps.build_changelog.outputs.changelog }}
|
||||
prerelease: ${{ inputs.prerelease }}
|
||||
make_latest: ${{ inputs.make_latest }}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
|
||||
|
||||
- name: Send Discord Notification
|
||||
env:
|
||||
DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
|
||||
GH_TOKEN: ${{ secrets.GH_TOKEN }}
|
||||
run: |
|
||||
RELEASE_INFO=$(gh release view "${{ github.ref_name }}" -R ${{ github.repository }} --json name,url,body,author)
|
||||
|
||||
RELEASE_TITLE=$(echo "$RELEASE_INFO" | jq -r .name)
|
||||
if [ -z "$RELEASE_TITLE" ] || [ "$RELEASE_TITLE" = "null" ]; then RELEASE_TITLE="${{ github.ref_name }}"; fi
|
||||
|
||||
RELEASE_URL=$(echo "$RELEASE_INFO" | jq -r .url)
|
||||
RELEASE_BODY=$(echo "$RELEASE_INFO" | jq -r .body)
|
||||
|
||||
AUTHOR_NAME="Portabase"
|
||||
AUTHOR_ICON="https://github.com/Portabase.png"
|
||||
|
||||
PAYLOAD=$(jq -n \
|
||||
--arg title "$RELEASE_TITLE" \
|
||||
--arg description "$RELEASE_BODY" \
|
||||
--arg url "$RELEASE_URL" \
|
||||
--arg author "$AUTHOR_NAME" \
|
||||
--arg icon "$AUTHOR_ICON" \
|
||||
--arg discord_title "${{ inputs.discord_title }}" \
|
||||
--arg discord_footer "${{ inputs.discord_footer }}" \
|
||||
--argjson discord_color ${{ inputs.discord_color }} \
|
||||
'{
|
||||
content: $discord_title,
|
||||
embeds: [{
|
||||
title: $title,
|
||||
url: $url,
|
||||
description: $description,
|
||||
color: $discord_color,
|
||||
author: {
|
||||
name: $author,
|
||||
icon_url: $icon
|
||||
},
|
||||
footer: {
|
||||
text: $discord_footer
|
||||
}
|
||||
}]
|
||||
}'
|
||||
)
|
||||
|
||||
curl -H "Content-Type: application/json" \
|
||||
-d "$PAYLOAD" \
|
||||
"$DISCORD_WEBHOOK"
|
||||
@@ -0,0 +1,37 @@
|
||||
name: Publish Helm Chart
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
version:
|
||||
required: true
|
||||
type: string
|
||||
secrets:
|
||||
GH_TOKEN:
|
||||
required: true
|
||||
|
||||
jobs:
|
||||
publish-helm:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Helm
|
||||
uses: azure/setup-helm@v4
|
||||
|
||||
- name: Package Helm chart
|
||||
run: |
|
||||
mkdir -p ./helm-packages
|
||||
helm package helm \
|
||||
--version ${{ inputs.version }} \
|
||||
--app-version ${{ inputs.version }} \
|
||||
--destination ./helm-packages
|
||||
|
||||
- name: Authenticate to GitHub Packages
|
||||
run: |
|
||||
echo "${{ secrets.GH_TOKEN }}" | helm registry login ghcr.io -u ${{ github.actor }} --password-stdin
|
||||
|
||||
- name: Push Helm chart to GitHub Packages (OCI)
|
||||
run: |
|
||||
helm push ./helm-packages/portabase-agent-${{ inputs.version }}.tgz oci://ghcr.io/portabase/charts
|
||||
+128
-12
@@ -1,32 +1,148 @@
|
||||
name: Publish Docker image for release
|
||||
name: Auto Release & Publish
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*.*.*'
|
||||
- '!*-*'
|
||||
pull_request:
|
||||
types: [ closed ]
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
docker_publish:
|
||||
|
||||
check-skip:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
skip: ${{ steps.set-skip.outputs.skip }}
|
||||
steps:
|
||||
- name: Determine if release should be skipped
|
||||
id: set-skip
|
||||
run: |
|
||||
TITLE="${{ github.event.pull_request.title }}"
|
||||
echo "PR title: $TITLE"
|
||||
|
||||
if [[ "$TITLE" == *"[skip-release]"* ]]; then
|
||||
echo "PR title contains [skip-release], skipping release jobs."
|
||||
echo "skip=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "skip=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
create-release:
|
||||
needs: check-skip
|
||||
if: ${{ needs.check-skip.outputs.skip == 'false' }}
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
draft_tag: ${{ steps.release_step.outputs.draft_tag }}
|
||||
version: ${{ steps.release_step.outputs.version }}
|
||||
steps:
|
||||
- uses: actions/create-github-app-token@v1
|
||||
id: app-token
|
||||
with:
|
||||
app-id: ${{ vars.APP_ID }}
|
||||
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ steps.app-token.outputs.token }}
|
||||
ref: main
|
||||
|
||||
- name: Setup Rust
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Install cargo-edit
|
||||
run: cargo install cargo-edit
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "lts/*"
|
||||
|
||||
- name: Install release-it globally
|
||||
run: |
|
||||
npm install -g release-it
|
||||
npm install -g @release-it/conventional-changelog
|
||||
npm install -g @release-it/bumper
|
||||
|
||||
- run: |
|
||||
git config --global user.name 'github-actions[bot]'
|
||||
git config --global user.email 'github-actions[bot]@users.noreply.github.com'
|
||||
|
||||
- name: Run release-it
|
||||
id: release_step
|
||||
run: |
|
||||
git pull origin main
|
||||
|
||||
VERSION=$(release-it --ci --release-version)
|
||||
echo $VERSION
|
||||
echo "version=$VERSION" >> $GITHUB_OUTPUT
|
||||
|
||||
OUTPUT=$(release-it --ci)
|
||||
echo "$OUTPUT"
|
||||
|
||||
DRAFT_TAG=$(echo "$OUTPUT" | grep -oE 'untagged-[a-z0-9]+')
|
||||
echo $DRAFT_TAG
|
||||
echo "draft_tag=$DRAFT_TAG" >> $GITHUB_OUTPUT
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}
|
||||
|
||||
publish-docker:
|
||||
needs: create-release
|
||||
if: ${{ needs.create-release.result == 'success' }}
|
||||
uses: ./.github/workflows/docker.yml
|
||||
with:
|
||||
version: ${{ needs.create-release.outputs.version }}
|
||||
ref: ${{ needs.create-release.outputs.version }}
|
||||
add_latest: true
|
||||
secrets:
|
||||
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
|
||||
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
github_release:
|
||||
needs: docker_publish
|
||||
uses: ./.github/workflows/github.yml
|
||||
publish-helm:
|
||||
needs: create-release
|
||||
if: ${{ needs.create-release.result == 'success' }}
|
||||
uses: ./.github/workflows/helm.yml
|
||||
with:
|
||||
prerelease: false
|
||||
make_latest: true
|
||||
version: ${{ needs.create-release.outputs.version }}
|
||||
secrets:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
finalize-release:
|
||||
needs:
|
||||
- create-release
|
||||
- publish-docker
|
||||
- publish-helm
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
release_tag: ${{ steps.publish_release_step.outputs.release_tag }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Publish GitHub Release
|
||||
id: publish_release_step
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
OUTPUT=$(gh release edit ${{ needs.create-release.outputs.draft_tag }} --draft=false)
|
||||
echo "$OUTPUT"
|
||||
RELEASE_TAG=$(echo "$OUTPUT" | sed -E 's|.*/releases/tag/||')
|
||||
echo "release_tag=$RELEASE_TAG" >> $GITHUB_OUTPUT
|
||||
|
||||
notify-discord:
|
||||
needs:
|
||||
- publish-docker
|
||||
- publish-helm
|
||||
- create-release
|
||||
- finalize-release
|
||||
uses: ./.github/workflows/discord.yml
|
||||
with:
|
||||
release_tag: ${{ needs.create-release.outputs.version }}
|
||||
discord_title: "||@release-agent|| New release published"
|
||||
discord_color: 5814783
|
||||
discord_color: 3066993
|
||||
discord_footer: "Portabase"
|
||||
secrets:
|
||||
DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
{
|
||||
"github": {
|
||||
"release": true,
|
||||
"draft": true,
|
||||
"tokenRef": "GITHUB_TOKEN"
|
||||
},
|
||||
"git": {
|
||||
"commit": true,
|
||||
"commitMessage": "chore: release ${version}",
|
||||
"requireCleanWorkingDir": true,
|
||||
"tag": true,
|
||||
"tagName": "${version}",
|
||||
"push": true
|
||||
},
|
||||
"hooks": {
|
||||
"before:bump": "cargo set-version ${version}"
|
||||
},
|
||||
"plugins": {
|
||||
"@release-it/conventional-changelog": {
|
||||
"preset": {
|
||||
"name": "conventionalcommits",
|
||||
"types": [
|
||||
{
|
||||
"type": "feat",
|
||||
"section": "✨ Features"
|
||||
},
|
||||
{
|
||||
"type": "fix",
|
||||
"section": "🐛 Bug Fixes"
|
||||
},
|
||||
{
|
||||
"type": "perf",
|
||||
"section": "⚡️ Performance Improvements"
|
||||
},
|
||||
{
|
||||
"type": "revert",
|
||||
"section": "⏪️ Reverts"
|
||||
},
|
||||
{
|
||||
"type": "docs",
|
||||
"section": "📝 Documentation",
|
||||
"hidden": true
|
||||
},
|
||||
{
|
||||
"type": "chore",
|
||||
"section": "🔧 Chores",
|
||||
"hidden": true
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"@release-it/bumper": {
|
||||
"out": [
|
||||
{
|
||||
"file": "CITATION.cff",
|
||||
"path": "version",
|
||||
"type": "text/yaml"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
+9
-4
@@ -1,6 +1,6 @@
|
||||
cff-version: 1.2.0
|
||||
title: Portabase Agent (Rust)
|
||||
message: "If you use this software, please cite it as below."
|
||||
message: If you use this software, please cite it as below.
|
||||
type: software
|
||||
authors:
|
||||
- family-names: Gauthereau
|
||||
@@ -9,7 +9,12 @@ authors:
|
||||
given-names: Killian
|
||||
repository-code: https://github.com/Portabase/agent-rust
|
||||
url: https://portabase.io
|
||||
abstract: "Portabase is a free, open-source, self-hosted solution for database administration, providing backup and restore capabilities, scheduling, retention policies, notifications, and support for multiple storage backends. Its headless agent architecture enables connection to multiple database instances securely and efficiently."
|
||||
abstract: >-
|
||||
Portabase is a free, open-source, self-hosted solution for database
|
||||
administration, providing backup and restore capabilities, scheduling,
|
||||
retention policies, notifications, and support for multiple storage backends.
|
||||
Its headless agent architecture enables connection to multiple database
|
||||
instances securely and efficiently.
|
||||
keywords:
|
||||
- database
|
||||
- administration
|
||||
@@ -22,5 +27,5 @@ keywords:
|
||||
- self-hosted
|
||||
- portabase
|
||||
license: Apache-2.0
|
||||
version: 1.1.0-rc.1
|
||||
date-released: "2026-02-10"
|
||||
version: 1.4.0
|
||||
date-released: '2026-02-24'
|
||||
|
||||
Generated
+1445
-859
File diff suppressed because it is too large
Load Diff
+16
-2
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "portabase-agent"
|
||||
version = "1.1.0-rc.1"
|
||||
version = "1.4.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
@@ -42,7 +42,21 @@ async-compression = { version = "0.4.37", features = ["tokio", "gzip"] }
|
||||
tokio-tar = "0.3.1"
|
||||
oauth2 = "5.0.0"
|
||||
hyper = "1.8.1"
|
||||
async-http-client = "0.2.0"
|
||||
aes-gcm = "0.11.0-rc.3"
|
||||
generic-array = "0.14.7"
|
||||
futures-util = "0.3.31"
|
||||
tokio-stream = "0.1.18"
|
||||
aes = "0.9.0-rc.4"
|
||||
typenum = "1.19.0"
|
||||
testcontainers = "0.27.1"
|
||||
testcontainers-modules = { version = "0.15.0", features = ["postgres"] }
|
||||
postgres = "0.19.12"
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
mockall = "0.13"
|
||||
testcontainers = "0.27.1"
|
||||
wiremock = "0.6"
|
||||
|
||||
|
||||
[[bin]]
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
include .env
|
||||
export $(shell sed 's/=.*//' .env)
|
||||
CLUSTER_SCRIPT=docker/entrypoints/app-dev-entrypoint.sh
|
||||
|
||||
.PHONY: seed-mongo seed-mysql seed-postgres
|
||||
|
||||
up:
|
||||
@bash $(CLUSTER_SCRIPT)
|
||||
|
||||
seed-mongo:
|
||||
@echo "Seeding MongoDB..."
|
||||
bash ./scripts/mongo/seed-mongo.sh
|
||||
@@ -30,4 +34,21 @@ seed-postgres-1gb:
|
||||
docker exec -i -e PGPASSWORD=$$PG_PASSWORD $$PG_CONTAINER \
|
||||
psql -U $$PG_USER -d $$PG_DB < ./scripts/postgres/seed-1gb.sql
|
||||
|
||||
|
||||
SQLITE_SEED_FILE := $(if $(filter big,$(SEED)),./scripts/sqlite/seed-big.sql,./scripts/sqlite/seed.sql)
|
||||
|
||||
seed-sqlite:
|
||||
@echo "Seeding Sqlite..."
|
||||
@echo "Run as root to fix permissions inside the volume"
|
||||
docker exec -u 0 -it db-sqlite sh -c "chmod -R 777 /workspace/data"
|
||||
@echo "Create the database file (if it doesn’t exist)"
|
||||
docker exec -u 0 -it db-sqlite sh -c "touch /workspace/data/app.db"
|
||||
@echo "Seed the database"
|
||||
docker exec -i db-sqlite sh -c "sqlite3 /workspace/data/app.db" < $(SQLITE_SEED_FILE)
|
||||
@echo "Verify"
|
||||
docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT name FROM sqlite_master WHERE type='table';"
|
||||
@echo "Done"
|
||||
|
||||
|
||||
|
||||
seed-all: seed-mongo seed-mysql seed-postgres seed-postgres-1gb
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
[](https://www.python.org/downloads/release/python-3120/)
|
||||
[](https://www.postgresql.org/)
|
||||
[](https://www.mysql.com/)
|
||||
[](https://sqlite.org/)
|
||||
[](https://mariadb.org/)
|
||||
[](https://www.mongodb.com/)
|
||||
[](https://github.com/Portabase/portabase)
|
||||
@@ -79,3 +80,5 @@ Distributed under the Apache License. See `LICENSE.txt` for more information.
|
||||
|
||||
[Docker-url]: https://www.docker.com/
|
||||
|
||||
|
||||
|
||||
|
||||
+12
-2
@@ -34,11 +34,21 @@
|
||||
"name": "Test database 5 - MongoDB",
|
||||
"database": "testdb",
|
||||
"type": "mongodb",
|
||||
"username": "",
|
||||
"password": "",
|
||||
"port": 27017,
|
||||
"host": "db-mongodb",
|
||||
"generated_id": "16678147-ff7e-4c97-8c83-0adeff214681"
|
||||
},
|
||||
{
|
||||
"name": "Test database 6 - SQLite DB",
|
||||
"type": "sqlite",
|
||||
"path": "/sqlite-data/workspace/data/app.db",
|
||||
"generated_id": "16678178-ff7e-4c97-8c83-0adeff214681"
|
||||
},
|
||||
{
|
||||
"name": "Test database 7 - SQLite DB",
|
||||
"type": "sqlite",
|
||||
"path": "/sqlite-data-2/workspace/data/app.db",
|
||||
"generated_id": "16678179-ff7e-4c97-8c83-0adeff214681"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
+32
-21
@@ -12,12 +12,13 @@ services:
|
||||
- cargo-registry:/usr/local/cargo/registry
|
||||
- cargo-git:/usr/local/cargo/git
|
||||
- cargo-target:/app/target
|
||||
# - sqlite-data:/sqlite-data/workspace/data
|
||||
# - ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data
|
||||
environment:
|
||||
APP_ENV: development
|
||||
LOG: debug
|
||||
TZ: "Europe/Paris"
|
||||
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiOTI0OGU1ZWUtYWU5Yi00ZGQ0LTk3MDUtYjEwYWQzNDU4YmNjIiwicHVibGljS2V5IjoiLS0tLS1CRUdJTiBSU0EgUFVCTElDIEtFWS0tLS0tXG5NSUlCQ2dLQ0FRRUE5TWV4M2pmdnVLdFB5YU1ERnh2Ulp2dmd3YkRJQ2JzQi81Wll5NDNSVVRBaXZRYjJiSDdYXG5qRHBQd1lJeCs4UFBrbHlRbDVMQzV1UWZEaCs4SVd4OG1LZ3FvMXpWMkdiZXdGbEdEWFYxVEdyU1ZEU25aSWR4XG52bWdYc29EeXhVMlJvWUFUMS9YMWxuc2YxenZKdkFMTkhXdEhRdk42SjVDZTFSMmFsendVRGFEVXlJNzRmSldQXG5tNTh0SDMrYklXL0VVTXdjaWNxM0oySWw3Vm9KNkZNUHJQL1ZSOWEvdFF1SU1qa200MXpFY2NscExPa2luRkxuXG54NmVUWkFSZUpya2UrbnRvZ2t4TGEyRWV5a1lUNzB4V3hKNWp5ZExBVnRvNkkyQlVLVVJoTkowTUFaU29NYUtvXG5iMGJRcnY1UzExZWllMnMrT2I3aTYzSFpkVUx0UmV1MVJ3SURBUUFCXG4tLS0tLUVORCBSU0EgUFVCTElDIEtFWS0tLS0tXG4ifQ"
|
||||
# EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiZWE2NTg1MDctZTA5My00NDUxLWIxZDAtMDgwZWZjMGNmNWYzIiwicHVibGljS2V5IjoiLS0tLS1CRUdJTiBSU0EgUFVCTElDIEtFWS0tLS0tXG5NSUlCQ2dLQ0FRRUE5TWV4M2pmdnVLdFB5YU1ERnh2Ulp2dmd3YkRJQ2JzQi81Wll5NDNSVVRBaXZRYjJiSDdYXG5qRHBQd1lJeCs4UFBrbHlRbDVMQzV1UWZEaCs4SVd4OG1LZ3FvMXpWMkdiZXdGbEdEWFYxVEdyU1ZEU25aSWR4XG52bWdYc29EeXhVMlJvWUFUMS9YMWxuc2YxenZKdkFMTkhXdEhRdk42SjVDZTFSMmFsendVRGFEVXlJNzRmSldQXG5tNTh0SDMrYklXL0VVTXdjaWNxM0oySWw3Vm9KNkZNUHJQL1ZSOWEvdFF1SU1qa200MXpFY2NscExPa2luRkxuXG54NmVUWkFSZUpya2UrbnRvZ2t4TGEyRWV5a1lUNzB4V3hKNWp5ZExBVnRvNkkyQlVLVVJoTkowTUFaU29NYUtvXG5iMGJRcnY1UzExZWllMnMrT2I3aTYzSFpkVUx0UmV1MVJ3SURBUUFCXG4tLS0tLUVORCBSU0EgUFVCTElDIEtFWS0tLS0tXG4ifQ"
|
||||
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiOGYwMmExZTAtNDY0NC00MWFmLWIzYjctYjZkYWNjNzQ4OWVhIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ=="
|
||||
#POOLING: 1
|
||||
#DATABASES_CONFIG_FILE: "config.toml"
|
||||
extra_hosts:
|
||||
@@ -38,22 +39,22 @@ services:
|
||||
- POSTGRES_PASSWORD=changeme
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
db-mariadb:
|
||||
container_name: db-mariadb
|
||||
image: mariadb:latest
|
||||
ports:
|
||||
- "3311:3306"
|
||||
environment:
|
||||
- MYSQL_DATABASE=mariadb
|
||||
- MYSQL_USER=mariadb
|
||||
- MYSQL_PASSWORD=changeme
|
||||
- MYSQL_RANDOM_ROOT_PASSWORD=yes
|
||||
volumes:
|
||||
- mariadb-data:/var/lib/mysql
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
#
|
||||
# db-mariadb:
|
||||
# container_name: db-mariadb
|
||||
# image: mariadb:latest
|
||||
# ports:
|
||||
# - "3311:3306"
|
||||
# environment:
|
||||
# - MYSQL_DATABASE=mariadb
|
||||
# - MYSQL_USER=mariadb
|
||||
# - MYSQL_PASSWORD=changeme
|
||||
# - MYSQL_RANDOM_ROOT_PASSWORD=yes
|
||||
# volumes:
|
||||
# - mariadb-data:/var/lib/mysql
|
||||
# networks:
|
||||
# - portabase
|
||||
#
|
||||
#
|
||||
# db-mongodb-auth:
|
||||
# container_name: db-mongodb-auth
|
||||
@@ -92,20 +93,30 @@ services:
|
||||
# networks:
|
||||
# - portabase
|
||||
|
||||
# sqlite:
|
||||
# container_name: db-sqlite
|
||||
# image: keinos/sqlite3
|
||||
# volumes:
|
||||
# - sqlite-data:/workspace/data
|
||||
# working_dir: /workspace
|
||||
# command: tail -f /dev/null
|
||||
# stdin_open: true
|
||||
# tty: true
|
||||
|
||||
|
||||
volumes:
|
||||
cargo-registry:
|
||||
cargo-git:
|
||||
cargo-target:
|
||||
|
||||
|
||||
postgres-data:
|
||||
mariadb-data:
|
||||
# mariadb-data:
|
||||
# mongodb-data:
|
||||
# mongodb-data-auth:
|
||||
# sqlite-data:
|
||||
|
||||
networks:
|
||||
portabase:
|
||||
name: portabase_network
|
||||
external: true
|
||||
|
||||
# docker network create portabase_network
|
||||
@@ -15,6 +15,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
zlib1g \
|
||||
curl \
|
||||
mariadb-client \
|
||||
sqlite3 \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
@@ -92,6 +93,7 @@ RUN apt-get update && apt-get install -y \
|
||||
libncurses6 \
|
||||
zlib1g \
|
||||
mariadb-client \
|
||||
sqlite3 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
|
||||
|
||||
Executable
+39
@@ -0,0 +1,39 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
check_docker() {
|
||||
if ! docker info > /dev/null 2>&1; then
|
||||
echo "Docker is not running. Attempting to start Docker..."
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
open -a Docker
|
||||
echo "Waiting for Docker to start..."
|
||||
until docker info > /dev/null 2>&1; do
|
||||
sleep 2
|
||||
done
|
||||
elif command -v systemctl >/dev/null 2>&1; then
|
||||
sudo systemctl start docker
|
||||
else
|
||||
echo "Cannot start Docker automatically. Please start Docker manually."
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "Docker is running."
|
||||
fi
|
||||
}
|
||||
|
||||
check_network() {
|
||||
local network_name="portabase_network"
|
||||
if ! docker network ls --format '{{.Name}}' | grep -q "^${network_name}$"; then
|
||||
echo "Docker network '${network_name}' not found. Creating..."
|
||||
docker network create "${network_name}"
|
||||
else
|
||||
echo "Docker network '${network_name}' already exists."
|
||||
fi
|
||||
}
|
||||
|
||||
check_docker
|
||||
check_network
|
||||
|
||||
echo "Starting docker-compose..."
|
||||
docker-compose -f ./docker-compose.yml up
|
||||
echo "Docker-compose started successfully."
|
||||
@@ -0,0 +1,17 @@
|
||||
.DS_Store
|
||||
.git/
|
||||
.gitignore
|
||||
.bzr/
|
||||
.bzrignore
|
||||
.hg/
|
||||
.hgignore
|
||||
.svn/
|
||||
*.swp
|
||||
*.bak
|
||||
*.tmp
|
||||
*.orig
|
||||
*~
|
||||
.project
|
||||
.idea/
|
||||
*.tmproj
|
||||
.vscode/
|
||||
@@ -0,0 +1,29 @@
|
||||
apiVersion: v2
|
||||
name: portabase-agent
|
||||
description: Helm chart for Portabase Agent
|
||||
type: application
|
||||
version: 0.0.0
|
||||
appVersion: "latest"
|
||||
keywords:
|
||||
- postgresql
|
||||
- mariadb
|
||||
- mongodb
|
||||
- mysql
|
||||
- sqlite
|
||||
- backup
|
||||
- database
|
||||
- restore
|
||||
- agent
|
||||
home: https://github.com/Portabase/agent
|
||||
|
||||
sources:
|
||||
- https://github.com/Portabase/agent
|
||||
- https://github.com/Portabase/agent/tree/main/helm
|
||||
|
||||
maintainers:
|
||||
- name: Charles Gauthereau
|
||||
url: https://github.com/RambokDev
|
||||
- name: Killian Larcher
|
||||
url: https://github.com/KillianLarcher
|
||||
|
||||
icon: https://raw.githubusercontent.com/Portabase/agent/main/.github/assets/logo.png
|
||||
@@ -0,0 +1,59 @@
|
||||
# Development Notes
|
||||
|
||||
## Check that Kubernetes is reachable locally
|
||||
|
||||
```bash
|
||||
kubectl get nodes
|
||||
```
|
||||
|
||||
## Install the local Portabase Agent Helm chart
|
||||
|
||||
```bash
|
||||
helm install portabase-agent . \
|
||||
--set env.EDGE_KEY=<your-edge-key>
|
||||
```
|
||||
|
||||
## Check the pods
|
||||
|
||||
```bash
|
||||
kubectl get pods
|
||||
```
|
||||
|
||||
## Check the services
|
||||
|
||||
```bash
|
||||
kubectl get svc
|
||||
```
|
||||
|
||||
## To update .env variables or JSON config:
|
||||
```bash
|
||||
kubectl rollout restart deployment portabase-agent
|
||||
```
|
||||
|
||||
## Install or upgrade the Helm chart
|
||||
```bash
|
||||
helm upgrade portabase-agent . \
|
||||
--reuse-values \
|
||||
--set env.EDGE_KEY="NEW_EDGE_KEY"
|
||||
```
|
||||
|
||||
## Rollout to restart
|
||||
```bash
|
||||
kubectl rollout restart deployment portabase-agent
|
||||
```
|
||||
|
||||
## List pods to get the pod name
|
||||
|
||||
```bash
|
||||
kubectl get pods -l app=portabase-agent
|
||||
```
|
||||
|
||||
## Get logs for the pod
|
||||
```bash
|
||||
kubectl logs portabase-agent-6f7d4f5c6b-abc12
|
||||
```
|
||||
|
||||
## Uninstall Agent
|
||||
``` bash
|
||||
helm uninstall portabase-agent
|
||||
```
|
||||
@@ -0,0 +1,7 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: portabase-agent-config
|
||||
data:
|
||||
config.json: |
|
||||
{{ .Values.volume.configFile.content | nindent 4 }}
|
||||
@@ -0,0 +1,62 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: portabase-agent
|
||||
labels:
|
||||
app: portabase-agent
|
||||
spec:
|
||||
replicas: {{ .Values.replicaCount }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: portabase-agent
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: portabase-agent
|
||||
spec:
|
||||
hostAliases:
|
||||
{{- range .Values.network.hostAliases }}
|
||||
- ip: {{ .ip }}
|
||||
hostnames:
|
||||
{{- range .hostnames }}
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: portabase-agent
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: portabase-agent-env
|
||||
volumeMounts:
|
||||
{{- if .Values.volume.configFile.enabled }}
|
||||
{{- if .Values.volume.configFile.hostPath }}
|
||||
- name: config
|
||||
mountPath: /config/config.json
|
||||
subPath: config.json
|
||||
readOnly: true
|
||||
# uses hostPath
|
||||
{{- else }}
|
||||
- name: config
|
||||
mountPath: /config/config.json
|
||||
subPath: config.json
|
||||
# uses ConfigMap content
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if .Values.volume.configFile.enabled }}
|
||||
{{- if .Values.volume.configFile.hostPath }}
|
||||
- name: config
|
||||
hostPath:
|
||||
path: {{ .Values.volume.configFile.hostPath }}
|
||||
type: File
|
||||
{{- else }}
|
||||
- name: config
|
||||
configMap:
|
||||
name: portabase-agent-config
|
||||
items:
|
||||
- key: config.json
|
||||
path: config.json
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,10 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: portabase-agent-env
|
||||
data:
|
||||
EDGE_KEY: {{ .Values.env.EDGE_KEY | quote }}
|
||||
TZ: {{ .Values.env.TZ | quote }}
|
||||
POLLING: {{ .Values.env.POLLING | quote }}
|
||||
APP_ENV: {{ .Values.env.APP_ENV | quote }}
|
||||
LOG: {{ .Values.env.LOG | quote }}
|
||||
@@ -0,0 +1,48 @@
|
||||
replicaCount: 1
|
||||
|
||||
image:
|
||||
repository: portabase/agent
|
||||
tag: latest
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
env:
|
||||
EDGE_KEY: "your_edge_key_here"
|
||||
TZ: "UTC"
|
||||
POLLING: "5"
|
||||
APP_ENV: "production"
|
||||
LOG: "info"
|
||||
|
||||
resources:
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
|
||||
volume:
|
||||
configFile:
|
||||
enabled: true
|
||||
hostPath: "" # Use host file if set, otherwise use `content`
|
||||
content: | # JSON content for config.json if no hostPath
|
||||
{
|
||||
"databases": [
|
||||
{
|
||||
"name": "my-site-prod (readable name)",
|
||||
"database": "devdb",
|
||||
"type": "postgresql",
|
||||
"host": "localhost",
|
||||
"port": 5432,
|
||||
"username": "admin_prod",
|
||||
"password": "super_secure_password",
|
||||
"generated_id": "550e8400-e29b-41d4-a716-446655440000"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
network:
|
||||
hostAliases:
|
||||
- ip: "127.0.0.1"
|
||||
hostnames:
|
||||
- "localhost"
|
||||
@@ -1,99 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
echo "Usage: ./release <version>"
|
||||
echo "Example: ./release v1.0.0"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
VERSION=$1
|
||||
CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
|
||||
|
||||
if [ "$CURRENT_BRANCH" = "main" ]; then
|
||||
if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "Error: On 'main' branch, only release tags (X.Y.Z) are allowed."
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+$ ]]; then
|
||||
echo "Error: On branch '$CURRENT_BRANCH', only RC tags matching X.Y.Z-rc.W are allowed (e.g., 1.0.0-rc.1)."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
CLEAN_VERSION=${VERSION#v}
|
||||
CURRENT_DATE=$(date +%Y-%m-%d)
|
||||
|
||||
echo "Preparing release $VERSION..."
|
||||
|
||||
|
||||
# package.json
|
||||
if [ -f package.json ]; then
|
||||
echo "Updating package.json..."
|
||||
if sed --version >/dev/null 2>&1; then
|
||||
sed -i "s/\"version\": \".*\"/\"version\": \"$CLEAN_VERSION\"/" package.json
|
||||
else
|
||||
sed -i '' "s/\"version\": \".*\"/\"version\": \"$CLEAN_VERSION\"/" package.json
|
||||
fi
|
||||
fi
|
||||
|
||||
# pyproject.toml
|
||||
if [ -f pyproject.toml ]; then
|
||||
echo "Updating pyproject.toml..."
|
||||
if sed --version >/dev/null 2>&1; then
|
||||
sed -i "s/^version = \".*\"/version = \"$CLEAN_VERSION\"/" pyproject.toml
|
||||
else
|
||||
sed -i '' "s/^version = \".*\"/version = \"$CLEAN_VERSION\"/" pyproject.toml
|
||||
fi
|
||||
fi
|
||||
|
||||
# Cargo.toml
|
||||
if [ -f Cargo.toml ]; then
|
||||
echo "Updating Cargo.toml..."
|
||||
if sed --version >/dev/null 2>&1; then
|
||||
sed -i "s/^version = \".*\"/version = \"$CLEAN_VERSION\"/" Cargo.toml
|
||||
else
|
||||
sed -i '' "s/^version = \".*\"/version = \"$CLEAN_VERSION\"/" Cargo.toml
|
||||
fi
|
||||
fi
|
||||
|
||||
# CITATION.cff
|
||||
if [ -f CITATION.cff ]; then
|
||||
echo "Updating CITATION.cff..."
|
||||
if sed --version >/dev/null 2>&1; then
|
||||
sed -i "s/^version: .*/version: $CLEAN_VERSION/" CITATION.cff
|
||||
sed -i "s/^date-released: .*/date-released: \"$CURRENT_DATE\"/" CITATION.cff
|
||||
else
|
||||
sed -i '' "s/^version: .*/version: $CLEAN_VERSION/" CITATION.cff
|
||||
sed -i '' "s/^date-released: .*/date-released: \"$CURRENT_DATE\"/" CITATION.cff
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
git add .
|
||||
|
||||
if ! git diff-index --quiet HEAD --; then
|
||||
echo "Committing changes..."
|
||||
git commit -m "chore(release): $VERSION"
|
||||
else
|
||||
echo "No changes to commit. Proceeding to tag..."
|
||||
fi
|
||||
|
||||
if git rev-parse "$VERSION" >/dev/null 2>&1; then
|
||||
echo "Tag $VERSION already exists. Aborting."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Creating tag $VERSION..."
|
||||
git tag -a "$VERSION" -m "Release $VERSION"
|
||||
|
||||
echo "Pushing changes and tags to remote..."
|
||||
git push
|
||||
git push origin "$VERSION"
|
||||
|
||||
echo "Successfully released $VERSION!"
|
||||
+15
-2
@@ -1,6 +1,5 @@
|
||||
# Seed instructions
|
||||
|
||||
## MongoDB
|
||||
|
||||
```bash
|
||||
make seed-mongo
|
||||
@@ -9,5 +8,19 @@ make seed-mysql
|
||||
make seed-mysql-1gb
|
||||
make seed-postgres
|
||||
make seed-postgres-1gb
|
||||
make seed-all
|
||||
make seed-all
|
||||
make seed-sqlite
|
||||
make seed-sqlite SEED=big
|
||||
```
|
||||
|
||||
## Verify commands
|
||||
|
||||
### Sqlite
|
||||
|
||||
```bash
|
||||
docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT * FROM users LIMIT 10;"
|
||||
```
|
||||
|
||||
```bash
|
||||
docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT name FROM sqlite_master WHERE type='table';"
|
||||
```
|
||||
|
||||
@@ -60,7 +60,7 @@ SELECT
|
||||
)
|
||||
FROM users u
|
||||
JOIN generate_series(1, 30) AS p(post_no)
|
||||
ON u.id <= 300000;
|
||||
ON u.id <= 500000;
|
||||
|
||||
-- ============================================================
|
||||
-- OPTIONAL: FORCE DISK MATERIALIZATION
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
PRAGMA foreign_keys = ON;
|
||||
BEGIN TRANSACTION;
|
||||
|
||||
-- Drop tables
|
||||
DROP TABLE IF EXISTS users;
|
||||
DROP TABLE IF EXISTS projects;
|
||||
DROP TABLE IF EXISTS tasks;
|
||||
|
||||
-- Users
|
||||
CREATE TABLE users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
email TEXT NOT NULL UNIQUE,
|
||||
full_name TEXT NOT NULL,
|
||||
role TEXT NOT NULL CHECK (role IN ('admin','manager','user')),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
-- Projects
|
||||
CREATE TABLE projects (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT,
|
||||
owner_id INTEGER NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
FOREIGN KEY (owner_id) REFERENCES users(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- Tasks
|
||||
CREATE TABLE tasks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
project_id INTEGER NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
status TEXT NOT NULL CHECK (status IN ('todo','in_progress','done')),
|
||||
priority INTEGER NOT NULL DEFAULT 3,
|
||||
due_date TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- Seed minimal users
|
||||
INSERT INTO users (email, full_name, role) VALUES
|
||||
('admin@example.com', 'System Admin', 'admin'),
|
||||
('manager@example.com', 'Project Manager', 'manager'),
|
||||
('user@example.com', 'Standard User', 'user');
|
||||
|
||||
-- Generate 10_000 projects
|
||||
WITH RECURSIVE numbers(x) AS (
|
||||
SELECT 1
|
||||
UNION ALL
|
||||
SELECT x+1 FROM numbers WHERE x<10000
|
||||
)
|
||||
INSERT INTO projects (name, description, owner_id)
|
||||
SELECT
|
||||
'Project #' || x,
|
||||
'Auto-generated project description for project #' || x,
|
||||
(1 + (x % 3)) -- cycle users 1..3
|
||||
FROM numbers;
|
||||
|
||||
-- Generate 100_000 tasks
|
||||
WITH RECURSIVE numbers(x) AS (
|
||||
SELECT 1
|
||||
UNION ALL
|
||||
SELECT x+1 FROM numbers WHERE x<100000
|
||||
)
|
||||
INSERT INTO tasks (project_id, title, status, priority, due_date)
|
||||
SELECT
|
||||
(1 + (x % 10000)), -- project id cycle
|
||||
'Task #' || x,
|
||||
CASE (x % 3) WHEN 0 THEN 'todo' WHEN 1 THEN 'in_progress' ELSE 'done' END,
|
||||
1 + (x % 5),
|
||||
date('now', '+' || (x % 30) || ' days')
|
||||
FROM numbers;
|
||||
|
||||
COMMIT;
|
||||
@@ -0,0 +1,59 @@
|
||||
PRAGMA foreign_keys = ON;
|
||||
|
||||
BEGIN TRANSACTION;
|
||||
|
||||
-- Drop existing tables (idempotent reset)
|
||||
DROP TABLE IF EXISTS users;
|
||||
DROP TABLE IF EXISTS projects;
|
||||
DROP TABLE IF EXISTS tasks;
|
||||
|
||||
-- Users
|
||||
CREATE TABLE users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
email TEXT NOT NULL UNIQUE,
|
||||
full_name TEXT NOT NULL,
|
||||
role TEXT NOT NULL CHECK (role IN ('admin','manager','user')),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
-- Projects
|
||||
CREATE TABLE projects (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT,
|
||||
owner_id INTEGER NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
FOREIGN KEY (owner_id) REFERENCES users(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- Tasks
|
||||
CREATE TABLE tasks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
project_id INTEGER NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
status TEXT NOT NULL CHECK (status IN ('todo','in_progress','done')),
|
||||
priority INTEGER NOT NULL DEFAULT 3,
|
||||
due_date TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- Seed users
|
||||
INSERT INTO users (email, full_name, role) VALUES
|
||||
('admin@example.com', 'System Admin', 'admin'),
|
||||
('manager@example.com', 'Project Manager', 'manager'),
|
||||
('user@example.com', 'Standard User', 'user');
|
||||
|
||||
-- Seed projects
|
||||
INSERT INTO projects (name, description, owner_id) VALUES
|
||||
('Internal Tooling', 'Backoffice automation platform', 2),
|
||||
('Client Portal', 'Customer-facing SaaS interface', 2);
|
||||
|
||||
-- Seed tasks
|
||||
INSERT INTO tasks (project_id, title, status, priority, due_date) VALUES
|
||||
(1, 'Define architecture', 'done', 1, date('now', '+3 days')),
|
||||
(1, 'Implement authentication', 'in_progress', 1, date('now', '+7 days')),
|
||||
(2, 'Design landing page', 'todo', 2, date('now', '+5 days')),
|
||||
(2, 'Setup CI/CD', 'todo', 2, date('now', '+10 days'));
|
||||
|
||||
COMMIT;
|
||||
Binary file not shown.
+1
-1
@@ -35,7 +35,7 @@ impl Context {
|
||||
panic!("Cannot initialize AgentContext due to invalid EDGE_KEY");
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
let server_url = format!("{}/api", edge_key.server_url);
|
||||
let api_client = ApiClient::new(server_url);
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ use crate::services::config::{DatabaseConfig, DbType};
|
||||
use anyhow::Result;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use crate::domain::sqlite::database::SqliteDatabase;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait Database: Send + Sync {
|
||||
@@ -27,6 +28,7 @@ impl DatabaseFactory {
|
||||
DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)),
|
||||
DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)),
|
||||
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
|
||||
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,6 +41,7 @@ impl DatabaseFactory {
|
||||
DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)),
|
||||
DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)),
|
||||
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
|
||||
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,4 +2,5 @@ pub mod factory;
|
||||
pub mod postgres;
|
||||
pub mod mysql;
|
||||
mod mongodb;
|
||||
mod sqlite;
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ pub async fn run(
|
||||
|
||||
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
|
||||
let mongodump = select_mongo_path().join("mongodump");
|
||||
let uri = get_mongo_uri(cfg.clone());
|
||||
let uri = get_mongo_uri(cfg.clone())?;
|
||||
|
||||
let output = Command::new(mongodump)
|
||||
.arg(format!("--uri={}", uri))
|
||||
|
||||
@@ -3,7 +3,7 @@ use anyhow::Result;
|
||||
use mongodb::Client;
|
||||
|
||||
pub async fn connect(cfg: DatabaseConfig) -> Result<Client> {
|
||||
let uri = get_mongo_uri(cfg);
|
||||
let uri = get_mongo_uri(cfg)?;
|
||||
let mut options = mongodb::options::ClientOptions::parse(&uri).await?;
|
||||
options.server_selection_timeout = Some(std::time::Duration::from_secs(3));
|
||||
options.connect_timeout = Some(std::time::Duration::from_secs(3));
|
||||
@@ -15,14 +15,16 @@ pub fn select_mongo_path() -> std::path::PathBuf {
|
||||
"/usr/local/mongodb/bin".to_string().into()
|
||||
}
|
||||
|
||||
pub fn get_mongo_uri(cfg: DatabaseConfig) -> String {
|
||||
if cfg.username.is_empty() {
|
||||
format!("mongodb://{}:{}/{}", cfg.host, cfg.port, cfg.database)
|
||||
pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result<String> {
|
||||
|
||||
if cfg.username.is_empty() || cfg.password.is_empty() {
|
||||
Ok(format!("mongodb://{}:{}/{}", cfg.host, cfg.port, cfg.database))
|
||||
} else {
|
||||
format!(
|
||||
Ok(format!(
|
||||
"mongodb://{}:{}@{}:{}/{}?authSource=admin",
|
||||
cfg.username, cfg.password, cfg.host, cfg.port, cfg.database
|
||||
)
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,14 +1,20 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use crate::domain::mongodb::connection::connect;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::Result;
|
||||
use mongodb::bson::doc;
|
||||
use tracing::{error};
|
||||
use crate::domain::mongodb::connection::connect;
|
||||
use tracing::error;
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
|
||||
let client = connect(cfg.clone()).await?;
|
||||
let db_name = if cfg.username.is_empty() { &cfg.database } else { "admin" };
|
||||
|
||||
let db_name = if cfg.username.is_empty() && cfg.password.is_empty() {
|
||||
&cfg.database
|
||||
} else {
|
||||
"admin"
|
||||
};
|
||||
|
||||
match client.database(db_name).run_command(doc! {"ping": 1}).await {
|
||||
Ok(_) => Ok(true),
|
||||
Err(e) => {
|
||||
|
||||
@@ -10,7 +10,7 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
|
||||
debug!("Starting MongoDB restore for database {}", cfg.name);
|
||||
|
||||
let mongorestore = select_mongo_path().join("mongorestore");
|
||||
let uri = get_mongo_uri(cfg.clone());
|
||||
let uri = get_mongo_uri(cfg.clone())?;
|
||||
|
||||
let output = Command::new(mongorestore)
|
||||
.arg(format!("--uri={}", uri))
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use crate::domain::mysql::connection::server_version;
|
||||
use crate::domain::mysql::connection::{server_version};
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -3,6 +3,7 @@ use std::process::Command;
|
||||
use anyhow::Result;
|
||||
|
||||
pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
|
||||
|
||||
let output = Command::new("mysql")
|
||||
.arg("--host").arg(&cfg.host)
|
||||
.arg("--port").arg(cfg.port.to_string())
|
||||
@@ -25,3 +26,4 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
|
||||
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ impl MySQLDatabase {
|
||||
|
||||
fn build_env(&self) -> HashMap<String, String> {
|
||||
let mut envs = std::env::vars().collect::<HashMap<_, _>>();
|
||||
envs.insert("MYSQL_PWD".to_string(), self.cfg.password.clone());
|
||||
envs.insert("MYSQL_PWD".to_string(), self.cfg.password.to_string());
|
||||
envs
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,19 +1,27 @@
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::Context;
|
||||
use std::collections::HashMap;
|
||||
use std::process::Command;
|
||||
use tokio::process::Command;
|
||||
use tokio::time::{Duration, timeout};
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::Result<bool> {
|
||||
let output = Command::new("mysqladmin")
|
||||
.arg("--host")
|
||||
|
||||
let mut cmd = Command::new("mysqladmin");
|
||||
cmd.arg("--host")
|
||||
.arg(cfg.host)
|
||||
.arg("--port")
|
||||
.arg(cfg.port.to_string())
|
||||
.arg("--user")
|
||||
.arg(cfg.username)
|
||||
.arg("ping")
|
||||
.envs(env)
|
||||
.output()
|
||||
.with_context(|| format!("Failed to ping MySQL server {}", cfg.name))?;
|
||||
Ok(output.status.success())
|
||||
.envs(env);
|
||||
|
||||
let result = timeout(Duration::from_secs(10), cmd.output()).await;
|
||||
|
||||
match result {
|
||||
Ok(output) => {
|
||||
let output = output?;
|
||||
Ok(output.status.success())
|
||||
}
|
||||
Err(_) => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::fs::File;
|
||||
use std::io::{Read, Write};
|
||||
use anyhow::{Context, Result};
|
||||
use tracing::{debug, error, info};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
|
||||
let handle = tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
@@ -55,7 +54,8 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
|
||||
.with_context(|| format!("Failed to start mysql restore for {}", cfg.name))?;
|
||||
|
||||
let mut stdin = child.stdin.take().context("Failed to open child stdin")?;
|
||||
stdin.write_all(sql_content.as_bytes())
|
||||
stdin
|
||||
.write_all(sql_content.as_bytes())
|
||||
.context("Failed to write SQL content to mysql stdin")?;
|
||||
stdin.flush()?;
|
||||
drop(stdin);
|
||||
@@ -74,8 +74,7 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
|
||||
Ok(())
|
||||
});
|
||||
|
||||
handle
|
||||
.await??;
|
||||
handle.await??;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use anyhow::Result;
|
||||
use tracing::{debug, error, info};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
use super::connection::{select_pg_path, server_version};
|
||||
use super::format::PostgresDumpFormat;
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
use std::path::Path;
|
||||
use crate::domain::postgres::format::PostgresDumpFormat;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::Result;
|
||||
use std::path::Path;
|
||||
use tokio_postgres::{Client, NoTls};
|
||||
use tracing::info;
|
||||
use tracing::{error, info};
|
||||
|
||||
pub async fn connect(cfg: &DatabaseConfig) -> Result<Client> {
|
||||
info!("Connecting to postgres database {}:{}", cfg.host, cfg.port);
|
||||
let dsn = format!(
|
||||
"host={} port={} user={} password={} dbname={}",
|
||||
cfg.host, cfg.port, cfg.username, cfg.password, cfg.database
|
||||
@@ -14,7 +15,7 @@ pub async fn connect(cfg: &DatabaseConfig) -> Result<Client> {
|
||||
let (client, connection) = tokio_postgres::connect(&dsn, NoTls).await?;
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = connection.await {
|
||||
tracing::error!("Postgres connection error: {}", e);
|
||||
error!("Postgres connection error: {}", e);
|
||||
}
|
||||
});
|
||||
Ok(client)
|
||||
@@ -34,7 +35,7 @@ pub fn select_pg_path(version: &str) -> std::path::PathBuf {
|
||||
|
||||
pub async fn terminate_connections(cfg: &DatabaseConfig) -> Result<()> {
|
||||
let mut admin = cfg.clone();
|
||||
admin.database = "postgres".into();
|
||||
admin.database = "postgres".to_string().into();
|
||||
|
||||
let client = connect(&admin).await?;
|
||||
|
||||
|
||||
@@ -39,7 +39,6 @@ impl Database for PostgresDatabase {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
let res = backup::run(self.cfg.clone(), self.format, dir.to_path_buf()).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
|
||||
res
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use anyhow::Result;
|
||||
use tracing::{debug, error, info};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
use super::connection::{select_pg_path, server_version, terminate_connections};
|
||||
use super::format::PostgresDumpFormat;
|
||||
@@ -14,6 +14,7 @@ pub async fn run(
|
||||
) -> Result<()> {
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
debug!("Starting restore for database {}", cfg.name);
|
||||
|
||||
let version = match futures::executor::block_on(server_version(&cfg)) {
|
||||
Ok(v) => {
|
||||
debug!("Postgres version detected: {}", v);
|
||||
@@ -91,8 +92,6 @@ pub async fn run(
|
||||
let dec = flate2::read::GzDecoder::new(tar_gz);
|
||||
let mut archive = tar::Archive::new(dec);
|
||||
|
||||
|
||||
|
||||
let tmp_dir = match tempfile::TempDir::new() {
|
||||
Ok(d) => d,
|
||||
Err(e) => {
|
||||
@@ -104,8 +103,6 @@ pub async fn run(
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
|
||||
if let Err(e) = archive.unpack(tmp_dir.path()) {
|
||||
error!("Failed to unpack FD archive for {}: {:?}", cfg.name, e);
|
||||
return Err(e.into());
|
||||
@@ -113,7 +110,6 @@ pub async fn run(
|
||||
|
||||
debug!("Listing contents of temp dir: {}", tmp_dir.path().display());
|
||||
|
||||
|
||||
for entry in std::fs::read_dir(tmp_dir.path())? {
|
||||
if let Ok(entry) = entry {
|
||||
let path = entry.path();
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
backup_dir: PathBuf,
|
||||
file_extension: &'static str,
|
||||
) -> Result<PathBuf> {
|
||||
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||
debug!("Starting SQLite backup for database {}", cfg.name);
|
||||
|
||||
let db_path_str = if cfg.path.is_empty() {
|
||||
anyhow::bail!("Database path not configured");
|
||||
} else {
|
||||
cfg.path.as_str().to_string()
|
||||
};
|
||||
|
||||
let db_path = PathBuf::from(db_path_str);
|
||||
|
||||
if !db_path.exists() {
|
||||
anyhow::bail!("SQLite database file not found: {}", db_path.display());
|
||||
}
|
||||
|
||||
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
|
||||
|
||||
let output = Command::new("sqlite3")
|
||||
.arg(db_path.as_os_str())
|
||||
.arg(format!(".backup '{}'", file_path.display()))
|
||||
.output()
|
||||
.context("SQLite backup command failed to start")?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
error!("SQLite backup failed for {}: {}", cfg.name, stderr);
|
||||
anyhow::bail!("SQLite backup failed for {}: {}", cfg.name, stderr);
|
||||
}
|
||||
|
||||
info!("SQLite backup completed for {}", cfg.name);
|
||||
Ok(file_path)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use super::{backup, ping, restore};
|
||||
use crate::domain::factory::Database;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::utils::locks::{DbOpLock, FileLock};
|
||||
|
||||
pub struct SqliteDatabase {
|
||||
cfg: DatabaseConfig,
|
||||
}
|
||||
|
||||
impl SqliteDatabase {
|
||||
pub fn new(cfg: DatabaseConfig) -> Self {
|
||||
Self { cfg }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Database for SqliteDatabase {
|
||||
fn file_extension(&self) -> &'static str {
|
||||
".backup"
|
||||
}
|
||||
|
||||
async fn ping(&self) -> Result<bool> {
|
||||
ping::run(self.cfg.clone()).await
|
||||
}
|
||||
|
||||
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
|
||||
async fn restore(&self, file: &Path) -> Result<()> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
mod backup;
|
||||
mod restore;
|
||||
mod ping;
|
||||
pub mod database;
|
||||
@@ -0,0 +1,5 @@
|
||||
use crate::services::config::DatabaseConfig;
|
||||
|
||||
pub async fn run(_cfg: DatabaseConfig) -> anyhow::Result<bool> {
|
||||
Ok(true)
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
debug!("Starting SQLite restore for database {}", cfg.name);
|
||||
|
||||
let db_path_str = if cfg.path.is_empty() {
|
||||
anyhow::bail!("Database path not configured");
|
||||
} else {
|
||||
cfg.path.as_str().to_string()
|
||||
};
|
||||
|
||||
let db_path = PathBuf::from(db_path_str);
|
||||
|
||||
if !restore_file.exists() {
|
||||
anyhow::bail!("Restore file not found: {}", restore_file.display());
|
||||
}
|
||||
|
||||
if db_path.exists() {
|
||||
std::fs::remove_file(&db_path)
|
||||
.with_context(|| format!("Failed to remove existing DB {}", db_path.display()))?;
|
||||
}
|
||||
|
||||
let output = Command::new("sqlite3")
|
||||
.arg(db_path.as_os_str())
|
||||
.arg(format!(".restore '{}'", restore_file.display()))
|
||||
.output()
|
||||
.with_context(|| format!("Failed to run sqlite3 restore for {}", cfg.name))?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
error!("SQLite restore failed for {}: {}", cfg.name, stderr);
|
||||
anyhow::bail!("SQLite restore failed for {}", cfg.name);
|
||||
}
|
||||
|
||||
info!("SQLite restore completed for {}", cfg.name);
|
||||
Ok(())
|
||||
})
|
||||
.await?
|
||||
}
|
||||
@@ -4,6 +4,8 @@ mod services;
|
||||
mod settings;
|
||||
mod tasks;
|
||||
mod utils;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
use crate::tasks::ping::ping_server;
|
||||
use crate::utils::locks::FileLock;
|
||||
|
||||
@@ -1,2 +1,3 @@
|
||||
pub mod status;
|
||||
pub mod backup;
|
||||
pub mod backup;
|
||||
pub mod restore;
|
||||
@@ -0,0 +1,33 @@
|
||||
use crate::services::api::models::agent::restore::ResultRestoreResponse;
|
||||
use crate::services::api::{ApiClient, ApiError};
|
||||
use anyhow::Result;
|
||||
use reqwest::Method;
|
||||
use serde::Serialize;
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct ResultRestoreRequest {
|
||||
#[serde(rename = "generatedId")]
|
||||
pub generated_id: String,
|
||||
pub status: String,
|
||||
}
|
||||
|
||||
impl ApiClient {
|
||||
pub async fn restore_result(
|
||||
&self,
|
||||
agent_id: impl Into<String>,
|
||||
generated_id: impl Into<String>,
|
||||
status: impl Into<String>,
|
||||
) -> Result<Option<ResultRestoreResponse>, ApiError> {
|
||||
let body = ResultRestoreRequest {
|
||||
generated_id: generated_id.into(),
|
||||
status: status.into(),
|
||||
};
|
||||
|
||||
let agent_id = agent_id.into();
|
||||
|
||||
let path = format!("/agent/{}/restore", agent_id);
|
||||
|
||||
self.request_with_body(Method::POST, path.as_str(), &body)
|
||||
.await
|
||||
}
|
||||
}
|
||||
@@ -1,2 +1,3 @@
|
||||
pub mod status;
|
||||
pub mod backup;
|
||||
pub mod backup;
|
||||
pub mod restore;
|
||||
@@ -0,0 +1,7 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct ResultRestoreResponse {
|
||||
pub message: String,
|
||||
pub status: bool,
|
||||
}
|
||||
@@ -1,388 +0,0 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use crate::core::context::Context as CoreContext;
|
||||
use crate::domain::factory::DatabaseFactory;
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::services::config::{DatabaseConfig, DatabasesConfig, DbType};
|
||||
use crate::services::storage;
|
||||
use crate::utils::common::BackupMethod;
|
||||
use crate::utils::compress::compress_to_tar_gz_large;
|
||||
use anyhow::Result;
|
||||
use futures::future::join_all;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use tempfile::TempDir;
|
||||
use tracing::{error, info};
|
||||
use crate::utils::locks::FileLock;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BackupResult {
|
||||
pub generated_id: String,
|
||||
pub db_type: DbType,
|
||||
pub status: String,
|
||||
pub backup_file: Option<PathBuf>,
|
||||
pub code: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct UploadResult {
|
||||
pub storage_id: String,
|
||||
pub success: bool,
|
||||
pub error: Option<String>,
|
||||
pub remote_file_path: Option<String>,
|
||||
pub total_size: Option<u64>,
|
||||
}
|
||||
|
||||
pub struct BackupService {
|
||||
ctx: Arc<CoreContext>,
|
||||
}
|
||||
|
||||
impl BackupService {
|
||||
pub fn new(ctx: Arc<CoreContext>) -> Self {
|
||||
Self { ctx }
|
||||
}
|
||||
|
||||
pub async fn dispatch(
|
||||
&self,
|
||||
generated_id: &String,
|
||||
config: &DatabasesConfig,
|
||||
method: BackupMethod,
|
||||
storages: &Vec<DatabaseStorage>,
|
||||
encrypt: bool,
|
||||
) {
|
||||
if let Some(cfg) = config
|
||||
.databases
|
||||
.iter()
|
||||
.find(|c| c.generated_id == generated_id.as_str())
|
||||
{
|
||||
let db_cfg = cfg.clone();
|
||||
let ctx = self.ctx.clone();
|
||||
let storages_clone = storages.clone();
|
||||
let generated_id_clone = generated_id.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
match TempDir::new() {
|
||||
Ok(temp_dir) => {
|
||||
match FileLock::is_locked(&generated_id_clone).await {
|
||||
Ok(true) => {
|
||||
error!("Backup already running for {}", &generated_id_clone);
|
||||
return;
|
||||
}
|
||||
Ok(false) => {
|
||||
|
||||
match ctx
|
||||
.api
|
||||
.backup_create(
|
||||
method.clone().to_string(),
|
||||
ctx.edge_key.agent_id.clone(),
|
||||
&generated_id_clone,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(backup_created_result) => {
|
||||
info!("Backup created successfully");
|
||||
let tmp_path = temp_dir.path().to_path_buf();
|
||||
info!("Created temp directory {}", tmp_path.display());
|
||||
match BackupService::run(db_cfg, &tmp_path).await {
|
||||
Ok(mut result) => {
|
||||
if let Some(backup_file) = result.backup_file.take() {
|
||||
match compress_to_tar_gz_large(&backup_file).await {
|
||||
Ok(compression_result) => {
|
||||
result.backup_file =
|
||||
Some(compression_result.compressed_path);
|
||||
let service = BackupService { ctx: ctx.clone() };
|
||||
let backup_id = backup_created_result.unwrap().backup.id;
|
||||
match service
|
||||
.upload(
|
||||
result.clone(),
|
||||
method,
|
||||
storages_clone.clone(),
|
||||
encrypt,
|
||||
&backup_id,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(upload_result) => {
|
||||
match service
|
||||
.send_result(
|
||||
result,
|
||||
upload_result,
|
||||
&backup_id
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
return;
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
"Failed to send backup result: {}",
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
"Failed to upload backup files: {}",
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
"Failed to compress backup file : {}",
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
error!("No backup file generated");
|
||||
}
|
||||
}
|
||||
Err(e) => error!("BackupService run failed: {}", e),
|
||||
}
|
||||
// TempDir is automatically deleted when dropped here
|
||||
}
|
||||
Err(e) => error!("Backup creation failed: {}", e),
|
||||
}
|
||||
},
|
||||
Err(e) => error!("An error occurred while checking lock : {}", e),
|
||||
}
|
||||
}
|
||||
Err(e) => error!("Failed to create temp dir: {}", e),
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, tmp_path: &Path) -> Result<BackupResult> {
|
||||
let db_instance = DatabaseFactory::create_for_backup(cfg.clone()).await;
|
||||
let generated_id = cfg.generated_id.clone();
|
||||
let db_type = cfg.db_type.clone();
|
||||
|
||||
let reachable = db_instance.ping().await.unwrap_or(false);
|
||||
info!("Reachable: {}", reachable);
|
||||
if !reachable {
|
||||
return Ok(BackupResult {
|
||||
generated_id,
|
||||
db_type,
|
||||
status: "failed".into(),
|
||||
backup_file: None,
|
||||
code: None,
|
||||
});
|
||||
}
|
||||
|
||||
match db_instance.backup(tmp_path).await {
|
||||
Ok(file) => Ok(BackupResult {
|
||||
generated_id,
|
||||
db_type,
|
||||
status: "success".into(),
|
||||
backup_file: Some(file),
|
||||
code: None,
|
||||
}),
|
||||
Err(e) => match e.to_string().as_str() {
|
||||
"backup_already_in_progress" => Ok(BackupResult {
|
||||
generated_id,
|
||||
db_type,
|
||||
status: "failed".into(),
|
||||
backup_file: None,
|
||||
code: Some(e.to_string()),
|
||||
}),
|
||||
_ => Ok(BackupResult {
|
||||
generated_id,
|
||||
db_type,
|
||||
status: "failed".into(),
|
||||
backup_file: None,
|
||||
code: None,
|
||||
}),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn upload(
|
||||
&self,
|
||||
result: BackupResult,
|
||||
method: BackupMethod,
|
||||
storages: Vec<DatabaseStorage>,
|
||||
encrypt: bool,
|
||||
backup_id: &String,
|
||||
) -> Result<Vec<UploadResult>> {
|
||||
if result.code.as_deref() == Some("backup_already_in_progress") {
|
||||
info!("Skipping send: backup already in progress");
|
||||
anyhow::bail!("backup_already_in_progres");
|
||||
}
|
||||
|
||||
let upload_futures = storages.into_iter().map(|storage| {
|
||||
info!(
|
||||
"Uploading storage -> {:?} for {:?}",
|
||||
storage.provider, storage.id
|
||||
);
|
||||
let provider = storage::get_provider(&storage);
|
||||
let result_clone = result.clone();
|
||||
let ctx_clone = self.ctx.clone();
|
||||
let storages_clone = storage.clone();
|
||||
let storage_id = storages_clone.id;
|
||||
let generated_id = result_clone.generated_id.clone();
|
||||
|
||||
async move {
|
||||
match self
|
||||
.ctx
|
||||
.api
|
||||
.backup_upload_init(
|
||||
self.ctx.edge_key.agent_id.clone(),
|
||||
generated_id.clone(),
|
||||
storage_id.clone(),
|
||||
backup_id,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(upload_init_result) => {
|
||||
info!("Uploading init result: {:#?}", upload_init_result);
|
||||
let backup_storage_id = upload_init_result.unwrap().backup_storage.id.clone();
|
||||
match provider {
|
||||
Some(provider) => {
|
||||
let upload_result = provider
|
||||
.upload(
|
||||
ctx_clone,
|
||||
result_clone,
|
||||
method,
|
||||
&storage,
|
||||
Some(encrypt),
|
||||
)
|
||||
.await;
|
||||
|
||||
let status = if upload_result.success {
|
||||
"success"
|
||||
} else {
|
||||
"failed"
|
||||
};
|
||||
info!("Storage {} uploaded to remote path {:?}", storage_id, upload_result.remote_file_path);
|
||||
|
||||
|
||||
let (remote_path, total_size) = match (
|
||||
&upload_result.remote_file_path,
|
||||
upload_result.total_size,
|
||||
) {
|
||||
(Some(path), Some(size)) => (path.clone(), size),
|
||||
_ => {
|
||||
return UploadResult {
|
||||
storage_id: storage_id.clone(),
|
||||
success: false,
|
||||
error: Some("remote_file_path or total_size missing".to_string()),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
|
||||
match self.ctx.api.backup_upload_status(
|
||||
self.ctx.edge_key.agent_id.clone(),
|
||||
generated_id.clone(),
|
||||
backup_storage_id,
|
||||
status,
|
||||
remote_path,
|
||||
total_size,
|
||||
backup_id
|
||||
).await {
|
||||
Ok(_) => {
|
||||
upload_result
|
||||
},
|
||||
Err(err)=> {
|
||||
error!(
|
||||
"backup_upload_status failed (generated_id={}, storage_id={}): {}",
|
||||
generated_id, storage_id, err
|
||||
);
|
||||
UploadResult {
|
||||
storage_id: storage_id.clone(),
|
||||
success: false,
|
||||
error: Some(err.to_string()),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None => {
|
||||
error!("Skipping storage due to missing provider");
|
||||
UploadResult {
|
||||
storage_id: storage_id.clone(),
|
||||
success: false,
|
||||
error: Some(
|
||||
"Skipping storage due to missing provider".to_string(),
|
||||
),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
"Unable to create the storage backup on remote server : {}",
|
||||
e
|
||||
);
|
||||
UploadResult {
|
||||
storage_id: storage_id.clone(),
|
||||
success: false,
|
||||
error: Some(
|
||||
"Unable to create the storage backup on remote server".to_string(),
|
||||
),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let results: Vec<UploadResult> = join_all(upload_futures).await;
|
||||
info!("Upload results: {:#?}", results);
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
pub async fn send_result(
|
||||
&self,
|
||||
result: BackupResult,
|
||||
upload_results: Vec<UploadResult>,
|
||||
backup_id: &String,
|
||||
) -> Result<()> {
|
||||
let status = if upload_results.iter().any(|r| r.success) {
|
||||
"success"
|
||||
} else {
|
||||
"failed"
|
||||
};
|
||||
|
||||
let file_size = upload_results
|
||||
.iter()
|
||||
.map(|r| r.total_size)
|
||||
.try_fold((0u64, 0u64), |(sum, count), v| {
|
||||
match v {
|
||||
Some(size) => Ok((sum + size, count + 1)),
|
||||
None => Err(()), // stop and return None
|
||||
}
|
||||
})
|
||||
.ok()
|
||||
.map(|(sum, count)| sum / count);
|
||||
|
||||
match self
|
||||
.ctx
|
||||
.api
|
||||
.backup_update(self.ctx.edge_key.agent_id.clone(), backup_id, status, file_size, &result.generated_id)
|
||||
.await
|
||||
{
|
||||
Ok(_result) => Ok(()),
|
||||
Err(e) => {
|
||||
error!(
|
||||
"backup_update failed (generated_id={}, backup_id={}): {}",
|
||||
&result.generated_id, &backup_id, e
|
||||
);
|
||||
Err(e.into())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
use super::service::BackupService;
|
||||
use crate::utils::compress::compress_to_tar_gz_large;
|
||||
use std::path::PathBuf;
|
||||
use anyhow::Result;
|
||||
|
||||
impl BackupService {
|
||||
|
||||
pub async fn compress_backup(
|
||||
&self,
|
||||
backup_file: Option<PathBuf>,
|
||||
) -> Result<PathBuf> {
|
||||
|
||||
let file = backup_file
|
||||
.ok_or_else(|| anyhow::anyhow!("No backup file generated"))?;
|
||||
|
||||
let compression = compress_to_tar_gz_large(&file).await?;
|
||||
|
||||
Ok(compression.compressed_path)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
use super::service::BackupService;
|
||||
use crate::services::config::DatabasesConfig;
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::utils::common::BackupMethod;
|
||||
use tracing::error;
|
||||
|
||||
impl BackupService {
|
||||
|
||||
pub async fn dispatch(
|
||||
&self,
|
||||
generated_id: &String,
|
||||
config: &DatabasesConfig,
|
||||
method: BackupMethod,
|
||||
storages: &Vec<DatabaseStorage>,
|
||||
encrypt: bool,
|
||||
) {
|
||||
|
||||
let Some(cfg) = config
|
||||
.databases
|
||||
.iter()
|
||||
.find(|c| c.generated_id == generated_id.as_str())
|
||||
else {
|
||||
error!("Database config not found for {}", generated_id);
|
||||
return;
|
||||
};
|
||||
|
||||
let service = Self {
|
||||
ctx: self.ctx.clone(),
|
||||
};
|
||||
|
||||
let db_cfg = cfg.clone();
|
||||
let storages = storages.clone();
|
||||
let generated_id = generated_id.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = service
|
||||
.execute_backup(generated_id, db_cfg, method, storages, encrypt)
|
||||
.await
|
||||
{
|
||||
error!("Backup execution failed: {}", e);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
use super::service::BackupService;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::utils::common::BackupMethod;
|
||||
use crate::utils::locks::FileLock;
|
||||
|
||||
use tempfile::TempDir;
|
||||
use anyhow::Result;
|
||||
|
||||
impl BackupService {
|
||||
|
||||
pub async fn execute_backup(
|
||||
&self,
|
||||
generated_id: String,
|
||||
db_cfg: DatabaseConfig,
|
||||
method: BackupMethod,
|
||||
storages: Vec<DatabaseStorage>,
|
||||
encrypt: bool,
|
||||
) -> Result<()> {
|
||||
|
||||
if FileLock::is_locked(&generated_id).await? {
|
||||
anyhow::bail!("backup already running");
|
||||
}
|
||||
|
||||
let backup = self.create_backup_record(&generated_id, &method).await?;
|
||||
let backup_id = backup.backup.id;
|
||||
|
||||
let temp_dir = TempDir::new()?;
|
||||
let tmp_path = temp_dir.path();
|
||||
|
||||
let mut result = Self::run(db_cfg, tmp_path).await?;
|
||||
|
||||
if result.status == "failed" {
|
||||
self.send_result(result, vec![], &backup_id).await?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let compressed = self.compress_backup(result.backup_file.take()).await?;
|
||||
result.backup_file = Some(compressed);
|
||||
|
||||
let uploads = self
|
||||
.upload(result.clone(), method, storages, encrypt, &backup_id)
|
||||
.await?;
|
||||
|
||||
self.send_result(result, uploads, &backup_id).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
use super::service::BackupService;
|
||||
use crate::utils::common::BackupMethod;
|
||||
use anyhow::{Result, anyhow};
|
||||
use crate::services::api::models::agent::backup::BackupResponse;
|
||||
|
||||
impl BackupService {
|
||||
|
||||
pub async fn create_backup_record(
|
||||
&self,
|
||||
generated_id: &str,
|
||||
method: &BackupMethod,
|
||||
) -> Result<BackupResponse> {
|
||||
|
||||
let response = self
|
||||
.ctx
|
||||
.api
|
||||
.backup_create(
|
||||
method.to_string(),
|
||||
self.ctx.edge_key.agent_id.clone(),
|
||||
generated_id,
|
||||
)
|
||||
.await?;
|
||||
|
||||
response.ok_or_else(|| anyhow!("backup_create returned empty response"))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
pub mod service;
|
||||
pub mod dispatcher;
|
||||
pub mod executor;
|
||||
pub mod compressor;
|
||||
pub mod uploader;
|
||||
pub mod result;
|
||||
pub mod models;
|
||||
pub mod helpers;
|
||||
pub mod runner;
|
||||
|
||||
pub use service::BackupService;
|
||||
@@ -0,0 +1,22 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::path::PathBuf;
|
||||
use crate::services::config::DbType;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BackupResult {
|
||||
pub generated_id: String,
|
||||
pub db_type: DbType,
|
||||
pub status: String,
|
||||
pub backup_file: Option<PathBuf>,
|
||||
pub code: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct UploadResult {
|
||||
pub storage_id: String,
|
||||
pub success: bool,
|
||||
pub error: Option<String>,
|
||||
pub remote_file_path: Option<String>,
|
||||
pub total_size: Option<u64>,
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
use super::models::{BackupResult, UploadResult};
|
||||
use super::service::BackupService;
|
||||
use crate::services::api::ApiError;
|
||||
use crate::services::api::models::agent::backup::BackupResponse;
|
||||
use anyhow::Result;
|
||||
use tracing::error;
|
||||
|
||||
impl BackupService {
|
||||
pub async fn send_result(
|
||||
&self,
|
||||
result: BackupResult,
|
||||
upload_results: Vec<UploadResult>,
|
||||
backup_id: &String,
|
||||
) -> Result<Option<BackupResponse>, ApiError> {
|
||||
let status = if upload_results.iter().any(|r| r.success) {
|
||||
"success"
|
||||
} else {
|
||||
"failed"
|
||||
};
|
||||
|
||||
let file_size = upload_results
|
||||
.iter()
|
||||
.filter_map(|r| r.total_size)
|
||||
.reduce(|a, b| a + b)
|
||||
.map(|sum| sum / upload_results.len() as u64);
|
||||
|
||||
self.ctx
|
||||
.api
|
||||
.backup_update(
|
||||
self.ctx.edge_key.agent_id.clone(),
|
||||
backup_id,
|
||||
status,
|
||||
file_size,
|
||||
&result.generated_id,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!(
|
||||
"backup_update failed (generated_id={}, backup_id={}): {}",
|
||||
result.generated_id, backup_id, e
|
||||
);
|
||||
e.into()
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
use super::models::BackupResult;
|
||||
use super::service::BackupService;
|
||||
|
||||
use crate::domain::factory::DatabaseFactory;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
|
||||
use anyhow::Result;
|
||||
use std::path::Path;
|
||||
use tracing::{error, info};
|
||||
|
||||
impl BackupService {
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
tmp_path: &Path,
|
||||
) -> Result<BackupResult> {
|
||||
|
||||
let db = DatabaseFactory::create_for_backup(cfg.clone()).await;
|
||||
|
||||
let generated_id = cfg.generated_id.clone();
|
||||
let db_type = cfg.db_type.clone();
|
||||
|
||||
let reachable = match db.ping().await {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
error!("Ping failed: {}", e);
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
|
||||
info!("Reachable: {}", reachable);
|
||||
|
||||
if !reachable {
|
||||
return Ok(BackupResult {
|
||||
generated_id,
|
||||
db_type,
|
||||
status: "failed".into(),
|
||||
backup_file: None,
|
||||
code: None,
|
||||
});
|
||||
}
|
||||
|
||||
match db.backup(tmp_path).await {
|
||||
|
||||
Ok(file) => Ok(BackupResult {
|
||||
generated_id,
|
||||
db_type,
|
||||
status: "success".into(),
|
||||
backup_file: Some(file),
|
||||
code: None,
|
||||
}),
|
||||
|
||||
Err(e) if e.to_string() == "backup_already_in_progress" => Ok(BackupResult {
|
||||
generated_id,
|
||||
db_type,
|
||||
status: "failed".into(),
|
||||
backup_file: None,
|
||||
code: Some("backup_already_in_progress".into()),
|
||||
}),
|
||||
|
||||
Err(_) => Ok(BackupResult {
|
||||
generated_id,
|
||||
db_type,
|
||||
status: "failed".into(),
|
||||
backup_file: None,
|
||||
code: None,
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
use std::sync::Arc;
|
||||
use crate::core::context::Context as CoreContext;
|
||||
|
||||
pub struct BackupService {
|
||||
pub ctx: Arc<CoreContext>,
|
||||
}
|
||||
|
||||
impl BackupService {
|
||||
pub fn new(ctx: Arc<CoreContext>) -> Self {
|
||||
Self { ctx }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
use super::service::BackupService;
|
||||
use super::models::{BackupResult, UploadResult};
|
||||
|
||||
use crate::services::storage;
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::utils::common::BackupMethod;
|
||||
|
||||
use futures::future::join_all;
|
||||
use anyhow::{Result, bail};
|
||||
use tracing::{info, error};
|
||||
|
||||
impl BackupService {
|
||||
|
||||
pub async fn upload(
|
||||
&self,
|
||||
result: BackupResult,
|
||||
method: BackupMethod,
|
||||
storages: Vec<DatabaseStorage>,
|
||||
encrypt: bool,
|
||||
backup_id: &String,
|
||||
) -> Result<Vec<UploadResult>> {
|
||||
|
||||
if result.code.as_deref() == Some("backup_already_in_progress") {
|
||||
info!("Skipping send: backup already in progress");
|
||||
bail!("backup_already_in_progress");
|
||||
}
|
||||
|
||||
let ctx = self.ctx.clone();
|
||||
|
||||
let futures = storages.into_iter().map(|storage| {
|
||||
|
||||
let ctx_clone = ctx.clone();
|
||||
let result_clone = result.clone();
|
||||
let provider = storage::get_provider(&storage);
|
||||
|
||||
let storage_id = storage.id.clone();
|
||||
let generated_id = result_clone.generated_id.clone();
|
||||
|
||||
async move {
|
||||
|
||||
info!("Uploading storage -> {:?} for {:?}", storage.provider, storage_id);
|
||||
|
||||
/*
|
||||
INIT STEP
|
||||
*/
|
||||
let init = match ctx_clone.api
|
||||
.backup_upload_init(
|
||||
ctx_clone.edge_key.agent_id.clone(),
|
||||
generated_id.clone(),
|
||||
storage_id.clone(),
|
||||
backup_id,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
error!("backup_upload_init failed: {}", e);
|
||||
|
||||
return UploadResult {
|
||||
storage_id,
|
||||
success: false,
|
||||
error: Some("backup_upload_init failed".into()),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let backup_storage_id = match init {
|
||||
Some(v) => v.backup_storage.id,
|
||||
None => {
|
||||
return UploadResult {
|
||||
storage_id,
|
||||
success: false,
|
||||
error: Some("backup_upload_init returned empty response".into()),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
/*
|
||||
PROVIDER CHECK
|
||||
*/
|
||||
let Some(provider) = provider else {
|
||||
error!("Skipping storage due to missing provider");
|
||||
|
||||
return UploadResult {
|
||||
storage_id,
|
||||
success: false,
|
||||
error: Some("missing provider".into()),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
};
|
||||
};
|
||||
|
||||
/*
|
||||
STORAGE UPLOAD
|
||||
*/
|
||||
let upload_result = provider
|
||||
.upload(
|
||||
ctx_clone.clone(),
|
||||
result_clone,
|
||||
method,
|
||||
&storage,
|
||||
Some(encrypt),
|
||||
)
|
||||
.await;
|
||||
|
||||
let status = if upload_result.success { "success" } else { "failed" };
|
||||
|
||||
if status != "success" {
|
||||
return upload_result;
|
||||
}
|
||||
|
||||
info!(
|
||||
"Storage {} uploaded to remote path {:?}",
|
||||
storage_id,
|
||||
upload_result.remote_file_path
|
||||
);
|
||||
|
||||
/*
|
||||
METADATA VALIDATION
|
||||
*/
|
||||
let (remote_path, total_size) = match (
|
||||
&upload_result.remote_file_path,
|
||||
upload_result.total_size,
|
||||
) {
|
||||
(Some(path), Some(size)) => (path.clone(), size),
|
||||
_ => {
|
||||
return UploadResult {
|
||||
storage_id,
|
||||
success: false,
|
||||
error: Some("remote_file_path or total_size missing".into()),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
/*
|
||||
STATUS UPDATE
|
||||
*/
|
||||
match ctx_clone.api.backup_upload_status(
|
||||
ctx_clone.edge_key.agent_id.clone(),
|
||||
generated_id,
|
||||
backup_storage_id,
|
||||
status,
|
||||
remote_path,
|
||||
total_size,
|
||||
backup_id,
|
||||
).await {
|
||||
|
||||
Ok(_) => upload_result,
|
||||
|
||||
Err(err) => {
|
||||
error!(
|
||||
"backup_upload_status failed (storage_id={}): {}",
|
||||
storage_id,
|
||||
err
|
||||
);
|
||||
|
||||
UploadResult {
|
||||
storage_id,
|
||||
success: false,
|
||||
error: Some(err.to_string()),
|
||||
remote_file_path: None,
|
||||
total_size: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let results: Vec<UploadResult> = join_all(futures).await;
|
||||
|
||||
info!("Upload results: {:#?}", results);
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
}
|
||||
+94
-6
@@ -9,6 +9,7 @@ use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use toml;
|
||||
use tracing::info;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
@@ -16,8 +17,8 @@ pub enum DbType {
|
||||
Mysql,
|
||||
Mariadb,
|
||||
Postgresql,
|
||||
MongoDB
|
||||
// Sqlite,
|
||||
MongoDB,
|
||||
Sqlite,
|
||||
// Add other DB types if needed
|
||||
}
|
||||
|
||||
@@ -28,7 +29,7 @@ impl DbType {
|
||||
DbType::Mariadb => "mysql",
|
||||
DbType::Postgresql => "postgresql",
|
||||
DbType::MongoDB => "mongodb",
|
||||
// DbType::Sqlite => "sqlite",
|
||||
DbType::Sqlite => "sqlite",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -45,6 +46,7 @@ pub struct DatabaseConfig {
|
||||
pub port: u16,
|
||||
pub host: String,
|
||||
pub generated_id: String,
|
||||
pub path: String,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
@@ -53,6 +55,29 @@ pub struct DatabasesConfig {
|
||||
pub databases: Vec<DatabaseConfig>,
|
||||
}
|
||||
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct InputDatabaseConfig {
|
||||
pub name: String,
|
||||
pub database: Option<String>,
|
||||
#[serde(rename = "type")]
|
||||
pub db_type: DbType,
|
||||
pub username: Option<String>,
|
||||
pub password: Option<String>,
|
||||
pub port: Option<u16>,
|
||||
pub host: Option<String>,
|
||||
pub generated_id: String,
|
||||
pub path: Option<String>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct InputDatabasesConfig {
|
||||
pub databases: Vec<InputDatabaseConfig>,
|
||||
}
|
||||
|
||||
|
||||
pub struct ConfigService {
|
||||
ctx: Arc<Context>,
|
||||
}
|
||||
@@ -95,7 +120,7 @@ impl ConfigService {
|
||||
file.read_to_string(&mut contents)
|
||||
.map_err(|e| format!("Failed to read config file: {}", e))?;
|
||||
|
||||
let config: DatabasesConfig = match extension {
|
||||
let input_config: InputDatabasesConfig = match extension {
|
||||
"json" => {
|
||||
serde_json::from_str(&contents).map_err(|e| format!("JSON parsing error: {}", e))?
|
||||
}
|
||||
@@ -105,8 +130,71 @@ impl ConfigService {
|
||||
_ => return Err("Unsupported config file format. Use .json or .toml".to_string()),
|
||||
};
|
||||
|
||||
info!("Databases : {:?} instances loaded", config.databases.len());
|
||||
fn required<T: Clone>(opt: &Option<T>, db_name: &str, field_name: &str) -> Result<T, String> {
|
||||
match opt {
|
||||
Some(v) => Ok(v.clone()),
|
||||
None => {
|
||||
let msg = format!("Missing required field '{}' for database '{}'", field_name, db_name);
|
||||
Err(msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(config)
|
||||
fn optional<T: Clone>(opt: &Option<T>) -> T where T: Default {
|
||||
opt.clone().unwrap_or_default()
|
||||
}
|
||||
|
||||
let mut databases = Vec::with_capacity(input_config.databases.len());
|
||||
|
||||
for db in input_config.databases {
|
||||
if Uuid::parse_str(&db.generated_id).is_err() {
|
||||
return Err(format!("Invalid UUID for database '{}'", db.name));
|
||||
}
|
||||
|
||||
let username = match db.db_type {
|
||||
DbType::Postgresql | DbType::Mysql | DbType::Mariadb => required(&db.username, &db.name, "username")?,
|
||||
_ => optional(&db.username),
|
||||
};
|
||||
|
||||
let password = match db.db_type {
|
||||
DbType::Postgresql | DbType::Mysql | DbType::Mariadb => required(&db.password, &db.name, "password")?,
|
||||
_ => optional(&db.password),
|
||||
};
|
||||
|
||||
let host = match db.db_type {
|
||||
DbType::Postgresql | DbType::Mysql | DbType::Mariadb | DbType::MongoDB => required(&db.host, &db.name, "host")?,
|
||||
DbType::Sqlite => optional(&db.host),
|
||||
};
|
||||
|
||||
let port = match db.db_type {
|
||||
DbType::Postgresql | DbType::Mysql | DbType::Mariadb | DbType::MongoDB => required(&db.port, &db.name, "port")?,
|
||||
DbType::Sqlite => db.port.unwrap_or(0),
|
||||
};
|
||||
|
||||
let database_name = match db.db_type {
|
||||
DbType::Sqlite => optional(&db.database),
|
||||
_ => required(&db.database, &db.name, "database")?
|
||||
};
|
||||
|
||||
let path_val = match db.db_type {
|
||||
DbType::Sqlite => required(&db.path, &db.name, "path")?,
|
||||
_ => optional(&db.path),
|
||||
};
|
||||
|
||||
databases.push(DatabaseConfig {
|
||||
name: db.name,
|
||||
database: database_name,
|
||||
db_type: db.db_type,
|
||||
username,
|
||||
password,
|
||||
host,
|
||||
port,
|
||||
generated_id: db.generated_id,
|
||||
path: path_val,
|
||||
});
|
||||
}
|
||||
|
||||
info!("Databases: {} instances loaded", databases.len());
|
||||
Ok(DatabasesConfig { databases })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,169 +0,0 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use crate::core::context::Context;
|
||||
use crate::domain::factory::DatabaseFactory;
|
||||
use crate::services::config::{DatabaseConfig, DatabasesConfig};
|
||||
use anyhow::Result;
|
||||
use serde::Serialize;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use tempfile::TempDir;
|
||||
use tracing::{error, info};
|
||||
use crate::services::api::models::agent::status::DatabaseStatus;
|
||||
use crate::utils::compress::decompress_large_tar_gz;
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct RestoreResult {
|
||||
#[serde(rename = "generatedId")]
|
||||
pub generated_id: String,
|
||||
pub status: String,
|
||||
}
|
||||
|
||||
pub struct RestoreService {
|
||||
ctx: Arc<Context>,
|
||||
}
|
||||
|
||||
impl RestoreService {
|
||||
pub fn new(ctx: Arc<Context>) -> Self {
|
||||
Self { ctx }
|
||||
}
|
||||
|
||||
pub async fn dispatch(&self, db: &DatabaseStatus, config: &DatabasesConfig) {
|
||||
if let Some(cfg) = config
|
||||
.databases
|
||||
.iter()
|
||||
.find(|c| c.generated_id == db.generated_id)
|
||||
{
|
||||
let db_cfg = cfg.clone();
|
||||
let ctx_clone = self.ctx.clone();
|
||||
let file_to_restore = db.data.restore.file.clone();
|
||||
if file_to_restore.is_none() {
|
||||
error!("restore file not found");
|
||||
return;
|
||||
}
|
||||
tokio::spawn(async move {
|
||||
match TempDir::new() {
|
||||
Ok(temp_dir) => {
|
||||
let tmp_path = temp_dir.path().to_path_buf();
|
||||
info!("Created temp directory {}", tmp_path.display());
|
||||
match RestoreService::run(db_cfg, &tmp_path, &file_to_restore.unwrap()).await {
|
||||
Ok(result) => {
|
||||
let service = RestoreService { ctx: ctx_clone };
|
||||
service.send_result(result).await;
|
||||
}
|
||||
Err(e) => error!("Restoration error {}", e),
|
||||
}
|
||||
// TempDir is automatically deleted when dropped
|
||||
}
|
||||
Err(e) => error!("Failed to create temp dir: {}", e),
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
tmp_path: &Path,
|
||||
file_url: &str,
|
||||
) -> Result<RestoreResult> {
|
||||
let generated_id = cfg.generated_id.clone();
|
||||
|
||||
info!("File url: {}", file_url);
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let response = client.get(file_url).send().await?;
|
||||
if !response.status().is_success() {
|
||||
error!("Backup download failed with status {}", response.status());
|
||||
return Ok(RestoreResult {
|
||||
generated_id,
|
||||
status: "failed".into(),
|
||||
});
|
||||
}
|
||||
|
||||
let bytes = response.bytes().await?;
|
||||
|
||||
let compressed_archive = tmp_path.join("compressed_archive_tmp.tar.gz");
|
||||
tokio::fs::write(&compressed_archive, &bytes).await?;
|
||||
info!("Backup downloaded to {}", compressed_archive.display());
|
||||
|
||||
let decompressed_files = decompress_large_tar_gz(compressed_archive.as_path(), tmp_path).await?;
|
||||
|
||||
info!("Decompressed_files {:#?}", decompressed_files);
|
||||
|
||||
if decompressed_files.is_empty() {
|
||||
return Ok(RestoreResult {
|
||||
generated_id,
|
||||
status: "failed".into(),
|
||||
});
|
||||
}
|
||||
|
||||
let backup_file_path = if decompressed_files.len() == 1 {
|
||||
decompressed_files.get(0).unwrap()
|
||||
}else {
|
||||
compressed_archive.as_path()
|
||||
};
|
||||
|
||||
info!("Decompressed file {}", backup_file_path.display());
|
||||
|
||||
let db_instance = DatabaseFactory::create_for_restore(cfg.clone(), &backup_file_path).await;
|
||||
let reachable = db_instance.ping().await.unwrap_or(false);
|
||||
info!("Reachable: {}", reachable);
|
||||
if !reachable {
|
||||
return Ok(RestoreResult {
|
||||
generated_id,
|
||||
status: "failed".into(),
|
||||
});
|
||||
}
|
||||
|
||||
match db_instance.restore(&backup_file_path).await {
|
||||
Ok(_) => Ok(RestoreResult {
|
||||
generated_id,
|
||||
status: "success".into(),
|
||||
}),
|
||||
Err(e) => {
|
||||
log::error!("Restore failed: {:?}", e);
|
||||
Ok(RestoreResult {
|
||||
generated_id,
|
||||
status: "failed".into(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO : update with ctx api manager
|
||||
pub async fn send_result(&self, result: RestoreResult) {
|
||||
info!(
|
||||
"[RestoreService] DB: {} | Status: {}",
|
||||
result.generated_id, result.status,
|
||||
);
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let url = format!(
|
||||
"{}/api/agent/{}/restore",
|
||||
self.ctx.edge_key.server_url, self.ctx.edge_key.agent_id
|
||||
);
|
||||
|
||||
let body = RestoreResult {
|
||||
generated_id: result.generated_id,
|
||||
status: result.status,
|
||||
};
|
||||
|
||||
match client.post(&url).json(&body).send().await {
|
||||
Ok(resp) => {
|
||||
let status = resp.status();
|
||||
if status.is_success() {
|
||||
info!("Restoration result sent successfully");
|
||||
} else {
|
||||
let text = resp.text().await.unwrap_or_default();
|
||||
error!(
|
||||
"Restoration result failed, status: {}, body: {}",
|
||||
status, text
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to send restoration result: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
use super::service::RestoreService;
|
||||
|
||||
use crate::utils::compress::decompress_large_tar_gz;
|
||||
use crate::utils::file::decrypt_file_stream_gcm;
|
||||
|
||||
use anyhow::Result;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
impl RestoreService {
|
||||
|
||||
pub async fn prepare_archive(
|
||||
&self,
|
||||
downloaded_file: PathBuf,
|
||||
tmp_path: &Path,
|
||||
) -> Result<PathBuf> {
|
||||
|
||||
let filename = downloaded_file
|
||||
.file_name()
|
||||
.unwrap()
|
||||
.to_string_lossy()
|
||||
.to_string();
|
||||
|
||||
let is_legacy = filename.ends_with(".sql") || filename.ends_with(".dump");
|
||||
|
||||
if is_legacy {
|
||||
return Ok(downloaded_file);
|
||||
}
|
||||
|
||||
let encrypted = filename.ends_with(".tar.gz.enc");
|
||||
|
||||
let mut archive = downloaded_file.clone();
|
||||
|
||||
if encrypted {
|
||||
|
||||
let new_name = filename.strip_suffix(".enc").unwrap();
|
||||
|
||||
let decrypted = tmp_path.join(new_name);
|
||||
|
||||
decrypt_file_stream_gcm(
|
||||
downloaded_file,
|
||||
decrypted.clone(),
|
||||
self.ctx.edge_key.master_key_b64.clone(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
archive = decrypted;
|
||||
}
|
||||
|
||||
let files = decompress_large_tar_gz(archive.as_path(), tmp_path).await?;
|
||||
|
||||
if files.is_empty() {
|
||||
anyhow::bail!("archive empty");
|
||||
}
|
||||
|
||||
if files.len() == 1 {
|
||||
Ok(files[0].clone())
|
||||
} else {
|
||||
Ok(archive)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
use super::service::RestoreService;
|
||||
use crate::services::config::DatabasesConfig;
|
||||
use crate::services::api::models::agent::status::DatabaseStatus;
|
||||
|
||||
use tracing::error;
|
||||
|
||||
impl RestoreService {
|
||||
|
||||
pub async fn dispatch(&self, db: &DatabaseStatus, config: &DatabasesConfig) {
|
||||
|
||||
let Some(cfg) = config
|
||||
.databases
|
||||
.iter()
|
||||
.find(|c| c.generated_id == db.generated_id)
|
||||
else {
|
||||
error!("Database config not found");
|
||||
return;
|
||||
};
|
||||
|
||||
let Some(file_to_restore) = db.data.restore.file.clone() else {
|
||||
error!("restore file not found");
|
||||
return;
|
||||
};
|
||||
|
||||
let service = Self {
|
||||
ctx: self.ctx.clone(),
|
||||
};
|
||||
|
||||
let db_cfg = cfg.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
|
||||
if let Err(e) = service
|
||||
.execute_restore(db_cfg, file_to_restore)
|
||||
.await
|
||||
{
|
||||
error!("Restore failed: {}", e);
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
use super::service::RestoreService;
|
||||
|
||||
use reqwest::{Client, Url};
|
||||
use anyhow::Result;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tracing::info;
|
||||
|
||||
impl RestoreService {
|
||||
|
||||
pub async fn download_backup(
|
||||
&self,
|
||||
file_url: &str,
|
||||
tmp_path: &Path,
|
||||
) -> Result<PathBuf> {
|
||||
|
||||
let client = Client::new();
|
||||
|
||||
let response = client.get(file_url).send().await?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
anyhow::bail!("download failed");
|
||||
}
|
||||
|
||||
let filename_from_header = response
|
||||
.headers()
|
||||
.get(reqwest::header::CONTENT_DISPOSITION)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|s| s.split("filename=").nth(1))
|
||||
.map(|f| f.trim_matches('"').to_string());
|
||||
|
||||
|
||||
let filename_from_url = Url::parse(file_url).ok().and_then(|u| {
|
||||
u.path_segments()?
|
||||
.last()
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(|s| s.to_string())
|
||||
});
|
||||
|
||||
let filename = filename_from_header
|
||||
.or(filename_from_url)
|
||||
.unwrap_or_else(|| "downloaded_file".to_string());
|
||||
|
||||
let path = tmp_path.join(&filename);
|
||||
|
||||
let bytes = response.bytes().await?;
|
||||
|
||||
tokio::fs::write(&path, &bytes).await?;
|
||||
|
||||
info!("Backup downloaded to {}", path.display());
|
||||
|
||||
Ok(path)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
use super::service::RestoreService;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use tempfile::TempDir;
|
||||
use anyhow::Result;
|
||||
use tracing::info;
|
||||
|
||||
impl RestoreService {
|
||||
|
||||
pub async fn execute_restore(
|
||||
&self,
|
||||
cfg: DatabaseConfig,
|
||||
file_url: String,
|
||||
) -> Result<()> {
|
||||
|
||||
let temp_dir = TempDir::new()?;
|
||||
let tmp_path = temp_dir.path();
|
||||
|
||||
info!("Created temp directory {}", tmp_path.display());
|
||||
|
||||
let downloaded = self.download_backup(&file_url, tmp_path).await?;
|
||||
|
||||
let backup_file = self.prepare_archive(downloaded, tmp_path).await?;
|
||||
|
||||
let result = self.run_restore(cfg, backup_file).await?;
|
||||
|
||||
self.send_result(result).await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
pub mod service;
|
||||
pub mod dispatcher;
|
||||
pub mod executor;
|
||||
pub mod downloader;
|
||||
pub mod archive;
|
||||
pub mod runner;
|
||||
pub mod result;
|
||||
pub mod models;
|
||||
|
||||
pub use service::RestoreService;
|
||||
@@ -0,0 +1,8 @@
|
||||
use serde::Serialize;
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct RestoreResult {
|
||||
#[serde(rename = "generatedId")]
|
||||
pub generated_id: String,
|
||||
pub status: String,
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
use super::service::RestoreService;
|
||||
use super::models::RestoreResult;
|
||||
|
||||
use tracing::{info, error};
|
||||
|
||||
impl RestoreService {
|
||||
pub async fn send_result(&self, result: RestoreResult) {
|
||||
|
||||
info!(
|
||||
"[RestoreService] DB: {} | Status: {}",
|
||||
result.generated_id, result.status
|
||||
);
|
||||
|
||||
match self.ctx
|
||||
.api
|
||||
.restore_result(
|
||||
self.ctx.edge_key.agent_id.clone(),
|
||||
&result.generated_id,
|
||||
&result.status,
|
||||
)
|
||||
.await {
|
||||
Ok(_) => {
|
||||
info!("Restoration result sent successfully");
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to send restoration result: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
use super::service::RestoreService;
|
||||
use super::models::RestoreResult;
|
||||
|
||||
use crate::domain::factory::DatabaseFactory;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
|
||||
use anyhow::Result;
|
||||
use std::path::PathBuf;
|
||||
use tracing::{info, error};
|
||||
|
||||
impl RestoreService {
|
||||
|
||||
pub async fn run_restore(
|
||||
&self,
|
||||
cfg: DatabaseConfig,
|
||||
backup_file: PathBuf,
|
||||
) -> Result<RestoreResult> {
|
||||
|
||||
let generated_id = cfg.generated_id.clone();
|
||||
|
||||
let db = DatabaseFactory::create_for_restore(cfg.clone(), &backup_file).await;
|
||||
|
||||
let reachable = db.ping().await.unwrap_or(false);
|
||||
|
||||
info!("Reachable: {}", reachable);
|
||||
|
||||
if !reachable {
|
||||
return Ok(RestoreResult {
|
||||
generated_id,
|
||||
status: "failed".into(),
|
||||
});
|
||||
}
|
||||
|
||||
match db.restore(&backup_file).await {
|
||||
|
||||
Ok(_) => Ok(RestoreResult {
|
||||
generated_id,
|
||||
status: "success".into(),
|
||||
}),
|
||||
|
||||
Err(e) => {
|
||||
error!("Restore failed: {:?}", e);
|
||||
|
||||
Ok(RestoreResult {
|
||||
generated_id,
|
||||
status: "failed".into(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
use std::sync::Arc;
|
||||
use crate::core::context::Context;
|
||||
|
||||
pub struct RestoreService {
|
||||
pub ctx: Arc<Context>,
|
||||
}
|
||||
|
||||
impl RestoreService {
|
||||
pub fn new(ctx: Arc<Context>) -> Self {
|
||||
Self { ctx }
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
pub mod providers;
|
||||
|
||||
use crate::core::context::Context;
|
||||
use crate::services::backup::{BackupResult, UploadResult};
|
||||
use crate::utils::common::BackupMethod;
|
||||
use async_trait::async_trait;
|
||||
use providers::local;
|
||||
@@ -10,6 +9,7 @@ use providers::google_drive;
|
||||
use std::sync::Arc;
|
||||
use tracing::{error, info};
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::services::backup::models::{BackupResult, UploadResult};
|
||||
|
||||
#[async_trait]
|
||||
pub trait StorageProvider: Send + Sync {
|
||||
|
||||
@@ -3,7 +3,6 @@ mod models;
|
||||
|
||||
use crate::core::context::Context;
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::services::backup::{BackupResult, UploadResult};
|
||||
use crate::services::storage::StorageProvider;
|
||||
use crate::utils::common::BackupMethod;
|
||||
use crate::utils::file::{full_file_name, full_file_path};
|
||||
@@ -12,6 +11,7 @@ use async_trait::async_trait;
|
||||
use std::sync::Arc;
|
||||
use tokio::fs;
|
||||
use tracing::{error, info};
|
||||
use crate::services::backup::models::{BackupResult, UploadResult};
|
||||
use crate::services::storage::providers::google_drive::helpers::{upload_stream_to_google_drive};
|
||||
use crate::services::storage::providers::google_drive::models::GoogleDriveProviderConfig;
|
||||
|
||||
@@ -56,7 +56,7 @@ impl StorageProvider for GoogleDriveProvider {
|
||||
let upload = match build_stream(
|
||||
&file_path,
|
||||
encrypt,
|
||||
encrypt.then(|| ctx.edge_key.public_key.as_bytes().to_vec()),
|
||||
&ctx.edge_key.master_key_b64
|
||||
)
|
||||
.await
|
||||
{
|
||||
|
||||
@@ -1,18 +1,16 @@
|
||||
use crate::core::context::Context;
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::services::backup::{BackupResult, UploadResult};
|
||||
use crate::services::storage::StorageProvider;
|
||||
use crate::utils::common::BackupMethod;
|
||||
use crate::utils::file::{full_file_name, full_file_path};
|
||||
use crate::utils::stream::build_stream;
|
||||
use crate::utils::tus::upload_to_tus_stream_with_headers;
|
||||
use async_trait::async_trait;
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose;
|
||||
use reqwest::header::{HeaderMap, HeaderValue};
|
||||
use std::sync::Arc;
|
||||
use tokio::fs;
|
||||
use tracing::error;
|
||||
use crate::services::backup::models::{BackupResult, UploadResult};
|
||||
|
||||
pub struct LocalProvider;
|
||||
|
||||
@@ -58,7 +56,7 @@ impl StorageProvider for LocalProvider {
|
||||
let upload = match build_stream(
|
||||
&file_path,
|
||||
encrypt,
|
||||
encrypt.then(|| ctx.edge_key.public_key.as_bytes().to_vec()),
|
||||
&ctx.edge_key.master_key_b64
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -92,30 +90,7 @@ impl StorageProvider for LocalProvider {
|
||||
"X-Method",
|
||||
HeaderValue::from_str(&method.to_string()).unwrap(),
|
||||
);
|
||||
|
||||
if let Some(enc) = upload.encryption {
|
||||
let mut meta_pairs = Vec::new();
|
||||
|
||||
meta_pairs.push(format!("version {}", "1"));
|
||||
meta_pairs.push(format!("cipher {}", "AES-256-CBC+RSA-OAEP-SHA256"));
|
||||
meta_pairs.push(format!(
|
||||
"encrypted_aes_key_b64 {}",
|
||||
general_purpose::STANDARD.encode(&enc.encrypted_aes_key)
|
||||
));
|
||||
meta_pairs.push(format!(
|
||||
"iv_b64 {}",
|
||||
general_purpose::STANDARD.encode(&enc.iv)
|
||||
));
|
||||
|
||||
let metadata_header_value = meta_pairs.join(",");
|
||||
|
||||
extra_headers.insert(
|
||||
"Upload-Metadata",
|
||||
HeaderValue::from_str(&*general_purpose::STANDARD.encode(metadata_header_value))
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
let tus_endpoint = format!("{}/tus/files", ctx.edge_key.server_url);
|
||||
|
||||
match upload_to_tus_stream_with_headers(upload.stream, &tus_endpoint, extra_headers, total_size).await {
|
||||
|
||||
@@ -2,11 +2,10 @@ mod models;
|
||||
|
||||
use crate::core::context::Context;
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::services::backup::{BackupResult, UploadResult};
|
||||
use crate::services::storage::StorageProvider;
|
||||
use crate::services::storage::providers::s3::models::S3ProviderConfig;
|
||||
use crate::utils::common::BackupMethod;
|
||||
use crate::utils::file::{EncryptionMetadataFile, full_file_name, full_file_path};
|
||||
use crate::utils::file::{full_file_name, full_file_path};
|
||||
use crate::utils::stream::build_stream;
|
||||
use async_trait::async_trait;
|
||||
use aws_sdk_s3 as s3;
|
||||
@@ -14,12 +13,12 @@ use aws_sdk_s3::config::BehaviorVersion;
|
||||
use aws_sdk_s3::config::Region;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use base64::{Engine as _, engine::general_purpose};
|
||||
use futures::StreamExt;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use tokio::fs;
|
||||
use tracing::{error, info};
|
||||
use crate::services::backup::models::{BackupResult, UploadResult};
|
||||
|
||||
pub struct S3Provider {}
|
||||
|
||||
@@ -59,13 +58,7 @@ impl StorageProvider for S3Provider {
|
||||
|
||||
let encrypt = encrypt.unwrap_or(false);
|
||||
|
||||
let upload = match build_stream(
|
||||
&file_path,
|
||||
encrypt,
|
||||
encrypt.then(|| ctx.edge_key.public_key.as_bytes().to_vec()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
let upload = match build_stream(&file_path, encrypt, &ctx.edge_key.master_key_b64).await {
|
||||
Ok(u) => u,
|
||||
Err(e) => {
|
||||
error!("Stream build failed: {}", e);
|
||||
@@ -100,17 +93,25 @@ impl StorageProvider for S3Provider {
|
||||
"static-creds",
|
||||
);
|
||||
|
||||
let region = Region::new(config.region.clone().unwrap_or("eu-central-3".to_string()));
|
||||
let region = Region::new(config.region.clone().unwrap_or("us-east-1".to_string()));
|
||||
|
||||
let endpoint = if let Some(port) = &config.port {
|
||||
if port.trim().is_empty() {
|
||||
format!("{}://{}", if config.ssl { "https" } else { "http" }, config.end_point_url)
|
||||
} else {
|
||||
format!("{}://{}:{}", if config.ssl { "https" } else { "http" }, config.end_point_url, port)
|
||||
}
|
||||
} else {
|
||||
format!("{}://{}", if config.ssl { "https" } else { "http" }, config.end_point_url)
|
||||
};
|
||||
|
||||
info!("S3 endpoint to {}", &endpoint);
|
||||
|
||||
let sdk_config = s3::config::Builder::new()
|
||||
.credentials_provider(credentials)
|
||||
.region(region)
|
||||
.force_path_style(true)
|
||||
.endpoint_url(format!(
|
||||
"{}://{}",
|
||||
if config.ssl { "https" } else { "http" },
|
||||
config.end_point_url
|
||||
))
|
||||
.endpoint_url(endpoint)
|
||||
.behavior_version(BehaviorVersion::latest())
|
||||
.build();
|
||||
|
||||
@@ -283,35 +284,6 @@ impl StorageProvider for S3Provider {
|
||||
"Successfully completed multipart upload: {}",
|
||||
remote_file_path
|
||||
);
|
||||
|
||||
if let Some(enc) = upload.encryption {
|
||||
let meta = EncryptionMetadataFile {
|
||||
version: 1,
|
||||
cipher: "AES-256-CBC+RSA-OAEP-SHA256".to_string(),
|
||||
encrypted_aes_key_b64: general_purpose::STANDARD
|
||||
.encode(enc.encrypted_aes_key),
|
||||
iv_b64: general_purpose::STANDARD.encode(enc.iv),
|
||||
};
|
||||
|
||||
let meta_toml = toml::to_string(&meta).expect("Serialization error");
|
||||
|
||||
let meta_key = format!("{}.meta", remote_file_path);
|
||||
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&meta_key)
|
||||
.body(ByteStream::from(meta_toml.into_bytes()))
|
||||
.content_type("application/toml")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!("Metadata upload failed: {}", e);
|
||||
e
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
UploadResult {
|
||||
storage_id: storage.id.clone(),
|
||||
success: true,
|
||||
|
||||
@@ -8,6 +8,5 @@ pub struct S3ProviderConfig {
|
||||
pub end_point_url: String,
|
||||
pub ssl: bool,
|
||||
pub region: Option<String>,
|
||||
pub port: Option<String>,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
mod postgres;
|
||||
@@ -0,0 +1,35 @@
|
||||
use oauth2::url;
|
||||
use testcontainers::runners::AsyncRunner;
|
||||
use testcontainers_modules::postgres::Postgres;
|
||||
use crate::services::config::{DatabaseConfig, DbType};
|
||||
use url::Host;
|
||||
use crate::domain::factory::DatabaseFactory;
|
||||
|
||||
#[tokio::test]
|
||||
async fn postgres_ping_test() {
|
||||
|
||||
let container = Postgres::default()
|
||||
.start()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let host = container.get_host().await.unwrap_or(Host::parse("127.0.0.1").unwrap());
|
||||
let port = container.get_host_port_ipv4(5432).await.unwrap_or(5432) ;
|
||||
|
||||
let config = DatabaseConfig {
|
||||
name: "My test Postgres Database".to_string(),
|
||||
database: "postgres".to_string(),
|
||||
db_type: DbType::Postgresql,
|
||||
username: "postgres".to_string(),
|
||||
password: "postgres".to_string(),
|
||||
port,
|
||||
host: host.to_string(),
|
||||
generated_id: "40875631-e3d2-4dfe-a26b-2a347ecc64fd".to_string(),
|
||||
path: "".to_string(),
|
||||
};
|
||||
|
||||
let db = DatabaseFactory::create_for_backup(config.clone()).await;
|
||||
let reachable = db.ping().await.unwrap_or_else(|_| false);
|
||||
|
||||
assert_eq!(reachable, true);
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
mod utils;
|
||||
mod domain;
|
||||
@@ -0,0 +1,46 @@
|
||||
use serde_json::json;
|
||||
use crate::utils::common::{vec_to_option_json, BackupMethod};
|
||||
|
||||
#[test]
|
||||
fn backup_method_to_string_automatic() {
|
||||
let method = BackupMethod::Automatic;
|
||||
assert_eq!(method.to_string(), "automatic");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backup_method_to_string_manual() {
|
||||
let method = BackupMethod::Manual;
|
||||
assert_eq!(method.to_string(), "manual");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vec_to_option_json_returns_none_when_empty() {
|
||||
let v: Vec<i32> = vec![];
|
||||
let result = vec_to_option_json(v);
|
||||
|
||||
assert!(result.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vec_to_option_json_serializes_vector() {
|
||||
let v = vec![1, 2, 3];
|
||||
let result = vec_to_option_json(v);
|
||||
|
||||
assert_eq!(result, Some(json!([1, 2, 3])));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vec_to_option_json_serializes_struct_vector() {
|
||||
#[derive(serde::Serialize)]
|
||||
struct Item {
|
||||
id: u32,
|
||||
}
|
||||
|
||||
let v = vec![Item { id: 1 }, Item { id: 2 }];
|
||||
let result = vec_to_option_json(v);
|
||||
|
||||
assert_eq!(result, Some(json!([
|
||||
{ "id": 1 },
|
||||
{ "id": 2 }
|
||||
])));
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
use tempfile::tempdir;
|
||||
use tokio::fs::{write, read};
|
||||
use anyhow::Result;
|
||||
use crate::utils::compress::{compress_to_tar_gz_large, decompress_large_tar_gz};
|
||||
|
||||
#[tokio::test]
|
||||
async fn compress_creates_tar_gz() -> Result<()> {
|
||||
let tmp = tempdir()?;
|
||||
let file_path = tmp.path().join("test.txt");
|
||||
write(&file_path, b"hello world").await?;
|
||||
|
||||
let result = compress_to_tar_gz_large(&file_path).await?;
|
||||
assert!(result.compressed_path.exists());
|
||||
assert_eq!(result.compressed_path.extension().unwrap(), "gz");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compress_skips_existing_tar_gz() -> Result<()> {
|
||||
let tmp = tempdir()?;
|
||||
let file_path = tmp.path().join("already.tar.gz");
|
||||
write(&file_path, b"compressed").await?;
|
||||
|
||||
let result = compress_to_tar_gz_large(&file_path).await?;
|
||||
// Should return same path without creating a new file
|
||||
assert_eq!(result.compressed_path, file_path);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn decompress_restores_file() -> Result<()> {
|
||||
let tmp = tempdir()?;
|
||||
let file_path = tmp.path().join("file.txt");
|
||||
write(&file_path, b"data for decompress").await?;
|
||||
|
||||
let compress_result = compress_to_tar_gz_large(&file_path).await?;
|
||||
let output_dir = tmp.path().join("out");
|
||||
tokio::fs::create_dir_all(&output_dir).await?;
|
||||
|
||||
let extracted_files = decompress_large_tar_gz(&compress_result.compressed_path, &output_dir).await?;
|
||||
assert_eq!(extracted_files.len(), 1);
|
||||
|
||||
let extracted_content = read(&extracted_files[0]).await?;
|
||||
assert_eq!(extracted_content, b"data for decompress");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn decompress_multiple_files() -> Result<()> {
|
||||
let tmp = tempdir()?;
|
||||
let file1 = tmp.path().join("file1.txt");
|
||||
let file2 = tmp.path().join("file2.txt");
|
||||
write(&file1, b"file1").await?;
|
||||
write(&file2, b"file2").await?;
|
||||
|
||||
// Compress both files individually (for simplicity in this test)
|
||||
let compress1 = compress_to_tar_gz_large(&file1).await?;
|
||||
let compress2 = compress_to_tar_gz_large(&file2).await?;
|
||||
|
||||
let output_dir = tmp.path().join("out_multi");
|
||||
tokio::fs::create_dir_all(&output_dir).await?;
|
||||
|
||||
let extracted1 = decompress_large_tar_gz(&compress1.compressed_path, &output_dir).await?;
|
||||
let extracted2 = decompress_large_tar_gz(&compress2.compressed_path, &output_dir).await?;
|
||||
|
||||
assert_eq!(extracted1.len(), 1);
|
||||
assert_eq!(extracted2.len(), 1);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde::Deserialize;
|
||||
use toml::map::Map;
|
||||
use toml::Value;
|
||||
use crate::utils::deserializer::{camel_to_snake, deserialize_snake_case, to_snake_case};
|
||||
|
||||
#[test]
|
||||
fn camel_to_snake_simple() {
|
||||
assert_eq!(camel_to_snake("CamelCase"), "camel_case");
|
||||
assert_eq!(camel_to_snake("simpleTest"), "simple_test");
|
||||
assert_eq!(camel_to_snake("already_snake"), "already_snake");
|
||||
assert_eq!(camel_to_snake("X"), "x");
|
||||
assert_eq!(camel_to_snake("ABTest"), "a_b_test");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn to_snake_case_nested_table() {
|
||||
let mut inner_table = Map::new();
|
||||
inner_table.insert("InnerKey".into(), Value::String("value".into()));
|
||||
|
||||
let mut outer_table = Map::new();
|
||||
outer_table.insert("OuterKey".into(), Value::Table(inner_table));
|
||||
|
||||
let value = Value::Table(outer_table);
|
||||
|
||||
// Expected snake_case
|
||||
let mut expected_inner = Map::new();
|
||||
expected_inner.insert("inner_key".into(), Value::String("value".into()));
|
||||
|
||||
let mut expected_outer = Map::new();
|
||||
expected_outer.insert("outer_key".into(), Value::Table(expected_inner));
|
||||
|
||||
let expected = Value::Table(expected_outer);
|
||||
|
||||
let result = to_snake_case(value);
|
||||
assert_eq!(result, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn to_snake_case_array_of_tables() {
|
||||
let mut table1 = Map::new();
|
||||
table1.insert("CamelKey".into(), Value::Integer(1));
|
||||
|
||||
let mut table2 = Map::new();
|
||||
table2.insert("AnotherKey".into(), Value::Integer(2));
|
||||
|
||||
let value = Value::Array(vec![
|
||||
Value::Table(table1),
|
||||
Value::Table(table2),
|
||||
]);
|
||||
|
||||
let mut expected_table1 = Map::new();
|
||||
expected_table1.insert("camel_key".into(), Value::Integer(1));
|
||||
|
||||
let mut expected_table2 = Map::new();
|
||||
expected_table2.insert("another_key".into(), Value::Integer(2));
|
||||
|
||||
let expected = Value::Array(vec![
|
||||
Value::Table(expected_table1),
|
||||
Value::Table(expected_table2),
|
||||
]);
|
||||
|
||||
let result = to_snake_case(value);
|
||||
assert_eq!(result, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deserialize_snake_case_works_with_struct() {
|
||||
#[derive(Deserialize, Debug, PartialEq)]
|
||||
struct Config {
|
||||
some_value: i32,
|
||||
nested_table: Nested,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, PartialEq)]
|
||||
struct Nested {
|
||||
inner_value: String,
|
||||
}
|
||||
|
||||
let toml_str = r#"
|
||||
SomeValue = 42
|
||||
|
||||
[NestedTable]
|
||||
InnerValue = "hello"
|
||||
"#;
|
||||
|
||||
let value: Value = toml::from_str(toml_str).unwrap();
|
||||
let snake_value = deserialize_snake_case(value).unwrap();
|
||||
|
||||
// Deserialize to struct
|
||||
let config: Config = snake_value.try_into().unwrap();
|
||||
|
||||
assert_eq!(config.some_value, 42);
|
||||
assert_eq!(config.nested_table.inner_value, "hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn to_snake_case_non_table_value() {
|
||||
let value = Value::String("unchanged".into());
|
||||
let result = to_snake_case(value.clone());
|
||||
assert_eq!(result, value);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
use base64::{engine::general_purpose, Engine as _};
|
||||
use serde_json::json;
|
||||
use crate::utils::edge_key::{decode_edge_key, EdgeKeyError};
|
||||
|
||||
#[test]
|
||||
fn decode_valid_edge_key() {
|
||||
let edge_key_b64 = "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiNjI1MDQzY2YtN2MwMC00M2M4LWJjYzktZDM1MTk5ODk2ZGNkIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ==";
|
||||
let decoded = decode_edge_key(edge_key_b64).unwrap();
|
||||
|
||||
assert_eq!(decoded.server_url, "http://localhost:8887");
|
||||
assert_eq!(decoded.agent_id, "625043cf-7c00-43c8-bcc9-d35199896dcd");
|
||||
assert_eq!(decoded.master_key_b64, "BXV3XolC656SV7dNgcWPGQlk++rpLI6lGDi7CPB5ieo=");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_edge_key_missing_field() {
|
||||
let incomplete_json = json!({
|
||||
"serverUrl": "http://localhost:8887",
|
||||
"agentId": "123"
|
||||
// masterKeyB64 is missing
|
||||
})
|
||||
.to_string();
|
||||
|
||||
let b64 = general_purpose::URL_SAFE.encode(incomplete_json);
|
||||
let result = decode_edge_key(&b64);
|
||||
|
||||
match result {
|
||||
Err(EdgeKeyError::InvalidKey) => {}
|
||||
_ => panic!("Expected InvalidKey error"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_edge_key_invalid_base64() {
|
||||
let invalid_b64 = "!!!notbase64!!!";
|
||||
let result = decode_edge_key(invalid_b64);
|
||||
|
||||
match result {
|
||||
Err(EdgeKeyError::Base64Error(_)) => {}
|
||||
_ => panic!("Expected Base64Error"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_edge_key_invalid_json() {
|
||||
let invalid_json_b64 = general_purpose::URL_SAFE.encode("not a json string");
|
||||
let result = decode_edge_key(&invalid_json_b64);
|
||||
|
||||
match result {
|
||||
Err(EdgeKeyError::JsonError(_)) => {}
|
||||
_ => panic!("Expected JsonError"),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
mod normalize_cron_tests;
|
||||
mod common_tests;
|
||||
mod compress_tests;
|
||||
mod deserializer;
|
||||
mod edge_key_tests;
|
||||
@@ -0,0 +1,49 @@
|
||||
use crate::utils::text::normalize_cron;
|
||||
use cron::Schedule;
|
||||
use std::str::FromStr;
|
||||
use crate::utils::task_manager::cron::next_run_timestamp;
|
||||
|
||||
#[test]
|
||||
fn normalize_adds_seconds_to_five_field_cron() {
|
||||
let input = "*/5 * * * *";
|
||||
let normalized = normalize_cron(input);
|
||||
|
||||
assert_eq!(normalized, "0 */5 * * * *");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_keeps_six_field_cron() {
|
||||
let input = "0 */5 * * * *";
|
||||
let normalized = normalize_cron(input);
|
||||
|
||||
assert_eq!(normalized, "0 */5 * * * *");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalized_expression_is_valid_for_cron_schedule() {
|
||||
let input = "*/5 * * * *";
|
||||
let normalized = normalize_cron(input);
|
||||
|
||||
let schedule = Schedule::from_str(&normalized);
|
||||
assert!(schedule.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn next_run_timestamp_returns_future_timestamp() {
|
||||
let expr = normalize_cron("*/1 * * * *");
|
||||
let ts = next_run_timestamp(&expr);
|
||||
|
||||
let now = chrono::Local::now().timestamp();
|
||||
assert!(ts > now);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalization_does_not_break_schedule_parsing() {
|
||||
let input = "0 */10 * * * *";
|
||||
let normalized = normalize_cron(input);
|
||||
|
||||
let schedule = Schedule::from_str(&normalized).unwrap();
|
||||
let next = schedule.upcoming(chrono::Local).next();
|
||||
|
||||
assert!(next.is_some());
|
||||
}
|
||||
@@ -90,7 +90,6 @@ pub async fn decompress_large_tar_gz(
|
||||
extracted_files.push(full_path);
|
||||
}
|
||||
|
||||
// remove_file(tar_gz_path).await?;
|
||||
info!("Decompressed {:?} into {:?}", tar_gz_path, output_dir);
|
||||
|
||||
Ok(extracted_files)
|
||||
|
||||
@@ -8,7 +8,8 @@ where
|
||||
let value = Value::deserialize(deserializer)?;
|
||||
Ok(to_snake_case(value))
|
||||
}
|
||||
fn to_snake_case(value: Value) -> Value {
|
||||
|
||||
pub fn to_snake_case(value: Value) -> Value {
|
||||
match value {
|
||||
Value::Table(table) => Value::Table(
|
||||
table
|
||||
@@ -16,14 +17,12 @@ fn to_snake_case(value: Value) -> Value {
|
||||
.map(|(k, v)| (camel_to_snake(&k), to_snake_case(v)))
|
||||
.collect(),
|
||||
),
|
||||
Value::Array(arr) => {
|
||||
Value::Array(arr.into_iter().map(to_snake_case).collect())
|
||||
}
|
||||
Value::Array(arr) => Value::Array(arr.into_iter().map(to_snake_case).collect()),
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
fn camel_to_snake(s: &str) -> String {
|
||||
pub fn camel_to_snake(s: &str) -> String {
|
||||
let mut out = String::new();
|
||||
for (i, c) in s.chars().enumerate() {
|
||||
if c.is_uppercase() {
|
||||
|
||||
@@ -2,7 +2,6 @@ use base64::{Engine as _, engine::general_purpose};
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
use thiserror::Error;
|
||||
use tracing::error;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[allow(dead_code)]
|
||||
@@ -11,8 +10,8 @@ pub struct EdgeKey {
|
||||
pub server_url: String,
|
||||
#[serde(rename = "agentId")]
|
||||
pub agent_id: String,
|
||||
#[serde(rename = "publicKey")]
|
||||
pub public_key: String,
|
||||
#[serde(rename = "masterKeyB64")]
|
||||
pub master_key_b64: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
@@ -35,14 +34,14 @@ pub fn decode_edge_key(edge_key: &str) -> Result<EdgeKey, EdgeKeyError> {
|
||||
let decoded_str = String::from_utf8_lossy(&decoded_bytes);
|
||||
|
||||
let parsed: Value = serde_json::from_str(&decoded_str)?;
|
||||
|
||||
if parsed.get("serverUrl").is_some()
|
||||
&& parsed.get("agentId").is_some()
|
||||
&& parsed.get("publicKey").is_some()
|
||||
&& parsed.get("masterKeyB64").is_some()
|
||||
{
|
||||
let edge_key: EdgeKey = serde_json::from_value(parsed)?;
|
||||
Ok(edge_key)
|
||||
} else {
|
||||
error!("EDGE_KEY INVALID");
|
||||
Err(EdgeKeyError::InvalidKey)
|
||||
}
|
||||
}
|
||||
|
||||
+123
-48
@@ -1,21 +1,26 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use anyhow::Result;
|
||||
use async_stream::try_stream;
|
||||
use bytes::Bytes;
|
||||
use chrono::Utc;
|
||||
use futures::Stream;
|
||||
use openssl::encrypt::Encrypter;
|
||||
use openssl::hash::MessageDigest;
|
||||
use openssl::pkey::PKey;
|
||||
use openssl::rsa::Padding;
|
||||
use openssl::symm::{Cipher, Crypter, Mode};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::{Path, PathBuf};
|
||||
use tokio::fs::File;
|
||||
use tokio::io::{AsyncReadExt, BufReader};
|
||||
use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader};
|
||||
use uuid::Uuid;
|
||||
|
||||
use aes_gcm::aead::{Aead, KeyInit};
|
||||
use aes_gcm::{Aes256Gcm, Key, Nonce};
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose;
|
||||
use bytes::Bytes;
|
||||
use futures::Stream;
|
||||
use rand::rngs::OsRng;
|
||||
use rand::TryRngCore;
|
||||
use tokio::io::{AsyncWriteExt, BufWriter};
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
use tracing::info;
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct EncryptionMetadataFile {
|
||||
pub version: u8,
|
||||
@@ -32,7 +37,7 @@ pub fn full_extension(path: &Path) -> String {
|
||||
.to_string()
|
||||
}
|
||||
|
||||
pub fn full_file_name( encrypt: bool) -> String {
|
||||
pub fn full_file_name(encrypt: bool) -> String {
|
||||
let uuid = Uuid::new_v4();
|
||||
let base_name = format!("{}.{}", uuid, "tar.gz");
|
||||
if encrypt {
|
||||
@@ -46,55 +51,125 @@ pub fn full_file_path(file_name: &String) -> String {
|
||||
format!("backups/{}/{}", Utc::now().format("%Y-%m-%d"), file_name)
|
||||
}
|
||||
|
||||
pub async fn encrypt_file_stream(
|
||||
const CHUNK_SIZE: usize = 16 * 1024 * 1024;
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
struct FileHeader {
|
||||
version: u8,
|
||||
cipher: String,
|
||||
chunk_size: usize,
|
||||
base_nonce: Vec<u8>,
|
||||
}
|
||||
|
||||
pub async fn encrypt_file_stream_gcm(
|
||||
file_path: PathBuf,
|
||||
aes_key: [u8; 32],
|
||||
iv: [u8; 16],
|
||||
pub_key_pem: Vec<u8>,
|
||||
) -> Result<(impl Stream<Item = Result<Bytes>> + Send + 'static, Vec<u8>)> {
|
||||
// ---------- Encrypt AES key with RSA ----------
|
||||
let pkey = PKey::public_key_from_pem(&pub_key_pem)?;
|
||||
let mut rsa = Encrypter::new(&pkey)?;
|
||||
rsa.set_rsa_padding(Padding::PKCS1_OAEP)?;
|
||||
rsa.set_rsa_oaep_md(MessageDigest::sha256())?;
|
||||
rsa.set_rsa_mgf1_md(MessageDigest::sha256())?;
|
||||
master_key_b64: String,
|
||||
) -> Result<impl Stream<Item = Result<Bytes>> + Send + 'static> {
|
||||
let master_key_bytes = general_purpose::STANDARD
|
||||
.decode(master_key_b64)
|
||||
.map_err(|_| anyhow::anyhow!("Invalid base64"))?;
|
||||
|
||||
let mut encrypted_key = vec![0u8; rsa.encrypt_len(&aes_key)?];
|
||||
let len = rsa.encrypt(&aes_key, &mut encrypted_key)?;
|
||||
encrypted_key.truncate(len);
|
||||
let (tx, rx) = mpsc::channel(8);
|
||||
|
||||
// ---------- Streaming AES encryption ----------
|
||||
let stream = try_stream! {
|
||||
let file = File::open(&file_path).await?;
|
||||
tokio::spawn(async move {
|
||||
let mut rng = OsRng;
|
||||
let mut base_nonce = [0u8; 8];
|
||||
rng.try_fill_bytes(&mut base_nonce).unwrap();
|
||||
|
||||
let key = Key::<Aes256Gcm>::try_from(master_key_bytes.as_slice())
|
||||
.map_err(|_| anyhow::anyhow!("Invalid AES-256 key length")).unwrap();
|
||||
|
||||
let cipher = Aes256Gcm::new(&key);
|
||||
|
||||
let header = FileHeader {
|
||||
version: 1,
|
||||
cipher: "AES-256-GCM".to_string(),
|
||||
chunk_size: CHUNK_SIZE,
|
||||
base_nonce: base_nonce.to_vec(),
|
||||
};
|
||||
let header_json = serde_json::to_string(&header).unwrap();
|
||||
tx.send(Ok(Bytes::from(header_json + "\n"))).await.unwrap();
|
||||
|
||||
let file = File::open(&file_path).await.unwrap();
|
||||
let mut reader = BufReader::new(file);
|
||||
|
||||
let cipher = Cipher::aes_256_cbc();
|
||||
let mut crypter = Crypter::new(cipher, Mode::Encrypt, &aes_key, Some(&iv))?;
|
||||
crypter.pad(true);
|
||||
|
||||
let mut buffer = vec![0u8; 1024 * 1024];
|
||||
let mut buffer = vec![0u8; CHUNK_SIZE];
|
||||
let mut chunk_index: u32 = 0;
|
||||
|
||||
loop {
|
||||
let n = reader.read(&mut buffer).await?;
|
||||
let n = reader.read(&mut buffer).await.unwrap();
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
let mut out = vec![0u8; n + cipher.block_size()];
|
||||
let count = crypter.update(&buffer[..n], &mut out)?;
|
||||
out.truncate(count);
|
||||
let mut nonce_bytes = [0u8; 12];
|
||||
nonce_bytes[..8].copy_from_slice(&base_nonce);
|
||||
nonce_bytes[8..].copy_from_slice(&chunk_index.to_be_bytes());
|
||||
let nonce = Nonce::try_from(&nonce_bytes[..])
|
||||
.map_err(|_| anyhow::anyhow!("Invalid nonce length")).unwrap();
|
||||
|
||||
yield Bytes::from(out);
|
||||
let ciphertext = cipher.encrypt(&nonce, &buffer[..n]).unwrap();
|
||||
let mut out = Vec::with_capacity(4 + ciphertext.len());
|
||||
out.extend_from_slice(&(ciphertext.len() as u32).to_be_bytes());
|
||||
out.extend_from_slice(&ciphertext);
|
||||
|
||||
tx.send(Ok(Bytes::from(out))).await.unwrap();
|
||||
chunk_index += 1;
|
||||
}
|
||||
});
|
||||
|
||||
let mut final_block = vec![0u8; cipher.block_size()];
|
||||
let rest = crypter.finalize(&mut final_block)?;
|
||||
final_block.truncate(rest);
|
||||
|
||||
if !final_block.is_empty() {
|
||||
yield Bytes::from(final_block);
|
||||
}
|
||||
};
|
||||
|
||||
Ok((stream, encrypted_key))
|
||||
Ok(ReceiverStream::new(rx))
|
||||
}
|
||||
|
||||
pub async fn decrypt_file_stream_gcm(
|
||||
encrypted_path: PathBuf,
|
||||
decrypted_path: PathBuf,
|
||||
master_key_b64: String,
|
||||
) -> Result<()> {
|
||||
info!("Decrypting {:?}", decrypted_path);
|
||||
|
||||
let master_key_bytes = general_purpose::STANDARD
|
||||
.decode(master_key_b64)
|
||||
.map_err(|_| anyhow::anyhow!("Invalid base64"))?;
|
||||
|
||||
let mut reader = BufReader::new(File::open(&encrypted_path).await?);
|
||||
|
||||
let mut header_line = Vec::new();
|
||||
reader.read_until(b'\n', &mut header_line).await?;
|
||||
let header: FileHeader = serde_json::from_slice(&header_line)?;
|
||||
|
||||
let key = Key::<Aes256Gcm>::try_from(master_key_bytes.as_slice())
|
||||
.map_err(|_| anyhow::anyhow!("Invalid AES-256 key length"))?;
|
||||
let cipher = Aes256Gcm::new(&key);
|
||||
|
||||
let mut writer = BufWriter::new(File::create(&decrypted_path).await?);
|
||||
let mut chunk_index: u32 = 0;
|
||||
|
||||
loop {
|
||||
let mut len_buf = [0u8; 4];
|
||||
match reader.read_exact(&mut len_buf).await {
|
||||
Ok(_) => {}
|
||||
Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => break,
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
let chunk_len = u32::from_be_bytes(len_buf) as usize;
|
||||
|
||||
let mut chunk_ciphertext = vec![0u8; chunk_len];
|
||||
reader.read_exact(&mut chunk_ciphertext).await?;
|
||||
|
||||
let mut nonce_bytes = [0u8; 12];
|
||||
nonce_bytes[..8].copy_from_slice(&header.base_nonce);
|
||||
nonce_bytes[8..].copy_from_slice(&chunk_index.to_be_bytes());
|
||||
let nonce = Nonce::try_from(&nonce_bytes[..])
|
||||
.map_err(|_| anyhow::anyhow!("Invalid nonce length"))?;
|
||||
|
||||
let plaintext = cipher
|
||||
.decrypt(&nonce, chunk_ciphertext.as_slice())
|
||||
.map_err(|e| anyhow::anyhow!("AES-GCM decryption failed: {:?}", e))?;
|
||||
|
||||
writer.write_all(&plaintext).await?;
|
||||
chunk_index += 1;
|
||||
}
|
||||
|
||||
writer.flush().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
+6
-31
@@ -1,51 +1,29 @@
|
||||
use crate::utils::file::encrypt_file_stream;
|
||||
use crate::utils::file::encrypt_file_stream_gcm;
|
||||
use anyhow::Result;
|
||||
use bytes::Bytes;
|
||||
use futures::{Stream, StreamExt};
|
||||
use rand::RngCore;
|
||||
use std::pin::Pin;
|
||||
use tokio_util::io::ReaderStream;
|
||||
|
||||
pub struct EncryptionMetadata {
|
||||
pub encrypted_aes_key: Vec<u8>,
|
||||
pub iv: [u8; 16],
|
||||
}
|
||||
|
||||
pub struct UploadStream {
|
||||
pub stream: Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>,
|
||||
pub encryption: Option<EncryptionMetadata>,
|
||||
}
|
||||
|
||||
pub async fn build_stream(
|
||||
file_path: &std::path::Path,
|
||||
encrypt: bool,
|
||||
public_key_pem: Option<Vec<u8>>,
|
||||
master_key_b64: &String,
|
||||
) -> Result<UploadStream> {
|
||||
if encrypt {
|
||||
let public_key =
|
||||
public_key_pem.ok_or_else(|| anyhow::anyhow!("Missing public key for encryption"))?;
|
||||
|
||||
let mut aes_key = [0u8; 32];
|
||||
rand::rng().fill_bytes(&mut aes_key);
|
||||
|
||||
let mut iv = [0u8; 16];
|
||||
rand::rng().fill_bytes(&mut iv);
|
||||
|
||||
let (encrypted_stream, encrypted_aes_key) =
|
||||
encrypt_file_stream(file_path.to_path_buf(), aes_key, iv, public_key).await?;
|
||||
let encrypted_stream =
|
||||
encrypt_file_stream_gcm(file_path.to_path_buf(), master_key_b64.to_string()).await?;
|
||||
|
||||
let stream = Box::pin(
|
||||
encrypted_stream
|
||||
.map(|r| r.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))),
|
||||
);
|
||||
|
||||
Ok(UploadStream {
|
||||
stream,
|
||||
encryption: Some(EncryptionMetadata {
|
||||
encrypted_aes_key,
|
||||
iv,
|
||||
}),
|
||||
})
|
||||
Ok(UploadStream { stream })
|
||||
} else {
|
||||
let file = tokio::fs::File::open(file_path).await?;
|
||||
let reader = ReaderStream::new(file);
|
||||
@@ -54,9 +32,6 @@ pub async fn build_stream(
|
||||
reader.map(|r| r.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))),
|
||||
);
|
||||
|
||||
Ok(UploadStream {
|
||||
stream,
|
||||
encryption: None,
|
||||
})
|
||||
Ok(UploadStream { stream })
|
||||
}
|
||||
}
|
||||
|
||||
+116
-24
@@ -1,8 +1,8 @@
|
||||
use anyhow::Result;
|
||||
use anyhow::{Context, Result};
|
||||
use bytes::Bytes;
|
||||
use futures::{Stream, StreamExt};
|
||||
use log::info;
|
||||
use reqwest::header::{CONTENT_TYPE, HeaderMap, HeaderValue};
|
||||
use tracing::{error, info};
|
||||
|
||||
const PATCH_CHUNK_SIZE: usize = 1 * 1024 * 1024;
|
||||
|
||||
@@ -18,39 +18,62 @@ where
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
info!("File size: {}", total_size);
|
||||
info!("Endpoint URL: {}", tus_endpoint);
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("Tus-Resumable", HeaderValue::from_static("1.0.0"));
|
||||
headers.insert("Upload-Defer-Length", HeaderValue::from_static("1"));
|
||||
let mut create_headers = HeaderMap::new();
|
||||
create_headers.insert("Tus-Resumable", HeaderValue::from_static("1.0.0"));
|
||||
create_headers.insert("Upload-Defer-Length", HeaderValue::from_static("1"));
|
||||
|
||||
let resp = client
|
||||
.post(tus_endpoint)
|
||||
.headers(headers.clone())
|
||||
.headers(create_headers.clone())
|
||||
.send()
|
||||
.await?;
|
||||
.await
|
||||
.context("Failed to send POST to create TUS upload")?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
anyhow::bail!("Failed to create upload: {}", resp.status());
|
||||
let status = resp.status();
|
||||
let headers = resp.headers().clone();
|
||||
let body = resp
|
||||
.text()
|
||||
.await
|
||||
.unwrap_or_else(|_| "<failed to read body>".into());
|
||||
|
||||
error!(
|
||||
"TUS creation failed | status={} | headers={:?} | body={}",
|
||||
status, headers, body
|
||||
);
|
||||
|
||||
anyhow::bail!(
|
||||
"Failed to create upload.\nStatus: {}\nHeaders: {:?}\nBody: {}",
|
||||
status,
|
||||
headers,
|
||||
body
|
||||
);
|
||||
}
|
||||
|
||||
let upload_url = resp
|
||||
.headers()
|
||||
.get("Location")
|
||||
.ok_or_else(|| anyhow::anyhow!("Missing Location header"))?
|
||||
.to_str()?
|
||||
.context("TUS creation response missing Location header")?
|
||||
.to_str()
|
||||
.context("Invalid Location header value")?
|
||||
.to_string();
|
||||
|
||||
let mut stream = Box::pin(
|
||||
encrypted_stream.map(|r| r.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))),
|
||||
);
|
||||
|
||||
let mut stream = Box::pin(encrypted_stream);
|
||||
let mut offset: u64 = 0;
|
||||
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk?;
|
||||
let chunk = chunk.context("Stream produced IO error")?;
|
||||
|
||||
for sub_chunk in chunk.chunks(PATCH_CHUNK_SIZE) {
|
||||
let mut patch_headers = extra_headers.clone();
|
||||
patch_headers.insert("Tus-Resumable", HeaderValue::from_static("1.0.0"));
|
||||
patch_headers.insert("Upload-Offset", HeaderValue::from_str(&offset.to_string())?);
|
||||
patch_headers.insert(
|
||||
"Upload-Offset",
|
||||
HeaderValue::from_str(&offset.to_string())
|
||||
.context("Invalid offset header value")?,
|
||||
);
|
||||
patch_headers.insert(
|
||||
CONTENT_TYPE,
|
||||
HeaderValue::from_static("application/offset+octet-stream"),
|
||||
@@ -61,24 +84,69 @@ where
|
||||
.headers(patch_headers)
|
||||
.body(sub_chunk.to_vec())
|
||||
.send()
|
||||
.await?;
|
||||
.await
|
||||
.with_context(|| format!("PATCH request failed at offset {}", offset))?;
|
||||
|
||||
if !patch_resp.status().is_success() {
|
||||
let status = patch_resp.status();
|
||||
let headers = patch_resp.headers().clone();
|
||||
let body = patch_resp
|
||||
.text()
|
||||
.await
|
||||
.unwrap_or_else(|_| "<failed to read body>".into());
|
||||
|
||||
error!(
|
||||
"TUS PATCH failure | offset={} | status={} | body={}",
|
||||
offset, status, body
|
||||
);
|
||||
|
||||
anyhow::bail!(
|
||||
"Chunk upload failed at offset {}: {}",
|
||||
"Chunk upload failed.\n\
|
||||
URL: {}\n\
|
||||
Offset: {}\n\
|
||||
Status: {}\n\
|
||||
Headers: {:?}\n\
|
||||
Body: {}",
|
||||
upload_url,
|
||||
offset,
|
||||
patch_resp.status()
|
||||
status,
|
||||
headers,
|
||||
body
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(server_offset) = patch_resp.headers().get("Upload-Offset") {
|
||||
let server_offset = server_offset
|
||||
.to_str()
|
||||
.context("Invalid Upload-Offset header")?
|
||||
.parse::<u64>()
|
||||
.context("Failed to parse Upload-Offset header")?;
|
||||
|
||||
let expected = offset + sub_chunk.len() as u64;
|
||||
|
||||
if server_offset != expected {
|
||||
anyhow::bail!(
|
||||
"Offset mismatch detected.\nLocal expected: {}\nServer returned: {}",
|
||||
expected,
|
||||
server_offset
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
offset += sub_chunk.len() as u64;
|
||||
// info!("Progress: {}/{}", offset, total_size);
|
||||
}
|
||||
}
|
||||
|
||||
let mut finalize_headers = extra_headers.clone();
|
||||
finalize_headers.insert("Tus-Resumable", HeaderValue::from_static("1.0.0"));
|
||||
finalize_headers.insert("Upload-Offset", HeaderValue::from_str(&offset.to_string())?);
|
||||
finalize_headers.insert("Upload-Length", HeaderValue::from_str(&offset.to_string())?);
|
||||
finalize_headers.insert(
|
||||
"Upload-Offset",
|
||||
HeaderValue::from_str(&offset.to_string()).context("Invalid finalize offset header")?,
|
||||
);
|
||||
finalize_headers.insert(
|
||||
"Upload-Length",
|
||||
HeaderValue::from_str(&offset.to_string()).context("Invalid finalize length header")?,
|
||||
);
|
||||
finalize_headers.insert(
|
||||
CONTENT_TYPE,
|
||||
HeaderValue::from_static("application/offset+octet-stream"),
|
||||
@@ -88,11 +156,35 @@ where
|
||||
.patch(&upload_url)
|
||||
.headers(finalize_headers)
|
||||
.send()
|
||||
.await?;
|
||||
.await
|
||||
.context("Finalize PATCH request failed")?;
|
||||
|
||||
if !finalize_resp.status().is_success() {
|
||||
anyhow::bail!("Failed to finalize upload");
|
||||
let status = finalize_resp.status();
|
||||
let body = finalize_resp
|
||||
.text()
|
||||
.await
|
||||
.unwrap_or_else(|_| "<failed to read body>".into());
|
||||
|
||||
error!(
|
||||
"TUS finalize failure | offset={} | status={} | body={}",
|
||||
offset, status, body
|
||||
);
|
||||
|
||||
anyhow::bail!(
|
||||
"Finalize upload failed.\n\
|
||||
URL: {}\n\
|
||||
Final offset: {}\n\
|
||||
Status: {}\n\
|
||||
Body: {}",
|
||||
upload_url,
|
||||
offset,
|
||||
status,
|
||||
body
|
||||
);
|
||||
}
|
||||
|
||||
info!("Upload completed successfully. Final size: {}", offset);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user