mirror of
https://github.com/Portabase/agent.git
synced 2026-09-11 02:27:10 +00:00
Compare commits
41 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 380cea79fb | |||
| e8bda60f16 | |||
| 84c6b765c8 | |||
| 9790837200 | |||
| 8827234344 | |||
| b3417304bb | |||
| 5f4006b1ca | |||
| f24b1ec3e4 | |||
| bb1b16d0f2 | |||
| 2aca036ee4 | |||
| 2a72a55e57 | |||
| ae63a0df50 | |||
| 560c5e02f2 | |||
| ca17bf9cb9 | |||
| 9fbaab6feb | |||
| 1289497d98 | |||
| 27c55dbfd8 | |||
| 45e3acd709 | |||
| baf2edc829 | |||
| f050fa35e0 | |||
| 90d55f7630 | |||
| 82ffb71496 | |||
| 20b3b58024 | |||
| 6cddc73565 | |||
| fd3b0076c0 | |||
| 334ab51125 | |||
| 4df3fe2d0c | |||
| ec7715ca26 | |||
| 32d4e22196 | |||
| f277c5485f | |||
| 585e0bea72 | |||
| ee97c6056a | |||
| b37a4f2180 | |||
| 1d15f40662 | |||
| 324a2ea3f2 | |||
| 9ec92af6c7 | |||
| 1dafdca2a0 | |||
| d12817a960 | |||
| 608b82a254 | |||
| bbecca92f9 | |||
| b2b2733c4d |
@@ -36,7 +36,7 @@ jobs:
|
||||
agent-test bash -c "
|
||||
mkdir -p /app/coverage &&
|
||||
rm -rf /app/target/* /app/coverage/* &&
|
||||
cargo test --verbose &&
|
||||
cargo test --verbose -- --test-threads=2 &&
|
||||
sync
|
||||
"
|
||||
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
name: GHCR Publish
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
version:
|
||||
required: true
|
||||
type: string
|
||||
ref:
|
||||
required: true
|
||||
type: string
|
||||
add_latest:
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
dockerfile:
|
||||
required: false
|
||||
type: string
|
||||
default: "./docker/Dockerfile"
|
||||
target:
|
||||
required: false
|
||||
type: string
|
||||
default: "prod"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build ${{ matrix.platform }}
|
||||
runs-on: ${{ matrix.platform == 'linux/amd64' && 'ubuntu-latest' || 'ubuntu-24.04-arm' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
platform: [ linux/amd64, linux/arm64 ]
|
||||
steps:
|
||||
- name: Prepare vars
|
||||
id: prep
|
||||
run: |
|
||||
ARCH="${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}"
|
||||
echo "arch=$ARCH" >> "$GITHUB_OUTPUT"
|
||||
echo "image=ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/agent" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and push by digest
|
||||
id: build
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: ${{ inputs.dockerfile }}
|
||||
platforms: ${{ matrix.platform }}
|
||||
target: ${{ inputs.target }}
|
||||
provenance: false
|
||||
outputs: type=image,name=${{ steps.prep.outputs.image }},push-by-digest=true,name-canonical=true,push=true
|
||||
cache-from: type=gha,scope=ghcr-${{ steps.prep.outputs.arch }}
|
||||
cache-to: type=gha,mode=max,scope=ghcr-${{ steps.prep.outputs.arch }},ignore-error=true
|
||||
|
||||
- name: Export digest
|
||||
env:
|
||||
DIGEST: ${{ steps.build.outputs.digest }}
|
||||
run: |
|
||||
mkdir -p /tmp/digests
|
||||
touch "/tmp/digests/${DIGEST#sha256:}"
|
||||
|
||||
- name: Upload digest
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: digest-${{ steps.prep.outputs.arch }}
|
||||
path: /tmp/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
merge:
|
||||
name: Create multi-arch manifest
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
steps:
|
||||
- name: Prepare vars
|
||||
id: prep
|
||||
run: echo "image=ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/agent" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Download digests
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: /tmp/digests
|
||||
pattern: digest-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract Docker metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ steps.prep.outputs.image }}
|
||||
tags: |
|
||||
type=semver,pattern={{version}},value=${{ inputs.version }}
|
||||
type=semver,pattern={{major}}.{{minor}},value=${{ inputs.version }}
|
||||
type=semver,pattern={{major}},value=${{ inputs.version }}
|
||||
type=raw,value=latest,enable=${{ inputs.add_latest }}
|
||||
|
||||
- name: Create and push manifest list
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
docker buildx imagetools create \
|
||||
$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
|
||||
$(printf '${{ steps.prep.outputs.image }}@sha256:%s ' *)
|
||||
|
||||
- name: Inspect
|
||||
run: docker buildx imagetools inspect ${{ steps.prep.outputs.image }}:${{ inputs.version }}
|
||||
@@ -100,6 +100,15 @@ jobs:
|
||||
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
|
||||
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
publish-docker-ghcr:
|
||||
needs: create-release
|
||||
if: ${{ needs.create-release.result == 'success' }}
|
||||
uses: ./.github/workflows/ghcr.yml
|
||||
with:
|
||||
version: ${{ needs.create-release.outputs.version }}
|
||||
ref: ${{ needs.create-release.outputs.version }}
|
||||
add_latest: true
|
||||
|
||||
publish-helm:
|
||||
needs: create-release
|
||||
if: ${{ needs.create-release.result == 'success' }}
|
||||
|
||||
+1
-1
@@ -27,5 +27,5 @@ keywords:
|
||||
- self-hosted
|
||||
- portabase
|
||||
license: Apache-2.0
|
||||
version: 1.15.0
|
||||
version: 1.17.1
|
||||
date-released: '2026-02-24'
|
||||
|
||||
Generated
+2
-1
@@ -3503,7 +3503,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "portabase-agent"
|
||||
version = "1.15.0"
|
||||
version = "1.17.1"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"aes-gcm",
|
||||
@@ -3516,6 +3516,7 @@ dependencies = [
|
||||
"azure_core",
|
||||
"azure_storage_blob",
|
||||
"base64 0.22.1",
|
||||
"bollard",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"cron",
|
||||
|
||||
+3
-2
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "portabase-agent"
|
||||
version = "1.15.0"
|
||||
version = "1.17.1"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
@@ -35,7 +35,7 @@ rand = "0.9.2"
|
||||
bytes = "1.11.0"
|
||||
async-stream = "0.3.6"
|
||||
uuid = { version = "1.20.0", features = ["v4"] }
|
||||
tokio-util = { version = "0.7.18", features = ["compat"] }
|
||||
tokio-util = { version = "0.7.18", features = ["compat", "io"] }
|
||||
tiberius = { version = "0.12", default-features = false, features = ["rustls", "chrono"] }
|
||||
aws-config = "1.8.13"
|
||||
aws-sdk-s3 = { version = "1.122.0", features = ["behavior-version-latest"] }
|
||||
@@ -57,6 +57,7 @@ testcontainers = "0.27.1"
|
||||
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "valkey", "mysql", "mariadb", "mongo"] }
|
||||
postgres = "0.19.12"
|
||||
url = "2.5.8"
|
||||
bollard = "0.20.0"
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
|
||||
@@ -124,6 +124,13 @@
|
||||
"port": 1433,
|
||||
"host": "db-mssql",
|
||||
"generated_id": "16706125-ff7e-4c97-8c83-0adeff214682"
|
||||
},
|
||||
{
|
||||
"name": "Test database 14 - Docker Volume",
|
||||
"type": "docker-volume",
|
||||
"volume_name": "databases_sqlite-data",
|
||||
"generated_id": "16706126-ff7e-4c97-8c83-0adeff214690",
|
||||
"container_name": "db-sqlite"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
+4
-2
@@ -11,15 +11,17 @@ services:
|
||||
- cargo-git:/usr/local/cargo/git
|
||||
- ./databases.json:/config/config.json
|
||||
#- ./databases.toml:/config/config.toml
|
||||
#- /var/run/docker.sock:/var/run/docker.sock
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
# - cargo-target:/app/target
|
||||
- databases_sqlite-data:/sqlite-data/workspace/data
|
||||
- ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data
|
||||
# - /bigdisk:/scratch
|
||||
environment:
|
||||
APP_ENV: development
|
||||
LOG: debug
|
||||
TZ: "Europe/Paris"
|
||||
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiNWE2YjcxMDgtMGJhYS00Yjg1LTgwMmMtNTNjNjJiMDAzZDgzIiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ=="
|
||||
# TMPDIR: /scratch
|
||||
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiNmM4NWE3ODQtODRkMi00YzUyLTgzYmUtZTc2MDZkZjg2YjM5IiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ=="
|
||||
#CHUNK_SIZE_MB: "1"
|
||||
#POOLING: 1
|
||||
#DATABASES_CONFIG_FILE: "config.toml"
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
use crate::domain::docker_volume::docker::{
|
||||
client, create_helper, remove_helper, resolve_helper_image, start_container, stop_container,
|
||||
};
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use bollard::query_parameters::DownloadFromContainerOptions;
|
||||
use futures_util::StreamExt;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tokio::fs::File;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, backup_dir: PathBuf, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||
futures::executor::block_on(async move {
|
||||
logger.log("info", format!("Starting docker-volume backup for {}", cfg.name));
|
||||
|
||||
let docker = client()?;
|
||||
let image = resolve_helper_image(&docker).await?;
|
||||
logger.log("debug", format!("Helper image: {image}"));
|
||||
|
||||
if let Some(name) = &cfg.container_name {
|
||||
logger.log("info", format!("Stopping container {name} for consistent backup"));
|
||||
stop_container(&docker, name).await?;
|
||||
}
|
||||
|
||||
let result = async {
|
||||
let helper = create_helper(&docker, &image, &cfg.volume_name, &cfg.generated_id, true, None).await?;
|
||||
|
||||
let file_path = backup_dir.join(format!("{}.tar", cfg.generated_id));
|
||||
let start = Instant::now();
|
||||
|
||||
let dl_opts = DownloadFromContainerOptions { path: "/vol".to_string() };
|
||||
let mut stream = docker.download_from_container(&helper.id, Some(dl_opts));
|
||||
|
||||
let mut out = File::create(&file_path)
|
||||
.await
|
||||
.with_context(|| format!("Failed to create backup file {}", file_path.display()))?;
|
||||
let mut bytes_written: u64 = 0;
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.context("Error streaming volume archive from Docker")?;
|
||||
bytes_written += chunk.len() as u64;
|
||||
out.write_all(&chunk).await?;
|
||||
}
|
||||
out.flush().await?;
|
||||
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
logger.log_command("docker download_from_container", None, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("Volume backup wrote {bytes_written} bytes to {}", file_path.display()));
|
||||
|
||||
remove_helper(&docker, &helper.id).await;
|
||||
anyhow::Ok(file_path)
|
||||
}
|
||||
.await;
|
||||
|
||||
if let Some(name) = &cfg.container_name {
|
||||
if let Err(e) = start_container(&docker, name).await {
|
||||
logger.log("error", format!("Failed to restart container {name}: {e}"));
|
||||
}
|
||||
}
|
||||
|
||||
result
|
||||
})
|
||||
})
|
||||
.await?
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::{backup, ping, restore};
|
||||
use crate::domain::factory::Database;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::utils::locks::{DbOpLock, FileLock};
|
||||
|
||||
pub struct DockerVolumeDatabase {
|
||||
cfg: DatabaseConfig,
|
||||
}
|
||||
|
||||
impl DockerVolumeDatabase {
|
||||
pub fn new(cfg: DatabaseConfig) -> Self {
|
||||
Self { cfg }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Database for DockerVolumeDatabase {
|
||||
fn file_extension(&self) -> &'static str {
|
||||
".tar"
|
||||
}
|
||||
|
||||
async fn ping(&self) -> Result<bool> {
|
||||
ping::run(self.cfg.clone()).await
|
||||
}
|
||||
|
||||
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), logger).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
|
||||
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use bollard::Docker;
|
||||
use bollard::models::{ContainerCreateBody, HostConfig};
|
||||
use bollard::query_parameters::{
|
||||
CreateContainerOptions, InspectContainerOptions, ListContainersOptions,
|
||||
RemoveContainerOptions, StartContainerOptions, StopContainerOptions,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use tracing::{info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub const EPHEMERAL_LABEL: &str = "io.portabase.ephemeral";
|
||||
const HELPER_MOUNT: &str = "/vol";
|
||||
|
||||
pub fn client() -> Result<Docker> {
|
||||
Docker::connect_with_unix_defaults().context("Failed to connect to Docker daemon socket")
|
||||
}
|
||||
|
||||
pub fn parse_container_id(mountinfo: &str, cgroup: &str) -> Option<String> {
|
||||
for src in [mountinfo, cgroup] {
|
||||
for line in src.lines() {
|
||||
for marker in ["/containers/", "/docker/"] {
|
||||
if let Some(idx) = line.find(marker) {
|
||||
let rest = &line[idx + marker.len()..];
|
||||
let id: String = rest.chars().take_while(|c| c.is_ascii_hexdigit()).collect();
|
||||
if id.len() >= 64 {
|
||||
return Some(id[..64].to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub async fn resolve_helper_image(docker: &Docker) -> Result<String> {
|
||||
if let Ok(img) = std::env::var("PORTABASE_HELPER_IMAGE") {
|
||||
if !img.trim().is_empty() {
|
||||
return Ok(img);
|
||||
}
|
||||
}
|
||||
let mountinfo = std::fs::read_to_string("/proc/self/mountinfo").unwrap_or_default();
|
||||
let cgroup = std::fs::read_to_string("/proc/self/cgroup").unwrap_or_default();
|
||||
let id = parse_container_id(&mountinfo, &cgroup).context(
|
||||
"Could not determine own container id; set PORTABASE_HELPER_IMAGE to a locally-present image",
|
||||
)?;
|
||||
let info = docker
|
||||
.inspect_container(&id, None::<InspectContainerOptions>)
|
||||
.await
|
||||
.with_context(|| format!("Failed to inspect self container {id}"))?;
|
||||
info.image
|
||||
.context("Self container inspection returned no image reference")
|
||||
}
|
||||
|
||||
pub struct Helper {
|
||||
pub id: String,
|
||||
}
|
||||
|
||||
pub async fn create_helper(
|
||||
docker: &Docker,
|
||||
image: &str,
|
||||
volume_name: &str,
|
||||
generated_id: &str,
|
||||
read_only: bool,
|
||||
cmd: Option<Vec<String>>,
|
||||
) -> Result<Helper> {
|
||||
let bind = format!(
|
||||
"{volume_name}:{HELPER_MOUNT}{}",
|
||||
if read_only { ":ro" } else { "" }
|
||||
);
|
||||
let mut labels = HashMap::new();
|
||||
labels.insert(EPHEMERAL_LABEL.to_string(), "true".to_string());
|
||||
labels.insert("com.docker.compose.project".to_string(), String::new());
|
||||
labels.insert("com.docker.compose.service".to_string(), String::new());
|
||||
labels.insert("com.docker.compose.oneoff".to_string(), String::new());
|
||||
|
||||
let name = format!(
|
||||
"portabase-vol-{generated_id}-{}",
|
||||
&Uuid::new_v4().to_string()[..8]
|
||||
);
|
||||
|
||||
let body = ContainerCreateBody {
|
||||
image: Some(image.to_string()),
|
||||
cmd,
|
||||
labels: Some(labels),
|
||||
host_config: Some(HostConfig {
|
||||
binds: Some(vec![bind]),
|
||||
auto_remove: Some(false),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let opts = CreateContainerOptions {
|
||||
name: Some(name),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let res = docker
|
||||
.create_container(Some(opts), body)
|
||||
.await
|
||||
.with_context(|| format!("Failed to create helper container for volume {volume_name}"))?;
|
||||
|
||||
Ok(Helper { id: res.id })
|
||||
}
|
||||
|
||||
|
||||
pub async fn remove_helper(docker: &Docker, id: &str) {
|
||||
let stop_opts = StopContainerOptions {
|
||||
t: Some(2),
|
||||
..Default::default()
|
||||
};
|
||||
let _ = docker.stop_container(id, Some(stop_opts)).await;
|
||||
|
||||
if let Ok(info) = docker
|
||||
.inspect_container(id, None::<InspectContainerOptions>)
|
||||
.await
|
||||
{
|
||||
let name = info.name.unwrap_or_default();
|
||||
let name = name.trim_start_matches('/');
|
||||
let code = info.state.and_then(|s| s.exit_code).unwrap_or_default();
|
||||
info!("Helper container {name} exited with code {code}");
|
||||
}
|
||||
|
||||
let opts = RemoveContainerOptions {
|
||||
force: true,
|
||||
..Default::default()
|
||||
};
|
||||
if let Err(e) = docker.remove_container(id, Some(opts)).await {
|
||||
warn!("Failed to remove helper container {id}: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn stop_container(docker: &Docker, name: &str) -> Result<()> {
|
||||
docker
|
||||
.stop_container(name, None::<StopContainerOptions>)
|
||||
.await
|
||||
.with_context(|| format!("Failed to stop container {name}"))
|
||||
}
|
||||
|
||||
pub async fn start_container(docker: &Docker, name: &str) -> Result<()> {
|
||||
docker
|
||||
.start_container(name, None::<StartContainerOptions>)
|
||||
.await
|
||||
.with_context(|| format!("Failed to start container {name}"))
|
||||
}
|
||||
|
||||
pub async fn sweep_ephemeral(docker: &Docker) -> Result<usize> {
|
||||
let mut filters = HashMap::new();
|
||||
filters.insert("label".to_string(), vec![format!("{EPHEMERAL_LABEL}=true")]);
|
||||
|
||||
let opts = ListContainersOptions {
|
||||
all: true,
|
||||
filters: Some(filters),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let list = docker.list_containers(Some(opts)).await?;
|
||||
let mut removed = 0;
|
||||
for c in list {
|
||||
if let Some(id) = c.id {
|
||||
remove_helper(docker, &id).await;
|
||||
removed += 1;
|
||||
}
|
||||
}
|
||||
Ok(removed)
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
pub mod backup;
|
||||
pub mod database;
|
||||
pub mod docker;
|
||||
pub mod ping;
|
||||
pub mod restore;
|
||||
@@ -0,0 +1,12 @@
|
||||
use crate::domain::docker_volume::docker::client;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::Result;
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
|
||||
let docker = client()?;
|
||||
match docker.inspect_volume(&cfg.volume_name).await {
|
||||
Ok(_) => Ok(true),
|
||||
Err(bollard::errors::Error::DockerResponseServerError { status_code: 404, .. }) => Ok(false),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
use crate::domain::docker_volume::docker::{
|
||||
client, create_helper, remove_helper, resolve_helper_image, start_container, stop_container,
|
||||
};
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use bollard::exec::StartExecResults;
|
||||
use bollard::models::ExecConfig;
|
||||
use bollard::query_parameters::UploadToContainerOptions;
|
||||
use futures_util::StreamExt;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, archive: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
futures::executor::block_on(async move {
|
||||
logger.log("info", format!("Starting docker-volume restore for {}", cfg.name));
|
||||
|
||||
let docker = client()?;
|
||||
let image = resolve_helper_image(&docker).await?;
|
||||
|
||||
logger.log("debug", format!("Restore archive: {}", archive.display()));
|
||||
|
||||
if let Some(name) = &cfg.container_name {
|
||||
logger.log("info", format!("Stopping container {name} for restore"));
|
||||
stop_container(&docker, name).await?;
|
||||
}
|
||||
|
||||
let result = async {
|
||||
let helper = create_helper(
|
||||
&docker,
|
||||
&image,
|
||||
&cfg.volume_name,
|
||||
&cfg.generated_id,
|
||||
false,
|
||||
Some(vec![
|
||||
"sh".into(),
|
||||
"-c".into(),
|
||||
"trap 'exit 0' TERM; sleep 2147483647 & wait".into(),
|
||||
]),
|
||||
)
|
||||
.await?;
|
||||
start_container(&docker, &helper.id).await?;
|
||||
|
||||
|
||||
let exec = docker
|
||||
.create_exec(
|
||||
&helper.id,
|
||||
ExecConfig {
|
||||
cmd: Some(vec![
|
||||
"sh".to_string(),
|
||||
"-c".to_string(),
|
||||
"rm -rf /vol/* /vol/.[!.]* 2>/dev/null || true".to_string(),
|
||||
]),
|
||||
attach_stdout: Some(true),
|
||||
attach_stderr: Some(true),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.context("Failed to create wipe exec")?;
|
||||
|
||||
if let StartExecResults::Attached { mut output, .. } =
|
||||
docker.start_exec(&exec.id, None).await.context("Failed to run wipe exec")?
|
||||
{
|
||||
while output.next().await.is_some() {}
|
||||
}
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
let file = tokio::fs::File::open(&archive)
|
||||
.await
|
||||
.with_context(|| format!("Failed to open {}", archive.display()))?;
|
||||
let stream = tokio_util::io::ReaderStream::new(file);
|
||||
|
||||
let up_opts = UploadToContainerOptions { path: "/".to_string(), ..Default::default() };
|
||||
docker
|
||||
.upload_to_container(&helper.id, Some(up_opts), bollard::body_try_stream(stream))
|
||||
.await
|
||||
.context("Failed to upload volume archive")?;
|
||||
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
logger.log_command("docker upload_to_container", None, Some(0), Some(duration_ms));
|
||||
|
||||
remove_helper(&docker, &helper.id).await;
|
||||
logger.log("info", format!("Volume restore completed for {}", cfg.name));
|
||||
anyhow::Ok(())
|
||||
}
|
||||
.await;
|
||||
|
||||
if let Some(name) = &cfg.container_name {
|
||||
if let Err(e) = start_container(&docker, name).await {
|
||||
logger.log("error", format!("Failed to restart container {name}: {e}"));
|
||||
}
|
||||
}
|
||||
|
||||
result
|
||||
})
|
||||
})
|
||||
.await?
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
use crate::domain::docker_volume::database::DockerVolumeDatabase;
|
||||
use crate::domain::mongodb::database::MongoDatabase;
|
||||
use crate::domain::mysql::database::MySQLDatabase;
|
||||
use crate::domain::postgres::cluster::database::PostgresClusterDatabase;
|
||||
@@ -41,6 +42,7 @@ impl DatabaseFactory {
|
||||
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)),
|
||||
DbType::Firebird => Arc::new(FirebirdDatabase::new(cfg)),
|
||||
DbType::Mssql => Arc::new(MssqlDatabase::new(cfg)),
|
||||
DbType::DockerVolume => Arc::new(DockerVolumeDatabase::new(cfg)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,6 +61,7 @@ impl DatabaseFactory {
|
||||
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)),
|
||||
DbType::Firebird => Arc::new(FirebirdDatabase::new(cfg)),
|
||||
DbType::Mssql => Arc::new(MssqlDatabase::new(cfg)),
|
||||
DbType::DockerVolume => Arc::new(DockerVolumeDatabase::new(cfg)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
pub mod docker_volume;
|
||||
pub mod factory;
|
||||
mod mongodb;
|
||||
pub mod mysql;
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
use crate::services::config::DatabaseConfig;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum RestoreCleanMode {
|
||||
None,
|
||||
Clean,
|
||||
DropSchemas,
|
||||
DropDatabase,
|
||||
}
|
||||
|
||||
impl RestoreCleanMode {
|
||||
pub fn from_config(cfg: &DatabaseConfig) -> (Self, Option<String>) {
|
||||
match cfg.options.get("clean_mode").and_then(|v| v.as_str()) {
|
||||
None | Some("clean") => (Self::Clean, None),
|
||||
Some("none") => (Self::None, None),
|
||||
Some("drop_schemas") => (Self::DropSchemas, None),
|
||||
Some("drop_database") => (Self::DropDatabase, None),
|
||||
Some(other) => (Self::Clean, Some(other.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn uses_pg_restore_clean(self) -> bool {
|
||||
matches!(self, Self::Clean)
|
||||
}
|
||||
}
|
||||
@@ -15,10 +15,11 @@ pub async fn run(
|
||||
env: HashMap<String, String>,
|
||||
logger: Arc<JobLogger>,
|
||||
) -> Result<()> {
|
||||
let handle = tokio::runtime::Handle::current();
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
logger.log("info", format!("Starting cluster restore for {}", cfg.name));
|
||||
|
||||
let version = match futures::executor::block_on(server_version(&cfg)) {
|
||||
let version = match handle.block_on(server_version(&cfg)) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
logger.log("error", format!("Failed to get server version for {}: {:?}", cfg.name, e));
|
||||
@@ -26,7 +27,7 @@ pub async fn run(
|
||||
}
|
||||
};
|
||||
|
||||
match futures::executor::block_on(is_superuser(&cfg)) {
|
||||
match handle.block_on(is_superuser(&cfg)) {
|
||||
Ok(true) => {}
|
||||
Ok(false) => {
|
||||
logger.log("error", format!("postgresql-cluster restore requires a superuser role for {}", cfg.name));
|
||||
@@ -40,7 +41,7 @@ pub async fn run(
|
||||
|
||||
let psql = select_pg_path(&version).join(psql_binary_name());
|
||||
|
||||
if let Err(e) = futures::executor::block_on(terminate_all_connections(&cfg)) {
|
||||
if let Err(e) = handle.block_on(terminate_all_connections(&cfg)) {
|
||||
logger.log("error", format!("Failed to terminate connections for cluster {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
|
||||
@@ -33,6 +33,14 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
pub async fn server_version_major(cfg: &DatabaseConfig) -> Result<u32> {
|
||||
let v = server_version(cfg).await?;
|
||||
Ok(v.split(['.', ' '])
|
||||
.next()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(17))
|
||||
}
|
||||
|
||||
pub async fn is_superuser(cfg: &DatabaseConfig) -> Result<bool> {
|
||||
let client = connect(cfg).await?;
|
||||
let is_super: bool = client
|
||||
@@ -43,6 +51,19 @@ pub async fn is_superuser(cfg: &DatabaseConfig) -> Result<bool> {
|
||||
Ok(is_super)
|
||||
}
|
||||
|
||||
pub async fn can_drop_database(cfg: &DatabaseConfig) -> Result<bool> {
|
||||
let client = connect(cfg).await?;
|
||||
let row = client
|
||||
.query_one(
|
||||
"SELECT r.rolsuper OR (r.rolcreatedb AND pg_catalog.pg_has_role(current_user, d.datdba, 'USAGE')) \
|
||||
FROM pg_roles r, pg_database d \
|
||||
WHERE r.rolname = current_user AND d.datname = current_database()",
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
Ok(row.get(0))
|
||||
}
|
||||
|
||||
|
||||
pub fn select_pg_path(version: &str) -> std::path::PathBuf {
|
||||
select_pg_path_with(version, &CONFIG.pg_bin_dir)
|
||||
@@ -114,6 +135,22 @@ pub(crate) fn psql_binary_name() -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn pg_restore_binary_name() -> &'static str {
|
||||
if cfg!(target_os = "windows") {
|
||||
"pg_restore.exe"
|
||||
} else {
|
||||
"pg_restore"
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn quote_ident(s: &str) -> String {
|
||||
format!("\"{}\"", s.replace('"', "\"\""))
|
||||
}
|
||||
|
||||
pub(crate) fn quote_literal(s: &str) -> String {
|
||||
format!("'{}'", s.replace('\'', "''"))
|
||||
}
|
||||
|
||||
pub(crate) fn pg_dump_exists_in(dir: &std::path::Path) -> bool {
|
||||
dir.join(pg_dump_binary_name()).is_file()
|
||||
}
|
||||
@@ -165,6 +202,107 @@ pub async fn terminate_all_connections(cfg: &DatabaseConfig) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn drop_and_recreate_database(cfg: &DatabaseConfig) -> Result<()> {
|
||||
let mut admin_cfg = cfg.clone();
|
||||
admin_cfg.database = "postgres".to_string();
|
||||
let admin = connect(&admin_cfg).await?;
|
||||
|
||||
let row = admin
|
||||
.query_opt(
|
||||
r#"
|
||||
SELECT pg_encoding_to_char(encoding), datcollate, datctype,
|
||||
pg_get_userbyid(datdba), datistemplate
|
||||
FROM pg_database WHERE datname = $1
|
||||
"#,
|
||||
&[&cfg.database],
|
||||
)
|
||||
.await?;
|
||||
|
||||
let (encoding, collate, ctype, owner) = match &row {
|
||||
Some(r) => (
|
||||
r.get::<_, String>(0),
|
||||
r.get::<_, String>(1),
|
||||
r.get::<_, String>(2),
|
||||
r.get::<_, String>(3),
|
||||
),
|
||||
None => ("UTF8".into(), "C".into(), "C".into(), cfg.username.clone()),
|
||||
};
|
||||
|
||||
if let Some(r) = &row {
|
||||
if r.get::<_, bool>(4) {
|
||||
anyhow::bail!("Refusing to drop template database {}", cfg.database);
|
||||
}
|
||||
}
|
||||
|
||||
let db = quote_ident(&cfg.database);
|
||||
|
||||
if let Err(e) = admin
|
||||
.batch_execute(&format!("ALTER DATABASE {db} WITH ALLOW_CONNECTIONS false"))
|
||||
.await
|
||||
{
|
||||
tracing::warn!("ALLOW_CONNECTIONS false failed for {}: {e}", cfg.database);
|
||||
}
|
||||
|
||||
let major = server_version_major(&admin_cfg).await?;
|
||||
let drop_stmt = if major >= 13 {
|
||||
format!("DROP DATABASE IF EXISTS {db} WITH (FORCE)")
|
||||
} else {
|
||||
format!("DROP DATABASE IF EXISTS {db}")
|
||||
};
|
||||
|
||||
let mut last_err = None;
|
||||
let mut dropped = false;
|
||||
for _ in 0..3 {
|
||||
let _ = terminate_connections(cfg).await;
|
||||
match admin.batch_execute(&drop_stmt).await {
|
||||
Ok(()) => {
|
||||
dropped = true;
|
||||
break;
|
||||
}
|
||||
Err(e) => {
|
||||
last_err = Some(e);
|
||||
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !dropped {
|
||||
let _ = admin
|
||||
.batch_execute(&format!("ALTER DATABASE {db} WITH ALLOW_CONNECTIONS true"))
|
||||
.await;
|
||||
return Err(last_err
|
||||
.map(anyhow::Error::from)
|
||||
.unwrap_or_else(|| anyhow::anyhow!("DROP DATABASE {} failed", cfg.database)));
|
||||
}
|
||||
|
||||
admin
|
||||
.batch_execute(&format!(
|
||||
"CREATE DATABASE {db} OWNER {} TEMPLATE template0 ENCODING {} LC_COLLATE {} LC_CTYPE {}",
|
||||
quote_ident(&owner),
|
||||
quote_literal(&encoding),
|
||||
quote_literal(&collate),
|
||||
quote_literal(&ctype),
|
||||
))
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn sniff_format(restore_file: &Path) -> Result<PostgresDumpFormat> {
|
||||
use std::io::Read;
|
||||
let mut f = std::fs::File::open(restore_file)?;
|
||||
let mut magic = [0u8; 5];
|
||||
let n = f.read(&mut magic)?;
|
||||
let head = &magic[..n];
|
||||
if head.starts_with(b"PGDMP") {
|
||||
Ok(PostgresDumpFormat::Fc)
|
||||
} else if head.starts_with(&[0x1f, 0x8b]) {
|
||||
Ok(PostgresDumpFormat::Fd)
|
||||
} else {
|
||||
anyhow::bail!("Unrecognized dump format for {:?}", restore_file)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn detect_format_from_file(restore_file: &Path) -> PostgresDumpFormat {
|
||||
match restore_file.extension().and_then(|e| e.to_str()) {
|
||||
Some("dump") => PostgresDumpFormat::Fc,
|
||||
@@ -174,6 +312,44 @@ pub fn detect_format_from_file(restore_file: &Path) -> PostgresDumpFormat {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn drop_all_schemas(cfg: &DatabaseConfig) -> Result<Vec<String>> {
|
||||
let client = connect(cfg).await?;
|
||||
let rows = client
|
||||
.query(
|
||||
r#"
|
||||
SELECT nspname FROM pg_namespace
|
||||
WHERE nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast')
|
||||
AND nspname NOT LIKE 'pg\_temp\_%'
|
||||
AND nspname NOT LIKE 'pg\_toast\_temp\_%'
|
||||
ORDER BY nspname
|
||||
"#,
|
||||
&[],
|
||||
)
|
||||
.await?;
|
||||
let schemas: Vec<String> = rows.iter().map(|r| r.get::<_, String>(0)).collect();
|
||||
for s in &schemas {
|
||||
client
|
||||
.batch_execute(&format!("DROP SCHEMA IF EXISTS {} CASCADE", quote_ident(s)))
|
||||
.await?;
|
||||
}
|
||||
client
|
||||
.batch_execute("SELECT lo_unlink(oid) FROM pg_largeobject_metadata")
|
||||
.await
|
||||
.ok();
|
||||
Ok(schemas)
|
||||
}
|
||||
|
||||
pub async fn recreate_public_schema(cfg: &DatabaseConfig, owner: &str) -> Result<()> {
|
||||
let client = connect(cfg).await?;
|
||||
client
|
||||
.batch_execute(&format!(
|
||||
"CREATE SCHEMA IF NOT EXISTS public AUTHORIZATION {}; GRANT USAGE ON SCHEMA public TO PUBLIC;",
|
||||
quote_ident(owner)
|
||||
))
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn detect_format_from_size(cfg: &DatabaseConfig) -> PostgresDumpFormat {
|
||||
info!(
|
||||
"Detecting database format {:?} - {:?}",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#[derive(Clone, Copy)]
|
||||
#[derive(Clone, Copy, PartialEq, Debug)]
|
||||
pub enum PostgresDumpFormat {
|
||||
Fc,
|
||||
Fd,
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
pub mod backup;
|
||||
pub(crate) mod cluster;
|
||||
pub(crate) mod clean_mode;
|
||||
pub(crate) mod connection;
|
||||
pub mod database;
|
||||
mod format;
|
||||
pub(crate) mod format;
|
||||
mod ping;
|
||||
mod restore;
|
||||
pub(crate) mod restore;
|
||||
|
||||
pub use connection::{detect_format_from_file, detect_format_from_size};
|
||||
|
||||
@@ -1,214 +0,0 @@
|
||||
use anyhow::Result;
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use super::connection::{select_pg_path, server_version, terminate_connections};
|
||||
use super::format::PostgresDumpFormat;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
format: PostgresDumpFormat,
|
||||
restore_file: PathBuf,
|
||||
env: HashMap<String, String>,
|
||||
logger: Arc<JobLogger>,
|
||||
) -> Result<()> {
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
logger.log("info", format!("Starting restore for database {}", cfg.name));
|
||||
|
||||
let version = match futures::executor::block_on(server_version(&cfg)) {
|
||||
Ok(v) => {
|
||||
logger.log("debug", format!("Postgres version detected: {}", v));
|
||||
v
|
||||
}
|
||||
Err(e) => {
|
||||
logger.log("error", format!("Failed to get server version for {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
|
||||
let pg_restore = select_pg_path(&version).join("pg_restore");
|
||||
|
||||
logger.log("debug", format!("Using pg_restore at {:?}", pg_restore));
|
||||
|
||||
if let Err(e) = futures::executor::block_on(terminate_connections(&cfg)) {
|
||||
logger.log("error", format!("Failed to terminate connections for {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
logger.log("info", format!("Connections terminated for database {}", cfg.name));
|
||||
|
||||
let keep_ownership = cfg.options
|
||||
.get("keep_ownership")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
|
||||
if keep_ownership {
|
||||
logger.log("info", format!("Restoring ownership and privileges for {}", cfg.name));
|
||||
} else {
|
||||
logger.log("info", format!("Stripping ownership and privileges for {} (--no-owner --no-privileges)", cfg.name));
|
||||
}
|
||||
|
||||
match format {
|
||||
PostgresDumpFormat::Fc => {
|
||||
logger.log("info", format!("Running FC restore for {}", cfg.name));
|
||||
let start = Instant::now();
|
||||
let mut cmd = Command::new(&pg_restore);
|
||||
if !keep_ownership {
|
||||
cmd.arg("--no-owner").arg("--no-privileges");
|
||||
}
|
||||
let output = cmd
|
||||
.arg("--clean")
|
||||
.arg("--if-exists")
|
||||
// .arg("--create")
|
||||
.arg("--host").arg(&cfg.host)
|
||||
.arg("--port").arg(cfg.port.to_string())
|
||||
.arg("--username").arg(&cfg.username)
|
||||
.arg("--dbname").arg(&cfg.database)
|
||||
.arg("-v")
|
||||
.arg(&restore_file)
|
||||
.envs(env)
|
||||
.output();
|
||||
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
|
||||
match output {
|
||||
Ok(o) => {
|
||||
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
|
||||
let stdout = String::from_utf8_lossy(&o.stdout).to_string();
|
||||
let combined = format!("{}{}", stdout, stderr);
|
||||
let exit_code = o.status.code().unwrap_or(-1);
|
||||
|
||||
if o.status.success() {
|
||||
logger.log_command("pg_restore", if combined.is_empty() { None } else { Some(combined) }, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("FC restore completed successfully for {}", cfg.name))
|
||||
} else {
|
||||
logger.log_command("pg_restore", if combined.is_empty() { None } else { Some(combined) }, Some(exit_code), Some(duration_ms));
|
||||
logger.log("error", format!("FC restore failed with status {:?} for {}", o.status, cfg.name));
|
||||
anyhow::bail!("Postgres restore failed for {}", cfg.name);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
logger.log_command("pg_restore", Some(e.to_string()), Some(-1), Some(duration_ms));
|
||||
logger.log("error", format!("Error executing pg_restore for {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
PostgresDumpFormat::Fd => {
|
||||
logger.log("info", format!("Running FD restore for {}", cfg.name));
|
||||
|
||||
let tar_gz = match std::fs::File::open(&restore_file) {
|
||||
Ok(f) => f,
|
||||
Err(e) => {
|
||||
logger.log("error", format!(
|
||||
"Failed to open restore file {:?} for {}: {:?}",
|
||||
restore_file, cfg.name, e
|
||||
));
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
|
||||
logger.log("info", format!("tar_gz {:?}", tar_gz));
|
||||
|
||||
let dec = flate2::read::GzDecoder::new(tar_gz);
|
||||
let mut archive = tar::Archive::new(dec);
|
||||
|
||||
let tmp_dir = match tempfile::TempDir::new() {
|
||||
Ok(d) => d,
|
||||
Err(e) => {
|
||||
logger.log("error", format!(
|
||||
"Failed to create temporary directory for FD restore of {}: {:?}",
|
||||
cfg.name, e
|
||||
));
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(e) = archive.unpack(tmp_dir.path()) {
|
||||
logger.log("error", format!("Failed to unpack FD archive for {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
|
||||
logger.log("debug", format!("Listing contents of temp dir: {}", tmp_dir.path().display()));
|
||||
|
||||
for entry in std::fs::read_dir(tmp_dir.path())? {
|
||||
if let Ok(entry) = entry {
|
||||
let path = entry.path();
|
||||
let file_type = entry.file_type()?;
|
||||
logger.log("debug", format!(
|
||||
" - {} | is_dir: {} | is_file: {}",
|
||||
path.display(),
|
||||
file_type.is_dir(),
|
||||
file_type.is_file()
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let dump_dir = if tmp_dir.path().join("toc.dat").exists() {
|
||||
tmp_dir.path().to_path_buf()
|
||||
} else {
|
||||
std::fs::read_dir(tmp_dir.path())?
|
||||
.filter_map(|e| e.ok())
|
||||
.find(|entry| entry.path().join("toc.dat").exists())
|
||||
.map(|e| e.path())
|
||||
.ok_or_else(|| anyhow::anyhow!("Invalid FD archive: toc.dat not found"))?
|
||||
};
|
||||
|
||||
let start = Instant::now();
|
||||
let mut cmd = Command::new(&pg_restore);
|
||||
if !keep_ownership {
|
||||
cmd.arg("--no-owner").arg("--no-privileges");
|
||||
}
|
||||
let output = cmd
|
||||
.arg("--clean")
|
||||
.arg("--if-exists")
|
||||
// .arg("--create")
|
||||
.arg("--host").arg(&cfg.host)
|
||||
.arg("--port").arg(cfg.port.to_string())
|
||||
.arg("--username").arg(&cfg.username)
|
||||
.arg("--dbname").arg(&cfg.database)
|
||||
.arg("-v")
|
||||
.arg("-j")
|
||||
.arg("4")
|
||||
.arg(dump_dir)
|
||||
.envs(env)
|
||||
.output();
|
||||
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
|
||||
match output {
|
||||
Ok(o) => {
|
||||
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
|
||||
let stdout = String::from_utf8_lossy(&o.stdout).to_string();
|
||||
let combined = format!("{}{}", stdout, stderr);
|
||||
let exit_code = o.status.code().unwrap_or(-1);
|
||||
|
||||
if o.status.success() {
|
||||
logger.log_command("pg_restore", if combined.is_empty() { None } else { Some(combined) }, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("FD restore completed successfully for {}", cfg.name))
|
||||
} else {
|
||||
logger.log_command("pg_restore", if combined.is_empty() { None } else { Some(combined) }, Some(exit_code), Some(duration_ms));
|
||||
logger.log("error", format!("FD restore failed with status {:?} for {}", o.status, cfg.name));
|
||||
anyhow::bail!("Postgres FD restore failed for {}", cfg.name);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
logger.log_command("pg_restore", Some(e.to_string()), Some(-1), Some(duration_ms));
|
||||
logger.log("error", format!("Error executing pg_restore for {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.log("info", format!("Restore finished for database {}", cfg.name));
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.await?
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
use anyhow::Result;
|
||||
use std::process::Command;
|
||||
use std::time::Instant;
|
||||
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
|
||||
pub(crate) fn run_pg_restore(
|
||||
mut cmd: Command,
|
||||
logger: &JobLogger,
|
||||
cfg: &DatabaseConfig,
|
||||
) -> Result<()> {
|
||||
let start = Instant::now();
|
||||
let output = cmd.output();
|
||||
let duration_ms = start.elapsed().as_millis() as f64;
|
||||
|
||||
match output {
|
||||
Ok(o) => {
|
||||
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
|
||||
let stdout = String::from_utf8_lossy(&o.stdout).to_string();
|
||||
let combined = format!("{}{}", stdout, stderr);
|
||||
let exit_code = o.status.code().unwrap_or(-1);
|
||||
let payload = if combined.is_empty() { None } else { Some(combined) };
|
||||
|
||||
if o.status.success() {
|
||||
logger.log_command("pg_restore", payload, Some(0), Some(duration_ms));
|
||||
logger.log("info", format!("Restore completed successfully for {}", cfg.name));
|
||||
Ok(())
|
||||
} else {
|
||||
logger.log_command("pg_restore", payload, Some(exit_code), Some(duration_ms));
|
||||
logger.log("error", format!("Restore failed with status {:?} for {}", o.status, cfg.name));
|
||||
anyhow::bail!("Postgres restore failed for {}", cfg.name);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
logger.log_command("pg_restore", Some(e.to_string()), Some(-1), Some(duration_ms));
|
||||
logger.log("error", format!("Error executing pg_restore for {}: {:?}", cfg.name, e));
|
||||
Err(e.into())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
mod command;
|
||||
mod prepare;
|
||||
mod run;
|
||||
mod toc;
|
||||
|
||||
pub use run::run;
|
||||
pub(crate) use command::run_pg_restore;
|
||||
pub(crate) use prepare::prepare_archive;
|
||||
pub(crate) use toc::toc_creates_public_schema;
|
||||
@@ -0,0 +1,67 @@
|
||||
use anyhow::Result;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
|
||||
use crate::domain::postgres::connection::sniff_format;
|
||||
use crate::domain::postgres::format::PostgresDumpFormat;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
|
||||
pub(crate) struct PreparedArchive {
|
||||
path: PathBuf,
|
||||
_tmp: Option<tempfile::TempDir>,
|
||||
toc: String,
|
||||
}
|
||||
|
||||
impl PreparedArchive {
|
||||
pub(crate) fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
pub(crate) fn toc(&self) -> &str {
|
||||
&self.toc
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_archive(
|
||||
format: PostgresDumpFormat,
|
||||
restore_file: &Path,
|
||||
pg_restore: &Path,
|
||||
logger: &JobLogger,
|
||||
) -> Result<PreparedArchive> {
|
||||
let sniffed = sniff_format(restore_file)?;
|
||||
if sniffed != format {
|
||||
logger.log("warn", format!("Declared format {:?} != sniffed {:?}; using sniffed", format, sniffed));
|
||||
}
|
||||
let format = sniffed;
|
||||
|
||||
let (path, tmp) = match format {
|
||||
PostgresDumpFormat::Fc => (restore_file.to_path_buf(), None),
|
||||
PostgresDumpFormat::Fd => {
|
||||
let tar_gz = std::fs::File::open(restore_file)?;
|
||||
let dec = flate2::read::GzDecoder::new(tar_gz);
|
||||
let mut archive = tar::Archive::new(dec);
|
||||
let tmp_dir = tempfile::TempDir::new()?;
|
||||
archive.unpack(tmp_dir.path())?;
|
||||
|
||||
let dump_dir = if tmp_dir.path().join("toc.dat").exists() {
|
||||
tmp_dir.path().to_path_buf()
|
||||
} else {
|
||||
std::fs::read_dir(tmp_dir.path())?
|
||||
.filter_map(|e| e.ok())
|
||||
.find(|entry| entry.path().join("toc.dat").exists())
|
||||
.map(|e| e.path())
|
||||
.ok_or_else(|| anyhow::anyhow!("Invalid FD archive: toc.dat not found"))?
|
||||
};
|
||||
(dump_dir, Some(tmp_dir))
|
||||
}
|
||||
};
|
||||
|
||||
let toc_out = Command::new(pg_restore).arg("-l").arg(&path).output()?;
|
||||
if !toc_out.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&toc_out.stderr).to_string();
|
||||
logger.log("error", format!("pg_restore -l failed: {}", stderr));
|
||||
anyhow::bail!("Archive validation failed (pg_restore -l): {}", stderr);
|
||||
}
|
||||
let toc = String::from_utf8_lossy(&toc_out.stdout).to_string();
|
||||
|
||||
Ok(PreparedArchive { path, _tmp: tmp, toc })
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
use anyhow::Result;
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::{prepare_archive, run_pg_restore, toc_creates_public_schema};
|
||||
use crate::domain::postgres::clean_mode::RestoreCleanMode;
|
||||
use crate::domain::postgres::connection::{
|
||||
can_drop_database, drop_all_schemas, drop_and_recreate_database, pg_restore_binary_name,
|
||||
recreate_public_schema, select_pg_path, server_version, terminate_connections,
|
||||
};
|
||||
use crate::domain::postgres::format::PostgresDumpFormat;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
format: PostgresDumpFormat,
|
||||
restore_file: PathBuf,
|
||||
env: HashMap<String, String>,
|
||||
logger: Arc<JobLogger>,
|
||||
) -> Result<()> {
|
||||
let handle = tokio::runtime::Handle::current();
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
logger.log("info", format!("Starting restore for database {}", cfg.name));
|
||||
|
||||
let version = match handle.block_on(server_version(&cfg)) {
|
||||
Ok(v) => {
|
||||
logger.log("debug", format!("Postgres version detected: {}", v));
|
||||
v
|
||||
}
|
||||
Err(e) => {
|
||||
logger.log("error", format!("Failed to get server version for {}: {:?}", cfg.name, e));
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
|
||||
let pg_restore = select_pg_path(&version).join(pg_restore_binary_name());
|
||||
|
||||
logger.log("debug", format!("Using pg_restore at {:?}", pg_restore));
|
||||
|
||||
let keep_ownership = cfg.options
|
||||
.get("keep_ownership")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
|
||||
if keep_ownership {
|
||||
logger.log("info", format!("Restoring ownership and privileges for {}", cfg.name));
|
||||
} else {
|
||||
logger.log("info", format!("Stripping ownership and privileges for {} (--no-owner --no-privileges)", cfg.name));
|
||||
}
|
||||
|
||||
let (mode, bad_value) = RestoreCleanMode::from_config(&cfg);
|
||||
if let Some(v) = bad_value {
|
||||
logger.log("warn", format!("Unknown clean_mode '{}' for {}, falling back to 'clean'", v, cfg.name));
|
||||
}
|
||||
|
||||
let prepared = prepare_archive(format, &restore_file, &pg_restore, &logger)?;
|
||||
|
||||
match mode {
|
||||
RestoreCleanMode::DropSchemas => {
|
||||
handle.block_on(terminate_connections(&cfg))?;
|
||||
let owner = cfg.username.clone();
|
||||
let dropped = handle.block_on(drop_all_schemas(&cfg))?;
|
||||
logger.log("warn", format!("clean_mode=drop_schemas dropped schemas {:?} in {}", dropped, cfg.database));
|
||||
if !toc_creates_public_schema(prepared.toc()) {
|
||||
handle.block_on(recreate_public_schema(&cfg, &owner))?;
|
||||
}
|
||||
}
|
||||
RestoreCleanMode::DropDatabase => {
|
||||
if !handle.block_on(can_drop_database(&cfg))? {
|
||||
anyhow::bail!(
|
||||
"clean_mode=drop_database requires CREATEDB + ownership on {}; use clean_mode=drop_schemas instead",
|
||||
cfg.database
|
||||
);
|
||||
}
|
||||
logger.log("warn", format!("clean_mode=drop_database DROPPING database {} before restore", cfg.database));
|
||||
handle.block_on(drop_and_recreate_database(&cfg))?;
|
||||
}
|
||||
RestoreCleanMode::Clean | RestoreCleanMode::None => {
|
||||
handle.block_on(terminate_connections(&cfg))?;
|
||||
}
|
||||
}
|
||||
|
||||
let mut cmd = Command::new(&pg_restore);
|
||||
if !keep_ownership {
|
||||
cmd.args(["--no-owner", "--no-privileges"]);
|
||||
}
|
||||
if mode.uses_pg_restore_clean() {
|
||||
cmd.args(["--clean", "--if-exists"]);
|
||||
}
|
||||
cmd.arg("--host").arg(&cfg.host)
|
||||
.arg("--port").arg(cfg.port.to_string())
|
||||
.arg("--username").arg(&cfg.username)
|
||||
.arg("--dbname").arg(&cfg.database)
|
||||
.arg("-v");
|
||||
if matches!(format, PostgresDumpFormat::Fd) {
|
||||
cmd.arg("-j").arg("4");
|
||||
}
|
||||
cmd.arg(prepared.path()).envs(env);
|
||||
|
||||
run_pg_restore(cmd, &logger, &cfg)?;
|
||||
logger.log("info", format!("Restore finished for database {}", cfg.name));
|
||||
Ok(())
|
||||
})
|
||||
.await?
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
pub(crate) fn toc_creates_public_schema(toc: &str) -> bool {
|
||||
toc.lines().any(|l| {
|
||||
l.split(" SCHEMA - ")
|
||||
.nth(1)
|
||||
.and_then(|rest| rest.split_whitespace().next())
|
||||
== Some("public")
|
||||
})
|
||||
}
|
||||
+10
@@ -22,6 +22,16 @@ async fn main() {
|
||||
eprintln!("Failed to clean locks on startup: {:?}", e);
|
||||
}
|
||||
|
||||
// Best-effort cleanup of ephemeral helper containers orphaned by a crash.
|
||||
match crate::domain::docker_volume::docker::client() {
|
||||
Ok(docker) => match crate::domain::docker_volume::docker::sweep_ephemeral(&docker).await {
|
||||
Ok(n) if n > 0 => tracing::info!("Removed {n} orphaned ephemeral helper container(s)"),
|
||||
Ok(_) => {}
|
||||
Err(e) => tracing::warn!("Ephemeral helper sweep failed: {e}"),
|
||||
},
|
||||
Err(e) => tracing::debug!("Docker socket unavailable, skipping helper sweep: {e}"),
|
||||
}
|
||||
|
||||
tokio::join!(ping_server(), async {
|
||||
let conn = redis_client::redis_connection().await;
|
||||
scheduler::scheduler_loop(conn).await;
|
||||
|
||||
@@ -24,6 +24,8 @@ pub struct DatabaseStorage {
|
||||
#[serde(deserialize_with = "deserialize_snake_case")]
|
||||
pub config: Value,
|
||||
pub provider: String,
|
||||
#[serde(default, rename = "folderName")]
|
||||
pub folder_name: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
@@ -31,7 +33,12 @@ pub struct DatabaseStatus {
|
||||
pub dbms: String,
|
||||
#[serde(rename = "generatedId")]
|
||||
pub generated_id: String,
|
||||
#[serde(default)]
|
||||
pub storages: Vec<DatabaseStorage>,
|
||||
#[serde(default)]
|
||||
pub storages_encrypted: Option<bool>,
|
||||
#[serde(default)]
|
||||
pub storages_ciphertext: Option<String>,
|
||||
pub encrypt: bool,
|
||||
pub data: DatabaseData,
|
||||
}
|
||||
|
||||
@@ -104,6 +104,7 @@ impl BackupService {
|
||||
method,
|
||||
&storage,
|
||||
Some(encrypt),
|
||||
&backup_storage_id,
|
||||
)
|
||||
.await;
|
||||
|
||||
|
||||
+20
-3
@@ -26,6 +26,8 @@ pub enum DbType {
|
||||
Valkey,
|
||||
Firebird,
|
||||
Mssql,
|
||||
#[serde(rename = "docker-volume")]
|
||||
DockerVolume,
|
||||
}
|
||||
|
||||
impl DbType {
|
||||
@@ -41,6 +43,7 @@ impl DbType {
|
||||
DbType::Valkey => "valkey",
|
||||
DbType::Firebird => "firebird",
|
||||
DbType::Mssql => "mssql",
|
||||
DbType::DockerVolume => "docker-volume",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -59,6 +62,8 @@ pub struct DatabaseConfig {
|
||||
pub generated_id: String,
|
||||
pub path: String,
|
||||
pub max_packet_size: String,
|
||||
pub volume_name: String,
|
||||
pub container_name: Option<String>,
|
||||
pub options: HashMap<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
@@ -82,6 +87,8 @@ pub struct InputDatabaseConfig {
|
||||
pub generated_id: String,
|
||||
pub path: Option<String>,
|
||||
pub max_packet_size: Option<String>,
|
||||
pub volume_name: Option<String>,
|
||||
pub container_name: Option<String>,
|
||||
pub options: Option<HashMap<String, serde_json::Value>>,
|
||||
}
|
||||
|
||||
@@ -202,7 +209,7 @@ impl ConfigService {
|
||||
| DbType::Firebird
|
||||
| DbType::Valkey
|
||||
| DbType::Mssql => required(&db.host, &db.name, "host")?,
|
||||
DbType::Sqlite => optional(&db.host),
|
||||
DbType::Sqlite | DbType::DockerVolume => optional(&db.host),
|
||||
};
|
||||
|
||||
let port = match db.db_type {
|
||||
@@ -215,11 +222,13 @@ impl ConfigService {
|
||||
| DbType::Firebird
|
||||
| DbType::Valkey
|
||||
| DbType::Mssql => required(&db.port, &db.name, "port")?,
|
||||
DbType::Sqlite => db.port.unwrap_or(0),
|
||||
DbType::Sqlite | DbType::DockerVolume => db.port.unwrap_or(0),
|
||||
};
|
||||
|
||||
let database_name = match db.db_type {
|
||||
DbType::Sqlite | DbType::Redis | DbType::Valkey => optional(&db.database),
|
||||
DbType::Sqlite | DbType::Redis | DbType::Valkey | DbType::DockerVolume => {
|
||||
optional(&db.database)
|
||||
}
|
||||
DbType::PostgresqlCluster => db
|
||||
.database
|
||||
.clone()
|
||||
@@ -239,6 +248,12 @@ impl ConfigService {
|
||||
_ => String::new(),
|
||||
};
|
||||
|
||||
let volume_name = match db.db_type {
|
||||
DbType::DockerVolume => required(&db.volume_name, &db.name, "volume_name")?,
|
||||
_ => optional(&db.volume_name),
|
||||
};
|
||||
let container_name = db.container_name.clone();
|
||||
|
||||
databases.push(DatabaseConfig {
|
||||
name: db.name,
|
||||
database: database_name,
|
||||
@@ -250,6 +265,8 @@ impl ConfigService {
|
||||
generated_id: db.generated_id,
|
||||
path: path_val,
|
||||
max_packet_size,
|
||||
volume_name,
|
||||
container_name,
|
||||
options: db.options.unwrap_or_default(),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,18 +1,19 @@
|
||||
use super::service::RestoreService;
|
||||
|
||||
use crate::utils::compress::decompress_large_tar_gz;
|
||||
use crate::utils::file::decrypt_file_stream_gcm;
|
||||
|
||||
use anyhow::Result;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::DbType;
|
||||
use crate::utils::common::choose_restore_path;
|
||||
|
||||
impl RestoreService {
|
||||
pub async fn prepare_archive(
|
||||
&self,
|
||||
downloaded_file: PathBuf,
|
||||
tmp_path: &Path,
|
||||
db_type: &DbType,
|
||||
logger: Arc<JobLogger>
|
||||
) -> Result<PathBuf> {
|
||||
logger.log("info", "Start preparing backup archive".to_string());
|
||||
@@ -59,6 +60,13 @@ impl RestoreService {
|
||||
archive = decrypted;
|
||||
}
|
||||
|
||||
if matches!(db_type, DbType::DockerVolume) {
|
||||
let raw_tar = tmp_path.join("volume.tar");
|
||||
crate::utils::compress::gunzip_to_file(archive.as_path(), &raw_tar).await?;
|
||||
logger.log("info", format!("Docker volume archive gunzipped to {}", raw_tar.display()));
|
||||
return Ok(raw_tar);
|
||||
}
|
||||
|
||||
logger.log("info", format!("Decompressing archive {}", archive.display()));
|
||||
|
||||
let files = match decompress_large_tar_gz(archive.as_path(), tmp_path).await {
|
||||
@@ -76,12 +84,8 @@ impl RestoreService {
|
||||
|
||||
logger.log("info", format!("Archive prepared, {} file(s) extracted", files.len()));
|
||||
|
||||
if files.len() == 1 {
|
||||
logger.log("debug", format!("Using single extracted file: {}", files[0].display()));
|
||||
Ok(files[0].clone())
|
||||
} else {
|
||||
logger.log("debug", format!("Multiple files extracted, using archive root: {}", archive.display()));
|
||||
Ok(archive)
|
||||
}
|
||||
let chosen = choose_restore_path(&files, tmp_path, &archive);
|
||||
logger.log("debug", format!("Restore source resolved to: {}", chosen.display()));
|
||||
Ok(chosen)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,9 @@ impl RestoreService {
|
||||
.download_backup(&file_url, tmp_path, Arc::clone(&logger), expected_size)
|
||||
.await?;
|
||||
|
||||
let backup_file = self.prepare_archive(downloaded, tmp_path, Arc::clone(&logger)).await?;
|
||||
let backup_file = self
|
||||
.prepare_archive(downloaded, tmp_path, &cfg.db_type, Arc::clone(&logger))
|
||||
.await?;
|
||||
|
||||
let result = self.run_restore(cfg, backup_file, Arc::clone(&logger)).await?;
|
||||
|
||||
|
||||
+25
-8
@@ -1,16 +1,18 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use crate::core::context::Context;
|
||||
use crate::domain::factory::DatabaseFactory;
|
||||
use crate::services::api::endpoints::status::DatabasePayload;
|
||||
use crate::services::api::models::agent::status::DatabaseStorage;
|
||||
use crate::services::api::models::agent::status::PingResult;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::settings::CONFIG;
|
||||
use crate::utils::file::decrypt_json_gcm;
|
||||
use futures_util::future::try_join_all;
|
||||
use reqwest::Client;
|
||||
use std::error::Error;
|
||||
use std::sync::Arc;
|
||||
use futures_util::future::try_join_all;
|
||||
use tracing::info;
|
||||
use crate::domain::factory::DatabaseFactory;
|
||||
|
||||
pub struct StatusService {
|
||||
ctx: Arc<Context>,
|
||||
@@ -28,12 +30,12 @@ impl StatusService {
|
||||
pub async fn ping(&self, databases: &[DatabaseConfig]) -> Result<PingResult, Box<dyn Error>> {
|
||||
let edge_key = &self.ctx.edge_key;
|
||||
|
||||
let databases_payload: Vec<DatabasePayload> = try_join_all(
|
||||
databases.into_iter().map(|db| async move {
|
||||
let databases_payload: Vec<DatabasePayload> =
|
||||
try_join_all(databases.into_iter().map(|db| async move {
|
||||
let db_engine = DatabaseFactory::create_for_backup(db.clone()).await;
|
||||
|
||||
let reachable = db_engine.ping().await?;
|
||||
info!("Ping {} => {:?}",db.name, reachable);
|
||||
info!("Ping {} => {:?}", db.name, reachable);
|
||||
|
||||
Ok::<DatabasePayload, anyhow::Error>(DatabasePayload {
|
||||
name: &db.name,
|
||||
@@ -41,16 +43,31 @@ impl StatusService {
|
||||
generated_id: &db.generated_id,
|
||||
ping_status: reachable,
|
||||
})
|
||||
})
|
||||
).await?;
|
||||
}))
|
||||
.await?;
|
||||
|
||||
let version_str = CONFIG.app_version.as_str();
|
||||
let result = self
|
||||
let mut result = self
|
||||
.ctx
|
||||
.api
|
||||
.agent_status(&edge_key.agent_id, &version_str, databases_payload)
|
||||
.await?
|
||||
.unwrap();
|
||||
|
||||
for db in result.databases.iter_mut() {
|
||||
if db.storages_encrypted == Some(true) {
|
||||
let ciphertext = db
|
||||
.storages_ciphertext
|
||||
.as_deref()
|
||||
.ok_or("storages_encrypted set but storages_ciphertext missing")?;
|
||||
|
||||
let plaintext = decrypt_json_gcm(ciphertext, &edge_key.master_key_b64)
|
||||
.map_err(|e| format!("Failed to decrypt storages: {e}"))?;
|
||||
|
||||
db.storages = serde_json::from_slice::<Vec<DatabaseStorage>>(&plaintext)
|
||||
.map_err(|e| format!("Failed to parse decrypted storages: {e}"))?;
|
||||
}
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ pub trait StorageProvider: Send + Sync {
|
||||
method: BackupMethod,
|
||||
config: &DatabaseStorage,
|
||||
encrypt: Option<bool>,
|
||||
backup_storage_id: &str,
|
||||
) -> UploadResult;
|
||||
}
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ impl StorageProvider for AzureBlobProvider {
|
||||
_method: BackupMethod,
|
||||
storage: &DatabaseStorage,
|
||||
encrypt: Option<bool>,
|
||||
_backup_storage_id: &str,
|
||||
) -> UploadResult {
|
||||
let Some(file_path) = result.backup_file else {
|
||||
return UploadResult {
|
||||
@@ -94,7 +95,7 @@ impl StorageProvider for AzureBlobProvider {
|
||||
};
|
||||
|
||||
let file_name = full_file_name(encrypt);
|
||||
let remote_file_path = full_file_path(&file_name);
|
||||
let remote_file_path = full_file_path(&file_name, storage.folder_name.as_deref());
|
||||
info!(
|
||||
"Starting block upload to azure blob {}/{}",
|
||||
config.container_name, remote_file_path
|
||||
|
||||
@@ -28,6 +28,7 @@ impl StorageProvider for GoogleCloudStorageProvider {
|
||||
_method: BackupMethod,
|
||||
storage: &DatabaseStorage,
|
||||
encrypt: Option<bool>,
|
||||
_backup_storage_id: &str,
|
||||
) -> UploadResult {
|
||||
let Some(file_path) = result.backup_file else {
|
||||
return UploadResult {
|
||||
@@ -84,7 +85,7 @@ impl StorageProvider for GoogleCloudStorageProvider {
|
||||
|
||||
let file_name = full_file_name(encrypt);
|
||||
info!("Uploading file {}", file_name);
|
||||
let remote_file_path = full_file_path(&file_name);
|
||||
let remote_file_path = full_file_path(&file_name, storage.folder_name.as_deref());
|
||||
|
||||
let client = match build_client(&config).await {
|
||||
Ok(c) => c,
|
||||
|
||||
@@ -26,6 +26,7 @@ impl StorageProvider for GoogleDriveProvider {
|
||||
_method: BackupMethod,
|
||||
storage: &DatabaseStorage,
|
||||
encrypt: Option<bool>,
|
||||
_backup_storage_id: &str,
|
||||
) -> UploadResult {
|
||||
let Some(file_path) = result.backup_file else {
|
||||
return UploadResult {
|
||||
@@ -84,7 +85,7 @@ impl StorageProvider for GoogleDriveProvider {
|
||||
|
||||
info!("Uploading file {}", file_name);
|
||||
|
||||
let remote_file_path = full_file_path(&file_name);
|
||||
let remote_file_path = full_file_path(&file_name, storage.folder_name.as_deref());
|
||||
|
||||
match upload_stream_to_google_drive(
|
||||
&config,
|
||||
|
||||
@@ -23,6 +23,7 @@ impl StorageProvider for LocalProvider {
|
||||
method: BackupMethod,
|
||||
storage: &DatabaseStorage,
|
||||
encrypt: Option<bool>,
|
||||
backup_storage_id: &str,
|
||||
) -> UploadResult {
|
||||
let Some(file_path) = result.backup_file else {
|
||||
return UploadResult {
|
||||
@@ -37,7 +38,7 @@ impl StorageProvider for LocalProvider {
|
||||
let encrypt = encrypt.unwrap_or(false);
|
||||
|
||||
let file_name = full_file_name(encrypt);
|
||||
let remote_file_path = full_file_path(&file_name);
|
||||
let remote_file_path = full_file_path(&file_name, storage.folder_name.as_deref());
|
||||
|
||||
let total_size = match fs::metadata(&file_path).await {
|
||||
Ok(meta) => meta.len(),
|
||||
@@ -82,6 +83,10 @@ impl StorageProvider for LocalProvider {
|
||||
"X-Generated-Id",
|
||||
HeaderValue::from_str(&result.generated_id).unwrap(),
|
||||
);
|
||||
extra_headers.insert(
|
||||
"X-Backup-Storage-Id",
|
||||
HeaderValue::from_str(backup_storage_id).unwrap(),
|
||||
);
|
||||
extra_headers.insert("X-Status", HeaderValue::from_str(&result.status).unwrap());
|
||||
extra_headers.insert(
|
||||
"X-Method",
|
||||
|
||||
@@ -34,6 +34,7 @@ impl StorageProvider for S3Provider {
|
||||
_method: BackupMethod,
|
||||
storage: &DatabaseStorage,
|
||||
encrypt: Option<bool>,
|
||||
_backup_storage_id: &str,
|
||||
) -> UploadResult {
|
||||
let Some(file_path) = result.backup_file else {
|
||||
return UploadResult {
|
||||
@@ -147,7 +148,7 @@ impl StorageProvider for S3Provider {
|
||||
info!("Uploading file {}", file_name);
|
||||
|
||||
let bucket = &config.bucket_name;
|
||||
let remote_file_path = full_file_path(&file_name);
|
||||
let remote_file_path = full_file_path(&file_name, storage.folder_name.as_deref());
|
||||
info!("S3 key {:}", remote_file_path);
|
||||
info!(
|
||||
"Starting multipart upload to s3://{}/{}",
|
||||
|
||||
@@ -14,6 +14,8 @@ fn cluster_config() -> DatabaseConfig {
|
||||
generated_id: "40875631-e3d2-4dfe-a26b-2a347ecc64fd".to_string(),
|
||||
path: String::new(),
|
||||
max_packet_size: String::new(),
|
||||
volume_name: String::new(),
|
||||
container_name: None,
|
||||
options: std::collections::HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,6 +36,8 @@ async fn start_cluster(user: &str) -> (ContainerAsync<Postgres>, DatabaseConfig)
|
||||
generated_id: "40875631-e3d2-4dfe-a26b-2a347ecc64fd".to_string(),
|
||||
path: "".to_string(),
|
||||
max_packet_size: "".to_string(),
|
||||
volume_name: "".to_string(),
|
||||
container_name: None,
|
||||
options: std::collections::HashMap::new(),
|
||||
};
|
||||
(container, config)
|
||||
|
||||
@@ -0,0 +1,279 @@
|
||||
use crate::domain::docker_volume::docker::parse_container_id;
|
||||
|
||||
static ENV_GUARD: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
|
||||
|
||||
#[test]
|
||||
fn parse_container_id_from_mountinfo_line() {
|
||||
let id = "a".repeat(64);
|
||||
let mountinfo = format!(
|
||||
"1234 1000 0:50 / /etc/hostname rw shared:1 - ext4 /var/lib/docker/containers/{id}/hostname rw"
|
||||
);
|
||||
assert_eq!(parse_container_id(&mountinfo, ""), Some(id));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_container_id_from_cgroup_v1() {
|
||||
let id = "b".repeat(64);
|
||||
let cgroup = format!("12:memory:/docker/{id}\n11:cpu:/docker/{id}\n");
|
||||
assert_eq!(parse_container_id("", &cgroup), Some(id));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_container_id_none_on_cgroup_v2() {
|
||||
assert_eq!(parse_container_id("", "0::/\n"), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn docker_volume_ping_true_for_existing_volume() {
|
||||
use crate::domain::docker_volume::docker::client;
|
||||
use bollard::models::VolumeCreateRequest;
|
||||
use bollard::query_parameters::RemoveVolumeOptions;
|
||||
|
||||
let docker = client().expect("docker daemon required for this test");
|
||||
let vol = format!("portabase-test-{}", uuid::Uuid::new_v4());
|
||||
docker
|
||||
.create_volume(VolumeCreateRequest { name: Some(vol.clone()), ..Default::default() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let cfg = volume_config(&vol);
|
||||
let reachable = crate::domain::docker_volume::ping::run(cfg).await.unwrap();
|
||||
assert!(reachable);
|
||||
|
||||
let missing = volume_config("portabase-does-not-exist-xyz");
|
||||
assert!(!crate::domain::docker_volume::ping::run(missing).await.unwrap());
|
||||
|
||||
docker.remove_volume(&vol, None::<RemoveVolumeOptions>).await.ok();
|
||||
}
|
||||
|
||||
fn volume_config(volume_name: &str) -> crate::services::config::DatabaseConfig {
|
||||
use crate::services::config::{DatabaseConfig, DbType};
|
||||
DatabaseConfig {
|
||||
name: "vol-test".to_string(),
|
||||
database: "".to_string(),
|
||||
db_type: DbType::DockerVolume,
|
||||
username: "".to_string(),
|
||||
password: "".to_string(),
|
||||
port: 0,
|
||||
host: "".to_string(),
|
||||
generated_id: uuid::Uuid::new_v4().to_string(),
|
||||
path: "".to_string(),
|
||||
max_packet_size: "".to_string(),
|
||||
volume_name: volume_name.to_string(),
|
||||
container_name: None,
|
||||
options: std::collections::HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn ensure_image(docker: &bollard::Docker, image: &str) {
|
||||
use bollard::query_parameters::CreateImageOptionsBuilder;
|
||||
use futures_util::StreamExt;
|
||||
|
||||
let (name, tag) = image.split_once(':').unwrap_or((image, "latest"));
|
||||
let opts = CreateImageOptionsBuilder::default()
|
||||
.from_image(name)
|
||||
.tag(tag)
|
||||
.build();
|
||||
let mut stream = docker.create_image(Some(opts), None, None);
|
||||
while let Some(item) = stream.next().await {
|
||||
item.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
async fn seed_volume(docker: &bollard::Docker, volume: &str, filename: &str, content: &str) {
|
||||
use bollard::models::{ContainerCreateBody, HostConfig};
|
||||
use bollard::query_parameters::{
|
||||
CreateContainerOptions, RemoveContainerOptions, StartContainerOptions,
|
||||
WaitContainerOptions,
|
||||
};
|
||||
use futures_util::StreamExt;
|
||||
|
||||
ensure_image(docker, "busybox").await;
|
||||
|
||||
let body = ContainerCreateBody {
|
||||
image: Some("busybox".to_string()),
|
||||
cmd: Some(vec![
|
||||
"sh".into(),
|
||||
"-c".into(),
|
||||
format!("printf '%s' '{content}' > /vol/{filename}"),
|
||||
]),
|
||||
host_config: Some(HostConfig {
|
||||
binds: Some(vec![format!("{volume}:/vol")]),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let created = docker
|
||||
.create_container(None::<CreateContainerOptions>, body)
|
||||
.await
|
||||
.unwrap();
|
||||
docker.start_container(&created.id, None::<StartContainerOptions>).await.unwrap();
|
||||
let mut wait = docker.wait_container(&created.id, None::<WaitContainerOptions>);
|
||||
while wait.next().await.is_some() {}
|
||||
docker
|
||||
.remove_container(&created.id, Some(RemoveContainerOptions { force: true, ..Default::default() }))
|
||||
.await
|
||||
.ok();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn docker_volume_backup_captures_files() {
|
||||
use crate::domain::docker_volume::docker::client;
|
||||
use bollard::models::VolumeCreateRequest;
|
||||
use bollard::query_parameters::RemoveVolumeOptions;
|
||||
|
||||
let _env_guard = ENV_GUARD.lock().await;
|
||||
unsafe { std::env::set_var("PORTABASE_HELPER_IMAGE", "busybox"); }
|
||||
|
||||
let docker = client().expect("docker daemon required");
|
||||
let vol = format!("portabase-test-{}", uuid::Uuid::new_v4());
|
||||
docker
|
||||
.create_volume(VolumeCreateRequest { name: Some(vol.clone()), ..Default::default() })
|
||||
.await
|
||||
.unwrap();
|
||||
seed_volume(&docker, &vol, "hello.txt", "backup-me").await;
|
||||
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let cfg = volume_config(&vol);
|
||||
let logger = std::sync::Arc::new(crate::services::backup::logger::JobLogger::new());
|
||||
let tar = crate::domain::docker_volume::backup::run(cfg, tmp.path().to_path_buf(), logger)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(tar.is_file());
|
||||
let names = tar_entry_names(&tar).await;
|
||||
assert!(names.iter().any(|n| n.ends_with("hello.txt")), "entries: {names:?}");
|
||||
|
||||
docker.remove_volume(&vol, None::<RemoveVolumeOptions>).await.ok();
|
||||
}
|
||||
|
||||
async fn tar_entry_names(tar_path: &std::path::Path) -> Vec<String> {
|
||||
use tokio_stream::StreamExt;
|
||||
let f = tokio::fs::File::open(tar_path).await.unwrap();
|
||||
let mut archive = tokio_tar::Archive::new(f);
|
||||
let mut names = Vec::new();
|
||||
let mut entries = archive.entries().unwrap();
|
||||
while let Some(e) = entries.next().await {
|
||||
let e = e.unwrap();
|
||||
names.push(e.path().unwrap().to_string_lossy().to_string());
|
||||
}
|
||||
names
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn docker_volume_restore_is_clean_replace() {
|
||||
use crate::domain::docker_volume::docker::client;
|
||||
use bollard::models::VolumeCreateRequest;
|
||||
use bollard::query_parameters::RemoveVolumeOptions;
|
||||
|
||||
let _env_guard = ENV_GUARD.lock().await;
|
||||
unsafe { std::env::set_var("PORTABASE_HELPER_IMAGE", "busybox"); }
|
||||
|
||||
let docker = client().expect("docker daemon required");
|
||||
let vol = format!("portabase-test-{}", uuid::Uuid::new_v4());
|
||||
docker
|
||||
.create_volume(VolumeCreateRequest { name: Some(vol.clone()), ..Default::default() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
seed_volume(&docker, &vol, "keeper.txt", "original").await;
|
||||
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let logger = std::sync::Arc::new(crate::services::backup::logger::JobLogger::new());
|
||||
let tar = crate::domain::docker_volume::backup::run(
|
||||
volume_config(&vol),
|
||||
tmp.path().to_path_buf(),
|
||||
logger.clone(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
seed_volume(&docker, &vol, "drift.txt", "added-later").await;
|
||||
|
||||
// Restore uploads the raw Docker tar directly.
|
||||
crate::domain::docker_volume::restore::run(volume_config(&vol), tar.clone(), logger)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let listing = list_volume(&docker, &vol).await;
|
||||
assert!(listing.contains("keeper.txt"), "listing: {listing}");
|
||||
assert!(!listing.contains("drift.txt"), "clean-replace failed, listing: {listing}");
|
||||
|
||||
docker.remove_volume(&vol, None::<RemoveVolumeOptions>).await.ok();
|
||||
}
|
||||
|
||||
async fn list_volume(docker: &bollard::Docker, volume: &str) -> String {
|
||||
use bollard::models::{ContainerCreateBody, HostConfig};
|
||||
use bollard::query_parameters::{
|
||||
CreateContainerOptions, LogsOptions, RemoveContainerOptions, StartContainerOptions,
|
||||
WaitContainerOptions,
|
||||
};
|
||||
use tokio_stream::StreamExt;
|
||||
|
||||
ensure_image(docker, "busybox").await;
|
||||
|
||||
let body = ContainerCreateBody {
|
||||
image: Some("busybox".to_string()),
|
||||
cmd: Some(vec!["sh".into(), "-c".into(), "ls -A /vol".into()]),
|
||||
host_config: Some(HostConfig {
|
||||
binds: Some(vec![format!("{volume}:/vol")]),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let created = docker.create_container(None::<CreateContainerOptions>, body).await.unwrap();
|
||||
docker.start_container(&created.id, None::<StartContainerOptions>).await.unwrap();
|
||||
let mut wait = docker.wait_container(&created.id, None::<WaitContainerOptions>);
|
||||
while wait.next().await.is_some() {}
|
||||
|
||||
let mut logs = docker.logs(
|
||||
&created.id,
|
||||
Some(LogsOptions { stdout: true, stderr: false, ..Default::default() }),
|
||||
);
|
||||
let mut out = String::new();
|
||||
while let Some(chunk) = logs.next().await {
|
||||
if let Ok(l) = chunk {
|
||||
out.push_str(&l.to_string());
|
||||
}
|
||||
}
|
||||
docker
|
||||
.remove_container(&created.id, Some(RemoveContainerOptions { force: true, ..Default::default() }))
|
||||
.await
|
||||
.ok();
|
||||
out
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sweep_removes_labeled_helpers() {
|
||||
use crate::domain::docker_volume::docker::{client, create_helper, sweep_ephemeral, EPHEMERAL_LABEL};
|
||||
use bollard::models::VolumeCreateRequest;
|
||||
use bollard::query_parameters::{ListContainersOptions, RemoveVolumeOptions};
|
||||
use std::collections::HashMap;
|
||||
|
||||
let _env_guard = ENV_GUARD.lock().await;
|
||||
unsafe { std::env::set_var("PORTABASE_HELPER_IMAGE", "busybox"); }
|
||||
let docker = client().expect("docker daemon required");
|
||||
|
||||
let vol = format!("portabase-test-{}", uuid::Uuid::new_v4());
|
||||
docker
|
||||
.create_volume(VolumeCreateRequest { name: Some(vol.clone()), ..Default::default() })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
ensure_image(&docker, "busybox").await;
|
||||
|
||||
let helper = create_helper(&docker, "busybox", &vol, "sweep-test", true, None).await.unwrap();
|
||||
|
||||
let removed = sweep_ephemeral(&docker).await.unwrap();
|
||||
assert!(removed >= 1);
|
||||
|
||||
let mut filters = HashMap::new();
|
||||
filters.insert("label".to_string(), vec![format!("{EPHEMERAL_LABEL}=true")]);
|
||||
let remaining = docker
|
||||
.list_containers(Some(ListContainersOptions { all: true, filters: Some(filters), ..Default::default() }))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(remaining.iter().all(|c| c.id.as_deref() != Some(helper.id.as_str())));
|
||||
|
||||
docker.remove_volume(&vol, None::<RemoveVolumeOptions>).await.ok();
|
||||
}
|
||||
@@ -40,6 +40,8 @@ async fn create_config() -> (ContainerAsync<GenericImage>, DatabaseConfig) {
|
||||
generated_id: "3c445eb4-c2c6-4bde-a423-ee1385dcf6d2".to_string(),
|
||||
path: "".to_string(),
|
||||
max_packet_size: "".to_string(),
|
||||
volume_name: "".to_string(),
|
||||
container_name: None,
|
||||
options: std::collections::HashMap::new(),
|
||||
};
|
||||
|
||||
|
||||
@@ -32,6 +32,8 @@ async fn create_config() -> (ContainerAsync<Mariadb>, DatabaseConfig) {
|
||||
generated_id: "3c4b4eb4-c2c6-4bde-a423-ee1385dcf6d2".to_string(),
|
||||
path: "".to_string(),
|
||||
max_packet_size: "512M".to_string(),
|
||||
volume_name: "".to_string(),
|
||||
container_name: None,
|
||||
options: std::collections::HashMap::new(),
|
||||
};
|
||||
|
||||
|
||||
@@ -7,3 +7,4 @@ mod redis;
|
||||
mod valkey;
|
||||
mod firebird;
|
||||
mod mssql;
|
||||
mod docker_volume;
|
||||
|
||||
@@ -30,6 +30,8 @@ async fn create_config() -> (ContainerAsync<Mongo>, DatabaseConfig) {
|
||||
generated_id: "96d30a9f-ff4b-47c9-aaab-f3147bb34f16".to_string(),
|
||||
path: "".to_string(),
|
||||
max_packet_size: "".to_string(),
|
||||
volume_name: "".to_string(),
|
||||
container_name: None,
|
||||
options: std::collections::HashMap::new(),
|
||||
};
|
||||
|
||||
|
||||
@@ -55,6 +55,8 @@ fn make_config(host: String, port: u16, database: &str, generated_id: &str) -> D
|
||||
generated_id: generated_id.to_string(),
|
||||
path: "".to_string(),
|
||||
max_packet_size: "".to_string(),
|
||||
volume_name: "".to_string(),
|
||||
container_name: None,
|
||||
options: std::collections::HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,6 +32,8 @@ async fn create_config() -> (ContainerAsync<Mysql>, DatabaseConfig) {
|
||||
generated_id: "0f1bb8f2-35a0-4c91-8098-e36873d3ce31".to_string(),
|
||||
path: "".to_string(),
|
||||
max_packet_size: "512M".to_string(),
|
||||
volume_name: "".to_string(),
|
||||
container_name: None,
|
||||
options: std::collections::HashMap::new(),
|
||||
};
|
||||
|
||||
|
||||
+704
-14
@@ -1,9 +1,14 @@
|
||||
use crate::domain::factory::DatabaseFactory;
|
||||
use crate::domain::postgres::connection::{pg_restore_binary_name, select_pg_path, server_version};
|
||||
use crate::domain::postgres::format::PostgresDumpFormat;
|
||||
use crate::domain::postgres::restore::prepare_archive;
|
||||
use crate::services::backup::logger::JobLogger;
|
||||
use crate::services::config::{DatabaseConfig, DbType};
|
||||
use crate::tests::init_tracing_for_test;
|
||||
use crate::utils::compress::{compress_to_tar_gz_large, decompress_large_tar_gz};
|
||||
use oauth2::url;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use tempfile::TempDir;
|
||||
use testcontainers::runners::AsyncRunner;
|
||||
use testcontainers::{ContainerAsync, ImageExt};
|
||||
@@ -39,6 +44,8 @@ async fn create_config() -> (ContainerAsync<Postgres>, DatabaseConfig) {
|
||||
generated_id: "40875631-e3d2-4dfe-a26b-2a347ecc64fd".to_string(),
|
||||
path: "".to_string(),
|
||||
max_packet_size: "".to_string(),
|
||||
volume_name: "".to_string(),
|
||||
container_name: None,
|
||||
options: std::collections::HashMap::new(),
|
||||
};
|
||||
|
||||
@@ -61,7 +68,6 @@ async fn postgres_ping_test() {
|
||||
async fn is_superuser_detects_superuser_role() {
|
||||
init_tracing_for_test();
|
||||
|
||||
// The testcontainer's POSTGRES_USER ("testuser") is the bootstrap superuser.
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let is_super = crate::domain::postgres::connection::is_superuser(&config)
|
||||
@@ -71,6 +77,31 @@ async fn is_superuser_detects_superuser_role() {
|
||||
assert!(is_super);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn can_drop_database_false_for_unprivileged_role() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, admin) = create_config().await;
|
||||
|
||||
let a = crate::domain::postgres::connection::connect(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
a.batch_execute("DROP ROLE IF EXISTS lowpriv; CREATE ROLE lowpriv LOGIN PASSWORD 'x';")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut low = admin.clone();
|
||||
low.username = "lowpriv".into();
|
||||
low.password = "x".into();
|
||||
|
||||
assert_eq!(
|
||||
crate::domain::postgres::connection::can_drop_database(&low)
|
||||
.await
|
||||
.unwrap(),
|
||||
false
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn postgres_backup_restore_test() {
|
||||
init_tracing_for_test();
|
||||
@@ -157,6 +188,8 @@ async fn postgres_password_with_slash_test() {
|
||||
generated_id: "5a1f0e3c-9b8a-4a8e-9b1b-0a1c2d3e4f5a".to_string(),
|
||||
path: "".to_string(),
|
||||
max_packet_size: "".to_string(),
|
||||
volume_name: "".to_string(),
|
||||
container_name: None,
|
||||
options: std::collections::HashMap::new(),
|
||||
};
|
||||
|
||||
@@ -166,17 +199,585 @@ async fn postgres_password_with_slash_test() {
|
||||
assert_eq!(reachable, true);
|
||||
}
|
||||
|
||||
fn pg_dump_env(config: &DatabaseConfig) -> std::collections::HashMap<String, String> {
|
||||
let mut env = std::env::vars().collect::<std::collections::HashMap<_, _>>();
|
||||
env.insert("PGPASSWORD".to_string(), config.password.clone());
|
||||
env
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepare_archive_fd_locates_toc_dir() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
let backup_path = crate::domain::postgres::backup::run(
|
||||
config.clone(),
|
||||
PostgresDumpFormat::Fd,
|
||||
temp_dir.path().to_path_buf(),
|
||||
pg_dump_env(&config),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let compression = compress_to_tar_gz_large(&backup_path, Arc::new(JobLogger::new()))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(compression.compressed_path.is_file());
|
||||
|
||||
let version = server_version(&config).await.unwrap();
|
||||
let pg_restore = select_pg_path(&version).join(pg_restore_binary_name());
|
||||
|
||||
let logger = JobLogger::new();
|
||||
|
||||
let prepared = prepare_archive(
|
||||
PostgresDumpFormat::Fd,
|
||||
&compression.compressed_path,
|
||||
&pg_restore,
|
||||
&logger,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert!(prepared.path().join("toc.dat").exists());
|
||||
assert!(!prepared.toc().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepare_archive_fc_returns_file_path_unchanged() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
let backup_path = crate::domain::postgres::backup::run(
|
||||
config.clone(),
|
||||
PostgresDumpFormat::Fc,
|
||||
temp_dir.path().to_path_buf(),
|
||||
pg_dump_env(&config),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(backup_path.is_file());
|
||||
|
||||
let version = server_version(&config).await.unwrap();
|
||||
let pg_restore = select_pg_path(&version).join(pg_restore_binary_name());
|
||||
|
||||
let logger = JobLogger::new();
|
||||
|
||||
let prepared = prepare_archive(PostgresDumpFormat::Fc, &backup_path, &pg_restore, &logger).unwrap();
|
||||
|
||||
assert_eq!(prepared.path(), backup_path.as_path());
|
||||
assert!(!prepared.toc().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restore_run_unified_fc_roundtrip() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
client.execute("CREATE TABLE t(id int);", &[]).await.unwrap();
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
let dump_file = crate::domain::postgres::backup::run(
|
||||
config.clone(),
|
||||
PostgresDumpFormat::Fc,
|
||||
temp_dir.path().to_path_buf(),
|
||||
pg_dump_env(&config),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(dump_file.is_file());
|
||||
|
||||
let format = crate::domain::postgres::connection::detect_format_from_file(&dump_file);
|
||||
|
||||
let result = crate::domain::postgres::restore::run(
|
||||
config.clone(),
|
||||
format,
|
||||
dump_file,
|
||||
pg_dump_env(&config),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(result.is_ok(), "restore::run failed: {:?}", result);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn drop_all_schemas_removes_user_schema() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
client
|
||||
.batch_execute("CREATE SCHEMA IF NOT EXISTS extra_ns; CREATE TABLE IF NOT EXISTS extra_ns.t(id int);")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let dropped = crate::domain::postgres::connection::drop_all_schemas(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(dropped.iter().any(|s| s == "extra_ns"));
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
let row = client
|
||||
.query_one(
|
||||
"SELECT count(*) FROM pg_namespace WHERE nspname = 'extra_ns'",
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let n: i64 = row.get(0);
|
||||
assert_eq!(n, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restore_drop_schemas_removes_extra_objects() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
client
|
||||
.batch_execute("CREATE TABLE base_t(id int);")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
let dump_file = crate::domain::postgres::backup::run(
|
||||
config.clone(),
|
||||
PostgresDumpFormat::Fc,
|
||||
temp_dir.path().to_path_buf(),
|
||||
pg_dump_env(&config),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
client
|
||||
.batch_execute("CREATE TABLE orphan_only_here(id int);")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut cfg = config.clone();
|
||||
cfg.options
|
||||
.insert("clean_mode".into(), serde_json::json!("drop_schemas"));
|
||||
|
||||
let format = crate::domain::postgres::connection::detect_format_from_file(&dump_file);
|
||||
|
||||
let result = crate::domain::postgres::restore::run(
|
||||
cfg.clone(),
|
||||
format,
|
||||
dump_file,
|
||||
pg_dump_env(&cfg),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(result.is_ok(), "restore::run failed: {:?}", result);
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let n: i64 = client
|
||||
.query_one(
|
||||
"SELECT count(*) FROM information_schema.tables WHERE table_name = 'orphan_only_here'",
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.get(0);
|
||||
assert_eq!(n, 0);
|
||||
|
||||
let n: i64 = client
|
||||
.query_one(
|
||||
"SELECT count(*) FROM information_schema.tables WHERE table_name = 'base_t'",
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.get(0);
|
||||
assert_eq!(n, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restore_clean_leaves_divergent_object() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
client
|
||||
.batch_execute("CREATE TABLE base_t(id int);")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
let dump_file = crate::domain::postgres::backup::run(
|
||||
config.clone(),
|
||||
PostgresDumpFormat::Fc,
|
||||
temp_dir.path().to_path_buf(),
|
||||
pg_dump_env(&config),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
client
|
||||
.batch_execute("CREATE TABLE survives_clean(id int);")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut cfg = config.clone();
|
||||
cfg.options
|
||||
.insert("clean_mode".into(), serde_json::json!("clean"));
|
||||
|
||||
let format = crate::domain::postgres::connection::detect_format_from_file(&dump_file);
|
||||
|
||||
let result = crate::domain::postgres::restore::run(
|
||||
cfg.clone(),
|
||||
format,
|
||||
dump_file,
|
||||
pg_dump_env(&cfg),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(result.is_ok(), "restore::run failed: {:?}", result);
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let n: i64 = client
|
||||
.query_one(
|
||||
"SELECT count(*) FROM information_schema.tables WHERE table_name = 'survives_clean'",
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.get(0);
|
||||
assert_eq!(n, 1, "clean mode is not a reset; divergent object survives");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn restore_unknown_clean_mode_falls_back() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
let dump_file = crate::domain::postgres::backup::run(
|
||||
config.clone(),
|
||||
PostgresDumpFormat::Fc,
|
||||
temp_dir.path().to_path_buf(),
|
||||
pg_dump_env(&config),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut cfg = config.clone();
|
||||
cfg.options
|
||||
.insert("clean_mode".into(), serde_json::json!("wat"));
|
||||
|
||||
let format = crate::domain::postgres::connection::detect_format_from_file(&dump_file);
|
||||
|
||||
let logger = Arc::new(JobLogger::new());
|
||||
|
||||
let result = crate::domain::postgres::restore::run(
|
||||
cfg.clone(),
|
||||
format,
|
||||
dump_file,
|
||||
pg_dump_env(&cfg),
|
||||
logger.clone(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(result.is_ok(), "restore::run failed: {:?}", result);
|
||||
|
||||
let entries = Arc::try_unwrap(logger)
|
||||
.expect("logger should have a single owner after run() completes")
|
||||
.into_entries();
|
||||
assert!(entries
|
||||
.iter()
|
||||
.any(|e| e.message.contains("Unknown clean_mode 'wat'")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn drop_database_preflight_preserves_data_when_unprivileged() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, admin) = create_config().await;
|
||||
|
||||
let a = crate::domain::postgres::connection::connect(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
a.batch_execute("DROP ROLE IF EXISTS lowpriv2; CREATE ROLE lowpriv2 LOGIN PASSWORD 'x';")
|
||||
.await
|
||||
.unwrap();
|
||||
a.batch_execute("CREATE TABLE IF NOT EXISTS keep_me(id int);")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
let dump_file = crate::domain::postgres::backup::run(
|
||||
admin.clone(),
|
||||
PostgresDumpFormat::Fc,
|
||||
temp_dir.path().to_path_buf(),
|
||||
pg_dump_env(&admin),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut low = admin.clone();
|
||||
low.username = "lowpriv2".into();
|
||||
low.password = "x".into();
|
||||
low.options
|
||||
.insert("clean_mode".into(), serde_json::json!("drop_database"));
|
||||
|
||||
let format = crate::domain::postgres::connection::detect_format_from_file(&dump_file);
|
||||
|
||||
let res = crate::domain::postgres::restore::run(
|
||||
low.clone(),
|
||||
format,
|
||||
dump_file,
|
||||
pg_dump_env(&low),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await;
|
||||
assert!(res.is_err(), "preflight must reject an unprivileged role");
|
||||
|
||||
let a = crate::domain::postgres::connection::connect(&admin)
|
||||
.await
|
||||
.unwrap();
|
||||
let n: i64 = a
|
||||
.query_one(
|
||||
"SELECT count(*) FROM information_schema.tables WHERE table_name = 'keep_me'",
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.get(0);
|
||||
assert_eq!(n, 1, "preflight must fail before dropping anything");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn drop_database_preserves_encoding_and_owner() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
client
|
||||
.batch_execute("CREATE TABLE base_t(id int);")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
let dump_file = crate::domain::postgres::backup::run(
|
||||
config.clone(),
|
||||
PostgresDumpFormat::Fc,
|
||||
temp_dir.path().to_path_buf(),
|
||||
pg_dump_env(&config),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let before = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap()
|
||||
.query_one(
|
||||
"SELECT pg_encoding_to_char(encoding), datcollate FROM pg_database WHERE datname = current_database()",
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let enc0: String = before.get(0);
|
||||
let coll0: String = before.get(1);
|
||||
|
||||
let mut cfg = config.clone();
|
||||
cfg.options
|
||||
.insert("clean_mode".into(), serde_json::json!("drop_database"));
|
||||
|
||||
let format = crate::domain::postgres::connection::detect_format_from_file(&dump_file);
|
||||
|
||||
let result = crate::domain::postgres::restore::run(
|
||||
cfg.clone(),
|
||||
format,
|
||||
dump_file,
|
||||
pg_dump_env(&cfg),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_ok(), "restore::run failed: {:?}", result);
|
||||
|
||||
let after = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap()
|
||||
.query_one(
|
||||
"SELECT pg_encoding_to_char(encoding), datcollate FROM pg_database WHERE datname = current_database()",
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let enc1: String = after.get(0);
|
||||
let coll1: String = after.get(1);
|
||||
assert_eq!(enc0, enc1);
|
||||
assert_eq!(coll0, coll1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn drop_database_force_wins_over_open_connection() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
let dump_file = crate::domain::postgres::backup::run(
|
||||
config.clone(),
|
||||
PostgresDumpFormat::Fc,
|
||||
temp_dir.path().to_path_buf(),
|
||||
pg_dump_env(&config),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut cfg = config.clone();
|
||||
cfg.options
|
||||
.insert("clean_mode".into(), serde_json::json!("drop_database"));
|
||||
|
||||
let squatter = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
let _keep = tokio::spawn(async move {
|
||||
let _ = squatter.query_one("SELECT pg_sleep(5)", &[]).await;
|
||||
});
|
||||
|
||||
let format = crate::domain::postgres::connection::detect_format_from_file(&dump_file);
|
||||
|
||||
let result = crate::domain::postgres::restore::run(
|
||||
cfg.clone(),
|
||||
format,
|
||||
dump_file,
|
||||
pg_dump_env(&cfg),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_ok(), "restore::run failed: {:?}", result);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sniff_format_detects_custom_and_gzip() {
|
||||
use crate::domain::postgres::connection::sniff_format;
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let fc = dir.path().join("a.dump");
|
||||
std::fs::write(&fc, b"PGDMP\x01\x0e\x00").unwrap();
|
||||
assert_eq!(sniff_format(&fc).unwrap(), PostgresDumpFormat::Fc);
|
||||
|
||||
let fd = dir.path().join("b.gz");
|
||||
std::fs::write(&fd, [0x1f, 0x8b, 0x08, 0x00]).unwrap();
|
||||
assert_eq!(sniff_format(&fd).unwrap(), PostgresDumpFormat::Fd);
|
||||
|
||||
let bad = dir.path().join("c.bin");
|
||||
std::fs::write(&bad, b"not a dump").unwrap();
|
||||
assert!(sniff_format(&bad).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn corrupt_archive_leaves_database_untouched() {
|
||||
init_tracing_for_test();
|
||||
|
||||
let (_container, config) = create_config().await;
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
client
|
||||
.batch_execute("CREATE TABLE must_survive(id int);")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut cfg = config.clone();
|
||||
cfg.options
|
||||
.insert("clean_mode".into(), serde_json::json!("drop_schemas"));
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let broken_file = dir.path().join("broken.tar.gz");
|
||||
std::fs::write(&broken_file, [0x1f, 0x8b, 0x08, 0x00, 0xde, 0xad, 0xbe, 0xef]).unwrap();
|
||||
|
||||
let result = crate::domain::postgres::restore::run(
|
||||
cfg.clone(),
|
||||
PostgresDumpFormat::Fd,
|
||||
broken_file,
|
||||
pg_dump_env(&cfg),
|
||||
Arc::new(JobLogger::new()),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(result.is_err(), "corrupt archive must be rejected before any destructive step");
|
||||
|
||||
let client = crate::domain::postgres::connection::connect(&config)
|
||||
.await
|
||||
.unwrap();
|
||||
let n: i64 = client
|
||||
.query_one(
|
||||
"SELECT count(*) FROM information_schema.tables WHERE table_name = 'must_survive'",
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.get(0);
|
||||
assert_eq!(n, 1, "corrupt archive must never trigger the schema drop");
|
||||
}
|
||||
|
||||
mod select_pg_path_tests {
|
||||
use crate::domain::postgres::connection::{
|
||||
pg_dump_binary_name, pg_dump_exists_in, pg_dumpall_binary_name, psql_binary_name,
|
||||
select_pg_path_with,
|
||||
pg_dump_binary_name, pg_dump_exists_in, pg_dumpall_binary_name, pg_restore_binary_name,
|
||||
psql_binary_name, select_pg_path_with,
|
||||
};
|
||||
|
||||
// `select_pg_path_with` takes the `PG_BIN_DIR` override as a plain
|
||||
// argument, so these tests never touch process-global env state or the
|
||||
// cached `CONFIG`. They stay deterministic regardless of whether — or at
|
||||
// which version — a real PostgreSQL install exists on the host.
|
||||
|
||||
#[test]
|
||||
fn respects_pg_bin_dir_override() {
|
||||
let custom = if cfg!(target_os = "windows") {
|
||||
@@ -190,8 +791,6 @@ mod select_pg_path_tests {
|
||||
|
||||
#[test]
|
||||
fn pg_bin_dir_override_ignores_requested_version() {
|
||||
// The override is taken as-is, regardless of which version was
|
||||
// requested — this documents/locks in that behavior.
|
||||
let custom = if cfg!(target_os = "windows") {
|
||||
r"C:\custom\pg\bin"
|
||||
} else {
|
||||
@@ -203,10 +802,6 @@ mod select_pg_path_tests {
|
||||
|
||||
#[test]
|
||||
fn empty_pg_bin_dir_falls_through_to_detection() {
|
||||
// An empty override means "unset" (matches `CONFIG.pg_bin_dir` when
|
||||
// `PG_BIN_DIR` is absent). It must not be returned as a literal empty
|
||||
// path — resolution falls through to platform defaults / PATH lookup
|
||||
// and yields a non-empty path.
|
||||
let path = select_pg_path_with("17", "");
|
||||
assert_ne!(path, std::path::PathBuf::from(""));
|
||||
}
|
||||
@@ -246,4 +841,99 @@ mod select_pg_path_tests {
|
||||
assert_eq!(name, "psql");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pg_restore_binary_name_is_platform_correct() {
|
||||
let name = pg_restore_binary_name();
|
||||
if cfg!(target_os = "windows") {
|
||||
assert_eq!(name, "pg_restore.exe");
|
||||
} else {
|
||||
assert_eq!(name, "pg_restore");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mod quoting_tests {
|
||||
use crate::domain::postgres::connection::{quote_ident, quote_literal};
|
||||
|
||||
#[test]
|
||||
fn quote_ident_escapes_double_quotes() {
|
||||
assert_eq!(quote_ident("devdb"), "\"devdb\"");
|
||||
assert_eq!(quote_ident("a\"b"), "\"a\"\"b\"");
|
||||
assert_eq!(quote_ident("drop\"; --"), "\"drop\"\"; --\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quote_literal_escapes_single_quotes() {
|
||||
assert_eq!(quote_literal("UTF8"), "'UTF8'");
|
||||
assert_eq!(quote_literal("O'Brien"), "'O''Brien'");
|
||||
}
|
||||
}
|
||||
|
||||
mod clean_mode_tests {
|
||||
use crate::domain::postgres::clean_mode::RestoreCleanMode as M;
|
||||
use crate::services::config::{DatabaseConfig, DbType};
|
||||
|
||||
fn cfg_with(clean_mode: Option<&str>) -> DatabaseConfig {
|
||||
let mut options = std::collections::HashMap::new();
|
||||
if let Some(v) = clean_mode {
|
||||
options.insert("clean_mode".to_string(), serde_json::json!(v));
|
||||
}
|
||||
DatabaseConfig {
|
||||
name: "t".into(),
|
||||
database: "testdb".into(),
|
||||
db_type: DbType::Postgresql,
|
||||
username: "testuser".into(),
|
||||
password: "changeme".into(),
|
||||
port: 5432,
|
||||
host: "localhost".into(),
|
||||
generated_id: "00000000-0000-0000-0000-000000000000".into(),
|
||||
path: "".into(),
|
||||
max_packet_size: "".into(),
|
||||
volume_name: "".into(),
|
||||
container_name: None,
|
||||
options,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clean_mode_parsing() {
|
||||
assert_eq!(M::from_config(&cfg_with(None)), (M::Clean, None));
|
||||
assert_eq!(M::from_config(&cfg_with(Some("clean"))), (M::Clean, None));
|
||||
assert_eq!(M::from_config(&cfg_with(Some("none"))), (M::None, None));
|
||||
assert_eq!(
|
||||
M::from_config(&cfg_with(Some("drop_schemas"))),
|
||||
(M::DropSchemas, None)
|
||||
);
|
||||
assert_eq!(
|
||||
M::from_config(&cfg_with(Some("drop_database"))),
|
||||
(M::DropDatabase, None)
|
||||
);
|
||||
assert_eq!(
|
||||
M::from_config(&cfg_with(Some("bogus"))),
|
||||
(M::Clean, Some("bogus".to_string()))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uses_pg_restore_clean_behavior() {
|
||||
assert!(M::Clean.uses_pg_restore_clean());
|
||||
assert!(!M::DropSchemas.uses_pg_restore_clean());
|
||||
assert!(!M::None.uses_pg_restore_clean());
|
||||
assert!(!M::DropDatabase.uses_pg_restore_clean());
|
||||
}
|
||||
}
|
||||
|
||||
mod toc_tests {
|
||||
use crate::domain::postgres::restore::toc_creates_public_schema;
|
||||
|
||||
#[test]
|
||||
fn toc_public_schema_detection() {
|
||||
let with = "215; 2615 2200 SCHEMA - public pg_database_owner";
|
||||
let without_table = "200; 1259 12346 TABLE devschema users devuser";
|
||||
let without_similar_schema = "216; 2615 2201 SCHEMA - publicish someowner";
|
||||
assert!(toc_creates_public_schema(with));
|
||||
assert!(!toc_creates_public_schema(without_table));
|
||||
assert!(!toc_creates_public_schema(without_similar_schema));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,6 +29,8 @@ async fn create_config() -> (ContainerAsync<Redis>, DatabaseConfig) {
|
||||
generated_id: "40875631-e3d2-4dfe-a26b-2a347ecc64fd".to_string(),
|
||||
path: "".to_string(),
|
||||
max_packet_size: "".to_string(),
|
||||
volume_name: "".to_string(),
|
||||
container_name: None,
|
||||
options: std::collections::HashMap::new(),
|
||||
};
|
||||
|
||||
|
||||
@@ -28,6 +28,8 @@ async fn create_config() -> (ContainerAsync<Valkey>, DatabaseConfig) {
|
||||
generated_id: "40875485-e3d2-4dfe-a26b-2a347ecc64fd".to_string(),
|
||||
path: "".to_string(),
|
||||
max_packet_size: "".to_string(),
|
||||
volume_name: "".to_string(),
|
||||
container_name: None,
|
||||
options: std::collections::HashMap::new(),
|
||||
};
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ use serde_json::json;
|
||||
|
||||
use crate::services::api::models::agent::backup::{BackupResponse, BackupUploadResponse};
|
||||
use crate::services::api::models::agent::restore::ResultRestoreResponse;
|
||||
use crate::services::api::models::agent::status::PingResult;
|
||||
use crate::services::api::models::agent::status::{DatabaseStatus, PingResult};
|
||||
|
||||
#[test]
|
||||
fn backup_response_deserializes_nested_backup_id() {
|
||||
@@ -110,3 +110,41 @@ fn ping_result_deserializes_and_normalizes_storage_config_keys() {
|
||||
assert!(result.databases[0].data.restore.file.is_none());
|
||||
assert!(result.databases[0].data.restore.meta_file.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn database_status_legacy_plaintext_storages() {
|
||||
let status: DatabaseStatus = serde_json::from_value(json!({
|
||||
"dbms": "postgres",
|
||||
"generatedId": "gen-1",
|
||||
"storages": [ { "id": "s1", "config": { "bucket": "b" }, "provider": "s3" } ],
|
||||
"encrypt": true,
|
||||
"data": {
|
||||
"backup": { "action": false, "cron": null },
|
||||
"restore": { "action": false, "file": null, "metaFile": null, "size": null }
|
||||
}
|
||||
})).unwrap();
|
||||
|
||||
assert_eq!(status.storages.len(), 1);
|
||||
assert_eq!(status.storages_encrypted, None);
|
||||
assert!(status.storages_ciphertext.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn database_status_encrypted_envelope() {
|
||||
let status: DatabaseStatus = serde_json::from_value(json!({
|
||||
"dbms": "postgres",
|
||||
"generatedId": "gen-1",
|
||||
"storages": [],
|
||||
"storages_encrypted": true,
|
||||
"storages_ciphertext": "AQIDBA==",
|
||||
"encrypt": true,
|
||||
"data": {
|
||||
"backup": { "action": true, "cron": null },
|
||||
"restore": { "action": false, "file": null, "metaFile": null, "size": null }
|
||||
}
|
||||
})).unwrap();
|
||||
|
||||
assert!(status.storages.is_empty());
|
||||
assert_eq!(status.storages_encrypted, Some(true));
|
||||
assert_eq!(status.storages_ciphertext.as_deref(), Some("AQIDBA=="));
|
||||
}
|
||||
|
||||
@@ -199,3 +199,68 @@ fn keep_ownership_extraction_logic() {
|
||||
let keep4 = opts4.get("keep_ownership").and_then(|v| v.as_bool()).unwrap_or(false);
|
||||
assert!(!keep4, "should strip when value is not bool");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_docker_volume_type() {
|
||||
let file = write_json(
|
||||
r#"{
|
||||
"databases": [
|
||||
{
|
||||
"name": "uploads",
|
||||
"type": "docker-volume",
|
||||
"volume_name": "myapp_uploads",
|
||||
"generated_id": "16678159-ff7e-4c97-8c83-0adeff214681",
|
||||
"container_name": "myapp"
|
||||
}
|
||||
]
|
||||
}"#,
|
||||
);
|
||||
|
||||
let service = ConfigService::new(test_context());
|
||||
let cfg = service.load(Some(file.path().to_str().unwrap())).unwrap();
|
||||
|
||||
assert_eq!(cfg.databases[0].db_type.as_str(), "docker-volume");
|
||||
assert_eq!(cfg.databases[0].volume_name, "myapp_uploads");
|
||||
assert_eq!(cfg.databases[0].container_name.as_deref(), Some("myapp"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn docker_volume_container_name_optional() {
|
||||
let file = write_json(
|
||||
r#"{
|
||||
"databases": [
|
||||
{
|
||||
"name": "uploads",
|
||||
"type": "docker-volume",
|
||||
"volume_name": "myapp_uploads",
|
||||
"generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"
|
||||
}
|
||||
]
|
||||
}"#,
|
||||
);
|
||||
|
||||
let service = ConfigService::new(test_context());
|
||||
let cfg = service.load(Some(file.path().to_str().unwrap())).unwrap();
|
||||
|
||||
assert_eq!(cfg.databases[0].volume_name, "myapp_uploads");
|
||||
assert!(cfg.databases[0].container_name.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn docker_volume_requires_volume_name() {
|
||||
let file = write_json(
|
||||
r#"{
|
||||
"databases": [
|
||||
{
|
||||
"name": "uploads",
|
||||
"type": "docker-volume",
|
||||
"generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"
|
||||
}
|
||||
]
|
||||
}"#,
|
||||
);
|
||||
|
||||
let service = ConfigService::new(test_context());
|
||||
let err = service.load(Some(file.path().to_str().unwrap())).unwrap_err();
|
||||
assert!(err.contains("volume_name"), "error was: {err}");
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use crate::utils::common::{BackupMethod, vec_to_option_json};
|
||||
use crate::utils::common::{BackupMethod, choose_restore_path, vec_to_option_json};
|
||||
use serde_json::json;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
#[test]
|
||||
fn backup_method_to_string_automatic() {
|
||||
@@ -47,3 +48,28 @@ fn vec_to_option_json_serializes_struct_vector() {
|
||||
]))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn choose_restore_path_single_file_returns_that_file() {
|
||||
let dir = Path::new("/tmp/extract");
|
||||
let archive = Path::new("/tmp/backup.tar.gz");
|
||||
let files = vec![PathBuf::from("/tmp/extract/dump.sql")];
|
||||
// Single extracted file: restore from that file directly.
|
||||
assert_eq!(
|
||||
choose_restore_path(&files, dir, archive),
|
||||
PathBuf::from("/tmp/extract/dump.sql")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn choose_restore_path_multi_file_non_docker_volume_returns_archive_path() {
|
||||
let dir = Path::new("/tmp/extract");
|
||||
let archive = Path::new("/tmp/backup.tar.gz");
|
||||
let files = vec![
|
||||
PathBuf::from("/tmp/extract/toc.dat"),
|
||||
PathBuf::from("/tmp/extract/3141.dat.gz"),
|
||||
];
|
||||
let chosen = choose_restore_path(&files, dir, archive);
|
||||
assert_eq!(chosen, PathBuf::from("/tmp/backup.tar.gz"));
|
||||
assert_eq!(chosen.extension().and_then(|e| e.to_str()), Some("gz"));
|
||||
}
|
||||
|
||||
@@ -71,3 +71,57 @@ async fn decompress_multiple_files() -> Result<()> {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compress_tar_is_not_double_wrapped() -> Result<()> {
|
||||
use tokio_tar::Builder as TarBuilder;
|
||||
|
||||
let tmp = tempdir()?;
|
||||
// Build a real tar containing a single entry "payload.txt".
|
||||
let payload = tmp.path().join("payload.txt");
|
||||
write(&payload, b"volume-bytes").await?;
|
||||
let tar_path = tmp.path().join("volume.tar");
|
||||
{
|
||||
let f = tokio::fs::File::create(&tar_path).await?;
|
||||
let mut b = TarBuilder::new(f);
|
||||
b.append_path_with_name(&payload, "payload.txt").await?;
|
||||
b.finish().await?;
|
||||
}
|
||||
|
||||
let result = compress_to_tar_gz_large(
|
||||
&tar_path,
|
||||
std::sync::Arc::new(crate::services::backup::logger::JobLogger::new()),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(result.compressed_path, tmp.path().join("volume.tar.gz"));
|
||||
|
||||
// Decompress and confirm the FIRST tar entry is "payload.txt" — i.e. our tar
|
||||
// was gzipped directly, not wrapped inside another tar named "volume.tar".
|
||||
let out = tmp.path().join("out");
|
||||
tokio::fs::create_dir_all(&out).await?;
|
||||
let files = decompress_large_tar_gz(&result.compressed_path, &out).await?;
|
||||
assert_eq!(files.len(), 1);
|
||||
assert_eq!(files[0].file_name().unwrap(), "payload.txt");
|
||||
assert_eq!(read(&files[0]).await?, b"volume-bytes");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn gunzip_to_file_restores_tar_byte_for_byte() -> Result<()> {
|
||||
use crate::utils::compress::gunzip_to_file;
|
||||
|
||||
let tmp = tempdir()?;
|
||||
let tar_path = tmp.path().join("input.tar");
|
||||
let original: Vec<u8> = (0u32..50_000).map(|n| (n % 256) as u8).collect();
|
||||
write(&tar_path, &original).await?;
|
||||
|
||||
let gz = compress_to_tar_gz_large(&tar_path, std::sync::Arc::new(crate::services::backup::logger::JobLogger::new())).await?;
|
||||
|
||||
let out_tar = tmp.path().join("out.tar");
|
||||
gunzip_to_file(&gz.compressed_path, &out_tar).await?;
|
||||
|
||||
assert_eq!(read(&out_tar).await?, original);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -30,12 +30,16 @@ fn full_file_name_matches_expected_suffix() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_file_path_prefixes_backups_directory_and_date() {
|
||||
fn full_file_path_uses_default_or_configured_folder_name() {
|
||||
let file_name = "backup.tar.gz".to_string();
|
||||
let full_path = full_file_path(&file_name);
|
||||
|
||||
assert!(full_path.starts_with("backups/"));
|
||||
assert!(full_path.ends_with("/backup.tar.gz"));
|
||||
let default_path = full_file_path(&file_name, None);
|
||||
assert!(default_path.starts_with("backups/"));
|
||||
assert!(default_path.ends_with("/backup.tar.gz"));
|
||||
|
||||
let configured_path = full_file_path(&file_name, Some("/portabase/"));
|
||||
assert!(configured_path.starts_with("portabase/"));
|
||||
assert!(configured_path.ends_with("/backup.tar.gz"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -86,3 +90,27 @@ async fn encrypt_stream_starts_with_json_header_line() -> Result<()> {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
use crate::utils::file::decrypt_json_gcm;
|
||||
|
||||
const VECTOR_KEY_B64: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=";
|
||||
const VECTOR_ENVELOPE_B64: &str = "AQIDBAUGBwgJCgsM4AfK9w7I2A7UDzMvpJaScfnUYAGDZgPWT5Chrp1pdzMPPQVpNjb6ZEiFea9YdWVFv1UEo9RGmmf+zYUv4I3gE4SU/SBrMwkCHEpJGJOzJtK3tSpJmzLVX3+7EeUNwp4qjZheL8p0pe1x6dRUtx3JmLjz1W/RhWd6zuReDItv6+0jg4CaPOHvFXBreaGNCTRslxbImD+lFBoEOvw8lsbH";
|
||||
const VECTOR_PLAINTEXT: &str = "[{\"id\":\"11111111-1111-1111-1111-111111111111\",\"config\":{\"bucket\":\"my-bucket\",\"accessKeyId\":\"AKIA\",\"secretAccessKey\":\"s3cr3t\"},\"provider\":\"s3\"}]";
|
||||
|
||||
#[test]
|
||||
fn decrypt_json_gcm_decrypts_node_vector() {
|
||||
let plaintext = decrypt_json_gcm(VECTOR_ENVELOPE_B64, VECTOR_KEY_B64).unwrap();
|
||||
assert_eq!(String::from_utf8(plaintext).unwrap(), VECTOR_PLAINTEXT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decrypt_json_gcm_rejects_wrong_key() {
|
||||
let wrong_key = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
|
||||
assert!(decrypt_json_gcm(VECTOR_ENVELOPE_B64, wrong_key).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decrypt_json_gcm_rejects_short_input() {
|
||||
// 8 bytes base64 -> shorter than nonce(12)+tag(16)
|
||||
assert!(decrypt_json_gcm("AAAAAAAAAAA=", VECTOR_KEY_B64).is_err());
|
||||
}
|
||||
|
||||
@@ -31,12 +31,60 @@ fn normalized_expression_is_valid_for_cron_schedule() {
|
||||
#[test]
|
||||
fn next_run_timestamp_returns_future_timestamp() {
|
||||
let expr = normalize_cron("*/1 * * * *");
|
||||
let ts = next_run_timestamp(&expr);
|
||||
let ts = next_run_timestamp(&expr).unwrap();
|
||||
|
||||
let now = chrono::Local::now().timestamp();
|
||||
assert!(ts > now);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_converts_unix_sunday_zero_to_crate_dow() {
|
||||
let input = "00 06 * * 0";
|
||||
let normalized = normalize_cron(input);
|
||||
|
||||
assert_eq!(normalized, "0 00 06 * * 1");
|
||||
|
||||
let schedule = Schedule::from_str(&normalized);
|
||||
assert!(schedule.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_converts_unix_saturday_to_crate_dow() {
|
||||
assert_eq!(normalize_cron("00 06 * * 6"), "0 00 06 * * 7");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalized_sunday_actually_fires_on_sunday() {
|
||||
use chrono::{Datelike, Timelike, Weekday};
|
||||
|
||||
let normalized = normalize_cron("00 03 * * 0");
|
||||
assert_eq!(normalized, "0 00 03 * * 1");
|
||||
|
||||
let schedule = Schedule::from_str(&normalized).unwrap();
|
||||
let next = schedule.upcoming(chrono::Utc).next().unwrap();
|
||||
|
||||
assert_eq!(next.weekday(), Weekday::Sun);
|
||||
assert_eq!(next.hour(), 3);
|
||||
assert_eq!(next.minute(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_converts_unix_dow_range() {
|
||||
assert_eq!(normalize_cron("00 06 * * 0-4"), "0 00 06 * * 1-5");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn next_run_timestamp_returns_none_for_invalid_cron() {
|
||||
assert!(next_run_timestamp("not a cron").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_leaves_out_of_range_dow_untouched() {
|
||||
let normalized = normalize_cron("* * * * 100");
|
||||
assert_eq!(normalized, "0 * * * * 100");
|
||||
assert!(Schedule::from_str(&normalized).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalization_does_not_break_schedule_parsing() {
|
||||
let input = "0 */10 * * * *";
|
||||
|
||||
@@ -1,5 +1,17 @@
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
pub(crate) fn choose_restore_path(
|
||||
extracted: &[PathBuf],
|
||||
_extraction_dir: &Path,
|
||||
archive: &Path,
|
||||
) -> PathBuf {
|
||||
match extracted.len() {
|
||||
1 => extracted[0].clone(),
|
||||
_ => archive.to_path_buf(),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub enum BackupMethod {
|
||||
|
||||
+54
-1
@@ -1,4 +1,4 @@
|
||||
use anyhow::Result;
|
||||
use anyhow::{Context, Result};
|
||||
use async_compression::tokio::bufread::GzipDecoder;
|
||||
use async_compression::tokio::write::GzipEncoder as AsyncGzipEncoder;
|
||||
use futures::StreamExt;
|
||||
@@ -32,6 +32,39 @@ pub async fn compress_to_tar_gz_large(file: &PathBuf, logger: Arc<JobLogger>) ->
|
||||
});
|
||||
}
|
||||
|
||||
if file
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.map(|n| n.ends_with(".tar"))
|
||||
.unwrap_or(false)
|
||||
{
|
||||
let gz_path = PathBuf::from(format!("{}.gz", file.display()));
|
||||
logger.log("info", format!("Input {:?} is a raw tar, gzipping directly", file));
|
||||
|
||||
let input = File::open(file)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Failed to open tar {:?}: {}", file, e))?;
|
||||
let mut reader = BufReader::with_capacity(8 * 1024 * 1024, input);
|
||||
|
||||
let output_file = File::create(&gz_path)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Failed to create {:?}: {}", gz_path, e))?;
|
||||
let mut encoder = AsyncGzipEncoder::new(output_file);
|
||||
|
||||
tokio::io::copy(&mut reader, &mut encoder)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Gzip copy failed: {}", e))?;
|
||||
encoder
|
||||
.shutdown()
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Gzip shutdown failed: {}", e))?;
|
||||
|
||||
logger.log("info", format!("Compressed {:?} to {:?}", file, gz_path));
|
||||
return Ok(CompressionResult {
|
||||
compressed_path: gz_path,
|
||||
});
|
||||
}
|
||||
|
||||
let tar_gz_path = file.with_extension("").with_extension("tar.gz");
|
||||
|
||||
let output_file = File::create(&tar_gz_path)
|
||||
@@ -119,3 +152,23 @@ pub async fn decompress_large_tar_gz(
|
||||
|
||||
Ok(extracted_files)
|
||||
}
|
||||
|
||||
pub async fn gunzip_to_file(gz_path: &Path, out_path: &Path) -> Result<()> {
|
||||
let file = File::open(gz_path)
|
||||
.await
|
||||
.with_context(|| format!("Failed to open {}", gz_path.display()))?;
|
||||
let buf_reader = BufReader::with_capacity(8 * 1024 * 1024, file);
|
||||
let mut decoder = GzipDecoder::new(buf_reader);
|
||||
|
||||
let out = File::create(out_path)
|
||||
.await
|
||||
.with_context(|| format!("Failed to create {}", out_path.display()))?;
|
||||
let mut writer = tokio::io::BufWriter::new(out);
|
||||
|
||||
tokio::io::copy(&mut decoder, &mut writer).await?;
|
||||
writer.shutdown().await?;
|
||||
|
||||
info!("Gunzipped {:?} into {:?}", gz_path, out_path);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
+37
-2
@@ -47,8 +47,14 @@ pub fn full_file_name(encrypt: bool) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn full_file_path(file_name: &String) -> String {
|
||||
format!("backups/{}/{}", Utc::now().format("%Y-%m-%d"), file_name)
|
||||
pub fn full_file_path(file_name: &String, folder_name: Option<&str>) -> String {
|
||||
let folder_name = folder_name
|
||||
.map(str::trim)
|
||||
.map(|folder_name| folder_name.trim_matches(char::from(47)))
|
||||
.filter(|folder_name| !folder_name.is_empty())
|
||||
.unwrap_or("backups");
|
||||
|
||||
format!("{}/{}/{}", folder_name, Utc::now().format("%Y-%m-%d"), file_name)
|
||||
}
|
||||
|
||||
const CHUNK_SIZE: usize = 16 * 1024 * 1024;
|
||||
@@ -175,3 +181,32 @@ pub async fn decrypt_file_stream_gcm(
|
||||
writer.flush().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Decrypt a base64 `nonce(12) ‖ ciphertext ‖ tag(16)` AES-256-GCM envelope
|
||||
/// using the raw master key (STANDARD base64). Returns the plaintext bytes.
|
||||
pub fn decrypt_json_gcm(ciphertext_b64: &str, master_key_b64: &str) -> Result<Vec<u8>> {
|
||||
let master_key_bytes = general_purpose::STANDARD
|
||||
.decode(master_key_b64)
|
||||
.map_err(|_| anyhow::anyhow!("Invalid base64 master key"))?;
|
||||
|
||||
let data = general_purpose::STANDARD
|
||||
.decode(ciphertext_b64)
|
||||
.map_err(|_| anyhow::anyhow!("Invalid base64 ciphertext"))?;
|
||||
|
||||
if data.len() < 12 + 16 {
|
||||
return Err(anyhow::anyhow!("Ciphertext too short"));
|
||||
}
|
||||
|
||||
let key = Key::<Aes256Gcm>::try_from(master_key_bytes.as_slice())
|
||||
.map_err(|_| anyhow::anyhow!("Invalid AES-256 key length"))?;
|
||||
let cipher = Aes256Gcm::new(&key);
|
||||
|
||||
let nonce = Nonce::try_from(&data[..12])
|
||||
.map_err(|_| anyhow::anyhow!("Invalid nonce length"))?;
|
||||
|
||||
let plaintext = cipher
|
||||
.decrypt(&nonce, &data[12..])
|
||||
.map_err(|e| anyhow::anyhow!("AES-GCM decryption failed: {:?}", e))?;
|
||||
|
||||
Ok(plaintext)
|
||||
}
|
||||
|
||||
@@ -10,9 +10,9 @@ use std::str::FromStr;
|
||||
use tracing::debug;
|
||||
use tracing::info;
|
||||
|
||||
pub fn next_run_timestamp(expr: &str) -> i64 {
|
||||
let schedule = Schedule::from_str(expr).unwrap();
|
||||
schedule.upcoming(Local).next().unwrap().timestamp()
|
||||
pub fn next_run_timestamp(expr: &str) -> Option<i64> {
|
||||
let schedule = Schedule::from_str(expr).ok()?;
|
||||
Some(schedule.upcoming(Local).next()?.timestamp())
|
||||
}
|
||||
|
||||
pub async fn check_and_update_cron(
|
||||
@@ -38,8 +38,9 @@ pub async fn check_and_update_cron(
|
||||
}
|
||||
|
||||
Some(cron) => {
|
||||
let cron = normalize_cron(&cron);
|
||||
debug!("Task cron (normalized): {:?}", cron);
|
||||
let raw_cron = cron;
|
||||
let cron = normalize_cron(&raw_cron);
|
||||
debug!("Task cron normalized: unix \"{}\" -> crate \"{}\"", raw_cron, cron);
|
||||
|
||||
if exists {
|
||||
let raw: String = conn.hget(&redis_key, "data").await.unwrap();
|
||||
@@ -50,24 +51,21 @@ pub async fn check_and_update_cron(
|
||||
let metadata_changed = stored.metadata != metadata;
|
||||
|
||||
if cron_changed || args_changed || metadata_changed {
|
||||
upsert_task(conn, &task_name, task, &cron, args.clone(), metadata)
|
||||
.await
|
||||
.unwrap_or_else(|e| {
|
||||
tracing::error!("Failed to update task {}: {:?}", task_name, e);
|
||||
});
|
||||
|
||||
info!(
|
||||
"Task {} updated (cron: {}, args: {}, metadata: {})",
|
||||
task_name, cron_changed, args_changed, metadata_changed
|
||||
);
|
||||
match upsert_task(conn, &task_name, task, &cron, args.clone(), metadata).await {
|
||||
Ok(()) => info!(
|
||||
"Task {} updated (cron: {}, args: {}, metadata: {})",
|
||||
task_name, cron_changed, args_changed, metadata_changed
|
||||
),
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to update task {}: {:?}", task_name, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
upsert_task(conn, &task_name, task, &cron, args, metadata)
|
||||
.await
|
||||
.unwrap_or_else(|e| {
|
||||
tracing::error!("Failed to create task {}: {:?}", task_name, e);
|
||||
});
|
||||
info!("Task {} created", task_name);
|
||||
match upsert_task(conn, &task_name, task, &cron, args, metadata).await {
|
||||
Ok(()) => info!("Task {} created", task_name),
|
||||
Err(e) => tracing::error!("Failed to create task {}: {:?}", task_name, e),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,8 +48,17 @@ pub async fn scheduler_loop(mut conn: MultiplexedConnection) {
|
||||
task_clone.task, e
|
||||
);
|
||||
}
|
||||
let next_ts = next_run_timestamp(&task_clone.cron);
|
||||
let _: () = conn_clone.zadd(SCHEDULE_KEY, &key, next_ts).await.unwrap();
|
||||
match next_run_timestamp(&task_clone.cron) {
|
||||
Some(next_ts) => {
|
||||
let _: () = conn_clone.zadd(SCHEDULE_KEY, &key, next_ts).await.unwrap();
|
||||
}
|
||||
None => {
|
||||
error!(
|
||||
"Invalid cron expression for task={}: {}",
|
||||
task_clone.task, task_clone.cron
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
||||
|
||||
@@ -16,7 +16,16 @@ pub async fn upsert_task(
|
||||
metadata: Option<Value>,
|
||||
) -> redis::RedisResult<()> {
|
||||
let key = format!("redbeat:{}", name);
|
||||
let next_ts = next_run_timestamp(cron);
|
||||
let next_ts = match next_run_timestamp(cron) {
|
||||
Some(ts) => ts,
|
||||
None => {
|
||||
return Err(redis::RedisError::from((
|
||||
redis::ErrorKind::Client,
|
||||
"invalid cron expression",
|
||||
cron.to_string(),
|
||||
)));
|
||||
}
|
||||
};
|
||||
|
||||
let entry = PeriodicTask {
|
||||
task: task.to_string(),
|
||||
|
||||
+51
-3
@@ -1,7 +1,55 @@
|
||||
pub fn normalize_cron(expr: &str) -> String {
|
||||
if expr.split_whitespace().count() == 5 {
|
||||
format!("0 {}", expr)
|
||||
let fields: Vec<&str> = expr.split_whitespace().collect();
|
||||
|
||||
let (sec, mut rest): (String, Vec<String>) = match fields.len() {
|
||||
5 => ("0".to_string(), fields.iter().map(|s| s.to_string()).collect()),
|
||||
6 => (
|
||||
fields[0].to_string(),
|
||||
fields[1..].iter().map(|s| s.to_string()).collect(),
|
||||
),
|
||||
_ => return expr.to_string(),
|
||||
};
|
||||
|
||||
if let Some(last) = rest.last_mut() {
|
||||
*last = convert_dow(last);
|
||||
}
|
||||
|
||||
format!("{} {}", sec, rest.join(" "))
|
||||
}
|
||||
|
||||
fn convert_dow(field: &str) -> String {
|
||||
field
|
||||
.split(',')
|
||||
.map(convert_dow_part)
|
||||
.collect::<Vec<_>>()
|
||||
.join(",")
|
||||
}
|
||||
|
||||
fn convert_dow_part(part: &str) -> String {
|
||||
let (base, step) = match part.split_once('/') {
|
||||
Some((b, s)) => (b, Some(s)),
|
||||
None => (part, None),
|
||||
};
|
||||
|
||||
let converted = if let Some((start, end)) = base.split_once('-') {
|
||||
match (start.parse::<u8>(), end.parse::<u8>()) {
|
||||
(Ok(a), Ok(b)) if a <= 7 && b <= 7 => {
|
||||
format!("{}-{}", (a % 7) + 1, (b % 7) + 1)
|
||||
}
|
||||
_ => base.to_string(),
|
||||
}
|
||||
} else if let Ok(n) = base.parse::<u8>() {
|
||||
if n <= 7 {
|
||||
((n % 7) + 1).to_string()
|
||||
} else {
|
||||
base.to_string()
|
||||
}
|
||||
} else {
|
||||
expr.to_string()
|
||||
base.to_string()
|
||||
};
|
||||
|
||||
match step {
|
||||
Some(s) => format!("{}/{}", converted, s),
|
||||
None => converted,
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user