mirror of
https://github.com/Portabase/agent.git
synced 2026-09-11 02:27:10 +00:00
Compare commits
34 Commits
1.1.0
...
1.1.6-rc.1
| Author | SHA1 | Date | |
|---|---|---|---|
| 58dd0259f9 | |||
| 5b7453208b | |||
| 35933aae52 | |||
| 178628cb1c | |||
| 3fc60e3f40 | |||
| e62de95827 | |||
| fc3612fa3f | |||
| 18fe211ad1 | |||
| 74ada01232 | |||
| 4fef360c06 | |||
| 2647941008 | |||
| b4bdc89888 | |||
| 829e5bade3 | |||
| fc9a4ef8fb | |||
| 3c5734ed4e | |||
| 4b77896ecb | |||
| 40c4a4e2bb | |||
| 8a05b93ff3 | |||
| c186932a79 | |||
| bf74173cac | |||
| 3dcebf4fcc | |||
| 4de2b81b33 | |||
| b06f9d8186 | |||
| f5a2182723 | |||
| e8a96980f1 | |||
| a64792bcc6 | |||
| 2d4542be5f | |||
| 4a00ed574d | |||
| fd220598fc | |||
| d8b239a123 | |||
| ebf0526366 | |||
| 6461c1523b | |||
| 2809373de1 | |||
| 2bcb393d60 |
+2
-2
@@ -22,5 +22,5 @@ keywords:
|
||||
- self-hosted
|
||||
- portabase
|
||||
license: Apache-2.0
|
||||
version: 1.1.0
|
||||
date-released: "2026-02-13"
|
||||
version: 1.1.6-rc.1
|
||||
date-released: "2026-02-23"
|
||||
Generated
+1
-1
@@ -2851,7 +2851,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "portabase-agent"
|
||||
version = "1.1.0-rc.2"
|
||||
version = "1.1.5"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"aes-gcm",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "portabase-agent"
|
||||
version = "1.1.0"
|
||||
version = "1.1.6-rc.1"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
|
||||
@@ -30,4 +30,21 @@ seed-postgres-1gb:
|
||||
docker exec -i -e PGPASSWORD=$$PG_PASSWORD $$PG_CONTAINER \
|
||||
psql -U $$PG_USER -d $$PG_DB < ./scripts/postgres/seed-1gb.sql
|
||||
|
||||
|
||||
SQLITE_SEED_FILE := $(if $(filter big,$(SEED)),./scripts/sqlite/seed-big.sql,./scripts/sqlite/seed.sql)
|
||||
|
||||
seed-sqlite:
|
||||
@echo "Seeding Sqlite..."
|
||||
@echo "Run as root to fix permissions inside the volume"
|
||||
docker exec -u 0 -it db-sqlite sh -c "chmod -R 777 /workspace/data"
|
||||
@echo "Create the database file (if it doesn’t exist)"
|
||||
docker exec -u 0 -it db-sqlite sh -c "touch /workspace/data/app.db"
|
||||
@echo "Seed the database"
|
||||
docker exec -i db-sqlite sh -c "sqlite3 /workspace/data/app.db" < $(SQLITE_SEED_FILE)
|
||||
@echo "Verify"
|
||||
docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT name FROM sqlite_master WHERE type='table';"
|
||||
@echo "Done"
|
||||
|
||||
|
||||
|
||||
seed-all: seed-mongo seed-mysql seed-postgres seed-postgres-1gb
|
||||
|
||||
+6
-2
@@ -34,11 +34,15 @@
|
||||
"name": "Test database 5 - MongoDB",
|
||||
"database": "testdb",
|
||||
"type": "mongodb",
|
||||
"username": "",
|
||||
"password": "",
|
||||
"port": 27017,
|
||||
"host": "db-mongodb",
|
||||
"generated_id": "16678147-ff7e-4c97-8c83-0adeff214681"
|
||||
},
|
||||
{
|
||||
"name": "Test database 6 - SQLite DB",
|
||||
"type": "sqlite",
|
||||
"path": "/sqlite-data/workspace/data/app.db",
|
||||
"generated_id": "16678178-ff7e-4c97-8c83-0adeff214681"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
+55
-44
@@ -11,13 +11,13 @@ services:
|
||||
# - ./databases.toml:/config/config.toml
|
||||
- cargo-registry:/usr/local/cargo/registry
|
||||
- cargo-git:/usr/local/cargo/git
|
||||
- cargo-target:/app/target
|
||||
# - cargo-target:/app/target
|
||||
- sqlite-data:/sqlite-data/workspace/data
|
||||
environment:
|
||||
APP_ENV: development
|
||||
LOG: debug
|
||||
TZ: "Europe/Paris"
|
||||
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiMzI3YTU4ODktYzE0MC00ODMzLTk1ZWMtNTBmMmU2NTFlZmJhIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ=="
|
||||
# EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiZWE2NTg1MDctZTA5My00NDUxLWIxZDAtMDgwZWZjMGNmNWYzIiwicHVibGljS2V5IjoiLS0tLS1CRUdJTiBSU0EgUFVCTElDIEtFWS0tLS0tXG5NSUlCQ2dLQ0FRRUE5TWV4M2pmdnVLdFB5YU1ERnh2Ulp2dmd3YkRJQ2JzQi81Wll5NDNSVVRBaXZRYjJiSDdYXG5qRHBQd1lJeCs4UFBrbHlRbDVMQzV1UWZEaCs4SVd4OG1LZ3FvMXpWMkdiZXdGbEdEWFYxVEdyU1ZEU25aSWR4XG52bWdYc29EeXhVMlJvWUFUMS9YMWxuc2YxenZKdkFMTkhXdEhRdk42SjVDZTFSMmFsendVRGFEVXlJNzRmSldQXG5tNTh0SDMrYklXL0VVTXdjaWNxM0oySWw3Vm9KNkZNUHJQL1ZSOWEvdFF1SU1qa200MXpFY2NscExPa2luRkxuXG54NmVUWkFSZUpya2UrbnRvZ2t4TGEyRWV5a1lUNzB4V3hKNWp5ZExBVnRvNkkyQlVLVVJoTkowTUFaU29NYUtvXG5iMGJRcnY1UzExZWllMnMrT2I3aTYzSFpkVUx0UmV1MVJ3SURBUUFCXG4tLS0tLUVORCBSU0EgUFVCTElDIEtFWS0tLS0tXG4ifQ"
|
||||
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiZjg4Y2E0MDMtNDgwOS00NGM4LTlkZjItY2VkNWYwYzhkNTM2IiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ=="
|
||||
#POOLING: 1
|
||||
#DATABASES_CONFIG_FILE: "config.toml"
|
||||
extra_hosts:
|
||||
@@ -54,54 +54,65 @@ services:
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
#
|
||||
# db-mongodb-auth:
|
||||
# container_name: db-mongodb-auth
|
||||
# image: mongo:latest
|
||||
# ports:
|
||||
# - "27082:27017"
|
||||
# environment:
|
||||
# MONGO_INITDB_ROOT_USERNAME: root
|
||||
# MONGO_INITDB_ROOT_PASSWORD: rootpassword
|
||||
# MONGO_INITDB_DATABASE: testdbauth
|
||||
# command: mongod --auth
|
||||
# networks:
|
||||
# - portabase
|
||||
# volumes:
|
||||
# - mongodb-data-auth:/data/db
|
||||
# healthcheck:
|
||||
# test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ]
|
||||
# interval: 5s
|
||||
# timeout: 5s
|
||||
# retries: 10
|
||||
#
|
||||
# db-mongodb:
|
||||
# container_name: db-mongodb
|
||||
# image: mongo:latest
|
||||
# ports:
|
||||
# - "27083:27017"
|
||||
# volumes:
|
||||
# - mongodb-data:/data/db
|
||||
# healthcheck:
|
||||
# test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ]
|
||||
# interval: 5s
|
||||
# timeout: 5s
|
||||
# retries: 10
|
||||
# environment:
|
||||
# MONGO_INITDB_DATABASE: testdb
|
||||
# networks:
|
||||
# - portabase
|
||||
|
||||
db-mongodb-auth:
|
||||
container_name: db-mongodb-auth
|
||||
image: mongo:latest
|
||||
ports:
|
||||
- "27082:27017"
|
||||
environment:
|
||||
MONGO_INITDB_ROOT_USERNAME: root
|
||||
MONGO_INITDB_ROOT_PASSWORD: rootpassword
|
||||
MONGO_INITDB_DATABASE: testdbauth
|
||||
command: mongod --auth
|
||||
networks:
|
||||
- portabase
|
||||
volumes:
|
||||
- mongodb-data-auth:/data/db
|
||||
healthcheck:
|
||||
test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
db-mongodb:
|
||||
container_name: db-mongodb
|
||||
image: mongo:latest
|
||||
ports:
|
||||
- "27083:27017"
|
||||
volumes:
|
||||
- mongodb-data:/data/db
|
||||
healthcheck:
|
||||
test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
environment:
|
||||
MONGO_INITDB_DATABASE: testdb
|
||||
networks:
|
||||
- portabase
|
||||
|
||||
sqlite:
|
||||
container_name: db-sqlite
|
||||
image: keinos/sqlite3
|
||||
volumes:
|
||||
- sqlite-data:/workspace/data
|
||||
working_dir: /workspace
|
||||
command: tail -f /dev/null
|
||||
stdin_open: true
|
||||
tty: true
|
||||
|
||||
|
||||
volumes:
|
||||
cargo-registry:
|
||||
cargo-git:
|
||||
cargo-target:
|
||||
|
||||
# cargo-target:
|
||||
|
||||
postgres-data:
|
||||
mariadb-data:
|
||||
# mongodb-data:
|
||||
# mongodb-data-auth:
|
||||
mongodb-data:
|
||||
mongodb-data-auth:
|
||||
sqlite-data:
|
||||
|
||||
networks:
|
||||
portabase:
|
||||
|
||||
@@ -15,6 +15,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
zlib1g \
|
||||
curl \
|
||||
mariadb-client \
|
||||
sqlite3 \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
@@ -92,6 +93,7 @@ RUN apt-get update && apt-get install -y \
|
||||
libncurses6 \
|
||||
zlib1g \
|
||||
mariadb-client \
|
||||
sqlite3 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
|
||||
|
||||
+15
-2
@@ -1,6 +1,5 @@
|
||||
# Seed instructions
|
||||
|
||||
## MongoDB
|
||||
|
||||
```bash
|
||||
make seed-mongo
|
||||
@@ -9,5 +8,19 @@ make seed-mysql
|
||||
make seed-mysql-1gb
|
||||
make seed-postgres
|
||||
make seed-postgres-1gb
|
||||
make seed-all
|
||||
make seed-all
|
||||
make seed-sqlite
|
||||
make seed-sqlite SEED=big
|
||||
```
|
||||
|
||||
## Verify commands
|
||||
|
||||
### Sqlite
|
||||
|
||||
```bash
|
||||
docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT * FROM users LIMIT 10;"
|
||||
```
|
||||
|
||||
```bash
|
||||
docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT name FROM sqlite_master WHERE type='table';"
|
||||
```
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
PRAGMA foreign_keys = ON;
|
||||
BEGIN TRANSACTION;
|
||||
|
||||
-- Drop tables
|
||||
DROP TABLE IF EXISTS users;
|
||||
DROP TABLE IF EXISTS projects;
|
||||
DROP TABLE IF EXISTS tasks;
|
||||
|
||||
-- Users
|
||||
CREATE TABLE users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
email TEXT NOT NULL UNIQUE,
|
||||
full_name TEXT NOT NULL,
|
||||
role TEXT NOT NULL CHECK (role IN ('admin','manager','user')),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
-- Projects
|
||||
CREATE TABLE projects (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT,
|
||||
owner_id INTEGER NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
FOREIGN KEY (owner_id) REFERENCES users(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- Tasks
|
||||
CREATE TABLE tasks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
project_id INTEGER NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
status TEXT NOT NULL CHECK (status IN ('todo','in_progress','done')),
|
||||
priority INTEGER NOT NULL DEFAULT 3,
|
||||
due_date TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- Seed minimal users
|
||||
INSERT INTO users (email, full_name, role) VALUES
|
||||
('admin@example.com', 'System Admin', 'admin'),
|
||||
('manager@example.com', 'Project Manager', 'manager'),
|
||||
('user@example.com', 'Standard User', 'user');
|
||||
|
||||
-- Generate 10_000 projects
|
||||
WITH RECURSIVE numbers(x) AS (
|
||||
SELECT 1
|
||||
UNION ALL
|
||||
SELECT x+1 FROM numbers WHERE x<10000
|
||||
)
|
||||
INSERT INTO projects (name, description, owner_id)
|
||||
SELECT
|
||||
'Project #' || x,
|
||||
'Auto-generated project description for project #' || x,
|
||||
(1 + (x % 3)) -- cycle users 1..3
|
||||
FROM numbers;
|
||||
|
||||
-- Generate 100_000 tasks
|
||||
WITH RECURSIVE numbers(x) AS (
|
||||
SELECT 1
|
||||
UNION ALL
|
||||
SELECT x+1 FROM numbers WHERE x<100000
|
||||
)
|
||||
INSERT INTO tasks (project_id, title, status, priority, due_date)
|
||||
SELECT
|
||||
(1 + (x % 10000)), -- project id cycle
|
||||
'Task #' || x,
|
||||
CASE (x % 3) WHEN 0 THEN 'todo' WHEN 1 THEN 'in_progress' ELSE 'done' END,
|
||||
1 + (x % 5),
|
||||
date('now', '+' || (x % 30) || ' days')
|
||||
FROM numbers;
|
||||
|
||||
COMMIT;
|
||||
@@ -0,0 +1,59 @@
|
||||
PRAGMA foreign_keys = ON;
|
||||
|
||||
BEGIN TRANSACTION;
|
||||
|
||||
-- Drop existing tables (idempotent reset)
|
||||
DROP TABLE IF EXISTS users;
|
||||
DROP TABLE IF EXISTS projects;
|
||||
DROP TABLE IF EXISTS tasks;
|
||||
|
||||
-- Users
|
||||
CREATE TABLE users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
email TEXT NOT NULL UNIQUE,
|
||||
full_name TEXT NOT NULL,
|
||||
role TEXT NOT NULL CHECK (role IN ('admin','manager','user')),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
-- Projects
|
||||
CREATE TABLE projects (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT,
|
||||
owner_id INTEGER NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
FOREIGN KEY (owner_id) REFERENCES users(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- Tasks
|
||||
CREATE TABLE tasks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
project_id INTEGER NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
status TEXT NOT NULL CHECK (status IN ('todo','in_progress','done')),
|
||||
priority INTEGER NOT NULL DEFAULT 3,
|
||||
due_date TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- Seed users
|
||||
INSERT INTO users (email, full_name, role) VALUES
|
||||
('admin@example.com', 'System Admin', 'admin'),
|
||||
('manager@example.com', 'Project Manager', 'manager'),
|
||||
('user@example.com', 'Standard User', 'user');
|
||||
|
||||
-- Seed projects
|
||||
INSERT INTO projects (name, description, owner_id) VALUES
|
||||
('Internal Tooling', 'Backoffice automation platform', 2),
|
||||
('Client Portal', 'Customer-facing SaaS interface', 2);
|
||||
|
||||
-- Seed tasks
|
||||
INSERT INTO tasks (project_id, title, status, priority, due_date) VALUES
|
||||
(1, 'Define architecture', 'done', 1, date('now', '+3 days')),
|
||||
(1, 'Implement authentication', 'in_progress', 1, date('now', '+7 days')),
|
||||
(2, 'Design landing page', 'todo', 2, date('now', '+5 days')),
|
||||
(2, 'Setup CI/CD', 'todo', 2, date('now', '+10 days'));
|
||||
|
||||
COMMIT;
|
||||
@@ -6,6 +6,7 @@ use crate::services::config::{DatabaseConfig, DbType};
|
||||
use anyhow::Result;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use crate::domain::sqlite::database::SqliteDatabase;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait Database: Send + Sync {
|
||||
@@ -27,6 +28,7 @@ impl DatabaseFactory {
|
||||
DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)),
|
||||
DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)),
|
||||
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
|
||||
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,6 +41,7 @@ impl DatabaseFactory {
|
||||
DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)),
|
||||
DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)),
|
||||
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
|
||||
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,4 +2,5 @@ pub mod factory;
|
||||
pub mod postgres;
|
||||
pub mod mysql;
|
||||
mod mongodb;
|
||||
mod sqlite;
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ pub async fn run(
|
||||
|
||||
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
|
||||
let mongodump = select_mongo_path().join("mongodump");
|
||||
let uri = get_mongo_uri(cfg.clone());
|
||||
let uri = get_mongo_uri(cfg.clone())?;
|
||||
|
||||
let output = Command::new(mongodump)
|
||||
.arg(format!("--uri={}", uri))
|
||||
|
||||
@@ -3,7 +3,7 @@ use anyhow::Result;
|
||||
use mongodb::Client;
|
||||
|
||||
pub async fn connect(cfg: DatabaseConfig) -> Result<Client> {
|
||||
let uri = get_mongo_uri(cfg);
|
||||
let uri = get_mongo_uri(cfg)?;
|
||||
let mut options = mongodb::options::ClientOptions::parse(&uri).await?;
|
||||
options.server_selection_timeout = Some(std::time::Duration::from_secs(3));
|
||||
options.connect_timeout = Some(std::time::Duration::from_secs(3));
|
||||
@@ -15,14 +15,16 @@ pub fn select_mongo_path() -> std::path::PathBuf {
|
||||
"/usr/local/mongodb/bin".to_string().into()
|
||||
}
|
||||
|
||||
pub fn get_mongo_uri(cfg: DatabaseConfig) -> String {
|
||||
if cfg.username.is_empty() {
|
||||
format!("mongodb://{}:{}/{}", cfg.host, cfg.port, cfg.database)
|
||||
pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result<String> {
|
||||
|
||||
if cfg.username.is_empty() && cfg.password.is_empty() {
|
||||
Ok(format!("mongodb://{}:{}/{}", cfg.host, cfg.port, cfg.database))
|
||||
} else {
|
||||
format!(
|
||||
Ok(format!(
|
||||
"mongodb://{}:{}@{}:{}/{}?authSource=admin",
|
||||
cfg.username, cfg.password, cfg.host, cfg.port, cfg.database
|
||||
)
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,14 +1,20 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use crate::domain::mongodb::connection::connect;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::Result;
|
||||
use mongodb::bson::doc;
|
||||
use tracing::{error};
|
||||
use crate::domain::mongodb::connection::connect;
|
||||
use tracing::error;
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
|
||||
let client = connect(cfg.clone()).await?;
|
||||
let db_name = if cfg.username.is_empty() { &cfg.database } else { "admin" };
|
||||
|
||||
let db_name = if cfg.username.is_empty() && cfg.password.is_empty() {
|
||||
&cfg.database
|
||||
} else {
|
||||
"admin"
|
||||
};
|
||||
|
||||
match client.database(db_name).run_command(doc! {"ping": 1}).await {
|
||||
Ok(_) => Ok(true),
|
||||
Err(e) => {
|
||||
|
||||
@@ -10,7 +10,7 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
|
||||
debug!("Starting MongoDB restore for database {}", cfg.name);
|
||||
|
||||
let mongorestore = select_mongo_path().join("mongorestore");
|
||||
let uri = get_mongo_uri(cfg.clone());
|
||||
let uri = get_mongo_uri(cfg.clone())?;
|
||||
|
||||
let output = Command::new(mongorestore)
|
||||
.arg(format!("--uri={}", uri))
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use crate::domain::mysql::connection::server_version;
|
||||
use crate::domain::mysql::connection::{server_version};
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -3,6 +3,7 @@ use std::process::Command;
|
||||
use anyhow::Result;
|
||||
|
||||
pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
|
||||
|
||||
let output = Command::new("mysql")
|
||||
.arg("--host").arg(&cfg.host)
|
||||
.arg("--port").arg(cfg.port.to_string())
|
||||
@@ -25,3 +26,4 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
|
||||
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ impl MySQLDatabase {
|
||||
|
||||
fn build_env(&self) -> HashMap<String, String> {
|
||||
let mut envs = std::env::vars().collect::<HashMap<_, _>>();
|
||||
envs.insert("MYSQL_PWD".to_string(), self.cfg.password.clone());
|
||||
envs.insert("MYSQL_PWD".to_string(), self.cfg.password.to_string());
|
||||
envs
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,19 +1,27 @@
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::Context;
|
||||
use std::collections::HashMap;
|
||||
use std::process::Command;
|
||||
use tokio::process::Command;
|
||||
use tokio::time::{Duration, timeout};
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::Result<bool> {
|
||||
let output = Command::new("mysqladmin")
|
||||
.arg("--host")
|
||||
|
||||
let mut cmd = Command::new("mysqladmin");
|
||||
cmd.arg("--host")
|
||||
.arg(cfg.host)
|
||||
.arg("--port")
|
||||
.arg(cfg.port.to_string())
|
||||
.arg("--user")
|
||||
.arg(cfg.username)
|
||||
.arg("ping")
|
||||
.envs(env)
|
||||
.output()
|
||||
.with_context(|| format!("Failed to ping MySQL server {}", cfg.name))?;
|
||||
Ok(output.status.success())
|
||||
.envs(env);
|
||||
|
||||
let result = timeout(Duration::from_secs(10), cmd.output()).await;
|
||||
|
||||
match result {
|
||||
Ok(output) => {
|
||||
let output = output?;
|
||||
Ok(output.status.success())
|
||||
}
|
||||
Err(_) => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::fs::File;
|
||||
use std::io::{Read, Write};
|
||||
use anyhow::{Context, Result};
|
||||
use tracing::{debug, error, info};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
|
||||
let handle = tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
@@ -55,7 +54,8 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
|
||||
.with_context(|| format!("Failed to start mysql restore for {}", cfg.name))?;
|
||||
|
||||
let mut stdin = child.stdin.take().context("Failed to open child stdin")?;
|
||||
stdin.write_all(sql_content.as_bytes())
|
||||
stdin
|
||||
.write_all(sql_content.as_bytes())
|
||||
.context("Failed to write SQL content to mysql stdin")?;
|
||||
stdin.flush()?;
|
||||
drop(stdin);
|
||||
@@ -74,8 +74,7 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
|
||||
Ok(())
|
||||
});
|
||||
|
||||
handle
|
||||
.await??;
|
||||
handle.await??;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use anyhow::Result;
|
||||
use tracing::{debug, error, info};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
use super::connection::{select_pg_path, server_version};
|
||||
use super::format::PostgresDumpFormat;
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
use std::path::Path;
|
||||
use crate::domain::postgres::format::PostgresDumpFormat;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::Result;
|
||||
use std::path::Path;
|
||||
use tokio_postgres::{Client, NoTls};
|
||||
use tracing::info;
|
||||
use tracing::{error, info};
|
||||
|
||||
pub async fn connect(cfg: &DatabaseConfig) -> Result<Client> {
|
||||
info!("Connecting to postgres database {}:{}", cfg.host, cfg.port);
|
||||
let dsn = format!(
|
||||
"host={} port={} user={} password={} dbname={}",
|
||||
cfg.host, cfg.port, cfg.username, cfg.password, cfg.database
|
||||
@@ -14,7 +15,7 @@ pub async fn connect(cfg: &DatabaseConfig) -> Result<Client> {
|
||||
let (client, connection) = tokio_postgres::connect(&dsn, NoTls).await?;
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = connection.await {
|
||||
tracing::error!("Postgres connection error: {}", e);
|
||||
error!("Postgres connection error: {}", e);
|
||||
}
|
||||
});
|
||||
Ok(client)
|
||||
@@ -34,7 +35,7 @@ pub fn select_pg_path(version: &str) -> std::path::PathBuf {
|
||||
|
||||
pub async fn terminate_connections(cfg: &DatabaseConfig) -> Result<()> {
|
||||
let mut admin = cfg.clone();
|
||||
admin.database = "postgres".into();
|
||||
admin.database = "postgres".to_string().into();
|
||||
|
||||
let client = connect(&admin).await?;
|
||||
|
||||
|
||||
@@ -39,7 +39,6 @@ impl Database for PostgresDatabase {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
let res = backup::run(self.cfg.clone(), self.format, dir.to_path_buf()).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
|
||||
res
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use anyhow::Result;
|
||||
use tracing::{debug, error, info};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
use super::connection::{select_pg_path, server_version, terminate_connections};
|
||||
use super::format::PostgresDumpFormat;
|
||||
@@ -14,6 +14,7 @@ pub async fn run(
|
||||
) -> Result<()> {
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
debug!("Starting restore for database {}", cfg.name);
|
||||
|
||||
let version = match futures::executor::block_on(server_version(&cfg)) {
|
||||
Ok(v) => {
|
||||
debug!("Postgres version detected: {}", v);
|
||||
@@ -91,8 +92,6 @@ pub async fn run(
|
||||
let dec = flate2::read::GzDecoder::new(tar_gz);
|
||||
let mut archive = tar::Archive::new(dec);
|
||||
|
||||
|
||||
|
||||
let tmp_dir = match tempfile::TempDir::new() {
|
||||
Ok(d) => d,
|
||||
Err(e) => {
|
||||
@@ -104,8 +103,6 @@ pub async fn run(
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
|
||||
if let Err(e) = archive.unpack(tmp_dir.path()) {
|
||||
error!("Failed to unpack FD archive for {}: {:?}", cfg.name, e);
|
||||
return Err(e.into());
|
||||
@@ -113,7 +110,6 @@ pub async fn run(
|
||||
|
||||
debug!("Listing contents of temp dir: {}", tmp_dir.path().display());
|
||||
|
||||
|
||||
for entry in std::fs::read_dir(tmp_dir.path())? {
|
||||
if let Ok(entry) = entry {
|
||||
let path = entry.path();
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
pub async fn run(
|
||||
cfg: DatabaseConfig,
|
||||
backup_dir: PathBuf,
|
||||
file_extension: &'static str,
|
||||
) -> Result<PathBuf> {
|
||||
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
|
||||
debug!("Starting SQLite backup for database {}", cfg.name);
|
||||
|
||||
let db_path_str = if cfg.path.is_empty() {
|
||||
anyhow::bail!("Database path not configured");
|
||||
} else {
|
||||
cfg.path.as_str().to_string()
|
||||
};
|
||||
|
||||
let db_path = PathBuf::from(db_path_str);
|
||||
|
||||
if !db_path.exists() {
|
||||
anyhow::bail!("SQLite database file not found: {}", db_path.display());
|
||||
}
|
||||
|
||||
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
|
||||
|
||||
let output = Command::new("sqlite3")
|
||||
.arg(db_path.as_os_str())
|
||||
.arg(format!(".backup '{}'", file_path.display()))
|
||||
.output()
|
||||
.context("SQLite backup command failed to start")?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
error!("SQLite backup failed for {}: {}", cfg.name, stderr);
|
||||
anyhow::bail!("SQLite backup failed for {}: {}", cfg.name, stderr);
|
||||
}
|
||||
|
||||
info!("SQLite backup completed for {}", cfg.name);
|
||||
Ok(file_path)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use super::{backup, ping, restore};
|
||||
use crate::domain::factory::Database;
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use crate::utils::locks::{DbOpLock, FileLock};
|
||||
|
||||
pub struct SqliteDatabase {
|
||||
cfg: DatabaseConfig,
|
||||
}
|
||||
|
||||
impl SqliteDatabase {
|
||||
pub fn new(cfg: DatabaseConfig) -> Self {
|
||||
Self { cfg }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Database for SqliteDatabase {
|
||||
fn file_extension(&self) -> &'static str {
|
||||
".backup"
|
||||
}
|
||||
|
||||
async fn ping(&self) -> Result<bool> {
|
||||
ping::run(self.cfg.clone()).await
|
||||
}
|
||||
|
||||
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
|
||||
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
|
||||
async fn restore(&self, file: &Path) -> Result<()> {
|
||||
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
|
||||
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
|
||||
FileLock::release(&self.cfg.generated_id).await?;
|
||||
res
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
mod backup;
|
||||
mod restore;
|
||||
mod ping;
|
||||
pub mod database;
|
||||
@@ -0,0 +1,5 @@
|
||||
use crate::services::config::DatabaseConfig;
|
||||
|
||||
pub async fn run(_cfg: DatabaseConfig) -> anyhow::Result<bool> {
|
||||
Ok(true)
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
use crate::services::config::DatabaseConfig;
|
||||
use anyhow::{Context, Result};
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
debug!("Starting SQLite restore for database {}", cfg.name);
|
||||
|
||||
let db_path_str = if cfg.path.is_empty() {
|
||||
anyhow::bail!("Database path not configured");
|
||||
} else {
|
||||
cfg.path.as_str().to_string()
|
||||
};
|
||||
|
||||
let db_path = PathBuf::from(db_path_str);
|
||||
|
||||
if !restore_file.exists() {
|
||||
anyhow::bail!("Restore file not found: {}", restore_file.display());
|
||||
}
|
||||
|
||||
if db_path.exists() {
|
||||
std::fs::remove_file(&db_path)
|
||||
.with_context(|| format!("Failed to remove existing DB {}", db_path.display()))?;
|
||||
}
|
||||
|
||||
let output = Command::new("sqlite3")
|
||||
.arg(db_path.as_os_str())
|
||||
.arg(format!(".restore '{}'", restore_file.display()))
|
||||
.output()
|
||||
.with_context(|| format!("Failed to run sqlite3 restore for {}", cfg.name))?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
error!("SQLite restore failed for {}: {}", cfg.name, stderr);
|
||||
anyhow::bail!("SQLite restore failed for {}", cfg.name);
|
||||
}
|
||||
|
||||
info!("SQLite restore completed for {}", cfg.name);
|
||||
Ok(())
|
||||
})
|
||||
.await?
|
||||
}
|
||||
+50
-21
@@ -69,7 +69,6 @@ impl BackupService {
|
||||
return;
|
||||
}
|
||||
Ok(false) => {
|
||||
|
||||
match ctx
|
||||
.api
|
||||
.backup_create(
|
||||
@@ -85,13 +84,24 @@ impl BackupService {
|
||||
info!("Created temp directory {}", tmp_path.display());
|
||||
match BackupService::run(db_cfg, &tmp_path).await {
|
||||
Ok(mut result) => {
|
||||
let backup_id = backup_created_result.unwrap().backup.id;
|
||||
|
||||
if result.status == "failed" {
|
||||
error!("Backup failed early for {}", result.generated_id);
|
||||
let service = BackupService { ctx: ctx.clone() };
|
||||
let _ = service
|
||||
.send_result(result, vec![], &backup_id)
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
if let Some(backup_file) = result.backup_file.take() {
|
||||
match compress_to_tar_gz_large(&backup_file).await {
|
||||
Ok(compression_result) => {
|
||||
result.backup_file =
|
||||
Some(compression_result.compressed_path);
|
||||
let service = BackupService { ctx: ctx.clone() };
|
||||
let backup_id = backup_created_result.unwrap().backup.id;
|
||||
match service
|
||||
.upload(
|
||||
result.clone(),
|
||||
@@ -107,7 +117,7 @@ impl BackupService {
|
||||
.send_result(
|
||||
result,
|
||||
upload_result,
|
||||
&backup_id
|
||||
&backup_id,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -147,7 +157,7 @@ impl BackupService {
|
||||
}
|
||||
Err(e) => error!("Backup creation failed: {}", e),
|
||||
}
|
||||
},
|
||||
}
|
||||
Err(e) => error!("An error occurred while checking lock : {}", e),
|
||||
}
|
||||
}
|
||||
@@ -162,7 +172,16 @@ impl BackupService {
|
||||
let generated_id = cfg.generated_id.clone();
|
||||
let db_type = cfg.db_type.clone();
|
||||
|
||||
let reachable = db_instance.ping().await.unwrap_or(false);
|
||||
|
||||
let reachable = match db_instance.ping().await {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
error!("Ping failed: {}", e);
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
info!("Reachable: {}", reachable);
|
||||
if !reachable {
|
||||
return Ok(BackupResult {
|
||||
@@ -258,8 +277,12 @@ impl BackupService {
|
||||
} else {
|
||||
"failed"
|
||||
};
|
||||
info!("Storage {} uploaded to remote path {:?}", storage_id, upload_result.remote_file_path);
|
||||
|
||||
if status != "success" {
|
||||
return upload_result;
|
||||
}
|
||||
|
||||
info!("Storage {} uploaded to remote path {:?}", storage_id, upload_result.remote_file_path);
|
||||
|
||||
let (remote_path, total_size) = match (
|
||||
&upload_result.remote_file_path,
|
||||
@@ -276,7 +299,7 @@ impl BackupService {
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
match self.ctx.api.backup_upload_status(
|
||||
self.ctx.edge_key.agent_id.clone(),
|
||||
generated_id.clone(),
|
||||
@@ -284,12 +307,12 @@ impl BackupService {
|
||||
status,
|
||||
remote_path,
|
||||
total_size,
|
||||
backup_id
|
||||
backup_id,
|
||||
).await {
|
||||
Ok(_) => {
|
||||
upload_result
|
||||
},
|
||||
Err(err)=> {
|
||||
}
|
||||
Err(err) => {
|
||||
error!(
|
||||
"backup_upload_status failed (generated_id={}, storage_id={}): {}",
|
||||
generated_id, storage_id, err
|
||||
@@ -355,17 +378,23 @@ impl BackupService {
|
||||
"failed"
|
||||
};
|
||||
|
||||
let file_size = upload_results
|
||||
.iter()
|
||||
.map(|r| r.total_size)
|
||||
.try_fold((0u64, 0u64), |(sum, count), v| {
|
||||
match v {
|
||||
Some(size) => Ok((sum + size, count + 1)),
|
||||
None => Err(()), // stop and return None
|
||||
}
|
||||
})
|
||||
.ok()
|
||||
.map(|(sum, count)| sum / count);
|
||||
let file_size = if status == "failed" {
|
||||
None
|
||||
} else {
|
||||
let mut sum = 0u64;
|
||||
let mut count = 0u64;
|
||||
|
||||
for size in upload_results.iter().filter_map(|r| r.total_size) {
|
||||
sum += size;
|
||||
count += 1;
|
||||
}
|
||||
|
||||
if count == 0 {
|
||||
None
|
||||
} else {
|
||||
Some(sum / count)
|
||||
}
|
||||
};
|
||||
|
||||
match self
|
||||
.ctx
|
||||
|
||||
+94
-6
@@ -9,6 +9,7 @@ use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use toml;
|
||||
use tracing::info;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
@@ -16,8 +17,8 @@ pub enum DbType {
|
||||
Mysql,
|
||||
Mariadb,
|
||||
Postgresql,
|
||||
MongoDB
|
||||
// Sqlite,
|
||||
MongoDB,
|
||||
Sqlite,
|
||||
// Add other DB types if needed
|
||||
}
|
||||
|
||||
@@ -28,7 +29,7 @@ impl DbType {
|
||||
DbType::Mariadb => "mysql",
|
||||
DbType::Postgresql => "postgresql",
|
||||
DbType::MongoDB => "mongodb",
|
||||
// DbType::Sqlite => "sqlite",
|
||||
DbType::Sqlite => "sqlite",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -45,6 +46,7 @@ pub struct DatabaseConfig {
|
||||
pub port: u16,
|
||||
pub host: String,
|
||||
pub generated_id: String,
|
||||
pub path: String,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
@@ -53,6 +55,29 @@ pub struct DatabasesConfig {
|
||||
pub databases: Vec<DatabaseConfig>,
|
||||
}
|
||||
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct InputDatabaseConfig {
|
||||
pub name: String,
|
||||
pub database: Option<String>,
|
||||
#[serde(rename = "type")]
|
||||
pub db_type: DbType,
|
||||
pub username: Option<String>,
|
||||
pub password: Option<String>,
|
||||
pub port: Option<u16>,
|
||||
pub host: Option<String>,
|
||||
pub generated_id: String,
|
||||
pub path: Option<String>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct InputDatabasesConfig {
|
||||
pub databases: Vec<InputDatabaseConfig>,
|
||||
}
|
||||
|
||||
|
||||
pub struct ConfigService {
|
||||
ctx: Arc<Context>,
|
||||
}
|
||||
@@ -95,7 +120,7 @@ impl ConfigService {
|
||||
file.read_to_string(&mut contents)
|
||||
.map_err(|e| format!("Failed to read config file: {}", e))?;
|
||||
|
||||
let config: DatabasesConfig = match extension {
|
||||
let input_config: InputDatabasesConfig = match extension {
|
||||
"json" => {
|
||||
serde_json::from_str(&contents).map_err(|e| format!("JSON parsing error: {}", e))?
|
||||
}
|
||||
@@ -105,8 +130,71 @@ impl ConfigService {
|
||||
_ => return Err("Unsupported config file format. Use .json or .toml".to_string()),
|
||||
};
|
||||
|
||||
info!("Databases : {:?} instances loaded", config.databases.len());
|
||||
fn required<T: Clone>(opt: &Option<T>, db_name: &str, field_name: &str) -> Result<T, String> {
|
||||
match opt {
|
||||
Some(v) => Ok(v.clone()),
|
||||
None => {
|
||||
let msg = format!("Missing required field '{}' for database '{}'", field_name, db_name);
|
||||
Err(msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(config)
|
||||
fn optional<T: Clone>(opt: &Option<T>) -> T where T: Default {
|
||||
opt.clone().unwrap_or_default()
|
||||
}
|
||||
|
||||
let mut databases = Vec::with_capacity(input_config.databases.len());
|
||||
|
||||
for db in input_config.databases {
|
||||
if Uuid::parse_str(&db.generated_id).is_err() {
|
||||
return Err(format!("Invalid UUID for database '{}'", db.name));
|
||||
}
|
||||
|
||||
let username = match db.db_type {
|
||||
DbType::Postgresql | DbType::Mysql | DbType::Mariadb => required(&db.username, &db.name, "username")?,
|
||||
_ => optional(&db.username),
|
||||
};
|
||||
|
||||
let password = match db.db_type {
|
||||
DbType::Postgresql | DbType::Mysql | DbType::Mariadb => required(&db.password, &db.name, "password")?,
|
||||
_ => optional(&db.password),
|
||||
};
|
||||
|
||||
let host = match db.db_type {
|
||||
DbType::Postgresql | DbType::Mysql | DbType::Mariadb | DbType::MongoDB => required(&db.host, &db.name, "host")?,
|
||||
DbType::Sqlite => optional(&db.host),
|
||||
};
|
||||
|
||||
let port = match db.db_type {
|
||||
DbType::Postgresql | DbType::Mysql | DbType::Mariadb | DbType::MongoDB => required(&db.port, &db.name, "port")?,
|
||||
DbType::Sqlite => db.port.unwrap_or(0),
|
||||
};
|
||||
|
||||
let database_name = match db.db_type {
|
||||
DbType::Sqlite => optional(&db.database),
|
||||
_ => required(&db.database, &db.name, "database")?
|
||||
};
|
||||
|
||||
let path_val = match db.db_type {
|
||||
DbType::Sqlite => required(&db.path, &db.name, "path")?,
|
||||
_ => optional(&db.path),
|
||||
};
|
||||
|
||||
databases.push(DatabaseConfig {
|
||||
name: db.name,
|
||||
database: database_name,
|
||||
db_type: db.db_type,
|
||||
username,
|
||||
password,
|
||||
host,
|
||||
port,
|
||||
generated_id: db.generated_id,
|
||||
path: path_val,
|
||||
});
|
||||
}
|
||||
|
||||
info!("Databases: {} instances loaded", databases.len());
|
||||
Ok(DatabasesConfig { databases })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,14 +58,7 @@ impl StorageProvider for S3Provider {
|
||||
|
||||
let encrypt = encrypt.unwrap_or(false);
|
||||
|
||||
let upload = match build_stream(
|
||||
&file_path,
|
||||
encrypt,
|
||||
&ctx.edge_key.master_key_b64
|
||||
// encrypt.then(|| ctx.edge_key.public_key.as_bytes().to_vec()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
let upload = match build_stream(&file_path, encrypt, &ctx.edge_key.master_key_b64).await {
|
||||
Ok(u) => u,
|
||||
Err(e) => {
|
||||
error!("Stream build failed: {}", e);
|
||||
@@ -100,17 +93,25 @@ impl StorageProvider for S3Provider {
|
||||
"static-creds",
|
||||
);
|
||||
|
||||
let region = Region::new(config.region.clone().unwrap_or("eu-central-3".to_string()));
|
||||
let region = Region::new(config.region.clone().unwrap_or("us-east-1".to_string()));
|
||||
|
||||
let endpoint = if let Some(port) = &config.port {
|
||||
if port.trim().is_empty() {
|
||||
format!("{}://{}", if config.ssl { "https" } else { "http" }, config.end_point_url)
|
||||
} else {
|
||||
format!("{}://{}:{}", if config.ssl { "https" } else { "http" }, config.end_point_url, port)
|
||||
}
|
||||
} else {
|
||||
format!("{}://{}", if config.ssl { "https" } else { "http" }, config.end_point_url)
|
||||
};
|
||||
|
||||
info!("S3 endpoint to {}", &endpoint);
|
||||
|
||||
let sdk_config = s3::config::Builder::new()
|
||||
.credentials_provider(credentials)
|
||||
.region(region)
|
||||
.force_path_style(true)
|
||||
.endpoint_url(format!(
|
||||
"{}://{}",
|
||||
if config.ssl { "https" } else { "http" },
|
||||
config.end_point_url
|
||||
))
|
||||
.endpoint_url(endpoint)
|
||||
.behavior_version(BehaviorVersion::latest())
|
||||
.build();
|
||||
|
||||
@@ -283,35 +284,6 @@ impl StorageProvider for S3Provider {
|
||||
"Successfully completed multipart upload: {}",
|
||||
remote_file_path
|
||||
);
|
||||
|
||||
// if let Some(enc) = upload.encryption {
|
||||
// let meta = EncryptionMetadataFile {
|
||||
// version: 1,
|
||||
// cipher: "AES-256-CBC+RSA-OAEP-SHA256".to_string(),
|
||||
// encrypted_aes_key_b64: general_purpose::STANDARD
|
||||
// .encode(enc.encrypted_aes_key),
|
||||
// iv_b64: general_purpose::STANDARD.encode(enc.iv),
|
||||
// };
|
||||
//
|
||||
// let meta_toml = toml::to_string(&meta).expect("Serialization error");
|
||||
//
|
||||
// let meta_key = format!("{}.meta", remote_file_path);
|
||||
//
|
||||
// client
|
||||
// .put_object()
|
||||
// .bucket(bucket)
|
||||
// .key(&meta_key)
|
||||
// .body(ByteStream::from(meta_toml.into_bytes()))
|
||||
// .content_type("application/toml")
|
||||
// .send()
|
||||
// .await
|
||||
// .map_err(|e| {
|
||||
// error!("Metadata upload failed: {}", e);
|
||||
// e
|
||||
// })
|
||||
// .unwrap();
|
||||
// }
|
||||
|
||||
UploadResult {
|
||||
storage_id: storage.id.clone(),
|
||||
success: true,
|
||||
|
||||
@@ -8,4 +8,5 @@ pub struct S3ProviderConfig {
|
||||
pub end_point_url: String,
|
||||
pub ssl: bool,
|
||||
pub region: Option<String>,
|
||||
pub port: Option<String>,
|
||||
}
|
||||
|
||||
+116
-24
@@ -1,8 +1,8 @@
|
||||
use anyhow::Result;
|
||||
use anyhow::{Context, Result};
|
||||
use bytes::Bytes;
|
||||
use futures::{Stream, StreamExt};
|
||||
use log::info;
|
||||
use reqwest::header::{CONTENT_TYPE, HeaderMap, HeaderValue};
|
||||
use tracing::{error, info};
|
||||
|
||||
const PATCH_CHUNK_SIZE: usize = 1 * 1024 * 1024;
|
||||
|
||||
@@ -18,39 +18,62 @@ where
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
info!("File size: {}", total_size);
|
||||
info!("Endpoint URL: {}", tus_endpoint);
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("Tus-Resumable", HeaderValue::from_static("1.0.0"));
|
||||
headers.insert("Upload-Defer-Length", HeaderValue::from_static("1"));
|
||||
let mut create_headers = HeaderMap::new();
|
||||
create_headers.insert("Tus-Resumable", HeaderValue::from_static("1.0.0"));
|
||||
create_headers.insert("Upload-Defer-Length", HeaderValue::from_static("1"));
|
||||
|
||||
let resp = client
|
||||
.post(tus_endpoint)
|
||||
.headers(headers.clone())
|
||||
.headers(create_headers.clone())
|
||||
.send()
|
||||
.await?;
|
||||
.await
|
||||
.context("Failed to send POST to create TUS upload")?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
anyhow::bail!("Failed to create upload: {}", resp.status());
|
||||
let status = resp.status();
|
||||
let headers = resp.headers().clone();
|
||||
let body = resp
|
||||
.text()
|
||||
.await
|
||||
.unwrap_or_else(|_| "<failed to read body>".into());
|
||||
|
||||
error!(
|
||||
"TUS creation failed | status={} | headers={:?} | body={}",
|
||||
status, headers, body
|
||||
);
|
||||
|
||||
anyhow::bail!(
|
||||
"Failed to create upload.\nStatus: {}\nHeaders: {:?}\nBody: {}",
|
||||
status,
|
||||
headers,
|
||||
body
|
||||
);
|
||||
}
|
||||
|
||||
let upload_url = resp
|
||||
.headers()
|
||||
.get("Location")
|
||||
.ok_or_else(|| anyhow::anyhow!("Missing Location header"))?
|
||||
.to_str()?
|
||||
.context("TUS creation response missing Location header")?
|
||||
.to_str()
|
||||
.context("Invalid Location header value")?
|
||||
.to_string();
|
||||
|
||||
let mut stream = Box::pin(
|
||||
encrypted_stream.map(|r| r.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))),
|
||||
);
|
||||
|
||||
let mut stream = Box::pin(encrypted_stream);
|
||||
let mut offset: u64 = 0;
|
||||
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk?;
|
||||
let chunk = chunk.context("Stream produced IO error")?;
|
||||
|
||||
for sub_chunk in chunk.chunks(PATCH_CHUNK_SIZE) {
|
||||
let mut patch_headers = extra_headers.clone();
|
||||
patch_headers.insert("Tus-Resumable", HeaderValue::from_static("1.0.0"));
|
||||
patch_headers.insert("Upload-Offset", HeaderValue::from_str(&offset.to_string())?);
|
||||
patch_headers.insert(
|
||||
"Upload-Offset",
|
||||
HeaderValue::from_str(&offset.to_string())
|
||||
.context("Invalid offset header value")?,
|
||||
);
|
||||
patch_headers.insert(
|
||||
CONTENT_TYPE,
|
||||
HeaderValue::from_static("application/offset+octet-stream"),
|
||||
@@ -61,24 +84,69 @@ where
|
||||
.headers(patch_headers)
|
||||
.body(sub_chunk.to_vec())
|
||||
.send()
|
||||
.await?;
|
||||
.await
|
||||
.with_context(|| format!("PATCH request failed at offset {}", offset))?;
|
||||
|
||||
if !patch_resp.status().is_success() {
|
||||
let status = patch_resp.status();
|
||||
let headers = patch_resp.headers().clone();
|
||||
let body = patch_resp
|
||||
.text()
|
||||
.await
|
||||
.unwrap_or_else(|_| "<failed to read body>".into());
|
||||
|
||||
error!(
|
||||
"TUS PATCH failure | offset={} | status={} | body={}",
|
||||
offset, status, body
|
||||
);
|
||||
|
||||
anyhow::bail!(
|
||||
"Chunk upload failed at offset {}: {}",
|
||||
"Chunk upload failed.\n\
|
||||
URL: {}\n\
|
||||
Offset: {}\n\
|
||||
Status: {}\n\
|
||||
Headers: {:?}\n\
|
||||
Body: {}",
|
||||
upload_url,
|
||||
offset,
|
||||
patch_resp.status()
|
||||
status,
|
||||
headers,
|
||||
body
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(server_offset) = patch_resp.headers().get("Upload-Offset") {
|
||||
let server_offset = server_offset
|
||||
.to_str()
|
||||
.context("Invalid Upload-Offset header")?
|
||||
.parse::<u64>()
|
||||
.context("Failed to parse Upload-Offset header")?;
|
||||
|
||||
let expected = offset + sub_chunk.len() as u64;
|
||||
|
||||
if server_offset != expected {
|
||||
anyhow::bail!(
|
||||
"Offset mismatch detected.\nLocal expected: {}\nServer returned: {}",
|
||||
expected,
|
||||
server_offset
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
offset += sub_chunk.len() as u64;
|
||||
// info!("Progress: {}/{}", offset, total_size);
|
||||
}
|
||||
}
|
||||
|
||||
let mut finalize_headers = extra_headers.clone();
|
||||
finalize_headers.insert("Tus-Resumable", HeaderValue::from_static("1.0.0"));
|
||||
finalize_headers.insert("Upload-Offset", HeaderValue::from_str(&offset.to_string())?);
|
||||
finalize_headers.insert("Upload-Length", HeaderValue::from_str(&offset.to_string())?);
|
||||
finalize_headers.insert(
|
||||
"Upload-Offset",
|
||||
HeaderValue::from_str(&offset.to_string()).context("Invalid finalize offset header")?,
|
||||
);
|
||||
finalize_headers.insert(
|
||||
"Upload-Length",
|
||||
HeaderValue::from_str(&offset.to_string()).context("Invalid finalize length header")?,
|
||||
);
|
||||
finalize_headers.insert(
|
||||
CONTENT_TYPE,
|
||||
HeaderValue::from_static("application/offset+octet-stream"),
|
||||
@@ -88,11 +156,35 @@ where
|
||||
.patch(&upload_url)
|
||||
.headers(finalize_headers)
|
||||
.send()
|
||||
.await?;
|
||||
.await
|
||||
.context("Finalize PATCH request failed")?;
|
||||
|
||||
if !finalize_resp.status().is_success() {
|
||||
anyhow::bail!("Failed to finalize upload");
|
||||
let status = finalize_resp.status();
|
||||
let body = finalize_resp
|
||||
.text()
|
||||
.await
|
||||
.unwrap_or_else(|_| "<failed to read body>".into());
|
||||
|
||||
error!(
|
||||
"TUS finalize failure | offset={} | status={} | body={}",
|
||||
offset, status, body
|
||||
);
|
||||
|
||||
anyhow::bail!(
|
||||
"Finalize upload failed.\n\
|
||||
URL: {}\n\
|
||||
Final offset: {}\n\
|
||||
Status: {}\n\
|
||||
Body: {}",
|
||||
upload_url,
|
||||
offset,
|
||||
status,
|
||||
body
|
||||
);
|
||||
}
|
||||
|
||||
info!("Upload completed successfully. Final size: {}", offset);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user