diff --git a/CITATION.cff b/CITATION.cff index 0eb1dc3..df6fef6 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -27,5 +27,5 @@ keywords: - self-hosted - portabase license: Apache-2.0 -version: 1.21.3 +version: 1.23.1 date-released: '2026-02-24' diff --git a/Cargo.lock b/Cargo.lock index 954596a..b2ceb93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3665,7 +3665,7 @@ dependencies = [ [[package]] name = "portabase-agent" -version = "1.22.0" +version = "1.23.1" dependencies = [ "aes", "aes-gcm", diff --git a/Cargo.toml b/Cargo.toml index 90f164c..3ac5691 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "portabase-agent" -version = "1.22.0" +version = "1.23.1" edition = "2024" license = "Apache-2.0" diff --git a/docker-compose.yml b/docker-compose.yml index aae6a57..ab4b9b7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,10 +12,10 @@ services: - ./databases.json:/config/config.json #- ./databases.toml:/config/config.toml - /var/run/docker.sock:/var/run/docker.sock -# - cargo-target:/app/target + # - cargo-target:/app/target - databases_sqlite-data:/sqlite-data/workspace/data - ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data -# - /bigdisk:/scratch + # - /bigdisk:/scratch - ./test-files/archive:/data/files-archive - ./test-files/snapshot:/data/files-snapshot - ./test-files/sync:/data/files-sync @@ -23,7 +23,7 @@ services: APP_ENV: development LOG: debug TZ: "Europe/Paris" -# TMPDIR: /scratch + # TMPDIR: /scratch EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiODBkZTY1NDktNzZjYS00NGZhLWIyNTItMTY0MWI2YTQzZjJkIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ==" #CHUNK_SIZE_MB: "1" #POOLING: 1 @@ -43,7 +43,7 @@ services: volumes: cargo-registry: cargo-git: -# cargo-target: + # cargo-target: databases_sqlite-data: external: true diff --git a/src/core/agent.rs b/src/core/agent.rs index aaf12a4..fb1bb17 100644 --- a/src/core/agent.rs +++ b/src/core/agent.rs @@ -9,6 +9,7 @@ use crate::services::restore::RestoreService; use crate::services::status::StatusService; use crate::settings::CONFIG; use crate::utils::common::BackupMethod; +use std::collections::HashSet; use std::path::PathBuf; use std::sync::Arc; use tracing::{error, info, warn}; @@ -60,6 +61,13 @@ impl Agent { error!("Failed to persist dashboard cache: {e}"); } + let live_ids: HashSet<&str> = ping_result + .databases + .iter() + .map(|d| d.generated_id.as_str()) + .collect(); + self.cron_service.prune(&live_ids).await; + let merged = merge(&local.databases, &self.dashboard_cache); for db in ping_result.databases.iter() { diff --git a/src/domain/mssql/backup.rs b/src/domain/mssql/backup.rs index be564e1..ccef82e 100644 --- a/src/domain/mssql/backup.rs +++ b/src/domain/mssql/backup.rs @@ -1,3 +1,4 @@ +use super::format::MssqlFormat; use crate::services::backup::logger::JobLogger; use crate::services::config::DatabaseConfig; use anyhow::{Context, Result}; @@ -13,7 +14,12 @@ pub async fn run( logger: Arc, ) -> Result { tokio::task::spawn_blocking(move || -> Result { - logger.log("info", format!("Starting MSSQL backup for database {}", cfg.name)); + let (format, unknown_format) = MssqlFormat::from_config(&cfg); + if let Some(v) = unknown_format { + logger.log("warn", format!("Unknown backup_format '{}' for {}, falling back to 'dacpac'", v, cfg.name)); + } + + logger.log("info", format!("Starting MSSQL {} backup for database {}", &format.extension()[1..], cfg.name)); let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension)); let connection_string = format!( @@ -24,8 +30,18 @@ pub async fn run( logger.log("info", format!("MSSQL backup: {}:{}/{} → {}", cfg.host, cfg.port, cfg.database, file_path.display())); let start = Instant::now(); - let output = Command::new("sqlpackage") - .arg("/a:Export") + let mut command = Command::new("sqlpackage"); + match format { + MssqlFormat::Dacpac => command.args([ + "/a:Extract", + "/p:ExtractAllTableData=True", + "/p:ExtractReferencedServerScopedElements=False", + "/p:IgnorePermissions=False", + "/p:IgnoreUserLoginMappings=False", + ]), + MssqlFormat::Bacpac => command.arg("/a:Export"), + }; + let output = command .arg(format!("/scs:{}", connection_string)) .arg(format!("/tf:{}", file_path.display())) .output() diff --git a/src/domain/mssql/connection.rs b/src/domain/mssql/connection.rs index aad6fad..92ee08d 100644 --- a/src/domain/mssql/connection.rs +++ b/src/domain/mssql/connection.rs @@ -1,13 +1,39 @@ use crate::services::config::DatabaseConfig; -use anyhow::Result; +use anyhow::{Context, Result}; use tiberius::{AuthMethod, Client, Config}; use tokio::net::TcpStream; use tokio_util::compat::{Compat, TokioAsyncWriteCompatExt}; -pub async fn build_client(cfg: &DatabaseConfig) -> Result>> { +const DROP_DATABASE_SQL: &str = "DECLARE @q nvarchar(258) = QUOTENAME(@P1); +IF DB_ID(@P1) IS NOT NULL +BEGIN + IF DB_ID(@P1) <= 4 + THROW 50000, N'Refusing to restore over a system database; nothing was dropped', 1; + IF ISNULL(HAS_PERMS_BY_NAME(NULL, 'DATABASE', 'CREATE DATABASE'), 0) = 0 + THROW 50000, N'Restore needs CREATE DATABASE permission (dbcreator) to recreate the target database; nothing was dropped', 1; + DECLARE @single bit = 0; + IF CAST(SERVERPROPERTY('EngineEdition') AS int) <> 5 + BEGIN + EXEC(N'ALTER DATABASE ' + @q + N' SET SINGLE_USER WITH ROLLBACK IMMEDIATE'); + SET @single = 1; + END + BEGIN TRY + EXEC(N'DROP DATABASE ' + @q); + END TRY + BEGIN CATCH + IF @single = 1 AND DB_ID(@P1) IS NOT NULL + EXEC(N'ALTER DATABASE ' + @q + N' SET MULTI_USER'); + THROW; + END CATCH +END"; + +pub async fn build_client(cfg: &DatabaseConfig, database: Option<&str>) -> Result>> { let mut config = Config::new(); config.host(&cfg.host); config.port(cfg.port); + if let Some(database) = database { + config.database(database); + } config.authentication(AuthMethod::sql_server(&cfg.username, &cfg.password)); config.trust_cert(); @@ -16,3 +42,14 @@ pub async fn build_client(cfg: &DatabaseConfig) -> Result Result<()> { + let mut client = build_client(cfg, Some("master")) + .await + .with_context(|| format!("Failed to connect to master for {}", cfg.name))?; + client + .execute(DROP_DATABASE_SQL, &[&cfg.database.as_str()]) + .await + .with_context(|| format!("Failed to drop MSSQL database {}", cfg.database))?; + Ok(()) +} diff --git a/src/domain/mssql/database.rs b/src/domain/mssql/database.rs index fda0652..b15d69b 100644 --- a/src/domain/mssql/database.rs +++ b/src/domain/mssql/database.rs @@ -1,3 +1,4 @@ +use super::format::MssqlFormat; use super::{backup, ping, restore}; use crate::domain::factory::Database; use crate::services::backup::logger::JobLogger; @@ -21,7 +22,7 @@ impl MssqlDatabase { #[async_trait] impl Database for MssqlDatabase { fn file_extension(&self) -> &'static str { - ".bacpac" + MssqlFormat::from_config(&self.cfg).0.extension() } async fn ping(&self) -> Result { diff --git a/src/domain/mssql/format.rs b/src/domain/mssql/format.rs new file mode 100644 index 0000000..dedb542 --- /dev/null +++ b/src/domain/mssql/format.rs @@ -0,0 +1,33 @@ +use crate::services::config::DatabaseConfig; +use std::path::Path; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MssqlFormat { + Dacpac, + Bacpac, +} + +impl MssqlFormat { + pub fn from_config(cfg: &DatabaseConfig) -> (Self, Option) { + match cfg.options.get("backup_format").and_then(|v| v.as_str()) { + None | Some("dacpac") => (Self::Dacpac, None), + Some("bacpac") => (Self::Bacpac, None), + Some(other) => (Self::Dacpac, Some(other.to_string())), + } + } + + pub fn from_path(path: &Path) -> Option { + match path.extension()?.to_str()?.to_ascii_lowercase().as_str() { + "dacpac" => Some(Self::Dacpac), + "bacpac" => Some(Self::Bacpac), + _ => None, + } + } + + pub fn extension(self) -> &'static str { + match self { + Self::Dacpac => ".dacpac", + Self::Bacpac => ".bacpac", + } + } +} diff --git a/src/domain/mssql/mod.rs b/src/domain/mssql/mod.rs index 143be20..c518aaf 100644 --- a/src/domain/mssql/mod.rs +++ b/src/domain/mssql/mod.rs @@ -1,4 +1,5 @@ pub mod database; +pub(crate) mod format; mod connection; mod ping; mod backup; diff --git a/src/domain/mssql/ping.rs b/src/domain/mssql/ping.rs index 34c33a5..14873bf 100644 --- a/src/domain/mssql/ping.rs +++ b/src/domain/mssql/ping.rs @@ -5,7 +5,7 @@ use tracing::{error, info}; pub async fn run(cfg: DatabaseConfig) -> Result { info!("Running ping for MSSQL database {}", cfg.name); - match super::connection::build_client(&cfg).await { + match super::connection::build_client(&cfg, None).await { Ok(mut client) => match client.simple_query("SELECT 1").await { Ok(_) => { info!("MSSQL ping succeeded for {}", cfg.name); diff --git a/src/domain/mssql/restore.rs b/src/domain/mssql/restore.rs index af56d6c..cce459e 100644 --- a/src/domain/mssql/restore.rs +++ b/src/domain/mssql/restore.rs @@ -1,3 +1,4 @@ +use super::format::MssqlFormat; use crate::services::backup::logger::JobLogger; use crate::services::config::DatabaseConfig; use anyhow::{Context, Result}; @@ -7,9 +8,23 @@ use std::sync::Arc; use std::time::Instant; pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc) -> Result<()> { - tokio::task::spawn_blocking(move || -> Result<()> { - logger.log("debug", format!("Starting MSSQL restore for database {}", cfg.name)); + logger.log("debug", format!("Starting MSSQL restore for database {}", cfg.name)); + let Some(format) = MssqlFormat::from_path(&restore_file) else { + let message = format!( + "Unsupported MSSQL backup file {}: expected .bacpac or .dacpac", + restore_file.display() + ); + logger.log("error", message.clone()); + anyhow::bail!(message); + }; + + logger.log("info", format!("Replacing MSSQL database {} (drop if it exists, then recreate from the backup)", cfg.database)); + let start = Instant::now(); + super::connection::drop_database(&cfg).await?; + logger.log_command(format!("DROP DATABASE IF EXISTS [{}]", cfg.database), None, Some(0), Some(start.elapsed().as_millis() as f64)); + + tokio::task::spawn_blocking(move || -> Result<()> { let connection_string = format!( "Server=tcp:{},{};Database={};User Id={};Password={};TrustServerCertificate=True;Encrypt=False", cfg.host, cfg.port, cfg.database, cfg.username, cfg.password @@ -24,8 +39,12 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc command.args(["/a:Publish", "/p:AllowIncompatiblePlatform=True"]), + MssqlFormat::Bacpac => command.arg("/a:Import"), + }; + let output = command .arg(format!("/tcs:{}", connection_string)) .arg(format!("/sf:{}", restore_file.display())) .output() diff --git a/src/domain/postgres/backup.rs b/src/domain/postgres/backup.rs index 906cfd9..0903a9e 100644 --- a/src/domain/postgres/backup.rs +++ b/src/domain/postgres/backup.rs @@ -34,6 +34,14 @@ pub async fn run( let pg_dump = select_pg_path(&version).join("pg_dump"); logger.log("debug", format!("Using pg_dump at {:?}", pg_dump)); + let (excludes, invalid) = exclude_args(&cfg); + for key in invalid { + logger.log("warn", format!("Ignoring options.{} for {}: expected a list of patterns", key, cfg.name)); + } + if !excludes.is_empty() { + logger.log("info", format!("Excluding from dump of {}: {}", cfg.name, excludes.join(" "))); + } + match format { PostgresDumpFormat::Fc => { logger.log("info", format!("Running FC backup for {}", cfg.name)); @@ -46,6 +54,7 @@ pub async fn run( .arg("--port").arg(cfg.port.to_string()) .arg("--username").arg(&cfg.username) .arg("--dbname").arg(&cfg.database) + .args(&excludes) .arg("-Fc") .arg("-f").arg(&file_path) .arg("-v") @@ -97,6 +106,7 @@ pub async fn run( .arg("--port").arg(cfg.port.to_string()) .arg("--username").arg(&cfg.username) .arg("--dbname").arg(&cfg.database) + .args(&excludes) .arg("-Fd") .arg("-j").arg("4") .arg("-f").arg(&dump_dir) @@ -151,3 +161,23 @@ pub async fn run( }) .await? } + +pub(crate) fn exclude_args(cfg: &DatabaseConfig) -> (Vec, Vec<&'static str>) { + let mut args = Vec::new(); + let mut invalid = Vec::new(); + for (key, flag) in [("exclude_tables", "--exclude-table"), ("exclude_schemas", "--exclude-schema")] { + match cfg.options.get(key).map(|v| v.as_array()) { + None => {} + Some(Some(items)) => args.extend( + items + .iter() + .filter_map(|v| v.as_str()) + .map(str::trim) + .filter(|p| !p.is_empty()) + .map(|p| format!("{flag}={p}")), + ), + Some(None) => invalid.push(key), + } + } + (args, invalid) +} diff --git a/src/services/cron.rs b/src/services/cron.rs index 69d9166..a74ac39 100644 --- a/src/services/cron.rs +++ b/src/services/cron.rs @@ -5,9 +5,31 @@ use crate::services::api::models::agent::status::DatabaseStatus; use crate::utils::common::vec_to_option_json; use crate::utils::redis_client; use crate::utils::task_manager::cron::check_and_update_cron; +use crate::utils::task_manager::tasks::{SCHEDULE_KEY, remove_task}; +use redis::AsyncCommands; use redis::aio::MultiplexedConnection; use serde_json::{Value, json}; +use std::collections::HashSet; use std::sync::Arc; +use tracing::{error, info}; + +const BACKUP_TASK_PREFIX: &str = "periodic.backup_"; + +pub fn backup_task_name(generated_id: &str) -> String { + format!("{BACKUP_TASK_PREFIX}{generated_id}") +} + +pub fn stale_backup_tasks(scheduled_keys: &[String], live_ids: &HashSet<&str>) -> Vec { + scheduled_keys + .iter() + .filter_map(|key| key.strip_prefix("redbeat:")) + .filter(|name| { + name.strip_prefix(BACKUP_TASK_PREFIX) + .is_some_and(|id| !live_ids.contains(id)) + }) + .map(str::to_string) + .collect() +} pub struct CronService { ctx: Arc, @@ -23,7 +45,7 @@ impl CronService { pub async fn sync(&mut self, database: &DatabaseStatus) -> Result { let generated_id = database.generated_id.as_str(); let dbms = database.dbms.as_str(); - let task_name = format!("periodic.backup_{}", generated_id); + let task_name = backup_task_name(generated_id); let args = vec![generated_id.to_string(), dbms.to_string()]; let storages: Option = vec_to_option_json(database.storages.clone()); let encrypt: bool = database.encrypt; @@ -45,4 +67,24 @@ impl CronService { Ok(true) } + + pub async fn prune(&mut self, live_ids: &HashSet<&str>) { + let keys: Vec = match self.conn.zrange(SCHEDULE_KEY, 0, -1).await { + Ok(keys) => keys, + Err(e) => { + error!("Failed to list scheduled tasks: {:?}", e); + return; + } + }; + + for task_name in stale_backup_tasks(&keys, live_ids) { + match remove_task(&mut self.conn, &task_name).await { + Ok(()) => info!( + "Task {} removed: database no longer returned by dashboard", + task_name + ), + Err(e) => error!("Failed to remove task {}: {:?}", task_name, e), + } + } + } } diff --git a/src/tests/domain/mssql.rs b/src/tests/domain/mssql.rs index 6022834..4aa6f6a 100644 --- a/src/tests/domain/mssql.rs +++ b/src/tests/domain/mssql.rs @@ -1,8 +1,10 @@ use crate::domain::factory::DatabaseFactory; +use crate::services::backup::logger::JobLogger; use crate::services::config::{DatabaseConfig, DbType}; use crate::tests::init_tracing_for_test; use crate::utils::compress::{compress_to_tar_gz_large, decompress_large_tar_gz}; use std::path::PathBuf; +use std::sync::Arc; use std::time::Duration; use tempfile::TempDir; use tiberius::{AuthMethod, Client, Config}; @@ -10,7 +12,7 @@ use testcontainers::runners::AsyncRunner; use testcontainers::{ContainerAsync, GenericImage, ImageExt}; use testcontainers::core::IntoContainerPort; use tokio::net::TcpStream; -use tokio_util::compat::TokioAsyncWriteCompatExt; +use tokio_util::compat::{Compat, TokioAsyncWriteCompatExt}; use tracing::{error, info}; const SA_PASSWORD: &str = "Test!Str0ng1"; @@ -61,6 +63,43 @@ fn make_config(host: String, port: u16, database: &str, generated_id: &str) -> D } } +async fn start() -> (ContainerAsync, String, u16) { + let container = start_container().await; + tokio::time::sleep(Duration::from_secs(30)).await; + let host = container.get_host().await.unwrap().to_string(); + let port = container.get_host_port_ipv4(1433).await.unwrap(); + (container, host, port) +} + +async fn connect_as(host: &str, port: u16, database: &str, user: &str, password: &str) -> Client> { + let mut config = Config::new(); + config.host(host); + config.port(port); + config.database(database); + config.authentication(AuthMethod::sql_server(user, password)); + config.trust_cert(); + + let tcp = TcpStream::connect(config.get_addr()).await.unwrap(); + tcp.set_nodelay(true).unwrap(); + Client::connect(config, tcp.compat_write()).await.unwrap() +} + +async fn exec(host: &str, port: u16, database: &str, sql: &str) { + let mut client = connect_as(host, port, database, "sa", SA_PASSWORD).await; + client.simple_query(sql).await.unwrap().into_results().await.unwrap(); +} + +async fn scalar(host: &str, port: u16, database: &str, user: &str, password: &str, sql: &str) -> i32 { + let mut client = connect_as(host, port, database, user, password).await; + let row = client.simple_query(sql).await.unwrap().into_row().await.unwrap().unwrap(); + row.get::(0).unwrap() +} + +fn with_backup_format(mut config: DatabaseConfig, format: &str) -> DatabaseConfig { + config.options.insert("backup_format".to_string(), serde_json::json!(format)); + config +} + #[tokio::test] async fn mssql_ping_test() { init_tracing_for_test(); @@ -102,8 +141,8 @@ async fn mssql_backup_test() { "backup file should be non-empty" ); assert!( - file_path.extension().and_then(|e| e.to_str()) == Some("bacpac"), - "backup file should have .bacpac extension" + file_path.extension().and_then(|e| e.to_str()) == Some("dacpac"), + "backup file should have .dacpac extension" ); } @@ -153,3 +192,303 @@ async fn mssql_backup_restore_test() { } } } + +#[tokio::test] +async fn mssql_backup_bacpac_test() { + init_tracing_for_test(); + let (_container, host, port) = start().await; + create_user_database(&host, port, "bacpacdb").await; + + let config = with_backup_format( + make_config(host, port, "bacpacdb", "5a445eb4-c2c6-4bde-a423-ee1385dcf6d6"), + "bacpac", + ); + let temp_dir = TempDir::new().unwrap(); + let file_path = DatabaseFactory::create_for_backup(config) + .await + .backup(temp_dir.path(), Arc::new(JobLogger::new())) + .await + .unwrap(); + + assert!(file_path.metadata().unwrap().len() > 0); + assert_eq!(file_path.extension().and_then(|e| e.to_str()), Some("bacpac")); +} + +#[tokio::test] +async fn mssql_backup_windows_principal_test() { + init_tracing_for_test(); + let (_container, host, port) = start().await; + create_user_database(&host, port, "windowsdb").await; + exec( + &host, + port, + "windowsdb", + "CREATE TABLE dbo.t (id INT PRIMARY KEY); INSERT dbo.t VALUES (1); \ + CREATE USER [NT AUTHORITY\\SYSTEM] FOR LOGIN [NT AUTHORITY\\SYSTEM];", + ) + .await; + + let config = make_config(host, port, "windowsdb", "5a445eb4-c2c6-4bde-a423-ee1385dcf6d7"); + let temp_dir = TempDir::new().unwrap(); + + let bacpac = DatabaseFactory::create_for_backup(with_backup_format(config.clone(), "bacpac")) + .await + .backup(temp_dir.path(), Arc::new(JobLogger::new())) + .await; + assert!(bacpac.unwrap_err().to_string().contains("SQL71627")); + + let file_path = DatabaseFactory::create_for_backup(config) + .await + .backup(temp_dir.path(), Arc::new(JobLogger::new())) + .await + .unwrap(); + assert_eq!(file_path.extension().and_then(|e| e.to_str()), Some("dacpac")); +} + +async fn restore_over_existing(format: &str, generated_id: &str) { + init_tracing_for_test(); + let (_container, host, port) = start().await; + create_user_database(&host, port, "appdb").await; + exec(&host, port, "master", "CREATE LOGIN app_login WITH PASSWORD = 'App!Str0ng1'").await; + exec( + &host, + port, + "appdb", + "CREATE TABLE dbo.users (id INT PRIMARY KEY, email NVARCHAR(100) NOT NULL); \ + INSERT dbo.users VALUES (1, 'a@x'), (2, 'b@x'), (3, 'c@x'); \ + CREATE USER app_user FOR LOGIN app_login; \ + GRANT SELECT ON dbo.users TO app_user;", + ) + .await; + + let config = with_backup_format(make_config(host.clone(), port, "appdb", generated_id), format); + let temp_dir = TempDir::new().unwrap(); + let file_path = DatabaseFactory::create_for_backup(config.clone()) + .await + .backup(temp_dir.path(), Arc::new(JobLogger::new())) + .await + .unwrap(); + assert_eq!(file_path.extension().and_then(|e| e.to_str()), Some(format)); + + exec( + &host, + port, + "appdb", + "DELETE FROM dbo.users WHERE id = 1; INSERT dbo.users VALUES (4, 'd@x'); CREATE TABLE dbo.extra (id INT);", + ) + .await; + + DatabaseFactory::create_for_restore(config, &file_path) + .await + .restore(&file_path, Arc::new(JobLogger::new())) + .await + .unwrap(); + + assert_eq!(scalar(&host, port, "appdb", "sa", SA_PASSWORD, "SELECT COUNT(*) FROM dbo.users").await, 3); + assert_eq!( + scalar(&host, port, "appdb", "sa", SA_PASSWORD, "SELECT COUNT(*) FROM dbo.users WHERE id IN (1, 2, 3)").await, + 3 + ); + assert_eq!( + scalar(&host, port, "appdb", "sa", SA_PASSWORD, "SELECT CASE WHEN OBJECT_ID('dbo.extra') IS NULL THEN 1 ELSE 0 END").await, + 1 + ); + assert_eq!( + scalar(&host, port, "appdb", "app_login", "App!Str0ng1", "SELECT COUNT(*) FROM dbo.users").await, + 3 + ); +} + +#[tokio::test] +async fn mssql_restore_over_existing_dacpac_test() { + restore_over_existing("dacpac", "5a445eb4-c2c6-4bde-a423-ee1385dcf6d8").await; +} + +#[tokio::test] +async fn mssql_restore_over_existing_bacpac_test() { + restore_over_existing("bacpac", "5a445eb4-c2c6-4bde-a423-ee1385dcf6d9").await; +} + +#[tokio::test] +async fn mssql_restore_rejects_unknown_extension_test() { + init_tracing_for_test(); + let (_container, host, port) = start().await; + create_user_database(&host, port, "keepdb").await; + + let temp_dir = TempDir::new().unwrap(); + let file_path = temp_dir.path().join("backup.dump"); + std::fs::write(&file_path, b"not a package").unwrap(); + + let config = make_config(host.clone(), port, "keepdb", "5a445eb4-c2c6-4bde-a423-ee1385dcf6da"); + let result = DatabaseFactory::create_for_restore(config, &file_path) + .await + .restore(&file_path, Arc::new(JobLogger::new())) + .await; + + assert!(result.unwrap_err().to_string().contains("expected .bacpac or .dacpac")); + assert_eq!( + scalar(&host, port, "master", "sa", SA_PASSWORD, "SELECT CASE WHEN DB_ID('keepdb') IS NULL THEN 0 ELSE 1 END").await, + 1 + ); +} + +#[tokio::test] +async fn mssql_restore_refuses_without_create_database_test() { + init_tracing_for_test(); + let (_container, host, port) = start().await; + create_user_database(&host, port, "ownerdb").await; + exec(&host, port, "master", "CREATE LOGIN owner_login WITH PASSWORD = 'Owner!Str0ng1'").await; + exec( + &host, + port, + "ownerdb", + "CREATE TABLE dbo.users (id INT PRIMARY KEY, email NVARCHAR(100) NOT NULL); \ + INSERT dbo.users VALUES (1, 'a@x'), (2, 'b@x'), (3, 'c@x'); \ + CREATE USER owner_user FOR LOGIN owner_login; \ + ALTER ROLE db_owner ADD MEMBER owner_user;", + ) + .await; + + let config = make_config(host.clone(), port, "ownerdb", "5a445eb4-c2c6-4bde-a423-ee1385dcf6db"); + let temp_dir = TempDir::new().unwrap(); + let file_path = DatabaseFactory::create_for_backup(config.clone()) + .await + .backup(temp_dir.path(), Arc::new(JobLogger::new())) + .await + .unwrap(); + + let mut owner_config = config; + owner_config.username = "owner_login".to_string(); + owner_config.password = "Owner!Str0ng1".to_string(); + let result = DatabaseFactory::create_for_restore(owner_config, &file_path) + .await + .restore(&file_path, Arc::new(JobLogger::new())) + .await; + + assert!(format!("{:#}", result.unwrap_err()).contains("CREATE DATABASE")); + assert_eq!( + scalar(&host, port, "master", "sa", SA_PASSWORD, "SELECT CASE WHEN DB_ID('ownerdb') IS NULL THEN 0 ELSE 1 END").await, + 1 + ); + assert_eq!(scalar(&host, port, "ownerdb", "sa", SA_PASSWORD, "SELECT COUNT(*) FROM dbo.users").await, 3); +} + +#[tokio::test] +async fn mssql_restore_failed_drop_keeps_multi_user_test() { + init_tracing_for_test(); + let (_container, host, port) = start().await; + create_user_database(&host, port, "alterdb").await; + exec( + &host, + port, + "master", + "CREATE LOGIN alter_login WITH PASSWORD = 'Alter!Str0ng1'; GRANT CREATE ANY DATABASE TO alter_login;", + ) + .await; + exec( + &host, + port, + "alterdb", + "CREATE TABLE dbo.t (id INT PRIMARY KEY); INSERT dbo.t VALUES (1); \ + CREATE USER alter_user FOR LOGIN alter_login; GRANT ALTER TO alter_user;", + ) + .await; + + let temp_dir = TempDir::new().unwrap(); + let file_path = temp_dir.path().join("backup.dacpac"); + std::fs::write(&file_path, b"not a package").unwrap(); + + let mut config = make_config(host.clone(), port, "alterdb", "5a445eb4-c2c6-4bde-a423-ee1385dcf6dc"); + config.username = "alter_login".to_string(); + config.password = "Alter!Str0ng1".to_string(); + let result = DatabaseFactory::create_for_restore(config, &file_path) + .await + .restore(&file_path, Arc::new(JobLogger::new())) + .await; + + assert!(!format!("{:#}", result.unwrap_err()).contains("nothing was dropped")); + assert_eq!( + scalar(&host, port, "master", "sa", SA_PASSWORD, "SELECT CASE WHEN DB_ID('alterdb') IS NULL THEN 0 ELSE 1 END").await, + 1 + ); + assert_eq!( + scalar( + &host, + port, + "master", + "sa", + SA_PASSWORD, + "SELECT CASE WHEN DATABASEPROPERTYEX('alterdb', 'UserAccess') = 'MULTI_USER' THEN 1 ELSE 0 END" + ) + .await, + 1 + ); +} + +#[tokio::test] +async fn mssql_restore_refuses_system_database_test() { + init_tracing_for_test(); + let (_container, host, port) = start().await; + + let temp_dir = TempDir::new().unwrap(); + let file_path = temp_dir.path().join("backup.dacpac"); + std::fs::write(&file_path, b"not a package").unwrap(); + + let config = make_config(host.clone(), port, "msdb", "5a445eb4-c2c6-4bde-a423-ee1385dcf6dd"); + let result = DatabaseFactory::create_for_restore(config, &file_path) + .await + .restore(&file_path, Arc::new(JobLogger::new())) + .await; + + assert!(format!("{:#}", result.unwrap_err()).contains("system database")); + assert_eq!( + scalar(&host, port, "master", "sa", SA_PASSWORD, "SELECT CASE WHEN DB_ID('msdb') IS NULL THEN 0 ELSE 1 END").await, + 1 + ); + assert_eq!( + scalar( + &host, + port, + "master", + "sa", + SA_PASSWORD, + "SELECT CASE WHEN DATABASEPROPERTYEX('msdb', 'UserAccess') = 'MULTI_USER' THEN 1 ELSE 0 END" + ) + .await, + 1 + ); +} + +mod format_tests { + use super::{make_config, with_backup_format}; + use crate::domain::mssql::format::MssqlFormat; + use std::path::Path; + + #[test] + fn from_config_defaults_to_dacpac() { + let cfg = make_config("localhost".into(), 1433, "db", "00000000-0000-0000-0000-000000000000"); + assert_eq!(MssqlFormat::from_config(&cfg), (MssqlFormat::Dacpac, None)); + assert_eq!( + MssqlFormat::from_config(&with_backup_format(cfg.clone(), "dacpac")), + (MssqlFormat::Dacpac, None) + ); + assert_eq!( + MssqlFormat::from_config(&with_backup_format(cfg.clone(), "bacpac")), + (MssqlFormat::Bacpac, None) + ); + assert_eq!( + MssqlFormat::from_config(&with_backup_format(cfg, "zip")), + (MssqlFormat::Dacpac, Some("zip".to_string())) + ); + } + + #[test] + fn from_path_reads_extension() { + assert_eq!(MssqlFormat::from_path(Path::new("/tmp/a.dacpac")), Some(MssqlFormat::Dacpac)); + assert_eq!(MssqlFormat::from_path(Path::new("/tmp/a.BACPAC")), Some(MssqlFormat::Bacpac)); + assert_eq!(MssqlFormat::from_path(Path::new("/tmp/backup.dump")), None); + assert_eq!(MssqlFormat::from_path(Path::new("/tmp/noext")), None); + assert_eq!(MssqlFormat::Dacpac.extension(), ".dacpac"); + assert_eq!(MssqlFormat::Bacpac.extension(), ".bacpac"); + } +} diff --git a/src/tests/domain/postgres.rs b/src/tests/domain/postgres.rs index ce33325..d34429a 100644 --- a/src/tests/domain/postgres.rs +++ b/src/tests/domain/postgres.rs @@ -316,6 +316,62 @@ async fn restore_run_unified_fc_roundtrip() { assert!(result.is_ok(), "restore::run failed: {:?}", result); } +#[tokio::test] +async fn backup_as_non_owner_succeeds_only_with_exclude_tables() { + init_tracing_for_test(); + + let (_container, admin) = create_config().await; + crate::domain::postgres::connection::connect(&admin) + .await + .unwrap() + .batch_execute( + "CREATE ROLE backup_ro LOGIN PASSWORD 'x'; + CREATE TABLE public.keep(id int); + CREATE TABLE public.search_index__x(id int); + GRANT SELECT ON public.keep TO backup_ro;", + ) + .await + .unwrap(); + + let mut ro = admin.clone(); + ro.username = "backup_ro".into(); + ro.password = "x".into(); + + let mut excluded = ro.clone(); + excluded + .options + .insert("exclude_tables".into(), serde_json::json!(["public.search_index__*"])); + + for format in [PostgresDumpFormat::Fc, PostgresDumpFormat::Fd] { + let dir = TempDir::new().unwrap(); + let denied = crate::domain::postgres::backup::run( + ro.clone(), format, dir.path().to_path_buf(), pg_dump_env(&ro), Arc::new(JobLogger::new()), + ) + .await; + assert!(denied.is_err(), "{format:?}: non-owner dump must fail without exclusions"); + + let dir = TempDir::new().unwrap(); + let logger = Arc::new(JobLogger::new()); + let dump = crate::domain::postgres::backup::run( + excluded.clone(), format, dir.path().to_path_buf(), pg_dump_env(&excluded), logger.clone(), + ) + .await + .unwrap_or_else(|e| panic!("{format:?}: dump with exclusions failed: {e:?}")); + + let entries = Arc::try_unwrap(logger).expect("single owner").into_entries(); + assert!(entries.iter().any(|e| e.message + == format!("Excluding from dump of {}: --exclude-table=public.search_index__*", excluded.name))); + + if format == PostgresDumpFormat::Fc { + let version = server_version(&admin).await.unwrap(); + let pg_restore = select_pg_path(&version).join(pg_restore_binary_name()); + let prepared = prepare_archive(format, &dump, &pg_restore, &JobLogger::new()).unwrap(); + assert!(prepared.toc().contains("TABLE public keep ")); + assert!(!prepared.toc().contains("search_index__x")); + } + } +} + #[tokio::test] async fn drop_all_schemas_removes_user_schema() { init_tracing_for_test(); @@ -924,6 +980,61 @@ mod clean_mode_tests { } } +mod exclude_args_tests { + use crate::domain::postgres::backup::exclude_args; + use crate::services::config::DatabaseConfig; + use serde_json::json; + + fn cfg(options: serde_json::Value) -> DatabaseConfig { + serde_json::from_value(json!({ + "name": "t", "database": "testdb", "type": "postgresql", + "username": "u", "password": "p", "port": 5432, "host": "localhost", + "generated_id": "00000000-0000-0000-0000-000000000000", + "path": "", "max_packet_size": "", "volume_name": "", + "options": options + })) + .unwrap() + } + + #[test] + fn no_options_yields_nothing() { + assert_eq!(exclude_args(&cfg(json!({}))), (vec![], vec![])); + assert_eq!( + exclude_args(&cfg(json!({"exclude_tables": [], "exclude_schemas": []}))), + (vec![], vec![]) + ); + } + + #[test] + fn lists_map_to_single_arg_flags_in_order() { + let (args, invalid) = exclude_args(&cfg(json!({ + "exclude_schemas": ["metabase_tmp"], + "exclude_tables": ["public.search_index__*", " audit.tmp_* ", "", " ", 42, "-f/tmp/x", "\"MyTable\""] + }))); + assert_eq!( + args, + vec![ + "--exclude-table=public.search_index__*", + "--exclude-table=audit.tmp_*", + "--exclude-table=-f/tmp/x", + "--exclude-table=\"MyTable\"", + "--exclude-schema=metabase_tmp", + ] + ); + assert!(invalid.is_empty()); + } + + #[test] + fn non_list_values_are_reported_and_ignored() { + let (args, invalid) = exclude_args(&cfg(json!({ + "exclude_tables": "public.search_index__*", + "exclude_schemas": {"a": 1} + }))); + assert!(args.is_empty()); + assert_eq!(invalid, vec!["exclude_tables", "exclude_schemas"]); + } +} + mod toc_tests { use crate::domain::postgres::restore::toc_creates_public_schema; diff --git a/src/tests/services/cron_tests.rs b/src/tests/services/cron_tests.rs new file mode 100644 index 0000000..7960d80 --- /dev/null +++ b/src/tests/services/cron_tests.rs @@ -0,0 +1,40 @@ +use crate::services::cron::{backup_task_name, stale_backup_tasks}; +use std::collections::HashSet; + +const ID_A: &str = "16678159-ff7e-4c97-8c83-0adeff214681"; +const ID_B: &str = "16678124-ff7e-4c97-8c83-0adeff214681"; + +fn key(id: &str) -> String { + format!("redbeat:{}", backup_task_name(id)) +} + +#[test] +fn backup_task_name_format() { + assert_eq!(backup_task_name(ID_A), format!("periodic.backup_{ID_A}")); +} + +#[test] +fn stale_backup_tasks_returns_ids_missing_from_response() { + let live: HashSet<&str> = [ID_A].into(); + assert_eq!( + stale_backup_tasks(&[key(ID_A), key(ID_B)], &live), + vec![backup_task_name(ID_B)] + ); +} + +#[test] +fn stale_backup_tasks_empty_response_prunes_all() { + assert_eq!( + stale_backup_tasks(&[key(ID_A), key(ID_B)], &HashSet::new()), + vec![backup_task_name(ID_A), backup_task_name(ID_B)] + ); +} + +#[test] +fn stale_backup_tasks_ignores_other_keys() { + let keys = vec![ + "redbeat:periodic.other_x".to_string(), + "periodic.backup_raw".to_string(), + ]; + assert!(stale_backup_tasks(&keys, &HashSet::new()).is_empty()); +} diff --git a/src/tests/services/mod.rs b/src/tests/services/mod.rs index 65ec678..e0e7408 100644 --- a/src/tests/services/mod.rs +++ b/src/tests/services/mod.rs @@ -2,6 +2,7 @@ mod api_models_tests; mod backup_runner_tests; pub(crate) mod backup_uploader_tests; mod config_tests; +mod cron_tests; mod dashboard_config_tests; pub(crate) mod local_storage_tests; mod restic_tests; diff --git a/src/utils/file.rs b/src/utils/file.rs index b39dd92..10bac58 100644 --- a/src/utils/file.rs +++ b/src/utils/file.rs @@ -1,7 +1,6 @@ #![allow(dead_code)] use anyhow::Result; -use chrono::Utc; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; use tokio::fs::File; @@ -54,7 +53,7 @@ pub fn full_file_path(file_name: &String, folder_name: Option<&str>) -> String { .filter(|folder_name| !folder_name.is_empty()) .unwrap_or("backups"); - format!("{}/{}/{}", folder_name, Utc::now().format("%Y-%m-%d"), file_name) + format!("{}/{}/{}", folder_name, chrono::Local::now().format("%Y-%m-%d"), file_name) } const CHUNK_SIZE: usize = 16 * 1024 * 1024;