From e9147d6c6089aebdc4de8fbcb4c176aa64ca777c Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Mon, 16 Mar 2026 08:57:00 +0100 Subject: [PATCH 01/11] fix: s3 port type mismatch, add string_or_number_to_string deserializer (#42) Co-authored-by: charlesgauthereau --- Cargo.lock | 105 ++++++++++---------- docker-compose.yml | 6 +- docker/Dockerfile | 2 +- src/services/storage/providers/s3/models.rs | 3 + src/utils/deserializer.rs | 17 ++++ 5 files changed, 76 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6e49eb4..dd3b6d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -82,9 +82,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anyhow" @@ -275,9 +275,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.125.0" +version = "1.126.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "223f5c95650d9557925a91f4c2db3def189e8f659452134a29e5cd2d37d708ed" +checksum = "7878050a2321d215eec9db8be09f8db59418b53860ae86cc7042b4094d6cb2bb" dependencies = [ "aws-credential-types", "aws-runtime", @@ -744,9 +744,9 @@ dependencies = [ [[package]] name = "bollard" -version = "0.20.1" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "227aa051deec8d16bd9c34605e7aaf153f240e35483dd42f6f78903847934738" +checksum = "ee04c4c84f1f811b017f2fbb7dd8815c976e7ca98593de9c1e2afad0f636bff4" dependencies = [ "async-stream", "base64 0.22.1", @@ -869,9 +869,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.56" +version = "1.2.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" +checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" dependencies = [ "find-msvc-tools", "jobserver", @@ -1199,9 +1199,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ "darling_core", "darling_macro", @@ -1209,11 +1209,10 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" dependencies = [ - "fnv", "ident_case", "proc-macro2", "quote", @@ -1223,9 +1222,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", @@ -1289,9 +1288,9 @@ dependencies = [ [[package]] name = "derive-where" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef941ded77d15ca19b40374869ac6000af1c9f2a4c0f3d4c70926287e6364a8f" +checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" dependencies = [ "proc-macro2", "quote", @@ -1450,7 +1449,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1957,9 +1956,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hybrid-array" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1b229d73f5803b562cc26e4da0396c8610a4ee209f4fac8fa4f8d709166dc45" +checksum = "8655f91cd07f2b9d0c24137bd650fe69617773435ee5ec83022377777ce65ef1" dependencies = [ "typenum", ] @@ -2089,7 +2088,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.3", "system-configuration", "tokio", "tower-service", @@ -2680,7 +2679,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -2821,9 +2820,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" dependencies = [ "critical-section", "portable-atomic", @@ -2831,9 +2830,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.75" +version = "0.10.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf" dependencies = [ "bitflags 2.11.0", "cfg-if", @@ -2863,9 +2862,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.111" +version = "0.9.112" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb" dependencies = [ "cc", "libc", @@ -3256,7 +3255,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls 0.23.37", - "socket2 0.5.10", + "socket2 0.6.3", "thiserror 2.0.18", "tokio", "tracing", @@ -3265,9 +3264,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.13" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ "aws-lc-rs", "bytes", @@ -3294,7 +3293,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.3", "tracing", "windows-sys 0.60.2", ] @@ -3647,7 +3646,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3718,7 +3717,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3772,9 +3771,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] @@ -3961,9 +3960,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.17.0" +version = "3.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "381b283ce7bc6b476d903296fb59d0d36633652b633b27f64db4fb46dcbfc3b9" +checksum = "dd5414fad8e6907dbdd5bc441a50ae8d6e26151a03b1de04d89a5576de61d01f" dependencies = [ "base64 0.22.1", "chrono", @@ -3980,9 +3979,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.17.0" +version = "3.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6d4e30573c8cb306ed6ab1dca8423eec9a463ea0e155f45399455e0368b27e0" +checksum = "d3db8978e608f1fe7357e211969fd9abdcae80bac1ba7a3369bb7eb6b404eb65" dependencies = [ "darling", "proc-macro2", @@ -4094,7 +4093,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4248,15 +4247,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.26.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4408,9 +4407,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ "tinyvec_macros", ] @@ -4720,9 +4719,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.22" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ "chrono", "matchers", @@ -5171,7 +5170,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -5736,18 +5735,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.40" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5" +checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.40" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953" +checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f" dependencies = [ "proc-macro2", "quote", diff --git a/docker-compose.yml b/docker-compose.yml index d6fbc5c..80cbcc1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,14 +11,14 @@ services: # - ./databases.toml:/config/config.toml - cargo-registry:/usr/local/cargo/registry - cargo-git:/usr/local/cargo/git - - cargo-target:/app/target +# - cargo-target:/app/target # - sqlite-data:/sqlite-data/workspace/data # - ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data environment: APP_ENV: development LOG: debug TZ: "Europe/Paris" - EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiOGYwMmExZTAtNDY0NC00MWFmLWIzYjctYjZkYWNjNzQ4OWVhIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ==" + EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiZWE5OWE5ZjMtNDhkYy00MjMyLTkzMTAtNTc5YWVkY2IyZjhmIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ==" #POOLING: 1 #DATABASES_CONFIG_FILE: "config.toml" extra_hosts: @@ -130,7 +130,7 @@ services: volumes: cargo-registry: cargo-git: - cargo-target: +# cargo-target: postgres-data: # mariadb-data: diff --git a/docker/Dockerfile b/docker/Dockerfile index 66e38d9..56437d5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # ========================= # Base image (shared) # ========================= -FROM rust:1.92.0 AS base +FROM rust:1.94.0 AS base RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ pkg-config \ diff --git a/src/services/storage/providers/s3/models.rs b/src/services/storage/providers/s3/models.rs index 2410fd6..ab3036c 100644 --- a/src/services/storage/providers/s3/models.rs +++ b/src/services/storage/providers/s3/models.rs @@ -1,4 +1,5 @@ use serde::{Deserialize, Serialize}; +use crate::utils::deserializer::string_or_number_to_string; #[derive(Debug, Deserialize, Serialize)] pub struct S3ProviderConfig { @@ -8,5 +9,7 @@ pub struct S3ProviderConfig { pub end_point_url: String, pub ssl: bool, pub region: Option, + #[serde(default, deserialize_with = "string_or_number_to_string")] pub port: Option, } + diff --git a/src/utils/deserializer.rs b/src/utils/deserializer.rs index 01e77f9..e4d094a 100644 --- a/src/utils/deserializer.rs +++ b/src/utils/deserializer.rs @@ -1,5 +1,7 @@ use serde::{Deserialize, Deserializer}; use toml::Value; +use serde_json::{Value as ValueJson, }; + pub fn deserialize_snake_case<'de, D>(deserializer: D) -> Result where @@ -36,3 +38,18 @@ pub fn camel_to_snake(s: &str) -> String { } out } + + +pub fn string_or_number_to_string<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let value = Option::::deserialize(deserializer)?; + + match value { + Some(ValueJson::String(s)) => Ok(Some(s)), + Some(ValueJson::Number(n)) => Ok(Some(n.to_string())), + Some(_) => Err(serde::de::Error::custom("port must be string or number")), + None => Ok(None), + } +} From 987c630c73e0370f9e95e2568a0743594340d4a6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 16 Mar 2026 07:59:25 +0000 Subject: [PATCH 02/11] chore: release 1.5.1 --- CITATION.cff | 2 +- Cargo.lock | 2 +- Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index 8a0de38..aa4651c 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -27,5 +27,5 @@ keywords: - self-hosted - portabase license: Apache-2.0 -version: 1.5.0 +version: 1.5.1 date-released: '2026-02-24' diff --git a/Cargo.lock b/Cargo.lock index dd3b6d8..cbe15dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3038,7 +3038,7 @@ dependencies = [ [[package]] name = "portabase-agent" -version = "1.5.0" +version = "1.5.1" dependencies = [ "aes", "aes-gcm", diff --git a/Cargo.toml b/Cargo.toml index d378fd3..d7ea3c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "portabase-agent" -version = "1.5.0" +version = "1.5.1" edition = "2024" [dependencies] From d6d2ef18f870686eec784368b3e54d8c8b128df6 Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Mon, 16 Mar 2026 11:24:30 +0100 Subject: [PATCH 03/11] fix: internal redis port --- docker-compose.yml | 2 +- docker/Dockerfile | 2 ++ docker/entrypoints/app-dev-entrypoint.sh | 2 +- entrypoint.sh | 2 +- src/settings.rs | 2 +- 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 80cbcc1..b4fd42a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,7 +18,7 @@ services: APP_ENV: development LOG: debug TZ: "Europe/Paris" - EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiZWE5OWE5ZjMtNDhkYy00MjMyLTkzMTAtNTc5YWVkY2IyZjhmIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ==" + EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiMmVkZWFlYjctNWYwNi00MjI1LWIzNWItOGYyM2MwOTMyMTA2IiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ==" #POOLING: 1 #DATABASES_CONFIG_FILE: "config.toml" extra_hosts: diff --git a/docker/Dockerfile b/docker/Dockerfile index 56437d5..5a714fa 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -17,6 +17,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ mariadb-client \ sqlite3 \ redis-tools \ + valkey \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -96,6 +97,7 @@ RUN apt-get update && apt-get install -y \ mariadb-client \ sqlite3 \ redis-tools \ + valkey \ && rm -rf /var/lib/apt/lists/* diff --git a/docker/entrypoints/app-dev-entrypoint.sh b/docker/entrypoints/app-dev-entrypoint.sh index 3e3fb46..be4a8f8 100755 --- a/docker/entrypoints/app-dev-entrypoint.sh +++ b/docker/entrypoints/app-dev-entrypoint.sh @@ -35,5 +35,5 @@ check_docker check_network echo "Starting docker-compose..." -docker-compose -f ./docker-compose.yml up +docker compose -f ./docker-compose.yml up echo "Docker-compose started successfully." \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh index b599ebb..9efdc0d 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -36,7 +36,7 @@ fi echo "[entrypoint] APP_ENV=$APP_ENV" echo "[entrypoint] Starting Redis..." -redis-server --daemonize yes +redis-server --port 84569 --daemonize yes echo "[entrypoint] Waiting for Redis to be ready..." until redis-cli ping >/dev/null 2>&1; do diff --git a/src/settings.rs b/src/settings.rs index 5270028..dae1a9b 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -31,7 +31,7 @@ impl Settings { app_version: env!("CARGO_PKG_VERSION").to_string(), app_env: env::var("APP_ENV").unwrap_or_else(|_| "development".into()), redis_url: env::var("CELERY_BROKER_URL") - .unwrap_or_else(|_| "redis://localhost:6379/".into()), + .unwrap_or_else(|_| "redis://localhost:84569/".into()), edge_key: env::var("EDGE_KEY").unwrap_or_default(), databases_config_file: env::var("DATABASES_CONFIG_FILE") .unwrap_or_else(|_| "config.json".into()), From cf5de3115c521f83053c4cee9ab3b731e28f85d4 Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Mon, 16 Mar 2026 12:11:01 +0100 Subject: [PATCH 04/11] feat: add valkey database engine for backup --- databases.json | 18 +++++++++- docker-compose.yml | 34 ++++++++++++++++-- entrypoint.sh | 14 ++++++-- src/domain/factory.rs | 4 +++ src/domain/mod.rs | 1 + src/domain/valkey/backup.rs | 65 +++++++++++++++++++++++++++++++++++ src/domain/valkey/database.rs | 45 ++++++++++++++++++++++++ src/domain/valkey/mod.rs | 3 ++ src/domain/valkey/ping.rs | 55 +++++++++++++++++++++++++++++ src/services/config.rs | 11 +++--- src/settings.rs | 2 +- 11 files changed, 239 insertions(+), 13 deletions(-) create mode 100644 src/domain/valkey/backup.rs create mode 100644 src/domain/valkey/database.rs create mode 100644 src/domain/valkey/mod.rs create mode 100644 src/domain/valkey/ping.rs diff --git a/databases.json b/databases.json index 90b7d73..cb701b0 100644 --- a/databases.json +++ b/databases.json @@ -60,11 +60,27 @@ { "name": "Test database 9 - Redis Auth", "type": "redis", - "password": "m6o2L0Ukw3Og5RJUKh", + "password": "supersecurepassword", "port": 6379, "username": "default", "host": "db-redis-auth", "generated_id": "16678160-ff7e-4c97-8c83-0adeff214681" + }, + { + "name": "Test database 10 - Valkey", + "type": "valkey", + "port": 6379, + "host": "db-valkey", + "generated_id": "16678560-ff7e-4c97-8c83-0adeff214681" + }, + { + "name": "Test database 11 - Valkey Auth", + "type": "valkey", + "password": "supersecurepassword", + "port": 6379, + "username": "default", + "host": "db-valkey-auth", + "generated_id": "16678561-ff7e-4c97-8c83-0adeff214681" } ] } diff --git a/docker-compose.yml b/docker-compose.yml index b4fd42a..c0847b4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -122,8 +122,35 @@ services: volumes: - redis-data-auth:/data environment: - - REDIS_PASSWORD=m6o2L0Ukw3Og5RJUKh - command: [ "redis-server", "--requirepass", "m6o2L0Ukw3Og5RJUKh", "--appendonly", "yes" ] + - REDIS_PASSWORD=supersecurepassword + command: [ "redis-server", "--requirepass", "supersecurepassword", "--appendonly", "yes" ] + networks: + - portabase + + db-valkey: + image: valkey/valkey + container_name: db-valkey + environment: + - ALLOW_EMPTY_PASSWORD=yes + - VALKEY_DISABLE_COMMANDS=FLUSHDB,FLUSHALL + ports: + - '6381:6379' + volumes: + - valkey-data:/bitnami/valkey/data + networks: + - portabase + + db-valkey-auth: + image: valkey/valkey + container_name: db-valkey-auth + command: > + --requirepass "supersecurepassword" + environment: + - VALKEY_DISABLE_COMMANDS=FLUSHDB,FLUSHALL + ports: + - '6382:6379' + volumes: + - valkey-data-auth:/bitnami/valkey/data networks: - portabase @@ -131,7 +158,6 @@ volumes: cargo-registry: cargo-git: # cargo-target: - postgres-data: # mariadb-data: # mongodb-data: @@ -139,6 +165,8 @@ volumes: # sqlite-data: redis-data: redis-data-auth: + valkey-data: + valkey-data-auth: networks: portabase: diff --git a/entrypoint.sh b/entrypoint.sh index 9efdc0d..7bb6438 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -34,16 +34,24 @@ if [ -n "$TZ" ]; then fi fi +REDIS_PORT=65515 echo "[entrypoint] APP_ENV=$APP_ENV" echo "[entrypoint] Starting Redis..." -redis-server --port 84569 --daemonize yes +redis-server --port $REDIS_PORT --daemonize yes & +sleep 1 echo "[entrypoint] Waiting for Redis to be ready..." -until redis-cli ping >/dev/null 2>&1; do +MAX_RETRIES=20 +COUNT=0 +until redis-cli -h localhost -p "$REDIS_PORT" ping >/dev/null 2>&1 ; do + COUNT=$((COUNT+1)) + if [ $COUNT -ge $MAX_RETRIES ]; then + echo "[ERROR] Redis did not start after $MAX_RETRIES attempts" + exit 1 + fi echo "[entrypoint] Redis not ready, sleeping 1s..." sleep 1 done - echo "[entrypoint] Redis is ready" diff --git a/src/domain/factory.rs b/src/domain/factory.rs index dc72d16..eadd24c 100644 --- a/src/domain/factory.rs +++ b/src/domain/factory.rs @@ -8,6 +8,7 @@ use crate::services::config::{DatabaseConfig, DbType}; use anyhow::Result; use std::path::{Path, PathBuf}; use std::sync::Arc; +use crate::domain::valkey::database::ValkeyDatabase; #[async_trait::async_trait] pub trait Database: Send + Sync { @@ -31,6 +32,7 @@ impl DatabaseFactory { DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)), DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)), DbType::Redis => Arc::new(RedisDatabase::new(cfg)), + DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)) } } @@ -45,6 +47,8 @@ impl DatabaseFactory { DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)), DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)), DbType::Redis => Arc::new(RedisDatabase::new(cfg)), + DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)) + } } } diff --git a/src/domain/mod.rs b/src/domain/mod.rs index 1663d08..ea49a82 100644 --- a/src/domain/mod.rs +++ b/src/domain/mod.rs @@ -4,4 +4,5 @@ pub mod mysql; mod mongodb; mod sqlite; mod redis; +mod valkey; diff --git a/src/domain/valkey/backup.rs b/src/domain/valkey/backup.rs new file mode 100644 index 0000000..fa9bd57 --- /dev/null +++ b/src/domain/valkey/backup.rs @@ -0,0 +1,65 @@ +use crate::services::config::DatabaseConfig; +use anyhow::{Context, Result}; +use std::path::PathBuf; +use std::process::Command; +use tracing::{debug, error, info}; + +pub async fn run( + cfg: DatabaseConfig, + backup_dir: PathBuf, + file_extension: &'static str, +) -> Result { + tokio::task::spawn_blocking(move || -> Result { + debug!("Starting Redis backup for database {}", cfg.name); + + let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension)); + + let mut cmd = Command::new("valkey-cli"); + + cmd.arg("-h") + .arg(&cfg.host) + .arg("-p") + .arg(cfg.port.to_string()); + + if !cfg.username.is_empty() { + cmd.arg("--user").arg(&cfg.username); + } + + if !cfg.password.is_empty() { + cmd.arg("-a").arg(&cfg.password); + } + + cmd.arg("--rdb").arg(&file_path); + + debug!("Command Backup: {:?}", cmd); + + let output = cmd.output().context("Valkey backup command failed")?; + + let stderr = String::from_utf8_lossy(&output.stderr); + let stdout = String::from_utf8_lossy(&output.stdout); + + if !output.status.success() { + if stderr.contains("NOAUTH") { + error!( + "Valkey backup failed for {}: Authentication required (NOAUTH)", + cfg.name + ); + anyhow::bail!( + "Valkey backup failed for {}: Authentication required", + cfg.name + ); + } else { + error!("Valkey backup failed for {}: {}", cfg.name, stderr); + anyhow::bail!("Valkey backup failed for {}: {}", cfg.name, stderr); + } + } + + info!( + "Valkey backup completed for {}. Output: {}", + cfg.name, stdout + ); + + Ok(file_path) + }) + .await? +} diff --git a/src/domain/valkey/database.rs b/src/domain/valkey/database.rs new file mode 100644 index 0000000..a0677ac --- /dev/null +++ b/src/domain/valkey/database.rs @@ -0,0 +1,45 @@ +use anyhow::{Result, bail}; +use async_trait::async_trait; +use std::path::{Path, PathBuf}; + +use crate::domain::factory::Database; +use crate::domain::valkey::{backup, ping}; +use crate::services::config::DatabaseConfig; +use crate::utils::locks::{DbOpLock, FileLock}; + +pub struct ValkeyDatabase { + cfg: DatabaseConfig, +} + +impl ValkeyDatabase { + pub fn new(cfg: DatabaseConfig) -> Self { + Self { cfg } + } +} + +#[async_trait] +impl Database for ValkeyDatabase { + fn file_extension(&self) -> &'static str { + ".rdb" + } + + async fn ping(&self) -> Result { + ping::run(self.cfg.clone()).await + } + + async fn backup(&self, dir: &Path, is_test: Option) -> Result { + let test_mode = is_test.unwrap_or(false); + if !test_mode { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; + } + let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await; + if !test_mode { + FileLock::release(&self.cfg.generated_id).await?; + } + res + } + + async fn restore(&self, _file: &Path, _is_test: Option) -> Result<()> { + bail!("Restore not supported for Valkey databases") + } +} diff --git a/src/domain/valkey/mod.rs b/src/domain/valkey/mod.rs new file mode 100644 index 0000000..3997ede --- /dev/null +++ b/src/domain/valkey/mod.rs @@ -0,0 +1,3 @@ +mod backup; +pub mod database; +mod ping; diff --git a/src/domain/valkey/ping.rs b/src/domain/valkey/ping.rs new file mode 100644 index 0000000..eb6f47c --- /dev/null +++ b/src/domain/valkey/ping.rs @@ -0,0 +1,55 @@ +use tracing::{debug, info}; +use crate::services::config::DatabaseConfig; +use tokio::process::Command; +use tokio::time::{timeout, Duration}; +use anyhow::{Result, Context}; + +pub async fn run(cfg: DatabaseConfig) -> Result { + let mut cmd = Command::new("valkey-cli"); + cmd.arg("-h") + .arg(&cfg.host) + .arg("-p") + .arg(cfg.port.to_string()); + + if !cfg.username.is_empty() { + cmd.arg("--user").arg(&cfg.username); + } + + if !cfg.password.is_empty() { + cmd.arg("-a").arg(&cfg.password); + } + + cmd.arg("PING"); + + debug!("Command Ping: {:?}", cmd); + + + let result = timeout(Duration::from_secs(10), cmd.output()).await; + + match result { + Ok(output) => { + let output = output.context("Failed to execute redis-cli")?; + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + + info!("Redis stdout: {}", stdout); + info!("Redis stderr: {}", stderr); + + if stderr.contains("NOAUTH") { + info!("Redis authentication failed (NOAUTH required)"); + return Ok(false); + } + + if !output.status.success() { + info!("Redis command failed with status: {:?}", output.status); + return Ok(false); + } + + Ok(stdout.contains("PONG")) + } + Err(_) => { + info!("Timeout connecting to Redis at {}:{}", cfg.host, cfg.port); + Ok(false) + } + } +} \ No newline at end of file diff --git a/src/services/config.rs b/src/services/config.rs index bdd990a..69460dd 100644 --- a/src/services/config.rs +++ b/src/services/config.rs @@ -19,8 +19,8 @@ pub enum DbType { Postgresql, MongoDB, Sqlite, - Redis - // Add other DB types if needed + Redis, + Valkey } impl DbType { @@ -32,6 +32,7 @@ impl DbType { DbType::MongoDB => "mongodb", DbType::Sqlite => "sqlite", DbType::Redis => "redis", + DbType::Valkey => "valkey", } } } @@ -164,17 +165,17 @@ impl ConfigService { }; let host = match db.db_type { - DbType::Postgresql | DbType::Mysql | DbType::Mariadb | DbType::MongoDB | DbType::Redis => required(&db.host, &db.name, "host")?, + DbType::Postgresql | DbType::Mysql | DbType::Mariadb | DbType::MongoDB | DbType::Redis | DbType::Valkey => required(&db.host, &db.name, "host")?, DbType::Sqlite => optional(&db.host), }; let port = match db.db_type { - DbType::Postgresql | DbType::Mysql | DbType::Mariadb | DbType::MongoDB | DbType::Redis => required(&db.port, &db.name, "port")?, + DbType::Postgresql | DbType::Mysql | DbType::Mariadb | DbType::MongoDB | DbType::Redis | DbType::Valkey => required(&db.port, &db.name, "port")?, DbType::Sqlite => db.port.unwrap_or(0), }; let database_name = match db.db_type { - DbType::Sqlite | DbType::Redis => optional(&db.database), + DbType::Sqlite | DbType::Redis | DbType::Valkey => optional(&db.database), _ => required(&db.database, &db.name, "database")? }; diff --git a/src/settings.rs b/src/settings.rs index dae1a9b..03d47cf 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -31,7 +31,7 @@ impl Settings { app_version: env!("CARGO_PKG_VERSION").to_string(), app_env: env::var("APP_ENV").unwrap_or_else(|_| "development".into()), redis_url: env::var("CELERY_BROKER_URL") - .unwrap_or_else(|_| "redis://localhost:84569/".into()), + .unwrap_or_else(|_| "redis://localhost:65515/".into()), edge_key: env::var("EDGE_KEY").unwrap_or_default(), databases_config_file: env::var("DATABASES_CONFIG_FILE") .unwrap_or_else(|_| "config.json".into()), From cddb9d78faed001e4203167203175534cadb910b Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Mon, 16 Mar 2026 12:16:03 +0100 Subject: [PATCH 05/11] fix: entrypoint.sh --- entrypoint.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 7bb6438..697ccc4 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -37,8 +37,7 @@ fi REDIS_PORT=65515 echo "[entrypoint] APP_ENV=$APP_ENV" echo "[entrypoint] Starting Redis..." -redis-server --port $REDIS_PORT --daemonize yes & -sleep 1 +redis-server --port $REDIS_PORT --daemonize yes echo "[entrypoint] Waiting for Redis to be ready..." MAX_RETRIES=20 From 45c33dd031c067ece4c91f19f32650e95ec6b87f Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Mon, 16 Mar 2026 12:30:22 +0100 Subject: [PATCH 06/11] feat: add valkey test sequences --- Cargo.toml | 4 +-- scripts/tests/requirements.sh | 3 +- src/tests/domain/mod.rs | 3 +- src/tests/domain/valkey.rs | 64 +++++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 src/tests/domain/valkey.rs diff --git a/Cargo.toml b/Cargo.toml index d7ea3c3..8e4d6b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ tokio-postgres = "0.7.15" futures = "0.3.31" tracing-appender = "0.2.4" time = { version = "0.3.44", features = ["macros"] } -mongodb = "3.5.0" +mongodb = "3.5.1" rand = "0.9.2" bytes = "1.11.0" async-stream = "0.3.6" @@ -49,7 +49,7 @@ tokio-stream = "0.1.18" aes = "0.9.0-rc.4" typenum = "1.19.0" testcontainers = "0.27.1" -testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis"] } +testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "valkey"] } postgres = "0.19.12" url = "2.5.8" diff --git a/scripts/tests/requirements.sh b/scripts/tests/requirements.sh index 2239327..324244e 100755 --- a/scripts/tests/requirements.sh +++ b/scripts/tests/requirements.sh @@ -58,7 +58,7 @@ if [[ "$OS_TYPE" == "Linux" ]]; then if command -v apt >/dev/null 2>&1; then echo "Linux detected with apt. Installing prerequisites..." sudo apt update - sudo apt install -y wget gnupg lsb-release redis-tools + sudo apt install -y wget gnupg lsb-release redis-tools valkey install_pg_binaries else echo "Unsupported Linux distribution. Only apt-based distros are supported." @@ -69,6 +69,7 @@ elif [[ "$OS_TYPE" == "Darwin" ]]; then if command -v brew >/dev/null 2>&1; then echo "macOS detected. Installing prerequisites..." brew install redis + brew install valkey sudo mkdir -p "$POSTGRES_BASE" sudo chown -R "$(whoami)" "$POSTGRES_BASE" diff --git a/src/tests/domain/mod.rs b/src/tests/domain/mod.rs index d68fe4f..7668a49 100644 --- a/src/tests/domain/mod.rs +++ b/src/tests/domain/mod.rs @@ -1,2 +1,3 @@ mod postgres; -mod redis; \ No newline at end of file +mod redis; +mod valkey; \ No newline at end of file diff --git a/src/tests/domain/valkey.rs b/src/tests/domain/valkey.rs new file mode 100644 index 0000000..b6d36bb --- /dev/null +++ b/src/tests/domain/valkey.rs @@ -0,0 +1,64 @@ +use tempfile::TempDir; +use testcontainers::runners::AsyncRunner; +use testcontainers::ContainerAsync; +use testcontainers_modules::valkey::{Valkey}; +use url::Host; +use crate::domain::factory::DatabaseFactory; +use crate::services::config::{DatabaseConfig, DbType}; +use crate::tests::init_tracing_for_test; + +async fn create_config() -> (ContainerAsync, DatabaseConfig) { + let container = Valkey::default().start().await.unwrap(); + + let host = container + .get_host() + .await + .unwrap_or(Host::parse("127.0.0.1").unwrap()); + + let port = container + .get_host_port_ipv4(6379) + .await + .unwrap_or(6379); + + let config = DatabaseConfig { + name: "Test Valey".to_string(), + database: "valkey".to_string(), + username: "".to_string(), + password: "".to_string(), + db_type: DbType::Valkey, + port, + host: host.to_string(), + generated_id: "40875485-e3d2-4dfe-a26b-2a347ecc64fd".to_string(), + path: "".to_string(), + }; + + (container, config) +} + +#[tokio::test] +async fn valkey_ping_test() { + init_tracing_for_test(); + + let (_container, config) = create_config().await; + + let db = DatabaseFactory::create_for_backup(config.clone()).await; + let reachable = db.ping().await.unwrap_or(false); + + assert!(reachable); +} + +#[tokio::test] +async fn valkey_backup_test() { + init_tracing_for_test(); + + let (_container, config) = create_config().await; + + let temp_dir = TempDir::new().unwrap(); + let backup_path = temp_dir.path(); + + let db = DatabaseFactory::create_for_backup(config.clone()).await; + + let file_path = db.backup(backup_path, Some(true)).await.unwrap(); + + assert!(file_path.is_file()); +} \ No newline at end of file From 5ec7bc9f285a78f9ca1276e8b3f016629c77a4a3 Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Mon, 16 Mar 2026 12:32:12 +0100 Subject: [PATCH 07/11] fix: docker-compose.yml volume for valkey --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c0847b4..706d1dd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -136,7 +136,7 @@ services: ports: - '6381:6379' volumes: - - valkey-data:/bitnami/valkey/data + - valkey-data:/data networks: - portabase @@ -150,7 +150,7 @@ services: ports: - '6382:6379' volumes: - - valkey-data-auth:/bitnami/valkey/data + - valkey-data-auth:/data networks: - portabase From fc4e4e3881dd3fcbd006d2ecd820822c1a934ecc Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Mon, 16 Mar 2026 12:32:57 +0100 Subject: [PATCH 08/11] fix: log in backup.rs --- src/domain/valkey/backup.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/valkey/backup.rs b/src/domain/valkey/backup.rs index fa9bd57..d064b19 100644 --- a/src/domain/valkey/backup.rs +++ b/src/domain/valkey/backup.rs @@ -10,7 +10,7 @@ pub async fn run( file_extension: &'static str, ) -> Result { tokio::task::spawn_blocking(move || -> Result { - debug!("Starting Redis backup for database {}", cfg.name); + debug!("Starting Valkey backup for database {}", cfg.name); let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension)); From cbc7517603350d198f00036f26e5f37b6a31881f Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Mon, 16 Mar 2026 14:28:24 +0100 Subject: [PATCH 09/11] fix: README.md with Valkey content --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 132f573..3cbef91 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,8 @@ [![PostgreSQL](https://img.shields.io/badge/PostgreSQL-336791?logo=postgresql&logoColor=white)](https://www.postgresql.org/) [![MySQL](https://img.shields.io/badge/MySQL-4479A1?logo=mysql&logoColor=white)](https://www.mysql.com/) [![SQLite](https://img.shields.io/badge/-SQLite-blue?logo=sqlite&logoColor=white)](https://sqlite.org/) +[![Redis](https://img.shields.io/badge/Redis-DC382D?style=flat&logo=Redis&logoColor=white)](https://redis.io/) +[![Valkey](https://img.shields.io/badge/Valkey-6284fc?style=flat&logo=Valkey&logoColor=white)](https://valkey.io/) [![MariaDB](https://img.shields.io/badge/MariaDB-003545?logo=mariadb&logoColor=white)](https://mariadb.org/) [![MongoDB](https://img.shields.io/badge/-MongoDB-13aa52?logo=mongodb&logoColor=white)](https://www.mongodb.com/) [![Self Hosted](https://img.shields.io/badge/self--hosted-yes-brightgreen)](https://github.com/Portabase/portabase) From e1bc77e481b8a923bdc373e38293447bd6faca08 Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Mon, 16 Mar 2026 14:31:11 +0100 Subject: [PATCH 10/11] fix: valkey.rs --- src/tests/domain/valkey.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/domain/valkey.rs b/src/tests/domain/valkey.rs index b6d36bb..840f35e 100644 --- a/src/tests/domain/valkey.rs +++ b/src/tests/domain/valkey.rs @@ -21,7 +21,7 @@ async fn create_config() -> (ContainerAsync, DatabaseConfig) { .unwrap_or(6379); let config = DatabaseConfig { - name: "Test Valey".to_string(), + name: "Test Valkey".to_string(), database: "valkey".to_string(), username: "".to_string(), password: "".to_string(), From f14a6dc318acaaab39e8a6e77e0848eb04e06a05 Mon Sep 17 00:00:00 2001 From: charles-gauthereau Date: Mon, 16 Mar 2026 14:32:33 +0100 Subject: [PATCH 11/11] fix: docker-compose.yml --- docker-compose.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 706d1dd..7adc57a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -132,7 +132,6 @@ services: container_name: db-valkey environment: - ALLOW_EMPTY_PASSWORD=yes - - VALKEY_DISABLE_COMMANDS=FLUSHDB,FLUSHALL ports: - '6381:6379' volumes: @@ -145,8 +144,6 @@ services: container_name: db-valkey-auth command: > --requirepass "supersecurepassword" - environment: - - VALKEY_DISABLE_COMMANDS=FLUSHDB,FLUSHALL ports: - '6382:6379' volumes: