From c348253015da08efb2559f91ce5a845aaa750e11 Mon Sep 17 00:00:00 2001 From: charlesgauthereau Date: Sun, 1 Feb 2026 15:49:33 +0100 Subject: [PATCH] fix: refactoring for big backup files. --- Cargo.lock | 132 ++++++++++++++++++++----------- Cargo.toml | 7 +- Makefile | 4 + docker-compose.prod.yml | 56 +++++++++++-- docker-compose.yml | 136 ++++++++++++++++---------------- docker/Dockerfile | 1 - scripts/mysql/seed-1gb.sql | 53 +++++++++++++ src/services/backup.rs | 156 ++++++++++++++++--------------------- src/utils/file.rs | 80 ++++++++++++++++--- 9 files changed, 402 insertions(+), 223 deletions(-) create mode 100644 scripts/mysql/seed-1gb.sql diff --git a/Cargo.lock b/Cargo.lock index 457ed44..6031425 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -51,6 +51,28 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d" +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -76,9 +98,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-lc-rs" -version = "1.15.3" +version = "1.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e84ce723ab67259cfeb9877c6a639ee9eb7a27b28123abd71db7f0d5d0cc9d86" +checksum = "7b7b6141e96a8c160799cc2d5adecd5cbbe5054cb8c7c4af53da0f83bb7ad256" dependencies = [ "aws-lc-sys", "zeroize", @@ -86,9 +108,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.36.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a442ece363113bd4bd4c8b18977a7798dd4d3c3383f34fb61936960e8f4ad8" +checksum = "5c34dda4df7017c8db52132f0f8a2e0f8161649d15723ed63fc00c82d0f2081a" dependencies = [ "cc", "cmake", @@ -172,9 +194,9 @@ checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" [[package]] name = "cc" -version = "1.2.53" +version = "1.2.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "755d2fce177175ffca841e9a06afdb2c4ab0f593d53b4dee48147dfaade85932" +checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29" dependencies = [ "find-msvc-tools", "jobserver", @@ -558,9 +580,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "flate2" @@ -927,7 +949,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.1", + "socket2 0.6.2", "system-configuration", "tokio", "tower-service", @@ -937,9 +959,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.64" +version = "0.1.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1321,9 +1343,9 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.12" +version = "0.12.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3dec6bd31b08944e08b58fd99373893a6c17054d6f3ea5006cc894f4f4eee2a" +checksum = "b4ac832c50ced444ef6be0767a008b02c106a909ba79d1d830501e94b96f6b7e" dependencies = [ "crossbeam-channel", "crossbeam-epoch", @@ -1387,7 +1409,7 @@ dependencies = [ "serde_with", "sha1", "sha2", - "socket2 0.6.1", + "socket2 0.6.2", "stringprep", "strsim", "take_mut", @@ -1433,9 +1455,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" [[package]] name = "num-integer" @@ -1598,8 +1620,10 @@ name = "portabase-agent" version = "1.0.4" dependencies = [ "anyhow", + "async-stream", "async-trait", "base64", + "bytes", "chrono", "cron", "dotenvy", @@ -1610,6 +1634,7 @@ dependencies = [ "mongodb", "once_cell", "openssl", + "rand", "redis", "reqwest", "serde", @@ -1620,10 +1645,10 @@ dependencies = [ "time", "tokio", "tokio-postgres", + "tokio-util", "toml", "tracing", "tracing-appender", - "tracing-log", "tracing-subscriber", ] @@ -1708,7 +1733,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.1", + "socket2 0.6.2", "thiserror 2.0.18", "tokio", "tracing", @@ -1746,16 +1771,16 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.1", + "socket2 0.6.2", "tracing", "windows-sys 0.60.2", ] [[package]] name = "quote" -version = "1.0.43" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc74d9a594b72ae6656596548f56f667211f8a97b3d4c3d467150794690dc40a" +checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" dependencies = [ "proc-macro2", ] @@ -1803,9 +1828,9 @@ dependencies = [ [[package]] name = "redis" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfe20977fe93830c0e9817a16fbf1ed1cfd8d4bba366087a1841d2c6033c251" +checksum = "e969d1d702793536d5fda739a82b88ad7cbe7d04f8386ee8cd16ad3eff4854a5" dependencies = [ "arcstr", "bytes", @@ -1818,7 +1843,7 @@ dependencies = [ "pin-project-lite", "ryu", "sha1_smol", - "socket2 0.6.1", + "socket2 0.6.2", "tokio", "tokio-util", "url", @@ -1894,12 +1919,14 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls", + "tokio-util", "tower", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", ] @@ -2249,9 +2276,9 @@ checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" [[package]] name = "siphasher" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" [[package]] name = "slab" @@ -2277,9 +2304,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" dependencies = [ "libc", "windows-sys 0.60.2", @@ -2459,9 +2486,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.45" +version = "0.3.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9e442fc33d7fdb45aa9bfeb312c095964abdf596f7567261062b2a7107aaabd" +checksum = "9da98b7d9b7dad93488a84b8248efc35352b0b2657397d4167e7ad67e5d535e5" dependencies = [ "deranged", "itoa", @@ -2476,15 +2503,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b36ee98fd31ec7426d599183e8fe26932a8dc1fb76ddb6214d05493377d34ca" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] name = "time-macros" -version = "0.2.25" +version = "0.2.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71e552d1249bf61ac2a52db88179fd0673def1e1ad8243a00d9ec9ed71fee3dd" +checksum = "78cc610bac2dcee56805c99642447d4c5dbde4d01f752ffea0199aee1f601dc4" dependencies = [ "num-conv", "time-core", @@ -2535,7 +2562,7 @@ dependencies = [ "mio", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.1", + "socket2 0.6.2", "tokio-macros", "windows-sys 0.61.2", ] @@ -2571,7 +2598,7 @@ dependencies = [ "postgres-protocol", "postgres-types", "rand", - "socket2 0.6.1", + "socket2 0.6.2", "tokio", "tokio-util", "whoami", @@ -2863,9 +2890,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" +checksum = "ee48d38b119b0cd71fe4141b30f5ba9c7c5d9f4e7a3a8b4a674e4b6ef789976f" dependencies = [ "getrandom 0.3.4", "js-sys", @@ -3002,6 +3029,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.85" @@ -3042,9 +3082,9 @@ dependencies = [ [[package]] name = "whoami" -version = "2.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace4d5c7b5ab3d99629156d4e0997edbe98a4beb6d5ba99e2cae830207a81983" +checksum = "8fae98cf96deed1b7572272dfc777713c249ae40aa1cf8862e091e8b745f5361" dependencies = [ "libredox", "wasite", @@ -3511,18 +3551,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.33" +version = "0.8.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668f5168d10b9ee831de31933dc111a459c97ec93225beb307aed970d1372dfd" +checksum = "7456cf00f0685ad319c5b1693f291a650eaf345e941d082fc4e03df8a03996ac" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.33" +version = "0.8.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c7962b26b0a8685668b671ee4b54d007a67d4eaf05fda79ac0ecf41e32270f1" +checksum = "1328722bbf2115db7e19d69ebcc15e795719e2d66b60827c6a69a117365e37a0" dependencies = [ "proc-macro2", "quote", @@ -3591,6 +3631,6 @@ dependencies = [ [[package]] name = "zmij" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfcd145825aace48cff44a8844de64bf75feec3080e0aa5cdbde72961ae51a65" +checksum = "02aae0f83f69aafc94776e879363e9771d7ecbffe2c7fbb6c14c5e00dfe88439" diff --git a/Cargo.toml b/Cargo.toml index 2098cab..fd13254 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ base64 = "0.22.1" thiserror = "2.0.17" log = "0.4.29" toml = "0.9.10" -reqwest = { version = "0.13.1", features = ["json", "blocking", "multipart"] } +reqwest = { version = "0.13.1", features = ["json", "blocking", "multipart", "stream"] } anyhow = "1.0.100" tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "fs"] } async-trait = "0.1.89" @@ -28,10 +28,13 @@ flate2 = "1.1.5" tar = "0.4.44" tokio-postgres = "0.7.15" futures = "0.3.31" -tracing-log = "0.2.0" tracing-appender = "0.2.4" time = { version = "0.3.44", features = ["macros"] } mongodb = "3.5.0" +rand = "0.9.2" +tokio-util = "0.7.18" +bytes = "1.11.0" +async-stream = "0.3.6" [[bin]] name = "app" diff --git a/Makefile b/Makefile index 77f5877..56a9060 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,10 @@ seed-mysql: @echo "Seeding MySQL..." mysql -h 127.0.0.1 -P "$$MYSQL_PORT" -u "$$MYSQL_USER" -p"$$MYSQL_PASSWORD" "$$MYSQL_DB" < ./scripts/mysql/seed-mysql.sql +seed-mysql-1gb: + @echo "Seeding MySQL..." + mysql -h 127.0.0.1 -P "$$MYSQL_PORT" -u "$$MYSQL_USER" -p"$$MYSQL_PASSWORD" "$$MYSQL_DB" < ./scripts/mysql/seed-1gb.sql + seed-postgres: @echo "Seeding Postgres..." diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 7edf591..768bb00 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -1,11 +1,11 @@ services: rust-app: - build: - context: . - dockerfile: docker/Dockerfile - target: prod -# image: portabase/agent:latest - platform: linux/arm64 +# build: +# context: . +# dockerfile: docker/Dockerfile +# target: prod + image: portabase/agent:latest +# platform: linux/arm64 container_name: rust-prod volumes: - ./databases.json:/config/config.json @@ -14,7 +14,7 @@ services: LOG: info TZ: "Europe/Paris" # DATABASES_CONFIG_FILE: "config.toml" - EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiY2VlZmNmNDQtOGE0YS00NjZlLTkwNDEtN2QzNDMzZjRjOTJkIiwicHVibGljS2V5IjoiLS0tLS1CRUdJTiBSU0EgUFVCTElDIEtFWS0tLS0tXG5NSUlCQ2dLQ0FRRUFuYUVKdnVBSExlNGF5d0RmNGplelVobm45VXRkdksyZ3pEMEg2cERJYXczYkJKRkpwVnVDXG5uVFV3MXA3Q2RnOXBzdjZhRnpyOXZPd0J2MjMzckxpdVpCT2lCb2p2Q0QrSlZid3hyTzBRRW5hN2dmaHV1ZGYwXG5VVlJOMkxmK1g1aTkvZzJTNm5xcExoTm1DaGFJNk8ybktYZUNlRmtubEErRUJrNnFoV1FCVGozb05TYTFTOFY1XG40UFRTT2I4NUo3a2k5YllEbXRiNWxrU3dCNXdXOTdtQjg0ZzI2WHAvU3FFcmhKc0NGK3YrN09vTWYzTzJqTTNoXG5XMUQ0MzBPRitWaklwUGdoV09rZy96NXZQUWFHRzhqQ0h4VDlJR0Q0bjhyS05LQ3FTOGNyN2diTGU0cWpNdmhvXG5BQVVvaHpHR2FRNkhlWlJ4S0UvM3J1a2JldnY5dnJ2TTNRSURBUUFCXG4tLS0tLUVORCBSU0EgUFVCTElDIEtFWS0tLS0tXG4ifQ" + EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiOGZmMDE4NTQtYjJhMS00ZTE0LTkwMjctZTJiOWIxZjQ1YzdlIiwicHVibGljS2V5IjoiLS0tLS1CRUdJTiBSU0EgUFVCTElDIEtFWS0tLS0tXG5NSUlCQ2dLQ0FRRUE5TWV4M2pmdnVLdFB5YU1ERnh2Ulp2dmd3YkRJQ2JzQi81Wll5NDNSVVRBaXZRYjJiSDdYXG5qRHBQd1lJeCs4UFBrbHlRbDVMQzV1UWZEaCs4SVd4OG1LZ3FvMXpWMkdiZXdGbEdEWFYxVEdyU1ZEU25aSWR4XG52bWdYc29EeXhVMlJvWUFUMS9YMWxuc2YxenZKdkFMTkhXdEhRdk42SjVDZTFSMmFsendVRGFEVXlJNzRmSldQXG5tNTh0SDMrYklXL0VVTXdjaWNxM0oySWw3Vm9KNkZNUHJQL1ZSOWEvdFF1SU1qa200MXpFY2NscExPa2luRkxuXG54NmVUWkFSZUpya2UrbnRvZ2t4TGEyRWV5a1lUNzB4V3hKNWp5ZExBVnRvNkkyQlVLVVJoTkowTUFaU29NYUtvXG5iMGJRcnY1UzExZWllMnMrT2I3aTYzSFpkVUx0UmV1MVJ3SURBUUFCXG4tLS0tLUVORCBSU0EgUFVCTElDIEtFWS0tLS0tXG4ifQ" extra_hosts: - "localhost:host-gateway" networks: @@ -22,6 +22,48 @@ services: + db-mongodb-auth: + container_name: db-mongodb-auth + image: mongo:latest + ports: + - "27082:27017" + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: rootpassword + MONGO_INITDB_DATABASE: testdbauth + command: mongod --auth + networks: + - portabase + volumes: + - mongodb-data-auth:/data/db + healthcheck: + test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] + interval: 5s + timeout: 5s + retries: 10 + + db-mongodb: + container_name: db-mongodb + image: mongo:latest + ports: + - "27083:27017" + volumes: + - mongodb-data:/data/db + healthcheck: + test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ] + interval: 5s + timeout: 5s + retries: 10 + environment: + MONGO_INITDB_DATABASE: testdb + networks: + - portabase + +volumes: + mongodb-data: + mongodb-data-auth: + + networks: portabase: name: portabase_network diff --git a/docker-compose.yml b/docker-compose.yml index dae3cdf..6a07ac3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,12 +11,12 @@ services: # - ./databases.toml:/config/config.toml - cargo-registry:/usr/local/cargo/registry - cargo-git:/usr/local/cargo/git - - cargo-target:/app/target +# - cargo-target:/app/target environment: APP_ENV: development LOG: debug TZ: "Europe/Paris" - EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiOGZmMDE4NTQtYjJhMS00ZTE0LTkwMjctZTJiOWIxZjQ1YzdlIiwicHVibGljS2V5IjoiLS0tLS1CRUdJTiBSU0EgUFVCTElDIEtFWS0tLS0tXG5NSUlCQ2dLQ0FRRUE5TWV4M2pmdnVLdFB5YU1ERnh2Ulp2dmd3YkRJQ2JzQi81Wll5NDNSVVRBaXZRYjJiSDdYXG5qRHBQd1lJeCs4UFBrbHlRbDVMQzV1UWZEaCs4SVd4OG1LZ3FvMXpWMkdiZXdGbEdEWFYxVEdyU1ZEU25aSWR4XG52bWdYc29EeXhVMlJvWUFUMS9YMWxuc2YxenZKdkFMTkhXdEhRdk42SjVDZTFSMmFsendVRGFEVXlJNzRmSldQXG5tNTh0SDMrYklXL0VVTXdjaWNxM0oySWw3Vm9KNkZNUHJQL1ZSOWEvdFF1SU1qa200MXpFY2NscExPa2luRkxuXG54NmVUWkFSZUpya2UrbnRvZ2t4TGEyRWV5a1lUNzB4V3hKNWp5ZExBVnRvNkkyQlVLVVJoTkowTUFaU29NYUtvXG5iMGJRcnY1UzExZWllMnMrT2I3aTYzSFpkVUx0UmV1MVJ3SURBUUFCXG4tLS0tLUVORCBSU0EgUFVCTElDIEtFWS0tLS0tXG4ifQ" + EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiZWE2NTg1MDctZTA5My00NDUxLWIxZDAtMDgwZWZjMGNmNWYzIiwicHVibGljS2V5IjoiLS0tLS1CRUdJTiBSU0EgUFVCTElDIEtFWS0tLS0tXG5NSUlCQ2dLQ0FRRUE5TWV4M2pmdnVLdFB5YU1ERnh2Ulp2dmd3YkRJQ2JzQi81Wll5NDNSVVRBaXZRYjJiSDdYXG5qRHBQd1lJeCs4UFBrbHlRbDVMQzV1UWZEaCs4SVd4OG1LZ3FvMXpWMkdiZXdGbEdEWFYxVEdyU1ZEU25aSWR4XG52bWdYc29EeXhVMlJvWUFUMS9YMWxuc2YxenZKdkFMTkhXdEhRdk42SjVDZTFSMmFsendVRGFEVXlJNzRmSldQXG5tNTh0SDMrYklXL0VVTXdjaWNxM0oySWw3Vm9KNkZNUHJQL1ZSOWEvdFF1SU1qa200MXpFY2NscExPa2luRkxuXG54NmVUWkFSZUpya2UrbnRvZ2t4TGEyRWV5a1lUNzB4V3hKNWp5ZExBVnRvNkkyQlVLVVJoTkowTUFaU29NYUtvXG5iMGJRcnY1UzExZWllMnMrT2I3aTYzSFpkVUx0UmV1MVJ3SURBUUFCXG4tLS0tLUVORCBSU0EgUFVCTElDIEtFWS0tLS0tXG4ifQ" #POOLING: 1 #DATABASES_CONFIG_FILE: "config.toml" extra_hosts: @@ -24,83 +24,83 @@ services: networks: - portabase -# db-postgres: -# container_name: db-postgres -# image: postgres:17-alpine -# ports: -# - "5436:5432" -# volumes: -# - postgres-data:/var/lib/postgresql/data -# environment: -# - POSTGRES_DB=devdb -# - POSTGRES_USER=devuser -# - POSTGRES_PASSWORD=changeme -# networks: -# - portabase + db-postgres: + container_name: db-postgres + image: postgres:17-alpine + ports: + - "5436:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + environment: + - POSTGRES_DB=devdb + - POSTGRES_USER=devuser + - POSTGRES_PASSWORD=changeme + networks: + - portabase + + db-mariadb: + container_name: db-mariadb + image: mariadb:latest + ports: + - "3311:3306" + environment: + - MYSQL_DATABASE=mariadb + - MYSQL_USER=mariadb + - MYSQL_PASSWORD=changeme + - MYSQL_RANDOM_ROOT_PASSWORD=yes + volumes: + - mariadb-data:/var/lib/mysql + networks: + - portabase + # -# db-mariadb: -# container_name: db-mariadb -# image: mariadb:latest +# db-mongodb-auth: +# container_name: db-mongodb-auth +# image: mongo:latest # ports: -# - "3311:3306" +# - "27082:27017" # environment: -# - MYSQL_DATABASE=mariadb -# - MYSQL_USER=mariadb -# - MYSQL_PASSWORD=changeme -# - MYSQL_RANDOM_ROOT_PASSWORD=yes -# volumes: -# - mariadb-data:/var/lib/mysql +# MONGO_INITDB_ROOT_USERNAME: root +# MONGO_INITDB_ROOT_PASSWORD: rootpassword +# MONGO_INITDB_DATABASE: testdbauth +# command: mongod --auth +# networks: +# - portabase +# volumes: +# - mongodb-data-auth:/data/db +# healthcheck: +# test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] +# interval: 5s +# timeout: 5s +# retries: 10 +# +# db-mongodb: +# container_name: db-mongodb +# image: mongo:latest +# ports: +# - "27083:27017" +# volumes: +# - mongodb-data:/data/db +# healthcheck: +# test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ] +# interval: 5s +# timeout: 5s +# retries: 10 +# environment: +# MONGO_INITDB_DATABASE: testdb # networks: # - portabase - - - db-mongodb-auth: - container_name: db-mongodb-auth - image: mongo:latest - ports: - - "27082:27017" - environment: - MONGO_INITDB_ROOT_USERNAME: root - MONGO_INITDB_ROOT_PASSWORD: rootpassword - MONGO_INITDB_DATABASE: testdbauth - command: mongod --auth - networks: - - portabase - volumes: - - mongodb-data-auth:/data/db - healthcheck: - test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ] - interval: 5s - timeout: 5s - retries: 10 - - db-mongodb: - container_name: db-mongodb - image: mongo:latest - ports: - - "27083:27017" - volumes: - - mongodb-data:/data/db - healthcheck: - test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ] - interval: 5s - timeout: 5s - retries: 10 - environment: - MONGO_INITDB_DATABASE: testdb - networks: - - portabase volumes: cargo-registry: cargo-git: - cargo-target: +# cargo-target: -# postgres-data: -# mariadb-data: - mongodb-data: - mongodb-data-auth: + postgres-data: + mariadb-data: +# mongodb-data: +# mongodb-data-auth: networks: portabase: diff --git a/docker/Dockerfile b/docker/Dockerfile index f65d402..d149b6d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -58,7 +58,6 @@ FROM base AS dev RUN cargo install cargo-watch -# Pre-cache dependencies COPY Cargo.toml Cargo.lock ./ RUN mkdir src && echo "fn main() {}" > src/main.rs RUN cargo build diff --git a/scripts/mysql/seed-1gb.sql b/scripts/mysql/seed-1gb.sql new file mode 100644 index 0000000..c285f30 --- /dev/null +++ b/scripts/mysql/seed-1gb.sql @@ -0,0 +1,53 @@ +CREATE DATABASE IF NOT EXISTS mariadb; +USE mariadb; + +DROP TABLE IF EXISTS users; +DROP TABLE IF EXISTS products; + +CREATE TABLE users ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + username VARCHAR(50) NOT NULL UNIQUE, + email VARCHAR(100) NOT NULL UNIQUE, + password VARCHAR(255) NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +) ENGINE=InnoDB; + +CREATE TABLE products ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(100) NOT NULL, + description MEDIUMTEXT, + price DECIMAL(10,2) NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +) ENGINE=InnoDB; + + +DELIMITER $$ + +DROP PROCEDURE IF EXISTS seed_data$$ +CREATE PROCEDURE seed_data() +BEGIN + DECLARE i BIGINT DEFAULT 1; + DECLARE j BIGINT; + DECLARE large_text TEXT; + + SET large_text = REPEAT('Lorem ipsum dolor sit amet, consectetur adipiscing elit. ', 300); + + WHILE i <= 200000 DO + INSERT INTO users (username, email, password) + VALUES (CONCAT('user', i), CONCAT('user', i, '@example.com'), 'changeme'); + SET i = i + 1; +END WHILE; + + SET i = 1; + WHILE i <= 200000 DO + INSERT INTO products (name, description, price) + VALUES (CONCAT('Product ', i), large_text, ROUND(RAND()*1000,2)); + SET i = i + 1; +END WHILE; +END$$ + +DELIMITER ; + +CALL seed_data(); + +DROP PROCEDURE IF EXISTS seed_data; diff --git a/src/services/backup.rs b/src/services/backup.rs index f049751..eedbc73 100644 --- a/src/services/backup.rs +++ b/src/services/backup.rs @@ -4,21 +4,16 @@ use crate::core::context::Context; use crate::domain::factory::DatabaseFactory; use crate::services::config::{DatabaseConfig, DatabasesConfig, DbType}; use crate::utils::common::BackupMethod; -use crate::utils::file::full_extension; +use crate::utils::file::{encrypt_file_stream, full_extension}; use anyhow::Result; use hex; -use openssl::encrypt::Encrypter; -use openssl::hash::MessageDigest; -use openssl::pkey::PKey; -use openssl::rand::rand_bytes; -use openssl::rsa::Padding; -use openssl::symm::{Cipher, Crypter, Mode}; -use reqwest::multipart::{Form, Part}; use std::path::{Path, PathBuf}; use std::sync::Arc; +use rand::RngCore; use tempfile::TempDir; -use tokio::fs; use tracing::{error, info}; +use reqwest::Body; +use futures::StreamExt; #[derive(Debug)] pub struct BackupResult { @@ -119,100 +114,81 @@ impl BackupService { pub async fn send_result(&self, result: BackupResult, method: BackupMethod) { if result.code.as_deref() == Some("backup_already_in_progress") { - info!( - "[BackupService] Skipping send for DB {}: backup already in progress", - result.generated_id - ); + info!("Skipping send: backup already in progress"); return; } - info!( - "[BackupService] DB: {} Type: {} Status: {} File: {:?}", - result.generated_id, - result.db_type.as_str(), - result.status, - result.backup_file + let Some(file_path) = result.backup_file else { + return; + }; + + let mut aes_key = [0u8; 32]; + rand::rng().fill_bytes(&mut aes_key); + + let mut iv = [0u8; 16]; + rand::rng().fill_bytes(&mut iv); + + let public_key_pem = self.ctx.edge_key.public_key.as_bytes().to_vec(); + + + let (encrypted_stream, encrypted_key_hex) = + match encrypt_file_stream( + file_path.clone(), + aes_key, + iv, + public_key_pem, + ) + .await + { + Ok(v) => v, + Err(e) => { + error!("Encryption failed: {}", e); + return; + } + }; + + let file_size = std::fs::metadata(&file_path) + .map(|m| m.len()) + .unwrap_or(0); + + let body = Body::wrap_stream( + encrypted_stream.map(|r| r.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))) + ); + + let url = format!( + "{}/services/v1/upload/{}", + self.ctx.edge_key.server_url, + self.ctx.edge_key.agent_id ); let client = reqwest::Client::new(); - let url = format!( - "{}/api/agent/{}/backup", - self.ctx.edge_key.server_url, self.ctx.edge_key.agent_id - ); - let mut form = Form::new() - .text("generatedId", result.generated_id.clone()) - .text("status", result.status.clone()) - .text("method", method.to_string()); - if let Some(file_path) = result.backup_file { - match fs::read(&file_path).await { - Ok(raw_data) => { - // AES key + IV - let mut aes_key = [0u8; 32]; - rand_bytes(&mut aes_key).unwrap(); + info!("file Size to {}", file_size); - let mut iv = [0u8; 16]; - rand_bytes(&mut iv).unwrap(); + let resp = client + .post(&url) + .header("X-Generated-Id", &result.generated_id) + .header("X-Status", &result.status) + .header("X-Method", method.to_string()) + .header("X-AES-Key", encrypted_key_hex) + .header("X-IV", hex::encode(iv)) + .header("X-Extension", full_extension(&file_path)) + .header("Transfer-Encoding", "chunked") + .header("X-File-Size", file_size) + .body(body) + .send() + .await; - // AES CBC PKCS7 encryption - let cipher = Cipher::aes_256_cbc(); - let mut encrypter = - Crypter::new(cipher, Mode::Encrypt, &aes_key, Some(&iv)).unwrap(); - encrypter.pad(true); - let mut encrypted = vec![0u8; raw_data.len() + cipher.block_size()]; - let count = encrypter.update(&raw_data, &mut encrypted).unwrap(); - let rest = encrypter.finalize(&mut encrypted[count..]).unwrap(); - encrypted.truncate(count + rest); - - // Encrypt AES key with RSA public key - let pub_key_pem = self.ctx.edge_key.public_key.as_bytes(); - let pkey = PKey::public_key_from_pem(pub_key_pem).unwrap(); - - let mut encrypter = Encrypter::new(&pkey).unwrap(); - // Set OAEP padding (default OAEP uses SHA1, so override) - encrypter.set_rsa_padding(Padding::PKCS1_OAEP).unwrap(); - // Set OAEP hash to SHA‑256 - encrypter.set_rsa_oaep_md(MessageDigest::sha256()).unwrap(); - encrypter.set_rsa_mgf1_md(MessageDigest::sha256()).unwrap(); - - let mut encrypted_key = vec![0u8; encrypter.encrypt_len(&aes_key).unwrap()]; - let encrypted_len = encrypter.encrypt(&aes_key, &mut encrypted_key).unwrap(); - encrypted_key.truncate(encrypted_len); - - let extension = full_extension(&file_path); - - // Attach file and AES info to multipart form - form = form - .part( - "file", - Part::bytes(encrypted) - .file_name(format!("{}.enc", result.generated_id)), - ) - .text("aes_key", hex::encode(encrypted_key)) - .text("iv", hex::encode(iv)) - .text("extension", extension); - } - Err(e) => { - error!("Failed to read backup file: {}", e); - } + match resp { + Ok(r) if r.status().is_success() => { + info!("Backup uploaded successfully"); } - } else { - form = form.text("file", ""); - } - - match client.post(&url).multipart(form).send().await { - Ok(resp) => { - let status = resp.status(); - if status.is_success() { - info!("Backup result sent successfully"); - } else { - let text = resp.text().await.unwrap_or_default(); // consumes resp - error!("Backup result failed, status: {}, body: {}", status, text); - } + Ok(r) => { + error!("Upload failed: {}", r.status()); } Err(e) => { - error!("Failed to send backup result: {}", e); + error!("Upload error: {}", e); } } } diff --git a/src/utils/file.rs b/src/utils/file.rs index 01c3c00..d0fcf61 100644 --- a/src/utils/file.rs +++ b/src/utils/file.rs @@ -1,15 +1,77 @@ -use std::path::Path; +use std::path::{Path, PathBuf}; +use anyhow::Result; +use tokio::fs::File; +use tokio::io::{AsyncReadExt, BufReader}; +use futures::{Stream}; +use bytes::Bytes; +use openssl::symm::{Cipher, Crypter, Mode}; +use openssl::pkey::PKey; +use openssl::encrypt::Encrypter; +use openssl::rsa::Padding; +use openssl::hash::MessageDigest; +use async_stream::try_stream; + +use hex; pub fn full_extension(path: &Path) -> String { path.file_name() - .and_then(|name| name.to_str()) - .map(|name| { - match name.find('.') { - Some(idx) => &name[idx..], - None => "", - } - }) - .unwrap_or_default() + .and_then(|n| n.to_str()) + .and_then(|n| n.find('.').map(|i| &n[i..])) + .unwrap_or("") .to_string() } +pub async fn encrypt_file_stream( + file_path: PathBuf, + aes_key: [u8; 32], + iv: [u8; 16], + pub_key_pem: Vec, +) -> Result<(impl Stream> + Send + 'static, String)> { + // ---------- Encrypt AES key with RSA ---------- + let pkey = PKey::public_key_from_pem(&pub_key_pem)?; + let mut rsa = Encrypter::new(&pkey)?; + rsa.set_rsa_padding(Padding::PKCS1_OAEP)?; + rsa.set_rsa_oaep_md(MessageDigest::sha256())?; + rsa.set_rsa_mgf1_md(MessageDigest::sha256())?; + + let mut encrypted_key = vec![0u8; rsa.encrypt_len(&aes_key)?]; + let len = rsa.encrypt(&aes_key, &mut encrypted_key)?; + encrypted_key.truncate(len); + + let encrypted_key_hex = hex::encode(encrypted_key); + + // ---------- Streaming AES encryption ---------- + let stream = try_stream! { + let file = File::open(&file_path).await?; + let mut reader = BufReader::new(file); + + let cipher = Cipher::aes_256_cbc(); + let mut crypter = Crypter::new(cipher, Mode::Encrypt, &aes_key, Some(&iv))?; + crypter.pad(true); + + let mut buffer = vec![0u8; 1024 * 1024]; // 1 MB chunks + + loop { + let n = reader.read(&mut buffer).await?; + if n == 0 { + break; + } + + let mut out = vec![0u8; n + cipher.block_size()]; + let count = crypter.update(&buffer[..n], &mut out)?; + out.truncate(count); + + yield Bytes::from(out); + } + + let mut final_block = vec![0u8; cipher.block_size()]; + let rest = crypter.finalize(&mut final_block)?; + final_block.truncate(rest); + + if !final_block.is_empty() { + yield Bytes::from(final_block); + } + }; + + Ok((stream, encrypted_key_hex)) +} \ No newline at end of file