From 5431b9273dafb8889efa0e76bf9bd7c2d8c86232 Mon Sep 17 00:00:00 2001 From: houseme Date: Thu, 7 May 2026 18:00:59 +0800 Subject: [PATCH] feat(targets): complete redis mysql postgres target wiring (#2842) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: jaehanbyun Signed-off-by: houseme Signed-off-by: Gunther Xing Signed-off-by: JaySon-Huang Co-authored-by: jaehanbyun Co-authored-by: Gunther Xing Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: cxymds Co-authored-by: JaySon Co-authored-by: 安正超 --- Cargo.lock | 747 ++++++++-- Cargo.toml | 28 +- crates/audit/src/factory.rs | 67 +- crates/audit/src/registry.rs | 6 +- crates/config/src/audit/mod.rs | 13 + crates/config/src/audit/mysql.rs | 53 + crates/config/src/audit/postgres.rs | 51 + crates/config/src/audit/redis.rs | 81 ++ crates/config/src/constants/targets.rs | 42 + crates/config/src/notify/mod.rs | 15 +- crates/config/src/notify/mysql.rs | 53 + crates/config/src/notify/postgres.rs | 51 + crates/config/src/notify/redis.rs | 82 ++ crates/ecstore/src/config/audit.rs | 250 +++- crates/ecstore/src/config/com.rs | 74 +- crates/ecstore/src/config/mod.rs | 12 +- crates/ecstore/src/config/notify.rs | 249 +++- crates/notify/src/factory.rs | 67 +- crates/notify/src/integration.rs | 20 +- crates/notify/src/registry.rs | 6 +- crates/targets/AGENTS.md | 62 + crates/targets/Cargo.toml | 9 + crates/targets/src/check.rs | 77 +- crates/targets/src/config/loader.rs | 33 +- crates/targets/src/config/mod.rs | 5 +- crates/targets/src/config/target_args.rs | 490 ++++++- crates/targets/src/lib.rs | 2 +- crates/targets/src/target/kafka.rs | 4 +- crates/targets/src/target/mod.rs | 92 +- crates/targets/src/target/mqtt.rs | 12 +- crates/targets/src/target/mysql.rs | 1226 +++++++++++++++++ crates/targets/src/target/nats.rs | 4 +- crates/targets/src/target/postgres.rs | 1095 +++++++++++++++ crates/targets/src/target/pulsar.rs | 4 +- crates/targets/src/target/redis.rs | 1205 ++++++++++++++++ crates/targets/src/target/webhook.rs | 4 +- crates/targets/tests/mysql_integration.rs | 252 ++++ crates/targets/tests/postgres_integration.rs | 305 ++++ rustfs/src/admin/handlers/audit.rs | 47 +- rustfs/src/admin/handlers/event.rs | 49 +- .../src/admin/handlers/target_descriptor.rs | 80 +- 41 files changed, 6787 insertions(+), 237 deletions(-) create mode 100644 crates/config/src/audit/mysql.rs create mode 100644 crates/config/src/audit/postgres.rs create mode 100644 crates/config/src/audit/redis.rs create mode 100644 crates/config/src/notify/mysql.rs create mode 100644 crates/config/src/notify/postgres.rs create mode 100644 crates/config/src/notify/redis.rs create mode 100644 crates/targets/AGENTS.md create mode 100644 crates/targets/src/target/mysql.rs create mode 100644 crates/targets/src/target/postgres.rs create mode 100644 crates/targets/src/target/redis.rs create mode 100644 crates/targets/tests/mysql_integration.rs create mode 100644 crates/targets/tests/postgres_integration.rs diff --git a/Cargo.lock b/Cargo.lock index de1793ac7..5716e3e33 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -230,6 +230,12 @@ dependencies = [ "rustversion", ] +[[package]] +name = "arcstr" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d" + [[package]] name = "argon2" version = "0.6.0-rc.8" @@ -570,9 +576,9 @@ dependencies = [ [[package]] name = "async-nats" -version = "0.47.0" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07d6f157065c3461096d51aacde0c326fa49f3f6e0199e204c566842cdaa5299" +checksum = "31811585c7c5bc2f60f8b80d5a6b0f737115611dac47567d7f7d94562ebb180b" dependencies = [ "base64 0.22.1", "bytes", @@ -582,7 +588,7 @@ dependencies = [ "nuid", "pin-project", "portable-atomic", - "rand 0.8.6", + "rand 0.10.1", "regex", "ring", "rustls-native-certs", @@ -592,7 +598,7 @@ dependencies = [ "serde_json", "serde_nanos", "serde_repr", - "thiserror 1.0.69", + "thiserror 2.0.18", "time", "tokio", "tokio-rustls", @@ -773,9 +779,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.131.0" +version = "1.132.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe1b8c5282bf859170836045296b3cd710b7573aceb909498366bb508a41058e" +checksum = "5575840a3a6b11f6011463ebe359320dfe5b67babb5e9b06fed6ddf809a9ab40" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1179,6 +1185,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", +] + [[package]] name = "backtrace" version = "0.3.76" @@ -1362,6 +1377,15 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "btoi" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b5ab9db53bcda568284df0fd39f6eac24ad6f7ba7ff1168b9e76eba6576b976" +dependencies = [ + "num-traits", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -1531,7 +1555,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" dependencies = [ "chrono", - "phf", + "phf 0.12.1", ] [[package]] @@ -1651,7 +1675,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" dependencies = [ "bytes", + "futures-core", "memchr", + "pin-project-lite", + "tokio", + "tokio-util", ] [[package]] @@ -2020,9 +2048,9 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.7" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", "typenum", @@ -2947,6 +2975,41 @@ dependencies = [ "xmltree", ] +[[package]] +name = "deadpool" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" +dependencies = [ + "deadpool-runtime", + "lazy_static", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-postgres" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d697d376cbfa018c23eb4caab1fd1883dd9c906a8c034e8d9a3cb06a7e0bef9" +dependencies = [ + "async-trait", + "deadpool", + "getrandom 0.2.17", + "tokio", + "tokio-postgres", + "tracing", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" +dependencies = [ + "tokio", +] + [[package]] name = "debugid" version = "0.8.0" @@ -2979,6 +3042,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ "const-oid 0.9.6", + "der_derive", + "flagset", "pem-rfc7468 0.7.0", "zeroize", ] @@ -3008,6 +3073,17 @@ dependencies = [ "rusticata-macros", ] +[[package]] +name = "der_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "deranged" version = "0.5.8" @@ -3156,7 +3232,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid 0.9.6", - "crypto-common 0.1.7", + "crypto-common 0.1.6", "subtle", ] @@ -3400,18 +3476,18 @@ checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" [[package]] name = "enumset" -version = "1.1.10" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25b07a8dfbbbfc0064c0a6bdf9edcf966de6b1c33ce344bdeca3b41615452634" +checksum = "7f96a4a12fe60ac746ae295a1a4ecb5bb02debc20856506c8635288065f142de" dependencies = [ "enumset_derive", ] [[package]] name = "enumset_derive" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43e744e4ea338060faee68ed933e46e722fb7f3617e722a5772d7e856d8b3ce" +checksum = "4bd536557b58c682b217b8fb199afdff47cd3eff260623f19e77074eb073d63a" dependencies = [ "darling 0.21.3", "proc-macro2", @@ -3504,6 +3580,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "fallible-iterator" version = "0.3.0" @@ -3587,6 +3669,12 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +[[package]] +name = "flagset" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" + [[package]] name = "flatbuffers" version = "25.12.19" @@ -3654,7 +3742,7 @@ checksum = "09a5a3f0acb82df800ca3aa50c0d60d286c5d13d4cfc3114b3a9663f13b032fe" dependencies = [ "arrayvec", "cfg-if", - "fallible-iterator", + "fallible-iterator 0.3.0", "gimli 0.31.1", "macho-unwind-info", "pe-unwind-info", @@ -3769,9 +3857,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.7" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" dependencies = [ "typenum", "version_check", @@ -3787,7 +3875,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -3848,7 +3936,7 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" dependencies = [ - "fallible-iterator", + "fallible-iterator 0.3.0", "stable_deref_trait", ] @@ -3866,9 +3954,9 @@ checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "google-cloud-auth" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a26c047222f874ea87177368ad07c65a9f66534ad3a3f9401f1322c802ccac" +checksum = "edd4f8c914f230834828771125168eaa39bc6602e32cb0316ceeff2add10d449" dependencies = [ "async-trait", "aws-lc-rs", @@ -3895,9 +3983,9 @@ dependencies = [ [[package]] name = "google-cloud-gax" -version = "1.9.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dc387965cc2efc28d73896d6707125815c16792c23c33a0c67794f3d6e31cc8" +checksum = "83d597e9e4758fc778a60d8c28a8677629675ae40d8652ec000ae5f53f5ae7ec" dependencies = [ "base64 0.22.1", "bytes", @@ -3915,9 +4003,9 @@ dependencies = [ [[package]] name = "google-cloud-gax-internal" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9529b797c68dc5c9d37effe7faa5d96d683a00e728287a9934bf6d3a86b58706" +checksum = "6e3e4d09e162fb71314e2c91879bdd495c1f8a1f69e71ccbae5767b8b6c833d6" dependencies = [ "bytes", "futures", @@ -3954,9 +4042,9 @@ dependencies = [ [[package]] name = "google-cloud-iam-v1" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "103d11cc828e7f33759c4dc6618a98415786fabba268147028e607c4fe1199aa" +checksum = "eab83affdded409153d2e616174054a780705016c288e43221a0d6a22dd78671" dependencies = [ "async-trait", "bytes", @@ -3972,9 +4060,9 @@ dependencies = [ [[package]] name = "google-cloud-longrunning" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa340b46d452fcd5e701b21dab6ed187454b6f1e4300e0bdb928fceefac847fa" +checksum = "df86ec067f13d858caf1a7fd9ed9315132aa0b0da2c1f0ba9b4357f2422d4bf9" dependencies = [ "async-trait", "bytes", @@ -3990,9 +4078,9 @@ dependencies = [ [[package]] name = "google-cloud-lro" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bfd3fa41649a3cd53675a94988b7d3789f2d8f2eb1b24af237450fe08ae9643" +checksum = "a4b3f2591e45a469e8ada961e92a62448b2b0522d8809554ad365cb964cb94d9" dependencies = [ "google-cloud-gax", "google-cloud-longrunning", @@ -4004,9 +4092,9 @@ dependencies = [ [[package]] name = "google-cloud-rpc" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3b123ea17ff20539fbdf145e6213e0464cc0a30b0d078a68bf90405ef17fb7" +checksum = "10b177796075b7bfc02bf2e405db665ee850a924fa44cedfc5282b473c5ab203" dependencies = [ "bytes", "google-cloud-wkt", @@ -4017,9 +4105,9 @@ dependencies = [ [[package]] name = "google-cloud-storage" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cf4d4e84fd3238cd5f5179b0a8e424c7ccc8106423a9fad627ae27ebc2b7b4" +checksum = "13a36095882d064b087d3171c1780505ff851a4292e04bd0d5cc6564d0a6d585" dependencies = [ "async-trait", "base64 0.22.1", @@ -4059,9 +4147,9 @@ dependencies = [ [[package]] name = "google-cloud-type" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbcae00eb50388f5127e65cf2d45679fc4ec0e3596975bb6ec41aec6319a51c9" +checksum = "d3658f2192252ba301a9d0a26e298bd56b55f0edb32de499a2f41ff244c09cf8" dependencies = [ "bytes", "google-cloud-wkt", @@ -4072,9 +4160,9 @@ dependencies = [ [[package]] name = "google-cloud-wkt" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b30ccefdb9276269bb0336afe207c5e0ba1a544a5eb0034763af051f2b9eb63" +checksum = "d5daa3084991800bcc5333d7e77bb19259a02b34ee35f35c27b49d602732306e" dependencies = [ "base64 0.22.1", "bytes", @@ -4733,16 +4821,6 @@ dependencies = [ "serde", ] -[[package]] -name = "iri-string" -version = "0.7.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "is-terminal" version = "0.4.17" @@ -4918,9 +4996,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.97" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1840c94c045fbcf8ba2812c95db44499f7c64910a912551aaaa541decebcacf" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" dependencies = [ "cfg-if", "futures-util", @@ -4960,6 +5038,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "keyed_priority_queue" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" +dependencies = [ + "indexmap 2.14.0", +] + [[package]] name = "konst" version = "0.2.20" @@ -5607,7 +5694,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.61.2", ] @@ -5633,9 +5720,9 @@ dependencies = [ [[package]] name = "mqttbytes-core-next" -version = "0.30.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bc8e9c3495ed805d2049920a74944a79533868159b0bcfdf585b961a155e46c" +checksum = "2558e8a9dd1e9ad2e771384ed983216741f60e21e5974b43b41f9980bff3e3d7" dependencies = [ "bytes", "thiserror 2.0.18", @@ -5653,6 +5740,82 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" +[[package]] +name = "mysql-common-derive" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66f62cad7623a9cb6f8f64037f0c4f69c8db8e82914334a83c9788201c2c1bfa" +dependencies = [ + "darling 0.20.11", + "heck", + "num-bigint", + "proc-macro-crate", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.117", + "termcolor", + "thiserror 2.0.18", +] + +[[package]] +name = "mysql_async" +version = "0.36.1" +source = "git+https://github.com/blackbeam/mysql_async?rev=98d3d8067efdf97d3e93cdca7b9231753c904aca#98d3d8067efdf97d3e93cdca7b9231753c904aca" +dependencies = [ + "bytes", + "crossbeam-queue", + "crossbeam-utils", + "flate2", + "futures-core", + "futures-sink", + "futures-util", + "keyed_priority_queue", + "lru 0.16.4", + "mysql_common", + "percent-encoding", + "rand 0.10.1", + "rustls", + "rustls-pemfile", + "serde", + "socket2", + "thiserror 2.0.18", + "tokio", + "tokio-rustls", + "tokio-util", + "tracing", + "twox-hash", + "url", + "webpki-roots 1.0.7", +] + +[[package]] +name = "mysql_common" +version = "0.37.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bffc2127d4035fa5a614935c663a15a4468e64e798473e0cc21c8df40a607588" +dependencies = [ + "base64 0.22.1", + "bitflags 2.11.1", + "btoi", + "byteorder", + "bytes", + "crc32fast", + "flate2", + "getrandom 0.3.4", + "mysql-common-derive", + "num-bigint", + "num-traits", + "regex", + "saturating", + "serde", + "serde_json", + "sha1 0.10.6", + "sha2 0.10.9", + "thiserror 2.0.18", + "uuid", +] + [[package]] name = "names" version = "0.14.0" @@ -6046,6 +6209,15 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + [[package]] name = "object" version = "0.29.0" @@ -6562,7 +6734,17 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" dependencies = [ - "phf_shared", + "phf_shared 0.12.1", +] + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared 0.13.1", + "serde", ] [[package]] @@ -6574,6 +6756,15 @@ dependencies = [ "siphasher", ] +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.12" @@ -6739,6 +6930,37 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "postgres-protocol" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56201207dac53e2f38e848e31b4b91616a6bb6e0c7205b77718994a7f49e70fc" +dependencies = [ + "base64 0.22.1", + "byteorder", + "bytes", + "fallible-iterator 0.2.0", + "hmac 0.13.0", + "md-5 0.11.0", + "memchr", + "rand 0.10.1", + "sha2 0.11.0", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dc729a129e682e8d24170cd30ae1aa01b336b096cbb56df6d534ffec133d186" +dependencies = [ + "bytes", + "fallible-iterator 0.2.0", + "postgres-protocol", + "serde_core", + "serde_json", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -6840,6 +7062,15 @@ dependencies = [ "elliptic-curve 0.13.8", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro-error-attr2" version = "2.0.0" @@ -7141,7 +7372,7 @@ dependencies = [ "libc", "once_cell", "raw-cpuid", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "web-sys", "winapi", ] @@ -7230,7 +7461,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -7439,6 +7670,37 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "redis" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d32a1ac9123f0d84fda64bfc02a271d9868483162dd2d9099b5c362ece064c" +dependencies = [ + "arc-swap", + "arcstr", + "async-lock", + "backon", + "bytes", + "cfg-if", + "combine", + "futures-channel", + "futures-util", + "itoa", + "num-bigint", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-native-certs", + "ryu", + "sha1_smol", + "socket2", + "tokio", + "tokio-rustls", + "tokio-util", + "url", + "xxhash-rust", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -7767,9 +8029,9 @@ dependencies = [ [[package]] name = "rumqttc-core-next" -version = "0.30.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8ba2cd7b3c1fece002d43a8a6f6d0374c2399c21e5c0ea54bf7f9e3e88c040a" +checksum = "c8a93655cd2d9d3faf7e039caa4eb39d9f4b49c336ade913c92655031c9fca71" dependencies = [ "async-tungstenite", "futures-io", @@ -7785,18 +8047,18 @@ dependencies = [ [[package]] name = "rumqttc-next" -version = "0.30.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8215de28413982ffd4c7a2beface3750426aaa4895016e3556a3279a762f9a7" +checksum = "9b6cd37028e449ffb206a1ddc60086a1a5bac578b45be91b487dccb47a930334" dependencies = [ "rumqttc-v5-next", ] [[package]] name = "rumqttc-v5-next" -version = "0.30.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c337cef1596844047b3e85db4fc3d0a5d6ab00f891e83abf32d0765b74ddd98" +checksum = "796245ad9ebd2b301ed20263ccf0ee430b6bc511b3ece7f815288a72d393e763" dependencies = [ "async-tungstenite", "bytes", @@ -8776,9 +9038,13 @@ version = "1.0.0-beta.2" dependencies = [ "async-nats", "async-trait", + "chrono", "criterion", + "deadpool-postgres", "hyper-rustls", + "mysql_async", "pulsar", + "redis", "reqwest 0.13.3", "rumqttc-next", "rustfs-config", @@ -8787,12 +9053,17 @@ dependencies = [ "rustfs-s3-common", "rustfs-utils", "rustls", + "rustls-native-certs", + "rustls-pki-types", "serde", "serde_json", "snap", "sysinfo", + "tempfile", "thiserror 2.0.18", "tokio", + "tokio-postgres", + "tokio-postgres-rustls", "tracing", "url", "urlencoding", @@ -8982,6 +9253,15 @@ dependencies = [ "security-framework", ] +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "rustls-pki-types" version = "1.14.1" @@ -9103,6 +9383,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "saturating" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71" + [[package]] name = "scc" version = "2.4.0" @@ -9418,6 +9704,12 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -9830,6 +10122,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" @@ -10031,6 +10334,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "test-case" version = "3.3.1" @@ -10233,6 +10545,27 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tls_codec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de2e01245e2bb89d6f05801c564fa27624dbd7b1846859876c7dad82e90bf6b" +dependencies = [ + "tls_codec_derive", + "zeroize", +] + +[[package]] +name = "tls_codec_derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "tokio" version = "1.52.2" @@ -10263,6 +10596,47 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tokio-postgres" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dd8df5ef180f6364759a6f00f7aadda4fbbac86cdee37480826a6ff9f3574ce" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator 0.2.0", + "futures-channel", + "futures-util", + "log", + "parking_lot 0.12.5", + "percent-encoding", + "phf 0.13.1", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand 0.10.1", + "socket2", + "tokio", + "tokio-util", + "whoami", +] + +[[package]] +name = "tokio-postgres-rustls" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27d684bad428a0f2481f42241f821db42c54e2dc81d8c00db8536c506b0a0144" +dependencies = [ + "const-oid 0.9.6", + "ring", + "rustls", + "tokio", + "tokio-postgres", + "tokio-rustls", + "x509-cert", +] + [[package]] name = "tokio-rustls" version = "0.26.4" @@ -10332,10 +10706,40 @@ dependencies = [ ] [[package]] -name = "tonic" -version = "0.14.5" +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.11+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" +dependencies = [ + "indexmap 2.14.0", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + +[[package]] +name = "tonic" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "axum", @@ -10365,9 +10769,9 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1882ac3bf5ef12877d7ed57aad87e75154c11931c2ba7e6cde5e22d63522c734" +checksum = "c68f61875ac5293cf72e6c8cf0158086428c82c37229e98c840878f1706b0322" dependencies = [ "prettyplease", "proc-macro2", @@ -10377,9 +10781,9 @@ dependencies = [ [[package]] name = "tonic-prost" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" dependencies = [ "bytes", "prost 0.14.3", @@ -10388,9 +10792,9 @@ dependencies = [ [[package]] name = "tonic-prost-build" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3144df636917574672e93d0f56d7edec49f90305749c668df5101751bb8f95a" +checksum = "654e5643eff75d7f8c99197ce1440ed19a3474eada74c12bbac488b2cafdae27" dependencies = [ "prettyplease", "proc-macro2", @@ -10423,9 +10827,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51" dependencies = [ "async-compression", "bitflags 2.11.1", @@ -10435,7 +10839,6 @@ dependencies = [ "http 1.4.0", "http-body 1.0.1", "http-body-util", - "iri-string", "pin-project-lite", "tokio", "tokio-util", @@ -10443,6 +10846,7 @@ dependencies = [ "tower-layer", "tower-service", "tracing", + "url", "uuid", ] @@ -10656,12 +11060,33 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-segmentation" version = "1.13.2" @@ -10808,6 +11233,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + [[package]] name = "wasip2" version = "1.0.3+wasi-0.2.9" @@ -10827,10 +11261,19 @@ dependencies = [ ] [[package]] -name = "wasm-bindgen" -version = "0.2.120" +name = "wasite" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df52b6d9b87e0c74c9edfa1eb2d9bf85e5d63515474513aa50fa181b3c4f5db1" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" dependencies = [ "cfg-if", "once_cell", @@ -10841,9 +11284,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.70" +version = "0.4.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af934872acec734c2d80e6617bbb5ff4f12b052dd8e6332b0817bce889516084" +checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" dependencies = [ "js-sys", "wasm-bindgen", @@ -10851,9 +11294,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.120" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b1041f495fb322e64aca85f5756b2172e35cd459376e67f2a6c9dffcedb103" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -10861,9 +11304,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.120" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dcd0ff20416988a18ac686d4d4d0f6aae9ebf08a389ff5d29012b05af2a1b41" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" dependencies = [ "bumpalo", "proc-macro2", @@ -10874,9 +11317,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.120" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49757b3c82ebf16c57d69365a142940b384176c24df52a087fb748e2085359ea" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" dependencies = [ "unicode-ident", ] @@ -10930,9 +11373,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.97" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eadbac71025cd7b0834f20d1fe8472e8495821b4e9801eb0a60bd1f19827602" +checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" dependencies = [ "js-sys", "wasm-bindgen", @@ -10987,6 +11430,19 @@ dependencies = [ "rustix 0.38.44", ] +[[package]] +name = "whoami" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "998767ef88740d1f5b0682a9c53c24431453923962269c2db68ee43788c5a40d" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite", + "web-sys", +] + [[package]] name = "wildmatch" version = "2.6.1" @@ -11145,7 +11601,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -11154,7 +11610,16 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", ] [[package]] @@ -11172,14 +11637,31 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] @@ -11197,48 +11679,105 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winnow" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.51.0" @@ -11351,6 +11890,18 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "x509-cert" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" +dependencies = [ + "const-oid 0.9.6", + "der 0.7.10", + "spki 0.7.3", + "tls_codec", +] + [[package]] name = "x509-parser" version = "0.18.1" diff --git a/Cargo.toml b/Cargo.toml index 4ce831713..66816b0d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -115,10 +115,11 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.2" } # Async Runtime and Networking async-channel = "2.5.0" +mysql_async = { version = "0.36.1", default-features = false, features = ["default-rustls", "tracing"], git = "https://github.com/blackbeam/mysql_async", rev = "98d3d8067efdf97d3e93cdca7b9231753c904aca" } async-compression = { version = "0.4.42" } async-recursion = "1.1.1" async-trait = "0.1.89" -async-nats = "0.47.0" +async-nats = "0.48.0" axum = "0.8.9" futures = "0.3.32" futures-core = "0.3.32" @@ -139,11 +140,11 @@ tokio-rustls = { version = "0.26.4", default-features = false, features = ["logg tokio-stream = { version = "0.1.18" } tokio-test = "0.4.5" tokio-util = { version = "0.7.18", features = ["io", "compat"] } -tonic = { version = "0.14.5", features = ["gzip"] } -tonic-prost = { version = "0.14.5" } -tonic-prost-build = { version = "0.14.5" } +tonic = { version = "0.14.6", features = ["gzip"] } +tonic-prost = { version = "0.14.6" } +tonic-prost-build = { version = "0.14.6" } tower = { version = "0.5.3", features = ["timeout"] } -tower-http = { version = "0.6.8", features = ["cors"] } +tower-http = { version = "0.6.10", features = ["cors"] } # Serialization and Data Formats bytes = { version = "1.11.1", features = ["serde"] } @@ -171,6 +172,7 @@ openidconnect = { version = "4.0", default-features = false } pbkdf2 = "0.13.0" rsa = { version = "0.10.0-rc.18" } rustls = { version = "0.23.40", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] } +rustls-native-certs = "0.8" rustls-pki-types = "1.14.1" sha1 = "0.11.0" sha2 = "0.11.0" @@ -183,6 +185,11 @@ humantime = "2.3.0" jiff = { version = "0.2.24", features = ["serde"] } time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros", "serde"] } +# Database +deadpool-postgres = { version = "0.14", features = ["rt_tokio_1"] } +tokio-postgres = { version = "0.7", default-features = false, features = ["runtime", "with-serde_json-1"] } +tokio-postgres-rustls = "0.13" + # Utilities and Tools anyhow = "1.0.102" arc-swap = "1.9.1" @@ -191,7 +198,7 @@ atoi = "2.0.0" atomic_enum = "0.3.0" aws-config = { version = "1.8.16" } aws-credential-types = { version = "1.2.14" } -aws-sdk-s3 = { version = "1.131.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] } +aws-sdk-s3 = { version = "1.132.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] } aws-smithy-http-client = { version = "1.1.12", default-features = false, features = ["default-client", "rustls-aws-lc"] } aws-smithy-types = { version = "1.4.7" } backtrace = "0.3.76" @@ -208,12 +215,12 @@ crossbeam-deque = "0.8.6" crossbeam-utils = "0.8.21" datafusion = "53.1.0" derive_builder = "0.20.2" -enumset = "1.1.10" +enumset = "1.1.12" faster-hex = "0.10.0" flate2 = "1.1.9" glob = "0.3.3" -google-cloud-storage = "1.11.0" -google-cloud-auth = "1.9.0" +google-cloud-storage = "1.12.0" +google-cloud-auth = "1.10.0" hashbrown = { version = "0.17.0", features = ["serde", "rayon"] } hex = "0.4.3" hex-simd = "0.8.0" @@ -246,7 +253,8 @@ rayon = "1.12.0" reed-solomon-erasure = { version = "6.0", default-features = false, features = ["std", "simd-accel"] } reed-solomon-simd = "3.1.0" regex = { version = "1.12.3" } -rumqttc = { package = "rumqttc-next", version = "0.30.1", features = ["websocket"] } +rumqttc = { package = "rumqttc-next", version = "0.33.0", features = ["websocket"] } +redis = { version = "1.2.1", features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] } rustix = { version = "1.1.4", features = ["fs"] } rust-embed = { version = "8.11.0" } rustc-hash = { version = "2.1.2" } diff --git a/crates/audit/src/factory.rs b/crates/audit/src/factory.rs index c27026268..ad6570459 100644 --- a/crates/audit/src/factory.rs +++ b/crates/audit/src/factory.rs @@ -15,13 +15,17 @@ use crate::AuditEntry; use async_trait::async_trait; use rustfs_config::AUDIT_DEFAULT_DIR; -use rustfs_config::audit::{AUDIT_KAFKA_KEYS, AUDIT_MQTT_KEYS, AUDIT_NATS_KEYS, AUDIT_PULSAR_KEYS, AUDIT_WEBHOOK_KEYS}; +use rustfs_config::audit::{ + AUDIT_KAFKA_KEYS, AUDIT_MQTT_KEYS, AUDIT_MYSQL_KEYS, AUDIT_NATS_KEYS, AUDIT_POSTGRES_KEYS, AUDIT_PULSAR_KEYS, + AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_REDIS_KEYS, AUDIT_WEBHOOK_KEYS, +}; use rustfs_ecstore::config::KVS; use rustfs_targets::{ Target, config::{ - build_kafka_args, build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_kafka_config, - validate_mqtt_config, validate_nats_config, validate_pulsar_config, validate_webhook_config, + build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args, build_pulsar_args, + build_redis_args, build_webhook_args, validate_kafka_config, validate_mqtt_config, validate_mysql_config, + validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, validate_webhook_config, }, error::TargetError, target::TargetType, @@ -138,3 +142,60 @@ impl TargetFactory for KafkaTargetFactory { AUDIT_KAFKA_KEYS.iter().map(|s| s.to_string()).collect() } } + +pub struct RedisTargetFactory; + +#[async_trait] +impl TargetFactory for RedisTargetFactory { + async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError> { + let args = build_redis_args(config, AUDIT_DEFAULT_DIR, AUDIT_REDIS_DEFAULT_CHANNEL, TargetType::AuditLog)?; + let target = rustfs_targets::target::redis::RedisTarget::new(id, args)?; + Ok(Box::new(target)) + } + + fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> { + validate_redis_config(config, AUDIT_DEFAULT_DIR, AUDIT_REDIS_DEFAULT_CHANNEL) + } + + fn get_valid_fields(&self) -> HashSet { + AUDIT_REDIS_KEYS.iter().map(|s| s.to_string()).collect() + } +} + +pub struct MySqlTargetFactory; + +#[async_trait] +impl TargetFactory for MySqlTargetFactory { + async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError> { + let args = build_mysql_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; + let target = rustfs_targets::target::mysql::MySqlTarget::new(id, args)?; + Ok(Box::new(target)) + } + + fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> { + validate_mysql_config(config, AUDIT_DEFAULT_DIR) + } + + fn get_valid_fields(&self) -> HashSet { + AUDIT_MYSQL_KEYS.iter().map(|s| s.to_string()).collect() + } +} + +pub struct PostgresTargetFactory; + +#[async_trait] +impl TargetFactory for PostgresTargetFactory { + async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError> { + let args = build_postgres_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; + let target = rustfs_targets::target::postgres::PostgresTarget::new(id, args)?; + Ok(Box::new(target)) + } + + fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> { + validate_postgres_config(config, AUDIT_DEFAULT_DIR) + } + + fn get_valid_fields(&self) -> HashSet { + AUDIT_POSTGRES_KEYS.iter().map(|s| s.to_string()).collect() + } +} diff --git a/crates/audit/src/registry.rs b/crates/audit/src/registry.rs index 4cd8356fb..82c3921b6 100644 --- a/crates/audit/src/registry.rs +++ b/crates/audit/src/registry.rs @@ -15,7 +15,8 @@ use crate::{ AuditEntry, AuditError, AuditResult, factory::{ - KafkaTargetFactory, MQTTTargetFactory, NATSTargetFactory, PulsarTargetFactory, TargetFactory, WebhookTargetFactory, + KafkaTargetFactory, MQTTTargetFactory, MySqlTargetFactory, NATSTargetFactory, PostgresTargetFactory, PulsarTargetFactory, + RedisTargetFactory, TargetFactory, WebhookTargetFactory, }, }; use futures::StreamExt; @@ -56,6 +57,9 @@ impl AuditRegistry { registry.register(ChannelTargetType::Nats.as_str(), Box::new(NATSTargetFactory)); registry.register(ChannelTargetType::Pulsar.as_str(), Box::new(PulsarTargetFactory)); registry.register(ChannelTargetType::Kafka.as_str(), Box::new(KafkaTargetFactory)); + registry.register(ChannelTargetType::Redis.as_str(), Box::new(RedisTargetFactory)); + registry.register(ChannelTargetType::MySql.as_str(), Box::new(MySqlTargetFactory)); + registry.register(ChannelTargetType::Postgres.as_str(), Box::new(PostgresTargetFactory)); registry } diff --git a/crates/config/src/audit/mod.rs b/crates/config/src/audit/mod.rs index 9390ec854..03e0834b3 100644 --- a/crates/config/src/audit/mod.rs +++ b/crates/config/src/audit/mod.rs @@ -18,14 +18,20 @@ mod kafka; mod mqtt; +mod mysql; mod nats; +mod postgres; mod pulsar; +mod redis; mod webhook; pub use kafka::*; pub use mqtt::*; +pub use mysql::*; pub use nats::*; +pub use postgres::*; pub use pulsar::*; +pub use redis::*; pub use webhook::*; use crate::DEFAULT_DELIMITER; @@ -37,15 +43,22 @@ pub const AUDIT_ROUTE_PREFIX: &str = const_str::concat!(AUDIT_PREFIX, DEFAULT_DE pub const AUDIT_WEBHOOK_SUB_SYS: &str = "audit_webhook"; pub const AUDIT_KAFKA_SUB_SYS: &str = "audit_kafka"; pub const AUDIT_MQTT_SUB_SYS: &str = "audit_mqtt"; +pub const AUDIT_MYSQL_SUB_SYS: &str = "audit_mysql"; pub const AUDIT_NATS_SUB_SYS: &str = "audit_nats"; +pub const AUDIT_POSTGRES_SUB_SYS: &str = "audit_postgres"; pub const AUDIT_PULSAR_SUB_SYS: &str = "audit_pulsar"; +pub const AUDIT_REDIS_SUB_SYS: &str = "audit_redis"; +pub const AUDIT_REDIS_DEFAULT_CHANNEL: &str = "rustfs_audit_channel"; pub const AUDIT_STORE_EXTENSION: &str = ".audit"; #[allow(dead_code)] pub const AUDIT_SUB_SYSTEMS: &[&str] = &[ AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, + AUDIT_MYSQL_SUB_SYS, AUDIT_NATS_SUB_SYS, + AUDIT_POSTGRES_SUB_SYS, AUDIT_PULSAR_SUB_SYS, + AUDIT_REDIS_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS, ]; diff --git a/crates/config/src/audit/mysql.rs b/crates/config/src/audit/mysql.rs new file mode 100644 index 000000000..551edf9c1 --- /dev/null +++ b/crates/config/src/audit/mysql.rs @@ -0,0 +1,53 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// A list of all valid configuration keys for an audit MySQL target. +pub const AUDIT_MYSQL_KEYS: &[&str] = &[ + crate::ENABLE_KEY, + crate::MYSQL_DSN_STRING, + crate::MYSQL_TABLE, + crate::MYSQL_FORMAT, + crate::MYSQL_TLS_CA, + crate::MYSQL_TLS_CLIENT_CERT, + crate::MYSQL_TLS_CLIENT_KEY, + crate::MYSQL_QUEUE_DIR, + crate::MYSQL_QUEUE_LIMIT, + crate::MYSQL_MAX_OPEN_CONNECTIONS, + crate::COMMENT_KEY, +]; + +// MySQL environment variables for audit target. +pub const ENV_AUDIT_MYSQL_ENABLE: &str = "RUSTFS_AUDIT_MYSQL_ENABLE"; +pub const ENV_AUDIT_MYSQL_DSN_STRING: &str = "RUSTFS_AUDIT_MYSQL_DSN_STRING"; +pub const ENV_AUDIT_MYSQL_TABLE: &str = "RUSTFS_AUDIT_MYSQL_TABLE"; +pub const ENV_AUDIT_MYSQL_FORMAT: &str = "RUSTFS_AUDIT_MYSQL_FORMAT"; +pub const ENV_AUDIT_MYSQL_TLS_CA: &str = "RUSTFS_AUDIT_MYSQL_TLS_CA"; +pub const ENV_AUDIT_MYSQL_TLS_CLIENT_CERT: &str = "RUSTFS_AUDIT_MYSQL_TLS_CLIENT_CERT"; +pub const ENV_AUDIT_MYSQL_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_MYSQL_TLS_CLIENT_KEY"; +pub const ENV_AUDIT_MYSQL_QUEUE_DIR: &str = "RUSTFS_AUDIT_MYSQL_QUEUE_DIR"; +pub const ENV_AUDIT_MYSQL_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_MYSQL_QUEUE_LIMIT"; +pub const ENV_AUDIT_MYSQL_MAX_OPEN_CONNECTIONS: &str = "RUSTFS_AUDIT_MYSQL_MAX_OPEN_CONNECTIONS"; + +pub const ENV_AUDIT_MYSQL_KEYS: &[&str; 10] = &[ + ENV_AUDIT_MYSQL_ENABLE, + ENV_AUDIT_MYSQL_DSN_STRING, + ENV_AUDIT_MYSQL_TABLE, + ENV_AUDIT_MYSQL_FORMAT, + ENV_AUDIT_MYSQL_TLS_CA, + ENV_AUDIT_MYSQL_TLS_CLIENT_CERT, + ENV_AUDIT_MYSQL_TLS_CLIENT_KEY, + ENV_AUDIT_MYSQL_QUEUE_DIR, + ENV_AUDIT_MYSQL_QUEUE_LIMIT, + ENV_AUDIT_MYSQL_MAX_OPEN_CONNECTIONS, +]; diff --git a/crates/config/src/audit/postgres.rs b/crates/config/src/audit/postgres.rs new file mode 100644 index 000000000..886cee4e6 --- /dev/null +++ b/crates/config/src/audit/postgres.rs @@ -0,0 +1,51 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub const AUDIT_POSTGRES_KEYS: &[&str] = &[ + crate::ENABLE_KEY, + crate::POSTGRES_DSN_STRING, + crate::POSTGRES_TABLE, + crate::POSTGRES_FORMAT, + crate::POSTGRES_TLS_REQUIRED, + crate::POSTGRES_TLS_CA, + crate::POSTGRES_TLS_CLIENT_CERT, + crate::POSTGRES_TLS_CLIENT_KEY, + crate::POSTGRES_QUEUE_DIR, + crate::POSTGRES_QUEUE_LIMIT, + crate::COMMENT_KEY, +]; + +pub const ENV_AUDIT_POSTGRES_ENABLE: &str = "RUSTFS_AUDIT_POSTGRES_ENABLE"; +pub const ENV_AUDIT_POSTGRES_DSN_STRING: &str = "RUSTFS_AUDIT_POSTGRES_DSN_STRING"; +pub const ENV_AUDIT_POSTGRES_TABLE: &str = "RUSTFS_AUDIT_POSTGRES_TABLE"; +pub const ENV_AUDIT_POSTGRES_FORMAT: &str = "RUSTFS_AUDIT_POSTGRES_FORMAT"; +pub const ENV_AUDIT_POSTGRES_TLS_REQUIRED: &str = "RUSTFS_AUDIT_POSTGRES_TLS_REQUIRED"; +pub const ENV_AUDIT_POSTGRES_TLS_CA: &str = "RUSTFS_AUDIT_POSTGRES_TLS_CA"; +pub const ENV_AUDIT_POSTGRES_TLS_CLIENT_CERT: &str = "RUSTFS_AUDIT_POSTGRES_TLS_CLIENT_CERT"; +pub const ENV_AUDIT_POSTGRES_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_POSTGRES_TLS_CLIENT_KEY"; +pub const ENV_AUDIT_POSTGRES_QUEUE_DIR: &str = "RUSTFS_AUDIT_POSTGRES_QUEUE_DIR"; +pub const ENV_AUDIT_POSTGRES_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_POSTGRES_QUEUE_LIMIT"; + +pub const ENV_AUDIT_POSTGRES_KEYS: &[&str; 10] = &[ + ENV_AUDIT_POSTGRES_ENABLE, + ENV_AUDIT_POSTGRES_DSN_STRING, + ENV_AUDIT_POSTGRES_TABLE, + ENV_AUDIT_POSTGRES_FORMAT, + ENV_AUDIT_POSTGRES_TLS_REQUIRED, + ENV_AUDIT_POSTGRES_TLS_CA, + ENV_AUDIT_POSTGRES_TLS_CLIENT_CERT, + ENV_AUDIT_POSTGRES_TLS_CLIENT_KEY, + ENV_AUDIT_POSTGRES_QUEUE_DIR, + ENV_AUDIT_POSTGRES_QUEUE_LIMIT, +]; diff --git a/crates/config/src/audit/redis.rs b/crates/config/src/audit/redis.rs new file mode 100644 index 000000000..a3dcaccbd --- /dev/null +++ b/crates/config/src/audit/redis.rs @@ -0,0 +1,81 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub const ENV_AUDIT_REDIS_ENABLE: &str = "RUSTFS_AUDIT_REDIS_ENABLE"; +pub const ENV_AUDIT_REDIS_URL: &str = "RUSTFS_AUDIT_REDIS_URL"; +pub const ENV_AUDIT_REDIS_CHANNEL: &str = "RUSTFS_AUDIT_REDIS_CHANNEL"; +pub const ENV_AUDIT_REDIS_USERNAME: &str = "RUSTFS_AUDIT_REDIS_USERNAME"; +pub const ENV_AUDIT_REDIS_PASSWORD: &str = "RUSTFS_AUDIT_REDIS_PASSWORD"; +pub const ENV_AUDIT_REDIS_KEEP_ALIVE_INTERVAL: &str = "RUSTFS_AUDIT_REDIS_KEEP_ALIVE_INTERVAL"; +pub const ENV_AUDIT_REDIS_QUEUE_DIR: &str = "RUSTFS_AUDIT_REDIS_QUEUE_DIR"; +pub const ENV_AUDIT_REDIS_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_REDIS_QUEUE_LIMIT"; +pub const ENV_AUDIT_REDIS_MAX_RETRY_ATTEMPTS: &str = "RUSTFS_AUDIT_REDIS_MAX_RETRY_ATTEMPTS"; +pub const ENV_AUDIT_REDIS_RECONNECT_RETRY_ATTEMPTS: &str = "RUSTFS_AUDIT_REDIS_RECONNECT_RETRY_ATTEMPTS"; +pub const ENV_AUDIT_REDIS_MIN_RETRY_DELAY: &str = "RUSTFS_AUDIT_REDIS_MIN_RETRY_DELAY"; +pub const ENV_AUDIT_REDIS_MAX_RETRY_DELAY: &str = "RUSTFS_AUDIT_REDIS_MAX_RETRY_DELAY"; +pub const ENV_AUDIT_REDIS_CONNECTION_TIMEOUT: &str = "RUSTFS_AUDIT_REDIS_CONNECTION_TIMEOUT"; +pub const ENV_AUDIT_REDIS_RESPONSE_TIMEOUT: &str = "RUSTFS_AUDIT_REDIS_RESPONSE_TIMEOUT"; +pub const ENV_AUDIT_REDIS_PIPELINE_BUFFER_SIZE: &str = "RUSTFS_AUDIT_REDIS_PIPELINE_BUFFER_SIZE"; +pub const ENV_AUDIT_REDIS_TLS_POLICY: &str = "RUSTFS_AUDIT_REDIS_TLS_POLICY"; +pub const ENV_AUDIT_REDIS_TLS_CA: &str = "RUSTFS_AUDIT_REDIS_TLS_CA"; +pub const ENV_AUDIT_REDIS_TLS_CLIENT_CERT: &str = "RUSTFS_AUDIT_REDIS_TLS_CLIENT_CERT"; +pub const ENV_AUDIT_REDIS_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_REDIS_TLS_CLIENT_KEY"; +pub const ENV_AUDIT_REDIS_TLS_ALLOW_INSECURE: &str = "RUSTFS_AUDIT_REDIS_TLS_ALLOW_INSECURE"; + +pub const ENV_AUDIT_REDIS_KEYS: &[&str; 20] = &[ + ENV_AUDIT_REDIS_ENABLE, + ENV_AUDIT_REDIS_URL, + ENV_AUDIT_REDIS_CHANNEL, + ENV_AUDIT_REDIS_USERNAME, + ENV_AUDIT_REDIS_PASSWORD, + ENV_AUDIT_REDIS_KEEP_ALIVE_INTERVAL, + ENV_AUDIT_REDIS_QUEUE_DIR, + ENV_AUDIT_REDIS_QUEUE_LIMIT, + ENV_AUDIT_REDIS_MAX_RETRY_ATTEMPTS, + ENV_AUDIT_REDIS_RECONNECT_RETRY_ATTEMPTS, + ENV_AUDIT_REDIS_MIN_RETRY_DELAY, + ENV_AUDIT_REDIS_MAX_RETRY_DELAY, + ENV_AUDIT_REDIS_CONNECTION_TIMEOUT, + ENV_AUDIT_REDIS_RESPONSE_TIMEOUT, + ENV_AUDIT_REDIS_PIPELINE_BUFFER_SIZE, + ENV_AUDIT_REDIS_TLS_POLICY, + ENV_AUDIT_REDIS_TLS_CA, + ENV_AUDIT_REDIS_TLS_CLIENT_CERT, + ENV_AUDIT_REDIS_TLS_CLIENT_KEY, + ENV_AUDIT_REDIS_TLS_ALLOW_INSECURE, +]; + +pub const AUDIT_REDIS_KEYS: &[&str] = &[ + crate::ENABLE_KEY, + crate::REDIS_URL, + crate::REDIS_CHANNEL, + crate::REDIS_USERNAME, + crate::REDIS_PASSWORD, + crate::REDIS_KEEP_ALIVE_INTERVAL, + crate::REDIS_QUEUE_DIR, + crate::REDIS_QUEUE_LIMIT, + crate::REDIS_MAX_RETRY_ATTEMPTS, + crate::REDIS_RECONNECT_RETRY_ATTEMPTS, + crate::REDIS_MIN_RETRY_DELAY, + crate::REDIS_MAX_RETRY_DELAY, + crate::REDIS_CONNECTION_TIMEOUT, + crate::REDIS_RESPONSE_TIMEOUT, + crate::REDIS_PIPELINE_BUFFER_SIZE, + crate::REDIS_TLS_POLICY, + crate::REDIS_TLS_CA, + crate::REDIS_TLS_CLIENT_CERT, + crate::REDIS_TLS_CLIENT_KEY, + crate::REDIS_TLS_ALLOW_INSECURE, + crate::COMMENT_KEY, +]; diff --git a/crates/config/src/constants/targets.rs b/crates/config/src/constants/targets.rs index f33a14a08..9c9800b0a 100644 --- a/crates/config/src/constants/targets.rs +++ b/crates/config/src/constants/targets.rs @@ -74,6 +74,48 @@ pub const PULSAR_TLS_HOSTNAME_VERIFICATION: &str = "tls_hostname_verification"; pub const PULSAR_QUEUE_DIR: &str = "queue_dir"; pub const PULSAR_QUEUE_LIMIT: &str = "queue_limit"; +pub const BASE_DSN_STRING: &str = "dsn_string"; + +pub const MYSQL_DSN_STRING: &str = BASE_DSN_STRING; +pub const MYSQL_TABLE: &str = "table"; +pub const MYSQL_FORMAT: &str = "format"; +pub const MYSQL_TLS_CA: &str = "tls_ca"; +pub const MYSQL_TLS_CLIENT_CERT: &str = "tls_client_cert"; +pub const MYSQL_TLS_CLIENT_KEY: &str = "tls_client_key"; +pub const MYSQL_QUEUE_DIR: &str = "queue_dir"; +pub const MYSQL_QUEUE_LIMIT: &str = "queue_limit"; +pub const MYSQL_MAX_OPEN_CONNECTIONS: &str = "max_open_connections"; + +pub const REDIS_URL: &str = "url"; +pub const REDIS_CHANNEL: &str = "channel"; +pub const REDIS_USERNAME: &str = "username"; +pub const REDIS_PASSWORD: &str = "password"; +pub const REDIS_KEEP_ALIVE_INTERVAL: &str = "keep_alive_interval"; +pub const REDIS_QUEUE_DIR: &str = "queue_dir"; +pub const REDIS_QUEUE_LIMIT: &str = "queue_limit"; +pub const REDIS_MAX_RETRY_ATTEMPTS: &str = "max_retry_attempts"; +pub const REDIS_RECONNECT_RETRY_ATTEMPTS: &str = "reconnect_retry_attempts"; +pub const REDIS_MIN_RETRY_DELAY: &str = "min_retry_delay"; +pub const REDIS_MAX_RETRY_DELAY: &str = "max_retry_delay"; +pub const REDIS_CONNECTION_TIMEOUT: &str = "connection_timeout"; +pub const REDIS_RESPONSE_TIMEOUT: &str = "response_timeout"; +pub const REDIS_PIPELINE_BUFFER_SIZE: &str = "pipeline_buffer_size"; +pub const REDIS_TLS_POLICY: &str = "tls_policy"; +pub const REDIS_TLS_CA: &str = "tls_ca"; +pub const REDIS_TLS_CLIENT_CERT: &str = "tls_client_cert"; +pub const REDIS_TLS_CLIENT_KEY: &str = "tls_client_key"; +pub const REDIS_TLS_ALLOW_INSECURE: &str = "tls_allow_insecure"; + +pub const POSTGRES_DSN_STRING: &str = BASE_DSN_STRING; +pub const POSTGRES_TABLE: &str = "table"; +pub const POSTGRES_FORMAT: &str = "format"; +pub const POSTGRES_TLS_REQUIRED: &str = "tls_required"; +pub const POSTGRES_TLS_CA: &str = "tls_ca"; +pub const POSTGRES_TLS_CLIENT_CERT: &str = "tls_client_cert"; +pub const POSTGRES_TLS_CLIENT_KEY: &str = "tls_client_key"; +pub const POSTGRES_QUEUE_DIR: &str = "queue_dir"; +pub const POSTGRES_QUEUE_LIMIT: &str = "queue_limit"; + /// Environment variable controlling whether target queue files are Snappy-compressed. /// Applies to both notify and audit target queue stores. pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS"; diff --git a/crates/config/src/notify/mod.rs b/crates/config/src/notify/mod.rs index 7a96dfebb..5f7262674 100644 --- a/crates/config/src/notify/mod.rs +++ b/crates/config/src/notify/mod.rs @@ -15,16 +15,22 @@ mod arn; mod kafka; mod mqtt; +mod mysql; mod nats; +mod postgres; mod pulsar; +mod redis; mod store; mod webhook; pub use arn::*; pub use kafka::*; pub use mqtt::*; +pub use mysql::*; pub use nats::*; +pub use postgres::*; pub use pulsar::*; +pub use redis::*; pub use store::*; pub use webhook::*; @@ -69,19 +75,20 @@ pub const ENV_NOTIFY_SEND_CONCURRENCY: &str = "RUSTFS_NOTIFY_SEND_CONCURRENCY"; /// Adjust this value based on your system's capabilities and expected load. pub const DEFAULT_NOTIFY_SEND_CONCURRENCY: usize = 64; -#[allow(dead_code)] pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[ NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, + NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS, + NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, + NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, ]; pub const NOTIFY_KAFKA_SUB_SYS: &str = "notify_kafka"; pub const NOTIFY_MQTT_SUB_SYS: &str = "notify_mqtt"; -#[allow(dead_code)] -pub const NOTIFY_MY_SQL_SUB_SYS: &str = "notify_mysql"; +pub const NOTIFY_MYSQL_SUB_SYS: &str = "notify_mysql"; #[allow(dead_code)] pub const NOTIFY_NATS_SUB_SYS: &str = "notify_nats"; #[allow(dead_code)] @@ -90,9 +97,9 @@ pub const NOTIFY_NSQ_SUB_SYS: &str = "notify_nsq"; pub const NOTIFY_ES_SUB_SYS: &str = "notify_elasticsearch"; #[allow(dead_code)] pub const NOTIFY_AMQP_SUB_SYS: &str = "notify_amqp"; -#[allow(dead_code)] pub const NOTIFY_POSTGRES_SUB_SYS: &str = "notify_postgres"; #[allow(dead_code)] pub const NOTIFY_REDIS_SUB_SYS: &str = "notify_redis"; +pub const NOTIFY_REDIS_DEFAULT_CHANNEL: &str = "rustfs_notify_channel"; pub const NOTIFY_PULSAR_SUB_SYS: &str = "notify_pulsar"; pub const NOTIFY_WEBHOOK_SUB_SYS: &str = "notify_webhook"; diff --git a/crates/config/src/notify/mysql.rs b/crates/config/src/notify/mysql.rs new file mode 100644 index 000000000..e49971b8e --- /dev/null +++ b/crates/config/src/notify/mysql.rs @@ -0,0 +1,53 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// A list of all valid configuration keys for a MySQL target. +pub const NOTIFY_MYSQL_KEYS: &[&str] = &[ + crate::ENABLE_KEY, + crate::MYSQL_DSN_STRING, + crate::MYSQL_TABLE, + crate::MYSQL_FORMAT, + crate::MYSQL_TLS_CA, + crate::MYSQL_TLS_CLIENT_CERT, + crate::MYSQL_TLS_CLIENT_KEY, + crate::MYSQL_QUEUE_DIR, + crate::MYSQL_QUEUE_LIMIT, + crate::MYSQL_MAX_OPEN_CONNECTIONS, + crate::COMMENT_KEY, +]; + +// MySQL Environment Variables +pub const ENV_NOTIFY_MYSQL_ENABLE: &str = "RUSTFS_NOTIFY_MYSQL_ENABLE"; +pub const ENV_NOTIFY_MYSQL_DSN_STRING: &str = "RUSTFS_NOTIFY_MYSQL_DSN_STRING"; +pub const ENV_NOTIFY_MYSQL_TABLE: &str = "RUSTFS_NOTIFY_MYSQL_TABLE"; +pub const ENV_NOTIFY_MYSQL_FORMAT: &str = "RUSTFS_NOTIFY_MYSQL_FORMAT"; +pub const ENV_NOTIFY_MYSQL_TLS_CA: &str = "RUSTFS_NOTIFY_MYSQL_TLS_CA"; +pub const ENV_NOTIFY_MYSQL_TLS_CLIENT_CERT: &str = "RUSTFS_NOTIFY_MYSQL_TLS_CLIENT_CERT"; +pub const ENV_NOTIFY_MYSQL_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_MYSQL_TLS_CLIENT_KEY"; +pub const ENV_NOTIFY_MYSQL_QUEUE_DIR: &str = "RUSTFS_NOTIFY_MYSQL_QUEUE_DIR"; +pub const ENV_NOTIFY_MYSQL_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_MYSQL_QUEUE_LIMIT"; +pub const ENV_NOTIFY_MYSQL_MAX_OPEN_CONNECTIONS: &str = "RUSTFS_NOTIFY_MYSQL_MAX_OPEN_CONNECTIONS"; + +pub const ENV_NOTIFY_MYSQL_KEYS: &[&str; 10] = &[ + ENV_NOTIFY_MYSQL_ENABLE, + ENV_NOTIFY_MYSQL_DSN_STRING, + ENV_NOTIFY_MYSQL_TABLE, + ENV_NOTIFY_MYSQL_FORMAT, + ENV_NOTIFY_MYSQL_TLS_CA, + ENV_NOTIFY_MYSQL_TLS_CLIENT_CERT, + ENV_NOTIFY_MYSQL_TLS_CLIENT_KEY, + ENV_NOTIFY_MYSQL_QUEUE_DIR, + ENV_NOTIFY_MYSQL_QUEUE_LIMIT, + ENV_NOTIFY_MYSQL_MAX_OPEN_CONNECTIONS, +]; diff --git a/crates/config/src/notify/postgres.rs b/crates/config/src/notify/postgres.rs new file mode 100644 index 000000000..4cdc649fd --- /dev/null +++ b/crates/config/src/notify/postgres.rs @@ -0,0 +1,51 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub const NOTIFY_POSTGRES_KEYS: &[&str] = &[ + crate::ENABLE_KEY, + crate::POSTGRES_DSN_STRING, + crate::POSTGRES_TABLE, + crate::POSTGRES_FORMAT, + crate::POSTGRES_TLS_REQUIRED, + crate::POSTGRES_TLS_CA, + crate::POSTGRES_TLS_CLIENT_CERT, + crate::POSTGRES_TLS_CLIENT_KEY, + crate::POSTGRES_QUEUE_DIR, + crate::POSTGRES_QUEUE_LIMIT, + crate::COMMENT_KEY, +]; + +pub const ENV_NOTIFY_POSTGRES_ENABLE: &str = "RUSTFS_NOTIFY_POSTGRES_ENABLE"; +pub const ENV_NOTIFY_POSTGRES_DSN_STRING: &str = "RUSTFS_NOTIFY_POSTGRES_DSN_STRING"; +pub const ENV_NOTIFY_POSTGRES_TABLE: &str = "RUSTFS_NOTIFY_POSTGRES_TABLE"; +pub const ENV_NOTIFY_POSTGRES_FORMAT: &str = "RUSTFS_NOTIFY_POSTGRES_FORMAT"; +pub const ENV_NOTIFY_POSTGRES_TLS_REQUIRED: &str = "RUSTFS_NOTIFY_POSTGRES_TLS_REQUIRED"; +pub const ENV_NOTIFY_POSTGRES_TLS_CA: &str = "RUSTFS_NOTIFY_POSTGRES_TLS_CA"; +pub const ENV_NOTIFY_POSTGRES_TLS_CLIENT_CERT: &str = "RUSTFS_NOTIFY_POSTGRES_TLS_CLIENT_CERT"; +pub const ENV_NOTIFY_POSTGRES_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_POSTGRES_TLS_CLIENT_KEY"; +pub const ENV_NOTIFY_POSTGRES_QUEUE_DIR: &str = "RUSTFS_NOTIFY_POSTGRES_QUEUE_DIR"; +pub const ENV_NOTIFY_POSTGRES_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_POSTGRES_QUEUE_LIMIT"; + +pub const ENV_NOTIFY_POSTGRES_KEYS: &[&str; 10] = &[ + ENV_NOTIFY_POSTGRES_ENABLE, + ENV_NOTIFY_POSTGRES_DSN_STRING, + ENV_NOTIFY_POSTGRES_TABLE, + ENV_NOTIFY_POSTGRES_FORMAT, + ENV_NOTIFY_POSTGRES_TLS_REQUIRED, + ENV_NOTIFY_POSTGRES_TLS_CA, + ENV_NOTIFY_POSTGRES_TLS_CLIENT_CERT, + ENV_NOTIFY_POSTGRES_TLS_CLIENT_KEY, + ENV_NOTIFY_POSTGRES_QUEUE_DIR, + ENV_NOTIFY_POSTGRES_QUEUE_LIMIT, +]; diff --git a/crates/config/src/notify/redis.rs b/crates/config/src/notify/redis.rs new file mode 100644 index 000000000..020a5937f --- /dev/null +++ b/crates/config/src/notify/redis.rs @@ -0,0 +1,82 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// A list of all valid configuration keys for a Redis target. +pub const NOTIFY_REDIS_KEYS: &[&str] = &[ + crate::ENABLE_KEY, + crate::REDIS_URL, + crate::REDIS_CHANNEL, + crate::REDIS_USERNAME, + crate::REDIS_PASSWORD, + crate::REDIS_KEEP_ALIVE_INTERVAL, + crate::REDIS_QUEUE_DIR, + crate::REDIS_QUEUE_LIMIT, + crate::REDIS_MAX_RETRY_ATTEMPTS, + crate::REDIS_RECONNECT_RETRY_ATTEMPTS, + crate::REDIS_MIN_RETRY_DELAY, + crate::REDIS_MAX_RETRY_DELAY, + crate::REDIS_CONNECTION_TIMEOUT, + crate::REDIS_RESPONSE_TIMEOUT, + crate::REDIS_PIPELINE_BUFFER_SIZE, + crate::REDIS_TLS_POLICY, + crate::REDIS_TLS_CA, + crate::REDIS_TLS_CLIENT_CERT, + crate::REDIS_TLS_CLIENT_KEY, + crate::REDIS_TLS_ALLOW_INSECURE, + crate::COMMENT_KEY, +]; + +pub const ENV_NOTIFY_REDIS_ENABLE: &str = "RUSTFS_NOTIFY_REDIS_ENABLE"; +pub const ENV_NOTIFY_REDIS_URL: &str = "RUSTFS_NOTIFY_REDIS_URL"; +pub const ENV_NOTIFY_REDIS_CHANNEL: &str = "RUSTFS_NOTIFY_REDIS_CHANNEL"; +pub const ENV_NOTIFY_REDIS_USERNAME: &str = "RUSTFS_NOTIFY_REDIS_USERNAME"; +pub const ENV_NOTIFY_REDIS_PASSWORD: &str = "RUSTFS_NOTIFY_REDIS_PASSWORD"; +pub const ENV_NOTIFY_REDIS_KEEP_ALIVE_INTERVAL: &str = "RUSTFS_NOTIFY_REDIS_KEEP_ALIVE_INTERVAL"; +pub const ENV_NOTIFY_REDIS_QUEUE_DIR: &str = "RUSTFS_NOTIFY_REDIS_QUEUE_DIR"; +pub const ENV_NOTIFY_REDIS_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_REDIS_QUEUE_LIMIT"; +pub const ENV_NOTIFY_REDIS_MAX_RETRY_ATTEMPTS: &str = "RUSTFS_NOTIFY_REDIS_MAX_RETRY_ATTEMPTS"; +pub const ENV_NOTIFY_REDIS_RECONNECT_RETRY_ATTEMPTS: &str = "RUSTFS_NOTIFY_REDIS_RECONNECT_RETRY_ATTEMPTS"; +pub const ENV_NOTIFY_REDIS_MIN_RETRY_DELAY: &str = "RUSTFS_NOTIFY_REDIS_MIN_RETRY_DELAY"; +pub const ENV_NOTIFY_REDIS_MAX_RETRY_DELAY: &str = "RUSTFS_NOTIFY_REDIS_MAX_RETRY_DELAY"; +pub const ENV_NOTIFY_REDIS_CONNECTION_TIMEOUT: &str = "RUSTFS_NOTIFY_REDIS_CONNECTION_TIMEOUT"; +pub const ENV_NOTIFY_REDIS_RESPONSE_TIMEOUT: &str = "RUSTFS_NOTIFY_REDIS_RESPONSE_TIMEOUT"; +pub const ENV_NOTIFY_REDIS_PIPELINE_BUFFER_SIZE: &str = "RUSTFS_NOTIFY_REDIS_PIPELINE_BUFFER_SIZE"; +pub const ENV_NOTIFY_REDIS_TLS_POLICY: &str = "RUSTFS_NOTIFY_REDIS_TLS_POLICY"; +pub const ENV_NOTIFY_REDIS_TLS_CA: &str = "RUSTFS_NOTIFY_REDIS_TLS_CA"; +pub const ENV_NOTIFY_REDIS_TLS_CLIENT_CERT: &str = "RUSTFS_NOTIFY_REDIS_TLS_CLIENT_CERT"; +pub const ENV_NOTIFY_REDIS_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_REDIS_TLS_CLIENT_KEY"; +pub const ENV_NOTIFY_REDIS_TLS_ALLOW_INSECURE: &str = "RUSTFS_NOTIFY_REDIS_TLS_ALLOW_INSECURE"; + +pub const ENV_NOTIFY_REDIS_KEYS: &[&str; 20] = &[ + ENV_NOTIFY_REDIS_ENABLE, + ENV_NOTIFY_REDIS_URL, + ENV_NOTIFY_REDIS_CHANNEL, + ENV_NOTIFY_REDIS_USERNAME, + ENV_NOTIFY_REDIS_PASSWORD, + ENV_NOTIFY_REDIS_KEEP_ALIVE_INTERVAL, + ENV_NOTIFY_REDIS_QUEUE_DIR, + ENV_NOTIFY_REDIS_QUEUE_LIMIT, + ENV_NOTIFY_REDIS_MAX_RETRY_ATTEMPTS, + ENV_NOTIFY_REDIS_RECONNECT_RETRY_ATTEMPTS, + ENV_NOTIFY_REDIS_MIN_RETRY_DELAY, + ENV_NOTIFY_REDIS_MAX_RETRY_DELAY, + ENV_NOTIFY_REDIS_CONNECTION_TIMEOUT, + ENV_NOTIFY_REDIS_RESPONSE_TIMEOUT, + ENV_NOTIFY_REDIS_PIPELINE_BUFFER_SIZE, + ENV_NOTIFY_REDIS_TLS_POLICY, + ENV_NOTIFY_REDIS_TLS_CA, + ENV_NOTIFY_REDIS_TLS_CLIENT_CERT, + ENV_NOTIFY_REDIS_TLS_CLIENT_KEY, + ENV_NOTIFY_REDIS_TLS_ALLOW_INSECURE, +]; diff --git a/crates/ecstore/src/config/audit.rs b/crates/ecstore/src/config/audit.rs index c55e90700..bcb5c694b 100644 --- a/crates/ecstore/src/config/audit.rs +++ b/crates/ecstore/src/config/audit.rs @@ -13,17 +13,25 @@ // limitations under the License. use crate::config::{KV, KVS}; +use rustfs_config::audit::AUDIT_REDIS_DEFAULT_CHANNEL; use rustfs_config::{ COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, - MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, - NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN, - PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, - PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA, - WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR, - WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY, + MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, + MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, + NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, + NATS_TOKEN, NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE, + POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER, + PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, + PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT, + REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD, + REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT, + REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL, + REDIS_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, + WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, + WEBHOOK_SKIP_TLS_VERIFY, }; use std::sync::LazyLock; @@ -339,6 +347,176 @@ pub static DEFAULT_AUDIT_PULSAR_KVS: LazyLock = LazyLock::new(|| { ]) }); +pub static DEFAULT_AUDIT_REDIS_KVS: LazyLock = LazyLock::new(|| { + KVS(vec![ + KV { + key: ENABLE_KEY.to_owned(), + value: EnableState::Off.to_string(), + hidden_if_empty: false, + }, + KV { + key: REDIS_URL.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_CHANNEL.to_owned(), + value: AUDIT_REDIS_DEFAULT_CHANNEL.to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_USERNAME.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_PASSWORD.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: REDIS_KEEP_ALIVE_INTERVAL.to_owned(), + value: "15".to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_QUEUE_DIR.to_owned(), + value: EVENT_DEFAULT_DIR.to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_QUEUE_LIMIT.to_owned(), + value: DEFAULT_LIMIT.to_string(), + hidden_if_empty: false, + }, + KV { + key: REDIS_MAX_RETRY_ATTEMPTS.to_owned(), + value: "3".to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_RECONNECT_RETRY_ATTEMPTS.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_MIN_RETRY_DELAY.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_MAX_RETRY_DELAY.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_CONNECTION_TIMEOUT.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_RESPONSE_TIMEOUT.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_PIPELINE_BUFFER_SIZE.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_TLS_POLICY.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: REDIS_TLS_CA.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: REDIS_TLS_CLIENT_CERT.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: REDIS_TLS_CLIENT_KEY.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: REDIS_TLS_ALLOW_INSECURE.to_owned(), + value: EnableState::Off.to_string(), + hidden_if_empty: false, + }, + KV { + key: COMMENT_KEY.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + ]) +}); + +pub static DEFAULT_AUDIT_POSTGRES_KVS: LazyLock = LazyLock::new(|| { + KVS(vec![ + KV { + key: ENABLE_KEY.to_owned(), + value: EnableState::Off.to_string(), + hidden_if_empty: false, + }, + KV { + key: POSTGRES_DSN_STRING.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: POSTGRES_TABLE.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: POSTGRES_FORMAT.to_owned(), + value: "namespace".to_owned(), + hidden_if_empty: false, + }, + KV { + key: POSTGRES_TLS_REQUIRED.to_owned(), + value: EnableState::Off.to_string(), + hidden_if_empty: false, + }, + KV { + key: POSTGRES_TLS_CA.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: POSTGRES_TLS_CLIENT_CERT.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: POSTGRES_TLS_CLIENT_KEY.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: POSTGRES_QUEUE_DIR.to_owned(), + value: EVENT_DEFAULT_DIR.to_owned(), + hidden_if_empty: false, + }, + KV { + key: POSTGRES_QUEUE_LIMIT.to_owned(), + value: DEFAULT_LIMIT.to_string(), + hidden_if_empty: false, + }, + KV { + key: COMMENT_KEY.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + ]) +}); + pub static DEFAULT_AUDIT_KAFKA_KVS: LazyLock = LazyLock::new(|| { KVS(vec![ KV { @@ -398,3 +576,63 @@ pub static DEFAULT_AUDIT_KAFKA_KVS: LazyLock = LazyLock::new(|| { }, ]) }); + +pub static DEFAULT_AUDIT_MYSQL_KVS: LazyLock = LazyLock::new(|| { + KVS(vec![ + KV { + key: ENABLE_KEY.to_owned(), + value: EnableState::Off.to_string(), + hidden_if_empty: false, + }, + KV { + key: MYSQL_DSN_STRING.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: MYSQL_TABLE.to_owned(), + value: "rustfs_audit_logs".to_owned(), + hidden_if_empty: false, + }, + KV { + key: MYSQL_FORMAT.to_owned(), + value: "access".to_owned(), + hidden_if_empty: false, + }, + KV { + key: MYSQL_TLS_CA.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: MYSQL_TLS_CLIENT_CERT.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: MYSQL_TLS_CLIENT_KEY.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: MYSQL_QUEUE_DIR.to_owned(), + value: EVENT_DEFAULT_DIR.to_owned(), + hidden_if_empty: false, + }, + KV { + key: MYSQL_QUEUE_LIMIT.to_owned(), + value: DEFAULT_LIMIT.to_string(), + hidden_if_empty: false, + }, + KV { + key: MYSQL_MAX_OPEN_CONNECTIONS.to_owned(), + value: "2".to_owned(), + hidden_if_empty: false, + }, + KV { + key: COMMENT_KEY.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + ]) +}); diff --git a/crates/ecstore/src/config/com.rs b/crates/ecstore/src/config/com.rs index 456a73051..32e25f5fa 100644 --- a/crates/ecstore/src/config/com.rs +++ b/crates/ecstore/src/config/com.rs @@ -19,12 +19,14 @@ use crate::global::is_first_cluster_node_local; use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI}; use http::HeaderMap; use rustfs_config::audit::{ - AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS, - AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS, + AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_MYSQL_KEYS, AUDIT_MYSQL_SUB_SYS, + AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS, AUDIT_POSTGRES_KEYS, AUDIT_POSTGRES_SUB_SYS, AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS, + AUDIT_REDIS_KEYS, AUDIT_REDIS_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS, }; use rustfs_config::notify::{ - NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, - NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS, + NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_KEYS, NOTIFY_MYSQL_SUB_SYS, + NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_KEYS, NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_KEYS, + NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_KEYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS, }; use rustfs_config::oidc::{IDENTITY_OPENID_KEYS, IDENTITY_OPENID_SUB_SYS, OIDC_REDIRECT_URI_DYNAMIC}; use rustfs_config::{COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, RUSTFS_REGION}; @@ -58,7 +60,7 @@ struct TargetConfigDescriptor { valid_keys: &'static [&'static str], } -fn notify_target_descriptors() -> [TargetConfigDescriptor; 5] { +fn notify_target_descriptors() -> [TargetConfigDescriptor; 8] { [ TargetConfigDescriptor { external_key: "webhook", @@ -78,12 +80,30 @@ fn notify_target_descriptors() -> [TargetConfigDescriptor; 5] { default_kvs: ¬ify::DEFAULT_NOTIFY_MQTT_KVS, valid_keys: NOTIFY_MQTT_KEYS, }, + TargetConfigDescriptor { + external_key: "mysql", + subsystem_key: NOTIFY_MYSQL_SUB_SYS, + default_kvs: ¬ify::DEFAULT_NOTIFY_MYSQL_KVS, + valid_keys: NOTIFY_MYSQL_KEYS, + }, TargetConfigDescriptor { external_key: "nats", subsystem_key: NOTIFY_NATS_SUB_SYS, default_kvs: ¬ify::DEFAULT_NOTIFY_NATS_KVS, valid_keys: NOTIFY_NATS_KEYS, }, + TargetConfigDescriptor { + external_key: "postgres", + subsystem_key: NOTIFY_POSTGRES_SUB_SYS, + default_kvs: ¬ify::DEFAULT_NOTIFY_POSTGRES_KVS, + valid_keys: NOTIFY_POSTGRES_KEYS, + }, + TargetConfigDescriptor { + external_key: "redis", + subsystem_key: NOTIFY_REDIS_SUB_SYS, + default_kvs: ¬ify::DEFAULT_NOTIFY_REDIS_KVS, + valid_keys: NOTIFY_REDIS_KEYS, + }, TargetConfigDescriptor { external_key: "pulsar", subsystem_key: NOTIFY_PULSAR_SUB_SYS, @@ -93,7 +113,7 @@ fn notify_target_descriptors() -> [TargetConfigDescriptor; 5] { ] } -fn audit_target_descriptors() -> [TargetConfigDescriptor; 5] { +fn audit_target_descriptors() -> [TargetConfigDescriptor; 8] { [ TargetConfigDescriptor { external_key: "webhook", @@ -113,18 +133,36 @@ fn audit_target_descriptors() -> [TargetConfigDescriptor; 5] { default_kvs: &audit::DEFAULT_AUDIT_MQTT_KVS, valid_keys: AUDIT_MQTT_KEYS, }, + TargetConfigDescriptor { + external_key: "mysql", + subsystem_key: AUDIT_MYSQL_SUB_SYS, + default_kvs: &audit::DEFAULT_AUDIT_MYSQL_KVS, + valid_keys: AUDIT_MYSQL_KEYS, + }, TargetConfigDescriptor { external_key: "nats", subsystem_key: AUDIT_NATS_SUB_SYS, default_kvs: &audit::DEFAULT_AUDIT_NATS_KVS, valid_keys: AUDIT_NATS_KEYS, }, + TargetConfigDescriptor { + external_key: "postgres", + subsystem_key: AUDIT_POSTGRES_SUB_SYS, + default_kvs: &audit::DEFAULT_AUDIT_POSTGRES_KVS, + valid_keys: AUDIT_POSTGRES_KEYS, + }, TargetConfigDescriptor { external_key: "pulsar", subsystem_key: AUDIT_PULSAR_SUB_SYS, default_kvs: &audit::DEFAULT_AUDIT_PULSAR_KVS, valid_keys: AUDIT_PULSAR_KEYS, }, + TargetConfigDescriptor { + external_key: "redis", + subsystem_key: AUDIT_REDIS_SUB_SYS, + default_kvs: &audit::DEFAULT_AUDIT_REDIS_KVS, + valid_keys: AUDIT_REDIS_KEYS, + }, ] } @@ -1132,9 +1170,11 @@ mod tests { }; use http::HeaderMap; use rustfs_config::audit::{AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS}; - use rustfs_config::notify::{NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS}; + use rustfs_config::notify::{NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS}; use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS; - use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState}; + use rustfs_config::{ + DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MYSQL_DSN_STRING, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_TABLE, + }; use rustfs_filemeta::FileInfo; use rustfs_lock::client::LockClient; use rustfs_lock::client::local::LocalClient; @@ -1775,6 +1815,15 @@ mod tests { "acks":"all", "tls_enable":true } + }, + "mysql":{ + "primary":{ + "enable":true, + "dsn_string":"rustfs:password@tcp(127.0.0.1:3306)/rustfs_events", + "table":"rustfs_events", + "queue_dir":"/tmp/mysql-queue", + "max_open_connections":"2" + } } } }"#; @@ -1811,6 +1860,15 @@ mod tests { assert_eq!(kafka.get(rustfs_config::KAFKA_TOPIC), "events-kafka"); assert_eq!(kafka.get(rustfs_config::KAFKA_ACKS), "all"); assert_eq!(kafka.get(rustfs_config::KAFKA_TLS_ENABLE), "true"); + + let mysql = cfg + .get_value(NOTIFY_MYSQL_SUB_SYS, "primary") + .expect("mysql target should be decoded"); + assert_eq!(mysql.get(ENABLE_KEY), EnableState::On.to_string()); + assert_eq!(mysql.get(MYSQL_DSN_STRING), "rustfs:password@tcp(127.0.0.1:3306)/rustfs_events"); + assert_eq!(mysql.get(MYSQL_TABLE), "rustfs_events"); + assert_eq!(mysql.get(MYSQL_QUEUE_DIR), "/tmp/mysql-queue"); + assert_eq!(mysql.get(MYSQL_MAX_OPEN_CONNECTIONS), "2"); } #[test] diff --git a/crates/ecstore/src/config/mod.rs b/crates/ecstore/src/config/mod.rs index 3aa6a9ec7..ddef5e638 100644 --- a/crates/ecstore/src/config/mod.rs +++ b/crates/ecstore/src/config/mod.rs @@ -26,10 +26,12 @@ use com::{STORAGE_CLASS_SUB_SYS, lookup_configs, read_config_without_migrate}; use rustfs_config::COMMENT_KEY; use rustfs_config::DEFAULT_DELIMITER; use rustfs_config::audit::{ - AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_SUB_SYS, AUDIT_PULSAR_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS, + AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_MYSQL_SUB_SYS, AUDIT_NATS_SUB_SYS, AUDIT_POSTGRES_SUB_SYS, + AUDIT_PULSAR_SUB_SYS, AUDIT_REDIS_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS, }; use rustfs_config::notify::{ - NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, + NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_SUB_SYS, + NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, }; use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS; use serde::{Deserialize, Serialize}; @@ -243,10 +245,16 @@ pub fn init() { kvs.insert(AUDIT_MQTT_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_MQTT_KVS.clone()); kvs.insert(NOTIFY_NATS_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_NATS_KVS.clone()); kvs.insert(AUDIT_NATS_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_NATS_KVS.clone()); + kvs.insert(NOTIFY_REDIS_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_REDIS_KVS.clone()); + kvs.insert(AUDIT_REDIS_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_REDIS_KVS.clone()); + kvs.insert(NOTIFY_POSTGRES_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_POSTGRES_KVS.clone()); + kvs.insert(AUDIT_POSTGRES_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_POSTGRES_KVS.clone()); kvs.insert(NOTIFY_PULSAR_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_PULSAR_KVS.clone()); kvs.insert(AUDIT_PULSAR_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_PULSAR_KVS.clone()); kvs.insert(NOTIFY_KAFKA_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_KAFKA_KVS.clone()); kvs.insert(AUDIT_KAFKA_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_KAFKA_KVS.clone()); + kvs.insert(NOTIFY_MYSQL_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_MYSQL_KVS.clone()); + kvs.insert(AUDIT_MYSQL_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_MYSQL_KVS.clone()); kvs.insert(IDENTITY_OPENID_SUB_SYS.to_owned(), oidc::DEFAULT_IDENTITY_OPENID_KVS.clone()); // Register all default configurations diff --git a/crates/ecstore/src/config/notify.rs b/crates/ecstore/src/config/notify.rs index d6fbdbe0b..2032bb4f3 100644 --- a/crates/ecstore/src/config/notify.rs +++ b/crates/ecstore/src/config/notify.rs @@ -13,16 +13,24 @@ // limitations under the License. use crate::config::{KV, KVS}; +use rustfs_config::notify::NOTIFY_REDIS_DEFAULT_CHANNEL; use rustfs_config::{ COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, - MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, - NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN, - PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, - PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, - WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY, + MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, + MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, + NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, + NATS_TOKEN, NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE, + POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER, + PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, + PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT, + REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD, + REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT, + REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL, + REDIS_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, + WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY, }; use std::sync::LazyLock; @@ -316,6 +324,176 @@ pub static DEFAULT_NOTIFY_PULSAR_KVS: LazyLock = LazyLock::new(|| { ]) }); +pub static DEFAULT_NOTIFY_REDIS_KVS: LazyLock = LazyLock::new(|| { + KVS(vec![ + KV { + key: ENABLE_KEY.to_owned(), + value: EnableState::Off.to_string(), + hidden_if_empty: false, + }, + KV { + key: REDIS_URL.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_CHANNEL.to_owned(), + value: NOTIFY_REDIS_DEFAULT_CHANNEL.to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_USERNAME.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_PASSWORD.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: REDIS_KEEP_ALIVE_INTERVAL.to_owned(), + value: "15".to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_QUEUE_DIR.to_owned(), + value: EVENT_DEFAULT_DIR.to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_QUEUE_LIMIT.to_owned(), + value: DEFAULT_LIMIT.to_string(), + hidden_if_empty: false, + }, + KV { + key: REDIS_MAX_RETRY_ATTEMPTS.to_owned(), + value: "3".to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_RECONNECT_RETRY_ATTEMPTS.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_MIN_RETRY_DELAY.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_MAX_RETRY_DELAY.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_CONNECTION_TIMEOUT.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_RESPONSE_TIMEOUT.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_PIPELINE_BUFFER_SIZE.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: REDIS_TLS_POLICY.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: REDIS_TLS_CA.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: REDIS_TLS_CLIENT_CERT.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: REDIS_TLS_CLIENT_KEY.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: REDIS_TLS_ALLOW_INSECURE.to_owned(), + value: EnableState::Off.to_string(), + hidden_if_empty: false, + }, + KV { + key: COMMENT_KEY.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + ]) +}); + +pub static DEFAULT_NOTIFY_POSTGRES_KVS: LazyLock = LazyLock::new(|| { + KVS(vec![ + KV { + key: ENABLE_KEY.to_owned(), + value: EnableState::Off.to_string(), + hidden_if_empty: false, + }, + KV { + key: POSTGRES_DSN_STRING.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: POSTGRES_TABLE.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: POSTGRES_FORMAT.to_owned(), + value: "namespace".to_owned(), + hidden_if_empty: false, + }, + KV { + key: POSTGRES_TLS_REQUIRED.to_owned(), + value: EnableState::Off.to_string(), + hidden_if_empty: false, + }, + KV { + key: POSTGRES_TLS_CA.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: POSTGRES_TLS_CLIENT_CERT.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: POSTGRES_TLS_CLIENT_KEY.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: POSTGRES_QUEUE_DIR.to_owned(), + value: EVENT_DEFAULT_DIR.to_owned(), + hidden_if_empty: false, + }, + KV { + key: POSTGRES_QUEUE_LIMIT.to_owned(), + value: DEFAULT_LIMIT.to_string(), + hidden_if_empty: false, + }, + KV { + key: COMMENT_KEY.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + ]) +}); + pub static DEFAULT_NOTIFY_KAFKA_KVS: LazyLock = LazyLock::new(|| { KVS(vec![ KV { @@ -375,3 +553,64 @@ pub static DEFAULT_NOTIFY_KAFKA_KVS: LazyLock = LazyLock::new(|| { }, ]) }); + +/// MySQL notification target default configuration +pub static DEFAULT_NOTIFY_MYSQL_KVS: LazyLock = LazyLock::new(|| { + KVS(vec![ + KV { + key: ENABLE_KEY.to_owned(), + value: EnableState::Off.to_string(), + hidden_if_empty: false, + }, + KV { + key: MYSQL_DSN_STRING.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: MYSQL_TABLE.to_owned(), + value: "rustfs_events".to_owned(), + hidden_if_empty: false, + }, + KV { + key: MYSQL_FORMAT.to_owned(), + value: "access".to_owned(), + hidden_if_empty: false, + }, + KV { + key: MYSQL_TLS_CA.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: MYSQL_TLS_CLIENT_CERT.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: MYSQL_TLS_CLIENT_KEY.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: MYSQL_QUEUE_DIR.to_owned(), + value: EVENT_DEFAULT_DIR.to_owned(), + hidden_if_empty: false, + }, + KV { + key: MYSQL_QUEUE_LIMIT.to_owned(), + value: DEFAULT_LIMIT.to_string(), + hidden_if_empty: false, + }, + KV { + key: MYSQL_MAX_OPEN_CONNECTIONS.to_owned(), + value: "2".to_owned(), + hidden_if_empty: false, + }, + KV { + key: COMMENT_KEY.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + ]) +}); diff --git a/crates/notify/src/factory.rs b/crates/notify/src/factory.rs index f9cdc9967..5a2b6bade 100644 --- a/crates/notify/src/factory.rs +++ b/crates/notify/src/factory.rs @@ -15,13 +15,17 @@ use crate::Event; use async_trait::async_trait; use rustfs_config::EVENT_DEFAULT_DIR; -use rustfs_config::notify::{NOTIFY_KAFKA_KEYS, NOTIFY_MQTT_KEYS, NOTIFY_NATS_KEYS, NOTIFY_PULSAR_KEYS, NOTIFY_WEBHOOK_KEYS}; +use rustfs_config::notify::{ + NOTIFY_KAFKA_KEYS, NOTIFY_MQTT_KEYS, NOTIFY_MYSQL_KEYS, NOTIFY_NATS_KEYS, NOTIFY_POSTGRES_KEYS, NOTIFY_PULSAR_KEYS, + NOTIFY_REDIS_DEFAULT_CHANNEL, NOTIFY_REDIS_KEYS, NOTIFY_WEBHOOK_KEYS, +}; use rustfs_ecstore::config::KVS; use rustfs_targets::{ Target, config::{ - build_kafka_args, build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_kafka_config, - validate_mqtt_config, validate_nats_config, validate_pulsar_config, validate_webhook_config, + build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args, build_pulsar_args, + build_redis_args, build_webhook_args, validate_kafka_config, validate_mqtt_config, validate_mysql_config, + validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, validate_webhook_config, }, error::TargetError, target::TargetType, @@ -120,6 +124,25 @@ impl TargetFactory for PulsarTargetFactory { } } +pub struct PostgresTargetFactory; + +#[async_trait] +impl TargetFactory for PostgresTargetFactory { + async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError> { + let args = build_postgres_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; + let target = rustfs_targets::target::postgres::PostgresTarget::new(id, args)?; + Ok(Box::new(target)) + } + + fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> { + validate_postgres_config(config, EVENT_DEFAULT_DIR) + } + + fn get_valid_fields(&self) -> HashSet { + NOTIFY_POSTGRES_KEYS.iter().map(|s| s.to_string()).collect() + } +} + pub struct KafkaTargetFactory; #[async_trait] @@ -138,3 +161,41 @@ impl TargetFactory for KafkaTargetFactory { NOTIFY_KAFKA_KEYS.iter().map(|s| s.to_string()).collect() } } + +pub struct MySqlTargetFactory; + +#[async_trait] +impl TargetFactory for MySqlTargetFactory { + async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError> { + let args = build_mysql_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; + let target = rustfs_targets::target::mysql::MySqlTarget::new(id, args)?; + Ok(Box::new(target)) + } + + fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> { + validate_mysql_config(config, EVENT_DEFAULT_DIR) + } + + fn get_valid_fields(&self) -> HashSet { + NOTIFY_MYSQL_KEYS.iter().map(|s| s.to_string()).collect() + } +} + +pub struct RedisTargetFactory; + +#[async_trait] +impl TargetFactory for RedisTargetFactory { + async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError> { + let args = build_redis_args(config, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL, TargetType::NotifyEvent)?; + let target = rustfs_targets::target::redis::RedisTarget::new(id, args)?; + Ok(Box::new(target)) + } + + fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> { + validate_redis_config(config, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL) + } + + fn get_valid_fields(&self) -> HashSet { + NOTIFY_REDIS_KEYS.iter().map(|s| s.to_string()).collect() + } +} diff --git a/crates/notify/src/integration.rs b/crates/notify/src/integration.rs index d6db1b438..b0f5b0e46 100644 --- a/crates/notify/src/integration.rs +++ b/crates/notify/src/integration.rs @@ -25,8 +25,8 @@ use crate::{ use hashbrown::HashMap; use rustfs_config::notify::{ DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_WEBHOOK_ENABLE, - ENV_NOTIFY_WEBHOOK_ENDPOINT, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, - NOTIFY_WEBHOOK_SUB_SYS, + ENV_NOTIFY_WEBHOOK_ENDPOINT, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS, + NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, }; use rustfs_config::{ENV_NOTIFY_ENABLE, EVENT_DEFAULT_DIR}; use rustfs_ecstore::config::{Config, KVS}; @@ -57,8 +57,11 @@ fn subsystem_target_type(target_type: &str) -> &str { NOTIFY_WEBHOOK_SUB_SYS => "webhook", NOTIFY_KAFKA_SUB_SYS => "kafka", NOTIFY_MQTT_SUB_SYS => "mqtt", + NOTIFY_MYSQL_SUB_SYS => "mysql", NOTIFY_NATS_SUB_SYS => "nats", + NOTIFY_POSTGRES_SUB_SYS => "postgres", NOTIFY_PULSAR_SUB_SYS => "pulsar", + NOTIFY_REDIS_SUB_SYS => "redis", _ => target_type, } } @@ -802,4 +805,17 @@ mod tests { assert_eq!(target_id.id, "ledger"); assert_eq!(target_id.name, "pulsar"); } + + #[test] + fn runtime_target_id_for_subsystem_maps_notify_redis_to_runtime_type() { + let target_id = runtime_target_id_for_subsystem(NOTIFY_REDIS_SUB_SYS, "Primary"); + assert_eq!(target_id.id, "primary"); + assert_eq!(target_id.name, "redis"); + } + #[test] + fn runtime_target_id_for_subsystem_maps_notify_postgres_to_runtime_type() { + let target_id = runtime_target_id_for_subsystem(NOTIFY_POSTGRES_SUB_SYS, "AuditTrail"); + assert_eq!(target_id.id, "audittrail"); + assert_eq!(target_id.name, "postgres"); + } } diff --git a/crates/notify/src/registry.rs b/crates/notify/src/registry.rs index edea07b89..35efbb268 100644 --- a/crates/notify/src/registry.rs +++ b/crates/notify/src/registry.rs @@ -14,7 +14,8 @@ use crate::Event; use crate::factory::{ - KafkaTargetFactory, MQTTTargetFactory, NATSTargetFactory, PulsarTargetFactory, TargetFactory, WebhookTargetFactory, + KafkaTargetFactory, MQTTTargetFactory, MySqlTargetFactory, NATSTargetFactory, PostgresTargetFactory, PulsarTargetFactory, + RedisTargetFactory, TargetFactory, WebhookTargetFactory, }; use futures::stream::{FuturesUnordered, StreamExt}; use hashbrown::HashMap; @@ -46,8 +47,11 @@ impl TargetRegistry { registry.register(ChannelTargetType::Webhook.as_str(), Box::new(WebhookTargetFactory)); registry.register(ChannelTargetType::Mqtt.as_str(), Box::new(MQTTTargetFactory)); registry.register(ChannelTargetType::Nats.as_str(), Box::new(NATSTargetFactory)); + registry.register(ChannelTargetType::Postgres.as_str(), Box::new(PostgresTargetFactory)); registry.register(ChannelTargetType::Pulsar.as_str(), Box::new(PulsarTargetFactory)); registry.register(ChannelTargetType::Kafka.as_str(), Box::new(KafkaTargetFactory)); + registry.register(ChannelTargetType::MySql.as_str(), Box::new(MySqlTargetFactory)); + registry.register(ChannelTargetType::Redis.as_str(), Box::new(RedisTargetFactory)); registry } diff --git a/crates/targets/AGENTS.md b/crates/targets/AGENTS.md new file mode 100644 index 000000000..79e6e5771 --- /dev/null +++ b/crates/targets/AGENTS.md @@ -0,0 +1,62 @@ +# Targets Crate Instructions + +Applies to `crates/targets/`. + +`rustfs-targets` provides the notification target abstraction layer: +the `Target` trait, built-in implementations (Webhook, Kafka, MQTT, +NATS, Pulsar, MySQL), persistent queue store, DSN/configuration +builders, and the `ChannelTargetType` registry that maps target types +to their runtime factories. + +## Library Design + +- Treat crate code as reusable library code by default. +- Prefer `thiserror` for library-facing error types. +- Do not use `unwrap()`, `expect()`, or panic-driven control flow outside tests. + +## Testing + +- Keep unit tests close to the module they test. +- Keep integration tests under `crates/targets/tests/` directory. +- Add regression tests for bug fixes and behavior changes. + +## Async and Performance + +- Keep async paths non-blocking. +- Move CPU-heavy operations out of async hot paths with `tokio::task::spawn_blocking` when appropriate. + +## Integration Tests + +### MySQL Integration Tests + +Integration tests in `tests/mysql_integration.rs` require a running MySQL 8.0+ +or TiDB 8.5+ instance. They are `#[ignore]` by default so CI never runs them. + +Start a test MySQL instance with Podman: + +```bash +podman run -d --name rustfs-mysql-test \ + -e MYSQL_ROOT_PASSWORD=testpass \ + -e MYSQL_DATABASE=testdb \ + -p 3306:3306 \ + docker.io/library/mysql:8.0.36 +``` + +Wait for MySQL to be ready (look for `ready for connections` in logs), +then run the integration tests: + +```bash +RUSTFS_MYSQL_TEST_DSN="root:testpass@tcp(127.0.0.1:3306)/testdb" \ + cargo test -p rustfs-targets -- --ignored +``` + +Clean up: + +```bash +podman rm -f rustfs-mysql-test +``` + +## Suggested Validation + +- `cargo test -p rustfs-targets` +- Full gate before commit: `make pre-commit` diff --git a/crates/targets/Cargo.toml b/crates/targets/Cargo.toml index 31b6aed6a..313509d5f 100644 --- a/crates/targets/Cargo.toml +++ b/crates/targets/Cargo.toml @@ -18,25 +18,34 @@ rustfs-utils = { workspace = true, features = ["notify", "tls"] } rustfs-s3-common = { workspace = true } async-trait = { workspace = true } async-nats = { workspace = true } +deadpool-postgres = { workspace = true } hyper-rustls = { workspace = true } pulsar = { workspace = true } reqwest = { workspace = true } rumqttc = { workspace = true } +redis = { workspace = true } rustls = { workspace = true } +rustls-native-certs = { workspace = true } +rustls-pki-types = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } snap = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time"] } +tokio-postgres = { workspace = true } +tokio-postgres-rustls = { workspace = true } tracing = { workspace = true } url = { workspace = true } urlencoding = { workspace = true } uuid = { workspace = true, features = ["v4", "serde"] } sysinfo = { workspace = true, features = ["multithread"] } rustfs-kafka-async = { workspace = true } +mysql_async = { workspace = true } +chrono = { workspace = true } [dev-dependencies] criterion = { workspace = true } +tempfile = { workspace = true } [[bench]] name = "queue_store_benchmark" diff --git a/crates/targets/src/check.rs b/crates/targets/src/check.rs index 4e61e1fa3..730613905 100644 --- a/crates/targets/src/check.rs +++ b/crates/targets/src/check.rs @@ -78,7 +78,7 @@ pub async fn check_mqtt_broker_available_with_tls( std::time::Duration::from_secs(5), None, )?; - let (client, mut eventloop) = AsyncClient::new(mqtt_options, 1); + let (client, mut eventloop) = AsyncClient::builder(mqtt_options).capacity(1).build(); // Try to connect and subscribe client @@ -94,7 +94,7 @@ pub async fn check_mqtt_broker_available_with_tls( } pub async fn check_nats_server_available(args: &crate::target::nats::NATSArgs) -> Result<(), crate::TargetError> { - match tokio::time::timeout(std::time::Duration::from_secs(5), async { + tokio::time::timeout(std::time::Duration::from_secs(5), async { let client = crate::target::nats::connect_nats(args).await?; client .flush() @@ -107,14 +107,11 @@ pub async fn check_nats_server_available(args: &crate::target::nats::NATSArgs) - Ok(()) }) .await - { - Ok(result) => result, - Err(_) => Err(crate::TargetError::Timeout("NATS connection timed out".to_string())), - } + .unwrap_or_else(|_| Err(crate::TargetError::Timeout("NATS connection timed out".to_string()))) } pub async fn check_pulsar_broker_available(args: &crate::target::pulsar::PulsarArgs) -> Result<(), crate::TargetError> { - match tokio::time::timeout(std::time::Duration::from_secs(5), async { + tokio::time::timeout(std::time::Duration::from_secs(5), async { let client = crate::target::pulsar::connect_pulsar(args).await?; client .lookup_partitioned_topic(args.topic.clone()) @@ -123,10 +120,52 @@ pub async fn check_pulsar_broker_available(args: &crate::target::pulsar::PulsarA Ok(()) }) .await - { - Ok(result) => result, - Err(_) => Err(crate::TargetError::Timeout("Pulsar connection timed out".to_string())), - } + .unwrap_or_else(|_| Err(crate::TargetError::Timeout("Pulsar connection timed out".to_string()))) +} + +/// Probes a PostgreSQL server for connectivity and verifies the configured +/// table is readable. +/// +/// Used by both the admin validation flow (pre-flight before persisting a +/// target) and `PostgresTarget::init()` (runtime startup check). The probe is +/// strictly read-only: +/// +/// 1. Build a deadpool pool from `args` (cheap, no actual connection yet). +/// 2. Check out a single connection. +/// 3. Run `SELECT 1` to confirm the credentials work. +/// 4. Run `SELECT 1 FROM . LIMIT 0` to confirm the relation +/// exists and the user has read permission. `LIMIT 0` ensures no rows are +/// actually returned and no DML side effects occur. +/// +/// The whole flow is wrapped in an 8s `tokio::time::timeout` so a stuck DNS +/// resolver or TLS handshake cannot exhaust the admin layer's outer 10s +/// timeout. +pub async fn check_postgres_server_available(args: &crate::target::postgres::PostgresArgs) -> Result<(), crate::TargetError> { + use crate::target::postgres::{build_pool, map_pg_error, map_pool_error, table_probe_sql}; + + args.validate()?; + + let timeout = std::time::Duration::from_secs(8); + tokio::time::timeout(timeout, async { + let pool = build_pool(args)?; + let client = pool + .get() + .await + .map_err(|e| map_pool_error(e, "PostgreSQL connectivity probe failed to acquire connection"))?; + client + .execute("SELECT 1", &[]) + .await + .map_err(|e| map_pg_error(&e, "PostgreSQL liveness probe failed"))?; + let probe_sql = table_probe_sql(&args.schema, &args.table); + client + .execute(probe_sql.as_str(), &[]) + .await + .map_err(|e| map_pg_error(&e, "PostgreSQL table probe failed"))?; + pool.close(); + Ok::<(), crate::TargetError>(()) + }) + .await + .unwrap_or_else(|_| Err(crate::TargetError::Timeout("PostgreSQL connectivity probe timed out".to_string()))) } pub async fn check_kafka_broker_available(args: &crate::target::kafka::KafkaArgs) -> Result<(), crate::TargetError> { @@ -163,15 +202,21 @@ pub async fn check_kafka_broker_available(args: &crate::target::kafka::KafkaArgs config = config.with_security(security); } - match tokio::time::timeout(Duration::from_secs(5), async { + tokio::time::timeout(Duration::from_secs(5), async { let _ = AsyncProducer::from_hosts_with_config(args.brokers.clone(), config) .await .map_err(|err| map_kafka_error(err, "Kafka broker check failed to create producer"))?; Ok(()) }) .await - { - Ok(result) => result, - Err(_) => Err(crate::TargetError::Timeout("Kafka connection timed out".to_string())), - } + .unwrap_or_else(|_| Err(crate::TargetError::Timeout("Kafka connection timed out".to_string()))) +} + +pub async fn check_redis_server_available(args: &crate::target::redis::RedisArgs) -> Result<(), crate::TargetError> { + tokio::time::timeout(std::time::Duration::from_secs(5), async { + let client = crate::target::redis::build_redis_client(args)?; + crate::target::redis::ping_redis_server(&client, args).await + }) + .await + .unwrap_or_else(|_| Err(crate::TargetError::Timeout("Redis connection timed out".to_string()))) } diff --git a/crates/targets/src/config/loader.rs b/crates/targets/src/config/loader.rs index a260954b7..3d715bcae 100644 --- a/crates/targets/src/config/loader.rs +++ b/crates/targets/src/config/loader.rs @@ -38,10 +38,24 @@ fn is_sensitive_target_field(field_name: &str) -> bool { || field_name.contains("client_key") || field_name.contains("access_key") || field_name.contains("auth") + || field_name.contains(rustfs_config::BASE_DSN_STRING) } fn redact_target_field_value(field_name: &str, value: &str) -> String { - if is_sensitive_target_field(field_name) && !value.is_empty() { + if value.is_empty() { + return value.to_string(); + } + // MySQL DSN fields need partial redaction instead of full masking so the + // remaining connection details (host, port, database) remain visible in + // debug logs while the password is hidden. + if field_name == rustfs_config::BASE_DSN_STRING { + let trimmed = value.trim_start(); + if trimmed.starts_with("postgres://") || trimmed.starts_with("postgresql://") { + return crate::target::postgres::redact_postgres_dsn(value); + } + return crate::target::mysql::redact_mysql_dsn(value); + } + if is_sensitive_target_field(field_name) { return "***redacted***".to_string(); } value.to_string() @@ -283,6 +297,23 @@ mod tests { assert_eq!(redact_target_field_value("queue_limit", "1000"), "1000"); } + #[test] + fn redact_dsn_string_partial_redaction() { + let dsn = "rustfs:secret123@tcp(mysql.example.com:3306)/rustfs_events"; + let redacted = redact_target_field_value(rustfs_config::MYSQL_DSN_STRING, dsn); + assert_eq!(redacted, "rustfs:***@tcp(mysql.example.com:3306)/rustfs_events"); + // empty dsn_string value + assert_eq!(redact_target_field_value(rustfs_config::MYSQL_DSN_STRING, ""), ""); + } + + #[test] + fn redact_postgres_dsn_string_partial_redaction() { + let dsn = "postgres://rustfs:secret123@pg.example.com:5432/rustfs_events?search_path=public"; + let redacted = redact_target_field_value(rustfs_config::POSTGRES_DSN_STRING, dsn); + assert_eq!(redacted, "postgres://rustfs:***@pg.example.com:5432/rustfs_events?search_path=public"); + assert_eq!(redact_target_field_value(rustfs_config::POSTGRES_DSN_STRING, ""), ""); + } + #[test] fn redacted_target_config_masks_sensitive_values_without_mutating_shape() { let mut config = KVS::new(); diff --git a/crates/targets/src/config/mod.rs b/crates/targets/src/config/mod.rs index ca27ef3e8..bfd9a37c2 100644 --- a/crates/targets/src/config/mod.rs +++ b/crates/targets/src/config/mod.rs @@ -21,6 +21,7 @@ pub use loader::{ collect_target_configs_from_env, }; pub use target_args::{ - build_kafka_args, build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_kafka_config, - validate_mqtt_config, validate_nats_config, validate_pulsar_config, validate_webhook_config, + build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args, build_pulsar_args, + build_redis_args, build_webhook_args, validate_kafka_config, validate_mqtt_config, validate_mysql_config, + validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, validate_webhook_config, }; diff --git a/crates/targets/src/config/target_args.rs b/crates/targets/src/config/target_args.rs index b6cee82e2..4d86ad297 100644 --- a/crates/targets/src/config/target_args.rs +++ b/crates/targets/src/config/target_args.rs @@ -18,8 +18,11 @@ use crate::target::{ TargetType, kafka::KafkaArgs, mqtt::{MQTTArgs, MQTTTlsConfig, validate_mqtt_broker_url}, + mysql::MySqlArgs, nats::{NATSArgs, validate_nats_address}, + postgres::{PostgresArgs, PostgresDsn, parse_postgres_format}, pulsar::{PulsarArgs, validate_pulsar_broker}, + redis::{RedisArgs, RedisTlsConfig, validate_redis_url}, webhook::WebhookArgs, }; use rumqttc::QoS; @@ -27,12 +30,19 @@ use rustfs_config::{ DEFAULT_LIMIT, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, - MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS, - NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, - NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD, - PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION, - PULSAR_TOPIC, PULSAR_USERNAME, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, - WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY, + MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, + MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA, + MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, + NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, + NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE, + POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER, + PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, + PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT, + REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD, + REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT, + REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL, + REDIS_USERNAME, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, + WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY, }; use rustfs_ecstore::config::KVS; use std::path::Path; @@ -286,6 +296,124 @@ pub fn validate_pulsar_config(config: &KVS, default_queue_dir: &str) -> Result<( validate_pulsar_broker_config(&broker, config, default_queue_dir) } +pub fn build_redis_args( + config: &KVS, + default_queue_dir: &str, + default_channel: &str, + target_type: TargetType, +) -> Result { + let url = config + .lookup(REDIS_URL) + .ok_or_else(|| TargetError::Configuration("Missing Redis URL".to_string()))?; + let url = parse_url(&url, "Redis URL")?; + + let channel = config + .lookup(REDIS_CHANNEL) + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| default_channel.to_string()); + + Ok(RedisArgs { + enable: true, + url, + channel, + username: config.lookup(REDIS_USERNAME).filter(|value| !value.trim().is_empty()), + password: config.lookup(REDIS_PASSWORD).filter(|value| !value.trim().is_empty()), + tls: RedisTlsConfig::from_values( + config.lookup(REDIS_TLS_POLICY).as_deref(), + config.lookup(REDIS_TLS_CA).as_deref(), + config.lookup(REDIS_TLS_CLIENT_CERT).as_deref(), + config.lookup(REDIS_TLS_CLIENT_KEY).as_deref(), + config.lookup(REDIS_TLS_ALLOW_INSECURE).as_deref(), + )?, + keep_alive: config + .lookup(REDIS_KEEP_ALIVE_INTERVAL) + .and_then(|v| v.parse::().ok()) + .map(Duration::from_secs) + .unwrap_or_else(|| Duration::from_secs(15)), + queue_dir: config + .lookup(REDIS_QUEUE_DIR) + .unwrap_or_else(|| default_queue_dir.to_string()), + queue_limit: config + .lookup(REDIS_QUEUE_LIMIT) + .and_then(|v| v.parse::().ok()) + .unwrap_or(DEFAULT_LIMIT), + max_retry_attempts: config + .lookup(REDIS_MAX_RETRY_ATTEMPTS) + .and_then(|v| v.parse::().ok()) + .unwrap_or(3), + reconnect_retry_attempts: config + .lookup(REDIS_RECONNECT_RETRY_ATTEMPTS) + .and_then(|v| v.parse::().ok()), + min_retry_delay: config + .lookup(REDIS_MIN_RETRY_DELAY) + .and_then(|v| v.parse::().ok()) + .map(Duration::from_millis), + max_retry_delay: config + .lookup(REDIS_MAX_RETRY_DELAY) + .and_then(|v| v.parse::().ok()) + .map(Duration::from_millis), + connection_timeout: config + .lookup(REDIS_CONNECTION_TIMEOUT) + .and_then(|v| v.parse::().ok()) + .map(Duration::from_secs), + response_timeout: config + .lookup(REDIS_RESPONSE_TIMEOUT) + .and_then(|v| v.parse::().ok()) + .map(Duration::from_secs), + pipeline_buffer_size: config + .lookup(REDIS_PIPELINE_BUFFER_SIZE) + .and_then(|v| v.parse::().ok()), + target_type, + }) +} + +pub fn build_postgres_args(config: &KVS, default_queue_dir: &str, target_type: TargetType) -> Result { + let dsn_string = config + .lookup(POSTGRES_DSN_STRING) + .ok_or_else(|| TargetError::Configuration("Missing PostgreSQL dsn_string".to_string()))?; + let table = config + .lookup(POSTGRES_TABLE) + .ok_or_else(|| TargetError::Configuration("Missing PostgreSQL table".to_string()))?; + + let schema = PostgresDsn::parse(&dsn_string)?.schema; + let format = parse_postgres_format(config.lookup(POSTGRES_FORMAT).as_deref())?; + + Ok(PostgresArgs { + enable: true, + dsn_string, + schema, + table, + format, + tls_required: parse_target_bool(config.lookup(POSTGRES_TLS_REQUIRED).as_deref()).unwrap_or(false), + tls_ca: config.lookup(POSTGRES_TLS_CA).unwrap_or_default(), + tls_client_cert: config.lookup(POSTGRES_TLS_CLIENT_CERT).unwrap_or_default(), + tls_client_key: config.lookup(POSTGRES_TLS_CLIENT_KEY).unwrap_or_default(), + queue_dir: config + .lookup(POSTGRES_QUEUE_DIR) + .unwrap_or_else(|| default_queue_dir.to_string()), + queue_limit: config + .lookup(POSTGRES_QUEUE_LIMIT) + .and_then(|v| v.parse::().ok()) + .unwrap_or(DEFAULT_LIMIT), + target_type, + }) +} + +pub fn validate_redis_config(config: &KVS, default_queue_dir: &str, default_channel: &str) -> Result<(), TargetError> { + let url = config + .lookup(REDIS_URL) + .ok_or_else(|| TargetError::Configuration("Missing Redis URL".to_string()))?; + let url = parse_url(&url, "Redis URL")?; + validate_redis_url(&url)?; + + let args = build_redis_args(config, default_queue_dir, default_channel, TargetType::NotifyEvent)?; + args.validate() +} +pub fn validate_postgres_config(config: &KVS, default_queue_dir: &str) -> Result<(), TargetError> { + let args = build_postgres_args(config, default_queue_dir, TargetType::NotifyEvent)?; + args.validate() +} + pub fn build_kafka_args(config: &KVS, default_queue_dir: &str, target_type: TargetType) -> Result { let brokers_raw = config .lookup(KAFKA_BROKERS) @@ -348,18 +476,81 @@ pub fn validate_kafka_config(config: &KVS, default_queue_dir: &str) -> Result<() let queue_dir = config .lookup(KAFKA_QUEUE_DIR) .unwrap_or_else(|| default_queue_dir.to_string()); - if !queue_dir.is_empty() && !std::path::Path::new(&queue_dir).is_absolute() { + if !queue_dir.is_empty() && !Path::new(&queue_dir).is_absolute() { return Err(TargetError::Configuration("Kafka queue directory must be an absolute path".to_string())); } Ok(()) } +/// Builds `MySqlArgs` from a KVS configuration. +/// +/// Parses all MySQL target configuration keys, applies defaults for +/// missing optional values, and validates that all required fields +/// are present and well-formed. +pub fn build_mysql_args(config: &KVS, default_queue_dir: &str, target_type: TargetType) -> Result { + let dsn_string = config + .lookup(MYSQL_DSN_STRING) + .ok_or_else(|| TargetError::Configuration("Missing MySQL dsn_string".to_string()))?; + + let table = config + .lookup(MYSQL_TABLE) + .ok_or_else(|| TargetError::Configuration("Missing MySQL table".to_string()))?; + + let args = MySqlArgs { + enable: true, + dsn_string, + table, + format: config.lookup(MYSQL_FORMAT).unwrap_or_else(|| "access".to_string()), + tls_ca: config.lookup(MYSQL_TLS_CA).unwrap_or_default(), + tls_client_cert: config.lookup(MYSQL_TLS_CLIENT_CERT).unwrap_or_default(), + tls_client_key: config.lookup(MYSQL_TLS_CLIENT_KEY).unwrap_or_default(), + queue_dir: config + .lookup(MYSQL_QUEUE_DIR) + .unwrap_or_else(|| default_queue_dir.to_string()), + queue_limit: config + .lookup(MYSQL_QUEUE_LIMIT) + .and_then(|v| v.parse::().ok()) + .unwrap_or(DEFAULT_LIMIT), + max_open_connections: config + .lookup(MYSQL_MAX_OPEN_CONNECTIONS) + .map(|value| { + value.trim().parse::().map_err(|_| { + TargetError::Configuration(format!("MySQL max_open_connections value '{}' is not a valid number", value)) + }) + }) + .transpose()? + .unwrap_or(2), + target_type, + }; + + args.validate()?; + Ok(args) +} + +/// Validates MySQL target configuration from a KVS without building args. +/// +/// Performs the same checks as `build_mysql_args` but discards the result, +/// used for pre-validation before target creation. +pub fn validate_mysql_config(config: &KVS, default_queue_dir: &str) -> Result<(), TargetError> { + let _ = build_mysql_args(config, default_queue_dir, TargetType::NotifyEvent)?; + Ok(()) +} + #[cfg(test)] mod tests { - use super::{build_kafka_args, validate_kafka_config}; - use crate::target::TargetType; - use rustfs_config::{KAFKA_ACKS, KAFKA_BROKERS, KAFKA_TOPIC}; + use super::{ + build_kafka_args, build_mysql_args, build_postgres_args, build_redis_args, validate_kafka_config, validate_mysql_config, + validate_postgres_config, validate_redis_config, + }; + use crate::target::{TargetType, postgres::PostgresFormat}; + use rustfs_config::{ + KAFKA_ACKS, KAFKA_BROKERS, KAFKA_TOPIC, MYSQL_DSN_STRING, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_TABLE, + MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, + POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, REDIS_CHANNEL, + REDIS_CONNECTION_TIMEOUT, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PIPELINE_BUFFER_SIZE, + REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT, REDIS_TLS_ALLOW_INSECURE, REDIS_URL, + }; use rustfs_ecstore::config::KVS; fn kafka_base_config() -> KVS { @@ -369,6 +560,16 @@ mod tests { config } + fn mysql_base_config() -> KVS { + let mut config = KVS::new(); + config.insert( + MYSQL_DSN_STRING.to_string(), + "rustfs:password@tcp(127.0.0.1:3306)/rustfs_events".to_string(), + ); + config.insert(MYSQL_TABLE.to_string(), "rustfs_events".to_string()); + config + } + #[test] fn build_kafka_args_accepts_all_ack_alias() { let mut config = kafka_base_config(); @@ -395,4 +596,273 @@ mod tests { let err = validate_kafka_config(&config, "").expect_err("invalid acks should fail"); assert!(err.to_string().contains("Kafka acks must be one of")); } + + #[test] + fn build_mysql_args_accepts_minimal_config() { + let args = build_mysql_args(&mysql_base_config(), "", TargetType::NotifyEvent).expect("valid mysql args"); + assert!(args.enable); + assert_eq!(args.dsn_string, "rustfs:password@tcp(127.0.0.1:3306)/rustfs_events"); + assert_eq!(args.table, "rustfs_events"); + assert_eq!(args.format, "access"); + assert_eq!(args.max_open_connections, 2); + assert_eq!(args.queue_limit, rustfs_config::DEFAULT_LIMIT); + } + + #[test] + fn build_mysql_args_applies_defaults() { + let args = build_mysql_args(&mysql_base_config(), "/custom/queue", TargetType::NotifyEvent).expect("valid mysql args"); + assert_eq!(args.queue_dir, "/custom/queue"); + assert_eq!(args.queue_limit, 100000); + assert_eq!(args.max_open_connections, 2); + } + + #[test] + fn build_mysql_args_rejects_missing_dsn() { + let mut config = KVS::new(); + config.insert(MYSQL_TABLE.to_string(), "events".to_string()); + + let err = build_mysql_args(&config, "", TargetType::NotifyEvent).expect_err("missing dsn should fail"); + assert!(err.to_string().contains("dsn_string")); + } + + #[test] + fn build_mysql_args_rejects_relative_queue_dir() { + let mut config = mysql_base_config(); + config.insert(MYSQL_QUEUE_DIR.to_string(), "relative/path".to_string()); + + let err = build_mysql_args(&config, "", TargetType::NotifyEvent).expect_err("relative path should fail"); + assert!(err.to_string().contains("absolute")); + } + + #[test] + fn validate_mysql_config_rejects_invalid_max_open_connections() { + let mut config = mysql_base_config(); + config.insert(MYSQL_MAX_OPEN_CONNECTIONS.to_string(), "not-a-number".to_string()); + + let err = validate_mysql_config(&config, "").expect_err("invalid max_open_connections should be rejected"); + assert!(err.to_string().contains("max_open_connections")); + } + + #[test] + fn validate_mysql_config_rejects_empty_dsn() { + let mut config = mysql_base_config(); + config.insert(MYSQL_DSN_STRING.to_string(), "".to_string()); + + let err = validate_mysql_config(&config, "").expect_err("empty dsn should fail"); + assert!(err.to_string().contains("empty")); + } + + #[test] + fn validate_mysql_config_rejects_unpaired_tls_client_fields() { + let mut config = mysql_base_config(); + config.insert(MYSQL_TLS_CLIENT_CERT.to_string(), "/etc/ssl/mysql/client.pem".to_string()); + + let err = validate_mysql_config(&config, "").expect_err("unpaired mysql TLS client cert should fail"); + assert!(err.to_string().contains("must be specified together")); + } + + #[test] + fn validate_mysql_config_rejects_relative_tls_paths() { + let mut config = mysql_base_config(); + config.insert(MYSQL_TLS_CA.to_string(), "ca.pem".to_string()); + + let err = validate_mysql_config(&config, "").expect_err("relative tls_ca should fail"); + assert!(err.to_string().contains("tls_ca must be an absolute path")); + + config.insert(MYSQL_TLS_CA.to_string(), "/etc/ssl/mysql/ca.pem".to_string()); + config.insert(MYSQL_TLS_CLIENT_CERT.to_string(), "client.pem".to_string()); + config.insert(MYSQL_TLS_CLIENT_KEY.to_string(), "client.key".to_string()); + + let err = validate_mysql_config(&config, "").expect_err("relative tls client paths should fail"); + assert!(err.to_string().contains("absolute path")); + } + + #[test] + fn build_mysql_args_accepts_absolute_tls_paths() { + let mut config = mysql_base_config(); + config.insert(MYSQL_TLS_CA.to_string(), "/etc/ssl/mysql/ca.pem".to_string()); + config.insert(MYSQL_TLS_CLIENT_CERT.to_string(), "/etc/ssl/mysql/client.pem".to_string()); + config.insert(MYSQL_TLS_CLIENT_KEY.to_string(), "/etc/ssl/mysql/client.key".to_string()); + + let args = build_mysql_args(&config, "", TargetType::NotifyEvent).expect("absolute mysql TLS paths should pass"); + assert_eq!(args.tls_ca, "/etc/ssl/mysql/ca.pem"); + assert_eq!(args.tls_client_cert, "/etc/ssl/mysql/client.pem"); + assert_eq!(args.tls_client_key, "/etc/ssl/mysql/client.key"); + } + + fn redis_base_config() -> KVS { + let mut config = KVS::new(); + config.insert(REDIS_URL.to_string(), "redis://127.0.0.1:6379/0".to_string()); + config.insert(REDIS_CHANNEL.to_string(), "events".to_string()); + config + } + fn postgres_base_config() -> KVS { + let mut config = KVS::new(); + config.insert( + POSTGRES_DSN_STRING.to_string(), + "postgres://postgres:rustfs@localhost:5432/rustfs_events?search_path=public".to_string(), + ); + config.insert(POSTGRES_TABLE.to_string(), "rustfs_events_namespace".to_string()); + config + } + + #[test] + fn build_redis_args_keeps_manager_tuning_fields_none_when_unset() { + let config = redis_base_config(); + + let args = build_redis_args(&config, "/tmp/queue", "default-channel", TargetType::NotifyEvent).expect("valid redis args"); + + assert_eq!(args.channel, "events"); + assert_eq!(args.reconnect_retry_attempts, None); + assert_eq!(args.min_retry_delay, None); + assert_eq!(args.max_retry_delay, None); + assert_eq!(args.connection_timeout, None); + assert_eq!(args.response_timeout, None); + assert_eq!(args.pipeline_buffer_size, None); + } + + #[test] + fn build_redis_args_uses_default_channel_when_missing() { + let mut config = KVS::new(); + config.insert(REDIS_URL.to_string(), "redis://127.0.0.1:6379/0".to_string()); + + let args = + build_redis_args(&config, "/tmp/queue", "fallback-channel", TargetType::NotifyEvent).expect("valid redis args"); + + assert_eq!(args.channel, "fallback-channel"); + } + + #[test] + fn build_redis_args_uses_default_channel_when_empty() { + let mut config = KVS::new(); + config.insert(REDIS_URL.to_string(), "redis://127.0.0.1:6379/0".to_string()); + config.insert(REDIS_CHANNEL.to_string(), " ".to_string()); + + let args = + build_redis_args(&config, "/tmp/queue", "fallback-channel", TargetType::NotifyEvent).expect("valid redis args"); + + assert_eq!(args.channel, "fallback-channel"); + } + + #[test] + fn build_redis_args_parses_optional_tuning_values_when_present() { + let mut config = redis_base_config(); + config.insert(REDIS_RECONNECT_RETRY_ATTEMPTS.to_string(), "9".to_string()); + config.insert(REDIS_MIN_RETRY_DELAY.to_string(), "250".to_string()); + config.insert(REDIS_MAX_RETRY_DELAY.to_string(), "5000".to_string()); + config.insert(REDIS_CONNECTION_TIMEOUT.to_string(), "7".to_string()); + config.insert(REDIS_RESPONSE_TIMEOUT.to_string(), "11".to_string()); + config.insert(REDIS_PIPELINE_BUFFER_SIZE.to_string(), "64".to_string()); + + let args = build_redis_args(&config, "/tmp/queue", "default-channel", TargetType::NotifyEvent).expect("valid redis args"); + + assert_eq!(args.reconnect_retry_attempts, Some(9)); + assert_eq!(args.min_retry_delay, Some(std::time::Duration::from_millis(250))); + assert_eq!(args.max_retry_delay, Some(std::time::Duration::from_millis(5000))); + assert_eq!(args.connection_timeout, Some(std::time::Duration::from_secs(7))); + assert_eq!(args.response_timeout, Some(std::time::Duration::from_secs(11))); + assert_eq!(args.pipeline_buffer_size, Some(64)); + } + + #[test] + fn build_redis_args_parses_tls_allow_insecure_when_present() { + let mut config = redis_base_config(); + config.insert(REDIS_URL.to_string(), "rediss://127.0.0.1:6379/0".to_string()); + config.insert(REDIS_TLS_ALLOW_INSECURE.to_string(), "on".to_string()); + + let args = build_redis_args(&config, "/tmp/queue", "default-channel", TargetType::NotifyEvent).expect("valid redis args"); + + assert!(args.tls.allow_insecure); + } + + #[test] + fn validate_redis_config_rejects_missing_url() { + let config = KVS::new(); + + let err = validate_redis_config(&config, "/tmp/queue", "default-channel").expect_err("missing redis url should fail"); + assert!(err.to_string().contains("Missing Redis URL")); + } + #[test] + fn build_postgres_args_accepts_minimal_config() { + let config = postgres_base_config(); + let args = build_postgres_args(&config, "", TargetType::NotifyEvent).expect("valid postgres args"); + assert_eq!( + args.dsn_string, + "postgres://postgres:rustfs@localhost:5432/rustfs_events?search_path=public" + ); + assert_eq!(args.format, PostgresFormat::Namespace); + } + + #[test] + fn build_postgres_args_parses_access_format() { + let mut config = postgres_base_config(); + config.insert(POSTGRES_FORMAT.to_string(), "access".to_string()); + let args = build_postgres_args(&config, "", TargetType::NotifyEvent).expect("valid postgres args"); + assert_eq!(args.format, PostgresFormat::Access); + } + + #[test] + fn validate_postgres_config_rejects_missing_dsn_string() { + let mut config = postgres_base_config(); + config.0.retain(|kv| kv.key != POSTGRES_DSN_STRING); + let err = validate_postgres_config(&config, "").expect_err("missing dsn_string should fail"); + assert!(err.to_string().contains("Missing PostgreSQL dsn_string")); + } + + #[test] + fn validate_postgres_config_rejects_empty_dsn_string() { + let mut config = postgres_base_config(); + config.insert(POSTGRES_DSN_STRING.to_string(), "".to_string()); + let err = validate_postgres_config(&config, "").expect_err("empty dsn_string should fail"); + assert!(err.to_string().contains("dsn_string cannot be empty")); + } + + #[test] + fn validate_postgres_config_rejects_missing_table() { + let mut config = postgres_base_config(); + config.0.retain(|kv| kv.key != POSTGRES_TABLE); + let err = validate_postgres_config(&config, "").expect_err("missing table should fail"); + assert!(err.to_string().contains("Missing PostgreSQL table")); + } + + #[test] + fn validate_postgres_config_rejects_invalid_dsn() { + let mut config = postgres_base_config(); + config.insert(POSTGRES_DSN_STRING.to_string(), "postgres://".to_string()); + let err = validate_postgres_config(&config, "").expect_err("invalid dsn should fail"); + assert!(err.to_string().contains("invalid PostgreSQL dsn_string")); + } + + #[test] + fn validate_postgres_config_rejects_relative_queue_dir() { + let mut config = postgres_base_config(); + config.insert(POSTGRES_QUEUE_DIR.to_string(), "relative/path".to_string()); + let err = validate_postgres_config(&config, "").expect_err("relative queue_dir should fail"); + assert!(err.to_string().contains("absolute path")); + } + + #[test] + fn validate_postgres_config_rejects_mtls_without_key() { + let mut config = postgres_base_config(); + config.insert(POSTGRES_TLS_CLIENT_CERT.to_string(), "/etc/ssl/cert.pem".to_string()); + let err = validate_postgres_config(&config, "").expect_err("missing key should fail"); + assert!(err.to_string().contains("must be specified together")); + } + + #[test] + fn validate_postgres_config_rejects_relative_tls_ca() { + let mut config = postgres_base_config(); + config.insert(POSTGRES_TLS_CA.to_string(), "relative/ca.pem".to_string()); + let err = validate_postgres_config(&config, "").expect_err("relative tls_ca should fail"); + assert!(err.to_string().contains("must be an absolute path")); + } + + #[test] + fn validate_postgres_config_rejects_relative_tls_client_cert() { + let mut config = postgres_base_config(); + config.insert(POSTGRES_TLS_CLIENT_CERT.to_string(), "relative/client.pem".to_string()); + config.insert(POSTGRES_TLS_CLIENT_KEY.to_string(), "/etc/ssl/client.key".to_string()); + let err = validate_postgres_config(&config, "").expect_err("relative tls_client_cert should fail"); + assert!(err.to_string().contains("must be an absolute path")); + } } diff --git a/crates/targets/src/lib.rs b/crates/targets/src/lib.rs index 483e9fd21..43b177f03 100644 --- a/crates/targets/src/lib.rs +++ b/crates/targets/src/lib.rs @@ -22,7 +22,7 @@ pub mod target; pub use check::{ check_kafka_broker_available, check_mqtt_broker_available, check_mqtt_broker_available_with_tls, check_nats_server_available, - check_pulsar_broker_available, + check_postgres_server_available, check_pulsar_broker_available, check_redis_server_available, }; pub use error::{StoreError, TargetError}; pub use rustfs_s3_common::EventName; diff --git a/crates/targets/src/target/kafka.rs b/crates/targets/src/target/kafka.rs index e1e14b5ea..3c3132ab8 100644 --- a/crates/targets/src/target/kafka.rs +++ b/crates/targets/src/target/kafka.rs @@ -19,7 +19,7 @@ use crate::{ store::{Key, QueueStore, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, - TargetType, + TargetType, queue_store_subdir_name, }, }; use async_trait::async_trait; @@ -136,7 +136,7 @@ where let queue_store = if !args.queue_dir.is_empty() { let queue_dir = - PathBuf::from(&args.queue_dir).join(format!("rustfs-{}-{}", ChannelTargetType::Kafka.as_str(), target_id.id)); + PathBuf::from(&args.queue_dir).join(queue_store_subdir_name(ChannelTargetType::Kafka.as_str(), &target_id.id)); let extension = match args.target_type { TargetType::AuditLog => AUDIT_STORE_EXTENSION, diff --git a/crates/targets/src/target/mod.rs b/crates/targets/src/target/mod.rs index ad1429deb..2e0ff4f95 100644 --- a/crates/targets/src/target/mod.rs +++ b/crates/targets/src/target/mod.rs @@ -14,7 +14,7 @@ use crate::arn::TargetID; use crate::store::{Key, Store}; -use crate::{StoreError, TargetError}; +use crate::{StoreError, TargetError, TargetLog}; use async_trait::async_trait; use rustfs_s3_common::EventName; use serde::de::DeserializeOwned; @@ -27,8 +27,11 @@ use tracing::warn; pub mod kafka; pub mod mqtt; +pub mod mysql; pub mod nats; +pub mod postgres; pub mod pulsar; +pub mod redis; pub mod webhook; /// A read-only snapshot of delivery counters for a target. @@ -289,8 +292,11 @@ pub enum ChannelTargetType { Webhook, Kafka, Mqtt, + MySql, Nats, + Postgres, Pulsar, + Redis, } impl ChannelTargetType { @@ -299,8 +305,11 @@ impl ChannelTargetType { ChannelTargetType::Webhook => "webhook", ChannelTargetType::Kafka => "kafka", ChannelTargetType::Mqtt => "mqtt", + ChannelTargetType::MySql => "mysql", ChannelTargetType::Nats => "nats", + ChannelTargetType::Postgres => "postgres", ChannelTargetType::Pulsar => "pulsar", + ChannelTargetType::Redis => "redis", } } } @@ -311,20 +320,15 @@ impl std::fmt::Display for ChannelTargetType { ChannelTargetType::Webhook => write!(f, "webhook"), ChannelTargetType::Kafka => write!(f, "kafka"), ChannelTargetType::Mqtt => write!(f, "mqtt"), + ChannelTargetType::MySql => write!(f, "mysql"), ChannelTargetType::Nats => write!(f, "nats"), + ChannelTargetType::Postgres => write!(f, "postgres"), ChannelTargetType::Pulsar => write!(f, "pulsar"), + ChannelTargetType::Redis => write!(f, "redis"), } } } -pub fn parse_bool(value: &str) -> Result { - match value.to_lowercase().as_str() { - "true" | "on" | "yes" | "1" => Ok(true), - "false" | "off" | "no" | "0" => Ok(false), - _ => Err(TargetError::ParseError(format!("Unable to parse boolean: {value}"))), - } -} - /// `TargetType` enum represents the type of target in the notification system. #[derive(Debug, Clone)] pub enum TargetType { @@ -350,6 +354,23 @@ impl std::fmt::Display for TargetType { } } +pub(crate) fn sanitize_queue_dir_component(component: &str) -> String { + let mut sanitized = String::with_capacity(component.len()); + for ch in component.chars() { + if ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.') { + sanitized.push(ch); + } else { + sanitized.push('_'); + } + } + + if sanitized.is_empty() { "_".to_string() } else { sanitized } +} + +pub(crate) fn queue_store_subdir_name(target_type: &str, target_id: &str) -> String { + format!("rustfs-{target_type}-{}", sanitize_queue_dir_component(target_id)) +} + /// Decodes a form-urlencoded object name to its original form. /// /// This function properly handles form-urlencoded strings where spaces are @@ -377,6 +398,31 @@ pub fn decode_object_name(encoded: &str) -> Result { .map_err(|e| TargetError::Encoding(format!("Failed to decode object key: {e}"))) } +pub(crate) fn build_queued_payload(event: &EntityTarget) -> Result +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + let object_name = decode_object_name(&event.object_name)?; + let key = format!("{}/{}", event.bucket_name, object_name); + + let log = TargetLog { + event_name: event.event_name, + key, + records: vec![event.data.clone()], + }; + + let body = serde_json::to_vec(&log).map_err(|err| TargetError::Serialization(format!("Failed to serialize event: {err}")))?; + let meta = QueuedPayloadMeta::new( + event.event_name, + event.bucket_name.clone(), + event.object_name.clone(), + "application/json", + body.len(), + ); + + Ok(QueuedPayload::new(meta, body)) +} + pub(crate) fn delete_stored_payload( store: &(dyn Store + Send + Sync), key: &Key, @@ -412,9 +458,37 @@ mod tests { assert_eq!(decoded.body, br#"{"ok":true}"#); } + #[test] + fn build_queued_payload_uses_event_data_shape() { + let event = EntityTarget { + object_name: "greeting+file+%282%29.csv".to_string(), + bucket_name: "bucket-a".to_string(), + event_name: EventName::ObjectCreatedPut, + data: "payload-data".to_string(), + }; + + let payload = build_queued_payload(&event).unwrap(); + let value: serde_json::Value = serde_json::from_slice(&payload.body).unwrap(); + + assert_eq!(value["Key"], "bucket-a/greeting file (2).csv"); + assert_eq!(value["Records"][0], "payload-data"); + } + #[test] fn queued_payload_decode_rejects_invalid_magic() { let err = QueuedPayload::decode(b"bad-payload").unwrap_err(); assert!(err.to_string().contains("magic") || err.to_string().contains("short")); } + + #[test] + fn sanitize_queue_dir_component_replaces_non_path_safe_characters() { + let sanitized = sanitize_queue_dir_component("tenant:alpha/beta\\gamma?*"); + assert_eq!(sanitized, "tenant_alpha_beta_gamma__"); + } + + #[test] + fn queue_store_subdir_name_sanitizes_target_id() { + let dir = queue_store_subdir_name("redis", "tenant:alpha"); + assert_eq!(dir, "rustfs-redis-tenant_alpha"); + } } diff --git a/crates/targets/src/target/mqtt.rs b/crates/targets/src/target/mqtt.rs index cb56ebb8d..a33611ada 100644 --- a/crates/targets/src/target/mqtt.rs +++ b/crates/targets/src/target/mqtt.rs @@ -19,7 +19,7 @@ use crate::{ store::{Key, QueueStore, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, - TargetType, + TargetType, queue_store_subdir_name, }, }; use async_trait::async_trait; @@ -465,7 +465,7 @@ impl MQTTArgs { } if !self.queue_dir.is_empty() { - let path = std::path::Path::new(&self.queue_dir); + let path = Path::new(&self.queue_dir); if !path.is_absolute() { return Err(TargetError::Configuration("mqtt queue_dir path should be absolute".to_string())); } @@ -512,7 +512,7 @@ where let target_id = TargetID::new(id, ChannelTargetType::Mqtt.as_str().to_string()); let queue_store = if !args.queue_dir.is_empty() { let base_path = PathBuf::from(&args.queue_dir); - let unique_dir_name = format!("rustfs-{}-{}", ChannelTargetType::Mqtt.as_str(), target_id.id).replace(":", "_"); + let unique_dir_name = queue_store_subdir_name(ChannelTargetType::Mqtt.as_str(), &target_id.id); // Ensure the directory name is valid for filesystem let specific_queue_path = base_path.join(unique_dir_name); debug!(target_id = %target_id, path = %specific_queue_path.display(), "Initializing queue store for MQTT target"); @@ -582,7 +582,7 @@ where Some(MAX_MQTT_PACKET_SIZE_BYTES), )?; - let (new_client, eventloop) = AsyncClient::new(mqtt_options, 10); + let (new_client, eventloop) = AsyncClient::builder(mqtt_options).capacity(10).build(); if let Err(e) = new_client.subscribe(&args_clone.topic, args_clone.qos).await { error!(target_id = %target_id_clone, error = %e, "Failed to subscribe to MQTT topic during init"); @@ -828,7 +828,7 @@ fn is_fatal_mqtt_error(err: &ConnectionError) -> bool { match state_err { // If StateError is caused by deserialization issues, check the underlying MqttBytesError rumqttc::StateError::Deserialization(mqtt_bytes_err) => { // The type of mqtt_bytes_err is &rumqttc::mqttbytes::Error - matches!( + matches!( mqtt_bytes_err, MqttBytesError::InvalidProtocol // Invalid agreement | MqttBytesError::InvalidProtocolLevel(_) // Invalid protocol level @@ -922,7 +922,7 @@ where Err(e) => { error!(target_id = %self.id, error = %e, "Failed to save event to store"); self.delivery_counters.record_final_failure(); - return Err(TargetError::Storage(format!("Failed to save event to store: {e}"))); + Err(TargetError::Storage(format!("Failed to save event to store: {e}"))) } } } else { diff --git a/crates/targets/src/target/mysql.rs b/crates/targets/src/target/mysql.rs new file mode 100644 index 000000000..12f57683a --- /dev/null +++ b/crates/targets/src/target/mysql.rs @@ -0,0 +1,1226 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + StoreError, Target, + arn::TargetID, + error::TargetError, + store::{Key, QueueStore, Store}, + target::{ + ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, + TargetType, build_queued_payload, delete_stored_payload, queue_store_subdir_name, + }, +}; +use async_trait::async_trait; +use mysql_async::{Conn, Opts, OptsBuilder, Pool, PoolConstraints, PoolOpts, SslOpts, prelude::Queryable}; +use rustfs_config::notify::NOTIFY_STORE_EXTENSION; +use rustfs_config::{MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY}; +use serde::Serialize; +use serde::de::DeserializeOwned; +use std::marker::PhantomData; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use tokio::sync::Mutex; +use tracing::{debug, error, info, warn}; + +/// Arguments for configuring a MySQL notification target. +/// +/// Contains all configuration values needed to connect to a MySQL/TiDB +/// database and write event notification records. +#[derive(Debug, Clone)] +pub struct MySqlArgs { + /// Whether the target is enabled + pub enable: bool, + /// MySQL data source name in format: `:@tcp(:)/` + pub dsn_string: String, + /// Target table name, accepts `identifier` or `database.identifier` + pub table: String, + /// Write format (currently only `access` is supported) + pub format: String, + /// Optional custom CA certificate file for TLS server verification + pub tls_ca: String, + /// Optional client certificate chain file for mutual TLS + pub tls_client_cert: String, + /// Optional client private key file for mutual TLS + pub tls_client_key: String, + /// Directory for persistent queue storage; must be an absolute path if non-empty + pub queue_dir: String, + /// Maximum number of events stored in the local queue + pub queue_limit: u64, + /// Maximum number of open MySQL connections in the pool (0 relies on the underlying library default) + pub max_open_connections: usize, + /// The target type (notify or audit) + pub target_type: TargetType, +} + +impl MySqlArgs { + /// Validates the MySQL target configuration. + pub fn validate(&self) -> Result<(), TargetError> { + // If the target is disabled, validation is skipped. + if !self.enable { + return Ok(()); + } + + if self.dsn_string.trim().is_empty() { + return Err(TargetError::Configuration("MySQL dsn_string cannot be empty".to_string())); + } + + let _ = MySqlDsn::parse(&self.dsn_string)?; + + validate_table_name(&self.table)?; + + if self.format != "access" { + return Err(TargetError::Configuration(format!( + "MySQL format '{}' is not supported; only 'access' is available", + self.format + ))); + } + + if self.tls_client_cert.is_empty() != self.tls_client_key.is_empty() { + return Err(TargetError::Configuration(format!( + "MySQL {MYSQL_TLS_CLIENT_CERT} and {MYSQL_TLS_CLIENT_KEY} must be specified together" + ))); + } + if !self.tls_ca.is_empty() && !Path::new(&self.tls_ca).is_absolute() { + return Err(TargetError::Configuration(format!("{MYSQL_TLS_CA} must be an absolute path"))); + } + if !self.tls_client_cert.is_empty() && !Path::new(&self.tls_client_cert).is_absolute() { + return Err(TargetError::Configuration(format!("{MYSQL_TLS_CLIENT_CERT} must be an absolute path"))); + } + if !self.tls_client_key.is_empty() && !Path::new(&self.tls_client_key).is_absolute() { + return Err(TargetError::Configuration(format!("{MYSQL_TLS_CLIENT_KEY} must be an absolute path"))); + } + + if !self.queue_dir.is_empty() { + let path = Path::new(&self.queue_dir); + if !path.is_absolute() { + return Err(TargetError::Configuration("MySQL queue_dir must be an absolute path".to_string())); + } + } + + Ok(()) + } +} + +/// Parsed representation of a MySQL DSN string. +/// +/// Produced by [`MySqlDsn::parse`] and consumed by the MySQL +/// target runtime to build connection options. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MySqlDsn { + /// MySQL user name + pub user: String, + /// MySQL password (plaintext, must be redacted before logging) + pub password: String, + /// MySQL server hostname or IP address + pub host: String, + /// MySQL server TCP port + pub port: u16, + /// Target database name + pub database: String, + /// Whether TLS is enabled + pub tls: bool, +} + +impl MySqlDsn { + /// Parses a MySQL DSN string into its components. + /// + /// Supported formats: + /// ```text + /// :@tcp(:)/ + /// mysql://:@tcp(:)/ + /// ``` + /// + /// Only `?tls=true`, `?tls=false`, and bare `?tls` are accepted; + /// other TLS query parameters (`verify_ca`, etc.) are rejected. + pub fn parse(dsn_string: &str) -> Result { + let input = dsn_string.trim(); + if input.is_empty() { + return Err(TargetError::Configuration("MySQL dsn_string cannot be empty".to_string())); + } + + let remainder = input + .strip_prefix("mysql://") + .or_else(|| input.strip_prefix("MYSQL://")) + .unwrap_or(input); + + let (body, query) = match remainder.split_once('?') { + Some((b, q)) => (b, Some(q)), + None => (remainder, None), + }; + + let mut tls = false; + if let Some(query) = query { + for param in query.split('&') { + let param = param.trim(); + if param.is_empty() { + continue; + } + let (key, value) = param.split_once('=').unwrap_or((param, "")); + match key.trim().to_ascii_lowercase().as_str() { + "tls" => { + let val = value.trim().to_ascii_lowercase(); + if val == "true" || val.is_empty() { + tls = true; + } else if val == "false" { + tls = false; + } else { + return Err(TargetError::Configuration(format!( + "unsupported value '{}' for TLS query parameter; use tls=true", + val + ))); + } + } + _ => { + return Err(TargetError::Configuration(format!("unsupported MySQL DSN query parameter '{}'", key))); + } + } + } + } + + let Some((credentials, host_part)) = body.split_once('@') else { + return Err(TargetError::Configuration( + "MySQL dsn_string must contain user:password@tcp(host:port)/database".to_string(), + )); + }; + + let Some((user, password)) = credentials.split_once(':') else { + return Err(TargetError::Configuration("MySQL dsn_string must contain user:password".to_string())); + }; + + let user = user.trim(); + let password = password.trim(); + + if user.is_empty() { + return Err(TargetError::Configuration("MySQL dsn_string user is empty".to_string())); + } + + let host_part = host_part.trim(); + + let Some(host_part_rest) = host_part.strip_prefix("tcp(") else { + return Err(TargetError::Configuration("MySQL dsn_string must use tcp(host:port) format".to_string())); + }; + + let Some((host_port, rest)) = host_part_rest.split_once(')') else { + return Err(TargetError::Configuration( + "MySQL dsn_string missing closing ')' after host:port".to_string(), + )); + }; + + let (host, port_str) = host_port + .split_once(':') + .ok_or_else(|| TargetError::Configuration("MySQL dsn_string host:port is required".to_string()))?; + + let host = host.trim(); + let port_str = port_str.trim(); + + if host.is_empty() { + return Err(TargetError::Configuration("MySQL dsn_string host is empty".to_string())); + } + + let port: u16 = port_str + .parse() + .map_err(|_| TargetError::Configuration(format!("MySQL dsn_string port '{}' is not a valid u16", port_str)))?; + + let database = rest + .strip_prefix('/') + .ok_or_else(|| TargetError::Configuration("MySQL dsn_string must include /database after host:port".to_string()))? + .trim(); + + if database.is_empty() { + return Err(TargetError::Configuration("MySQL dsn_string database is empty".to_string())); + } + + Ok(MySqlDsn { + user: user.to_string(), + password: password.to_string(), + host: host.to_string(), + port, + database: database.to_string(), + tls, + }) + } +} + +/// Returns a redacted version of the DSN string with the password replaced by `***`. +pub(crate) fn redact_mysql_dsn(dsn_string: &str) -> String { + let input = dsn_string.trim(); + if input.is_empty() { + return String::new(); + } + + let remainder = input + .strip_prefix("mysql://") + .or_else(|| input.strip_prefix("MYSQL://")) + .unwrap_or(input); + + let prefix = if input.starts_with("mysql://") { + "mysql://" + } else if input.starts_with("MYSQL://") { + "MYSQL://" + } else { + "" + }; + + match remainder.split_once('@') { + Some((credentials, host_part)) => match credentials.split_once(':') { + Some((user, _)) => format!("{}{}:***@{}", prefix, user.trim(), host_part.trim()), + None => format!("{prefix}***@{host_part}"), + }, + None => format!("{prefix}***"), + } +} + +fn is_valid_identifier_segment(segment: &str) -> bool { + if segment.is_empty() { + return false; + } + + let mut chars = segment.chars(); + let Some(first) = chars.next() else { + return false; + }; + if !first.is_ascii_alphabetic() && first != '_' { + return false; + } + + for ch in chars { + if !ch.is_ascii_alphanumeric() && ch != '_' { + return false; + } + } + + true +} + +fn ensure_rustls_provider_installed() { + if rustls::crypto::CryptoProvider::get_default().is_some() { + return; + } + + if let Err(err) = rustls::crypto::aws_lc_rs::default_provider().install_default() { + debug!("rustls provider already installed or unavailable for mysql target: {err:?}"); + } +} + +pub(crate) fn validate_table_name(table: &str) -> Result<(), TargetError> { + let table = table.trim(); + + if table.is_empty() { + return Err(TargetError::Configuration("MySQL table name is empty".to_string())); + } + + if table.contains('.') { + let parts: Vec<&str> = table.splitn(2, '.').collect(); + if parts.len() != 2 || parts[0].is_empty() || parts[1].is_empty() { + return Err(TargetError::Configuration(format!( + "MySQL table name '{}' is invalid; use identifier or database.identifier", + table + ))); + } + + if !is_valid_identifier_segment(parts[0]) { + return Err(TargetError::Configuration(format!( + "MySQL database name '{}' in '{}' is not a valid identifier", + parts[0], table + ))); + } + + if !is_valid_identifier_segment(parts[1]) { + return Err(TargetError::Configuration(format!( + "MySQL table name '{}' in '{}' is not a valid identifier", + parts[1], table + ))); + } + } else if !is_valid_identifier_segment(table) { + return Err(TargetError::Configuration(format!( + "MySQL table name '{}' is not a valid identifier", + table + ))); + } + + Ok(()) +} + +pub(crate) fn quote_table_name(table: &str) -> Result { + let table = table.trim(); + + if table.contains('.') { + let parts: Vec<&str> = table.splitn(2, '.').collect(); + Ok(format!("`{}`.`{}`", parts[0].trim(), parts[1].trim())) + } else { + Ok(format!("`{}`", table)) + } +} + +/// Extracts `event_time` from a serialized event JSON body. +/// +/// Reads `Records[0].eventTime` from the JSON payload, parses it as an +/// RFC 3339 timestamp, and returns it formatted as a MySQL DATETIME(6) +/// string (`YYYY-MM-DD HH:MM:SS.ffffff`). +/// +/// Returns an error if the field is missing, not a string, or cannot +/// be parsed; never falls back to the current time. +pub(crate) fn extract_event_time(body: &[u8]) -> Result { + let value: serde_json::Value = + serde_json::from_slice(body).map_err(|e| TargetError::Serialization(format!("Failed to parse event_data JSON: {e}")))?; + + let event_time = value + .get("Records") + .and_then(|r| r.get(0)) + .and_then(|r| r.get("eventTime")) + .and_then(|v| v.as_str()) + .ok_or_else(|| TargetError::Serialization("event_data is missing Records[0].eventTime".to_string()))?; + + let dt = chrono::DateTime::parse_from_rfc3339(event_time) + .map_err(|e| TargetError::Serialization(format!("Failed to parse eventTime '{}': {}", event_time, e)))?; + + Ok(dt.format("%Y-%m-%d %H:%M:%S%.6f").to_string()) +} + +async fn validate_existing_schema(conn: &mut Conn, table: &str) -> Result<(), TargetError> { + let quoted = quote_table_name(table)?; + let sql = format!("SHOW COLUMNS FROM {quoted}"); + + let columns: Vec = conn + .query(sql) + .await + .map_err(|e| TargetError::Initialization(format!("Failed to check MySQL table schema: {e}")))?; + + let mut has_event_time = false; + let mut has_event_data = false; + + for row in &columns { + let field: String = row.get(0).unwrap_or_default(); + let col_type: String = row.get(1).unwrap_or_default(); + let nullable: String = row.get(2).unwrap_or_default(); + + if field == "event_time" { + has_event_time = true; + if col_type.to_lowercase() != "datetime(6)" { + return Err(TargetError::Initialization( + "MySQL table column 'event_time' must be DATETIME(6) to match insert precision".to_string(), + )); + } + if nullable.to_lowercase() != "no" { + return Err(TargetError::Initialization( + "MySQL table column 'event_time' must be NOT NULL".to_string(), + )); + } + } else if field == "event_data" { + has_event_data = true; + if col_type.to_lowercase() != "json" { + return Err(TargetError::Initialization( + "MySQL table column 'event_data' must be JSON type".to_string(), + )); + } + if nullable.to_lowercase() != "no" { + return Err(TargetError::Initialization( + "MySQL table column 'event_data' must be NOT NULL".to_string(), + )); + } + } + } + + if !has_event_time { + return Err(TargetError::Initialization( + "MySQL table is missing required column 'event_time'".to_string(), + )); + } + if !has_event_data { + return Err(TargetError::Initialization( + "MySQL table is missing required column 'event_data'".to_string(), + )); + } + + Ok(()) +} + +/// A notification target that writes events to a MySQL/TiDB table. +/// +/// Each event is appended as a new row with `event_time` and `event_data` +/// columns. The target supports at-least-once delivery semantics via a +/// local `QueueStore` that replays events after transient MySQL outages. +/// +/// # Configuration example using `rc` +/// +/// ```bash +/// rc admin config set ALIAS notify_mysql:primary \ +/// enable=on \ +/// dsn_string="rustfs:password@tcp(mysql.example.com:3306)/rustfs_events?tls=true" \ +/// table="rustfs_events" \ +/// tls_ca="/etc/ssl/mysql/ca.pem" \ +/// tls_client_cert="/etc/ssl/mysql/client.pem" \ +/// tls_client_key="/etc/ssl/mysql/client.key" \ +/// queue_dir="/var/lib/rustfs/events" \ +/// queue_limit="100000" \ +/// max_open_connections="2" +/// ``` +/// +/// # Environment variables +/// +/// ```bash +/// RUSTFS_NOTIFY_MYSQL_ENABLE=on +/// RUSTFS_NOTIFY_MYSQL_DSN_STRING=rustfs:password@tcp(127.0.0.1:3306)/rustfs_events +/// RUSTFS_NOTIFY_MYSQL_TABLE=rustfs_events +/// RUSTFS_NOTIFY_MYSQL_TLS_CA=/etc/ssl/mysql/ca.pem +/// RUSTFS_NOTIFY_MYSQL_TLS_CLIENT_CERT=/etc/ssl/mysql/client.pem +/// RUSTFS_NOTIFY_MYSQL_TLS_CLIENT_KEY=/etc/ssl/mysql/client.key +/// RUSTFS_NOTIFY_MYSQL_QUEUE_DIR=/opt/rustfs/events +/// RUSTFS_NOTIFY_MYSQL_QUEUE_LIMIT=100000 +/// RUSTFS_NOTIFY_MYSQL_MAX_OPEN_CONNECTIONS=2 +/// ``` +pub struct MySqlTarget +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + /// Unique target identifier (name + type) + id: TargetID, + /// Parsed configuration for this MySQL target + args: MySqlArgs, + /// Optional persistent queue store for at-least-once delivery + store: Option + Send + Sync>>, + /// Lazily-initialized MySQL connection pool + pool: Arc>>, + /// Success/failure counters exposed via `delivery_snapshot` + delivery_counters: Arc, + /// Zero-sized marker for the event type `E` + _phantom: PhantomData, +} + +impl MySqlTarget +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + /// Creates a new MySqlTarget. + pub fn new(id: String, args: MySqlArgs) -> Result { + args.validate()?; + + let target_id = TargetID::new(id, ChannelTargetType::MySql.as_str().to_string()); + + // If `queue_dir` is non-empty, a `QueueStore` is created for persistent at-least-once delivery. + let queue_store = if !args.queue_dir.is_empty() { + let queue_dir = + PathBuf::from(&args.queue_dir).join(queue_store_subdir_name(ChannelTargetType::MySql.as_str(), &target_id.id)); + + let extension = match args.target_type { + TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION, + TargetType::NotifyEvent => NOTIFY_STORE_EXTENSION, + }; + + let store = QueueStore::::new(queue_dir, args.queue_limit, extension); + if let Err(e) = store.open() { + return Err(TargetError::Storage(format!("Failed to open MySQL queue store: {e}"))); + } + + Some(Box::new(store) as Box + Send + Sync>) + } else { + None + }; + + info!(target_id = %target_id.id, table = %args.table, "MySQL target created"); + + Ok(MySqlTarget { + id: target_id, + args, + store: queue_store, + // Pool is lazily initialized on first use to avoid unnecessary connections at startup and allow for better error handling + pool: Arc::new(Mutex::new(None)), + delivery_counters: Arc::new(TargetDeliveryCounters::default()), + _phantom: PhantomData, + }) + } + + /// Returns or lazily initializes the MySQL connection pool. + /// + /// # Errors + /// + /// | Scenario | Error variant | + /// |---|---| + /// | Connection refused / host unreachable / TLS handshake failed | `NotConnected` | + /// | `SELECT 1` health check failed | `NotConnected` | + /// | DDL permission denied / `CREATE TABLE` failed | `Initialization` | + /// | Existing table has incompatible schema | `Initialization` | + /// | DSN parse failure / invalid config | `Configuration` | + async fn get_or_init_pool(&self) -> Result { + { + let guard = self.pool.lock().await; + if let Some(pool) = guard.as_ref() { + return Ok(pool.clone()); + } + } + + let dsn = MySqlDsn::parse(&self.args.dsn_string)?; + + let mut builder = OptsBuilder::default() + .user(Some(dsn.user.clone())) + .pass(Some(dsn.password.clone())) + .ip_or_hostname(dsn.host.clone()) + .tcp_port(dsn.port) + .db_name(Some(dsn.database.clone())); + + if dsn.tls { + ensure_rustls_provider_installed(); + let mut ssl_opts = SslOpts::default(); + if !self.args.tls_ca.is_empty() { + ssl_opts = ssl_opts.with_root_certs(vec![PathBuf::from(self.args.tls_ca.clone()).into()]); + } + if !self.args.tls_client_cert.is_empty() && !self.args.tls_client_key.is_empty() { + let identity = mysql_async::ClientIdentity::new( + PathBuf::from(self.args.tls_client_cert.clone()).into(), + PathBuf::from(self.args.tls_client_key.clone()).into(), + ); + ssl_opts = ssl_opts.with_client_identity(Some(identity)); + } + builder = builder.ssl_opts(Some(ssl_opts)); + } else { + warn!( + "MySQL target '{}' is configured without TLS. This is insecure and should not be used in production.", + self.id + ); + } + + // When max_open_connections is 0, no explicit upper bound is set — + // mysql_async uses its default pool constraints (10–100). + if self.args.max_open_connections > 0 { + let constraints = PoolConstraints::new(1, self.args.max_open_connections).ok_or_else(|| { + TargetError::Configuration(format!( + "MySQL max_open_connections must be >= 1, got {}", + self.args.max_open_connections + )) + })?; + builder = builder.pool_opts(PoolOpts::default().with_constraints(constraints)); + } + + let opts = Opts::from(builder); + let pool = Pool::new(opts); + + // Uses a double-check pattern: the mutex guard is only held for + // short reads/writes to the pool cache. All I/O (connecting, + // DDL, schema validation) happens outside the lock so that + // concurrent callers are not blocked by a slow MySQL server. + let mut conn = pool.get_conn().await.map_err(|_| TargetError::NotConnected)?; + + conn.query_drop("SELECT 1").await.map_err(|_| TargetError::NotConnected)?; + + let ddl = format!( + "CREATE TABLE IF NOT EXISTS {} (event_time DATETIME(6) NOT NULL, event_data JSON NOT NULL)", + quote_table_name(&self.args.table)? + ); + conn.query_drop(ddl) + .await + .map_err(|e| TargetError::Initialization(format!("Failed to create MySQL table: {e}")))?; + + validate_existing_schema(&mut conn, &self.args.table).await?; + + // Double-check: another caller may have initialized the pool + // while we were doing I/O. + let mut guard = self.pool.lock().await; + if let Some(existing) = guard.as_ref() { + debug!( + "MySQL pool for target '{}' was initialized by another task during setup; using existing pool", + self.id + ); + return Ok(existing.clone()); + } + *guard = Some(pool.clone()); + Ok(pool) + } + + /// Inserts an event directly into the MySQL table. + async fn insert_event(&self, body: &[u8], meta: &QueuedPayloadMeta) -> Result<(), TargetError> { + debug!( + target_id = %self.id, + bucket = %meta.bucket_name, + object = %meta.object_name, + event = %meta.event_name, + payload_len = body.len(), + "Inserting MySQL event" + ); + + let pool = self.get_or_init_pool().await?; + // At this point the pool has already been initialized (get_or_init_pool + // succeeded above), so get_conn() failures are always transient: the + // connection was lost or the pool is temporarily exhausted. + let mut conn = pool.get_conn().await.map_err(|_| TargetError::NotConnected)?; + + let event_time = extract_event_time(body)?; + let event_data = + std::str::from_utf8(body).map_err(|e| TargetError::Serialization(format!("Event body is not valid UTF-8: {e}")))?; + + let sql = format!( + "INSERT INTO {} (event_time, event_data) VALUES (?, CAST(? AS JSON))", + quote_table_name(&self.args.table)? + ); + + conn.exec_drop(sql, (event_time.as_str(), event_data)) + .await + .map_err(map_mysql_error)?; + + self.delivery_counters.record_success(); + debug!(target_id = %self.id, "MySQL event inserted"); + Ok(()) + } + + fn clone_box(&self) -> Box + Send + Sync> { + Box::new(MySqlTarget:: { + id: self.id.clone(), + args: self.args.clone(), + store: self.store.as_ref().map(|s| s.boxed_clone()), + pool: Arc::clone(&self.pool), + delivery_counters: Arc::clone(&self.delivery_counters), + _phantom: PhantomData, + }) + } +} + +/// Maps a mysql_async error to `TargetError`: +/// - `Io`/`Driver` → `NotConnected` (connection lost, fixed-delay retry) +/// - `Server(1213|1205|1040)` → `Timeout` (deadlock/lock timeout/too +/// many connections, exponential-backoff retry) +/// - everything else → `Request` (permanent failure) +fn map_mysql_error(err: mysql_async::Error) -> TargetError { + match &err { + mysql_async::Error::Io(_) | mysql_async::Error::Driver(_) => TargetError::NotConnected, + mysql_async::Error::Server(server_err) => match server_err.code { + 1213 | 1205 | 1040 => { + TargetError::Timeout(format!("MySQL transient server error {}: {}", server_err.code, server_err.message)) + } + _ => TargetError::Request(format!("Failed to insert event: {err}")), + }, + _ => TargetError::Request(format!("Failed to insert event: {err}")), + } +} + +#[async_trait] +impl Target for MySqlTarget +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + fn id(&self) -> TargetID { + self.id.clone() + } + + async fn is_active(&self) -> Result { + if !self.args.enable { + return Ok(false); + } + + let pool = self.get_or_init_pool().await?; + + let health_result = tokio::time::timeout(tokio::time::Duration::from_secs(10), async { + let mut conn = pool.get_conn().await?; + conn.query_drop("SELECT 1").await + }) + .await; + + match health_result { + Ok(Ok(())) => { + debug!("MySQL target '{}' is reachable", self.id); + Ok(true) + } + // get_or_init_pool has already verified connectivity, DDL, and + // schema, so a SELECT 1 failure here is always transient + // (connection lost). No need to classify error codes. + Ok(Err(_)) => Err(TargetError::NotConnected), + Err(_elapsed) => Err(TargetError::Timeout("MySQL is_active health check timed out after 10s".to_string())), + } + } + + async fn save(&self, event: Arc>) -> Result<(), TargetError> { + let queued = match build_queued_payload(event.as_ref()) { + Ok(queued) => queued, + Err(err) => { + self.delivery_counters.record_final_failure(); + return Err(err); + } + }; + + if let Some(store) = &self.store { + // persist the event to a local queue before attempting to insert into MySQL. This will allow us to guarantee at-least-once delivery even if the database is temporarily unreachable or if the process crashes after acknowledging receipt but before writing to the database. + let encoded = match queued.encode() { + Ok(encoded) => encoded, + Err(err) => { + self.delivery_counters.record_final_failure(); + return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}"))); + } + }; + + if let Err(e) = store.put_raw(&encoded) { + self.delivery_counters.record_final_failure(); + return Err(TargetError::Storage(format!("Failed to save event to store: {e}"))); + } + + debug!("Event saved to queue store for MySQL target: {}", self.id); + Ok(()) + } else { + if let Err(err) = self.insert_event(&queued.body, &queued.meta).await { + self.delivery_counters.record_final_failure(); + return Err(err); + } + + Ok(()) + } + } + + async fn send_raw_from_store(&self, key: Key, body: Vec, meta: QueuedPayloadMeta) -> Result<(), TargetError> { + debug!(target_id = %self.id, key = %key, payload_len = body.len(), "Sending queued payload from store to MySQL target"); + + match extract_event_time(&body) { + Ok(_) => {} + Err(_) => { + // If the payload is missing the required eventTime field or it + // cannot be parsed, we consider it corrupted and drop it to + // avoid blocking the queue with undeliverable entries. + error!( + target_id = %self.id, + key = %key, + "Corrupted queued MySQL payload: missing or invalid Records[0].eventTime; dropping entry" + ); + + // attempt to delete the corrupted entry from the store if possible + if let Some(store) = &self.store + && let Err(e) = delete_stored_payload(store.as_ref(), &key) + { + error!(target_id = %self.id, key=%key, error = %e, "Failed to delete corrupted queue entry"); + } + + self.delivery_counters.record_final_failure(); + return Err(TargetError::Dropped(format!( + "Dropped corrupted queued MySQL payload {key}: missing or invalid Records[0].eventTime" + ))); + } + } + + if let Err(e) = self.insert_event(&body, &meta).await { + if matches!(e, TargetError::NotConnected) { + warn!(target_id = %self.id, "MySQL not reachable, event remains in queue store"); + return Err(TargetError::NotConnected); + } + if matches!(e, TargetError::Timeout(_)) { + warn!(target_id = %self.id, "MySQL timeout, event remains in queue store"); + return Err(e); + } + error!(target_id = %self.id, error = %e, "Failed to send event from store"); + return Err(e); + } + + debug!(target_id = %self.id, key = %key, "MySQL event replayed from store"); + Ok(()) + } + + async fn close(&self) -> Result<(), TargetError> { + let pool = { + let mut guard = self.pool.lock().await; + guard.take() + }; + + if let Some(pool) = pool { + pool.disconnect() + .await + .map_err(|err| TargetError::Network(format!("Failed to disconnect MySQL pool: {err}")))?; + } + + info!("MySQL target closed: {}", self.id); + Ok(()) + } + + fn store(&self) -> Option<&(dyn Store + Send + Sync)> { + self.store.as_deref() + } + + fn clone_dyn(&self) -> Box + Send + Sync> { + self.clone_box() + } + + async fn init(&self) -> Result<(), TargetError> { + if !self.args.enable { + debug!("MySQL target '{}' is disabled, skipping initialization", self.id); + return Ok(()); + } + self.get_or_init_pool().await?; + Ok(()) + } + + fn is_enabled(&self) -> bool { + self.args.enable + } + + fn delivery_snapshot(&self) -> TargetDeliverySnapshot { + self.delivery_counters + .snapshot(self.store.as_deref().map_or(0, |store| store.len() as u64)) + } + + fn record_final_failure(&self) { + self.delivery_counters.record_final_failure(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_dsn_format() { + let dsn = MySqlDsn::parse("rustfs:secret123@tcp(mysql.example.com:3306)/rustfs_events").expect("valid DSN"); + assert_eq!(dsn.user, "rustfs"); + assert_eq!(dsn.password, "secret123"); + assert_eq!(dsn.host, "mysql.example.com"); + assert_eq!(dsn.port, 3306); + assert_eq!(dsn.database, "rustfs_events"); + assert!(!dsn.tls); + } + + #[test] + fn parse_dsn_with_mysql_prefix() { + let dsn = MySqlDsn::parse("mysql://rustfs:password@tcp(127.0.0.1:3306)/mydb").expect("valid DSN with prefix"); + assert_eq!(dsn.user, "rustfs"); + assert_eq!(dsn.password, "password"); + assert_eq!(dsn.host, "127.0.0.1"); + assert_eq!(dsn.port, 3306); + assert_eq!(dsn.database, "mydb"); + } + + #[test] + fn parse_dsn_with_tls_true() { + let dsn = MySqlDsn::parse("rustfs:password@tcp(127.0.0.1:3306)/mydb?tls=true").expect("valid DSN with TLS"); + assert!(dsn.tls); + } + + #[test] + fn parse_dsn_with_tls_bare() { + let dsn = MySqlDsn::parse("rustfs:password@tcp(127.0.0.1:3306)/mydb?tls").expect("bare tls param"); + assert!(dsn.tls); + } + + #[test] + fn parse_dsn_rejects_unsupported_tls_params() { + let err = + MySqlDsn::parse("rustfs:password@tcp(127.0.0.1:3306)/mydb?verify_ca=true").expect_err("verify_ca should be rejected"); + assert!(err.to_string().contains("verify_ca")); + + let err = MySqlDsn::parse("rustfs:password@tcp(127.0.0.1:3306)/mydb?verify_identity=true") + .expect_err("verify_identity should be rejected"); + assert!(err.to_string().contains("verify_identity")); + + let err = MySqlDsn::parse("rustfs:password@tcp(127.0.0.1:3306)/mydb?built_in_roots=true") + .expect_err("built_in_roots should be rejected"); + assert!(err.to_string().contains("built_in_roots")); + } + + #[test] + fn parse_dsn_rejects_empty() { + let err = MySqlDsn::parse("").expect_err("empty DSN"); + assert!(err.to_string().contains("empty")); + } + + #[test] + fn parse_dsn_rejects_missing_at() { + let err = MySqlDsn::parse("rustfs:password").expect_err("missing @"); + assert!(err.to_string().contains("must contain user:password@")); + } + + #[test] + fn parse_dsn_rejects_non_tcp() { + let err = MySqlDsn::parse("rustfs:password@unix(/tmp/mysql.sock)/mydb").expect_err("non-tcp should be rejected"); + assert!(err.to_string().contains("tcp(")); + } + + #[test] + fn redact_dsn_masks_password() { + let redacted = redact_mysql_dsn("rustfs:secret123@tcp(mysql.example.com:3306)/rustfs_events"); + assert_eq!(redacted, "rustfs:***@tcp(mysql.example.com:3306)/rustfs_events"); + } + + #[test] + fn redact_dsn_with_mysql_prefix() { + let redacted = redact_mysql_dsn("mysql://rustfs:secret123@tcp(127.0.0.1:3306)/mydb"); + assert_eq!(redacted, "mysql://rustfs:***@tcp(127.0.0.1:3306)/mydb"); + } + + #[test] + fn redact_dsn_empty_password() { + let redacted = redact_mysql_dsn("root:@tcp(127.0.0.1:4000)/testdb"); + assert_eq!(redacted, "root:***@tcp(127.0.0.1:4000)/testdb"); + } + + #[test] + fn validate_table_name_accepts_valid_identifier() { + validate_table_name("rustfs_events").expect("valid table name"); + validate_table_name("my_db.events").expect("valid db.table"); + validate_table_name("_events").expect("valid starting underscore"); + validate_table_name("table_2").expect("valid with numbers"); + } + + #[test] + fn validate_table_name_rejects_invalid() { + let err = validate_table_name("").expect_err("empty"); + assert!(err.to_string().contains("empty")); + + let err = validate_table_name("1table").expect_err("starts with digit"); + assert!(err.to_string().contains("not a valid identifier")); + + let err = validate_table_name("my-table").expect_err("contains dash"); + assert!(err.to_string().contains("not a valid identifier")); + + let err = validate_table_name(".table").expect_err("empty db part"); + assert!(err.to_string().contains("invalid")); + + let err = validate_table_name("db.").expect_err("empty table part"); + assert!(err.to_string().contains("invalid")); + } + + #[test] + fn quote_table_name_quotes_simple() { + let quoted = quote_table_name("rustfs_events").expect("valid"); + assert_eq!(quoted, "`rustfs_events`"); + } + + #[test] + fn quote_table_name_quotes_database_table() { + let quoted = quote_table_name("my_db.events").expect("valid"); + assert_eq!(quoted, "`my_db`.`events`"); + } + + #[test] + fn extract_event_time_parses_valid_rfc3339() { + let body = + br#"{"EventName":"s3:ObjectCreated:Put","Key":"bucket/obj.txt","Records":[{"eventTime":"2026-05-03T10:00:00Z"}]}"#; + let result = extract_event_time(body).expect("valid event_time"); + assert!(result.starts_with("2026-05-03 10:00:00")); + } + + #[test] + fn extract_event_time_missing_field_errors() { + let body = br#"{"EventName":"s3:ObjectCreated:Put","Key":"bucket/obj.txt","Records":[]}"#; + let err = extract_event_time(body).expect_err("missing eventTime should fail"); + assert!(err.to_string().contains("missing Records[0].eventTime")); + } + + #[test] + fn extract_event_time_non_string_errors() { + let body = br#"{"EventName":"s3:ObjectCreated:Put","Records":[{"eventTime":123}]}"#; + let err = extract_event_time(body).expect_err("non-string eventTime should fail"); + assert!(err.to_string().contains("missing Records[0].eventTime")); + } + + #[test] + fn extract_event_time_malformed_rfc3339_errors() { + let body = br#"{"Records":[{"eventTime":"not-a-date"}]}"#; + let err = extract_event_time(body).expect_err("malformed date should fail"); + assert!(err.to_string().contains("Failed to parse eventTime")); + } + + #[test] + fn extract_event_time_missing_records_errors() { + let body = br#"{"EventName":"s3:ObjectCreated:Put"}"#; + let err = extract_event_time(body).expect_err("missing Records should fail"); + assert!(err.to_string().contains("missing Records[0].eventTime")); + } + + #[test] + fn queued_payload_round_trip_preserves_event_data() { + let entity = EntityTarget { + object_name: "bucket%2Fobj.txt".to_string(), + bucket_name: "testbucket".to_string(), + event_name: rustfs_s3_common::EventName::ObjectCreatedPut, + data: serde_json::json!({"eventTime": "2026-05-03T10:00:00Z"}), + }; + + let payload = build_queued_payload(&entity).expect("build payload"); + let encoded = payload.encode().expect("encode"); + let decoded = QueuedPayload::decode(&encoded).expect("decode"); + + assert_eq!(decoded.meta.event_name, payload.meta.event_name); + assert_eq!(decoded.meta.bucket_name, "testbucket"); + assert_eq!(decoded.meta.object_name, "bucket%2Fobj.txt"); + assert_eq!(decoded.meta.content_type, "application/json"); + + let body_str = std::str::from_utf8(&decoded.body).expect("utf8 body"); + assert!(body_str.contains("\"EventName\"")); + assert!(body_str.contains("\"Key\"")); + assert!(body_str.contains("testbucket")); + assert!(body_str.contains("\"Records\"")); + assert!(body_str.contains("\"eventTime\"")); + } + + #[test] + fn send_raw_from_store_drops_corrupted_payload() { + let tmpdir = tempfile::TempDir::new().expect("temp dir"); + let queue_dir = tmpdir.path().to_str().expect("valid path").to_string(); + + let target: MySqlTarget = MySqlTarget::new( + "test-corrupted".to_string(), + MySqlArgs { + enable: false, + dsn_string: "rustfs:pass@tcp(127.0.0.1:3306)/db".to_string(), + table: "events".to_string(), + format: "access".to_string(), + tls_ca: String::new(), + tls_client_cert: String::new(), + tls_client_key: String::new(), + queue_dir, + queue_limit: 10, + max_open_connections: 2, + target_type: TargetType::NotifyEvent, + }, + ) + .expect("valid args"); + + let body = br#"{"Records":[]}"#.to_vec(); + let meta = QueuedPayloadMeta::new( + rustfs_s3_common::EventName::ObjectCreatedPut, + "testbucket".to_string(), + "obj.txt".to_string(), + "application/json", + body.len(), + ); + + let encoded = QueuedPayload::new(meta.clone(), body.clone()) + .encode() + .expect("encode queued payload"); + + let stored_key = target.store().unwrap().put_raw(&encoded).expect("put raw"); + + let rt = tokio::runtime::Runtime::new().expect("runtime"); + let result = rt.block_on(target.send_raw_from_store(stored_key.clone(), body, meta)); + + match result { + Err(TargetError::Dropped(msg)) => { + assert!(msg.contains("Dropped")); + assert!(msg.contains("eventTime")); + } + other => panic!("expected TargetError::Dropped, got {:?}", other), + } + + assert!( + target.store().unwrap().get_raw(&stored_key).is_err(), + "corrupted entry should have been deleted from store" + ); + + assert_eq!(target.delivery_snapshot().failed_messages, 1); + } + + #[test] + fn send_raw_from_store_replays_valid_payload() { + let tmpdir = tempfile::TempDir::new().expect("temp dir"); + let queue_dir = tmpdir.path().to_str().expect("valid path").to_string(); + + let target: MySqlTarget = MySqlTarget::new( + "test-valid-replay".to_string(), + MySqlArgs { + enable: false, + dsn_string: "rustfs:pass@tcp(127.0.0.1:3306)/db".to_string(), + table: "events".to_string(), + format: "access".to_string(), + tls_ca: String::new(), + tls_client_cert: String::new(), + tls_client_key: String::new(), + queue_dir, + queue_limit: 10, + max_open_connections: 2, + target_type: TargetType::NotifyEvent, + }, + ) + .expect("valid args"); + + let body = + br#"{"EventName":"s3:ObjectCreated:Put","Key":"bucket/obj.txt","Records":[{"eventTime":"2026-05-03T10:00:00Z"}]}"# + .to_vec(); + let meta = QueuedPayloadMeta::new( + rustfs_s3_common::EventName::ObjectCreatedPut, + "testbucket".to_string(), + "obj.txt".to_string(), + "application/json", + body.len(), + ); + + let encoded = QueuedPayload::new(meta.clone(), body.clone()) + .encode() + .expect("encode queued payload"); + + let stored_key = target.store().unwrap().put_raw(&encoded).expect("put raw"); + + // With enable=false and no real MySQL, the insert will fail at + // pool init. But send_raw_from_store validates event_time before + // insert, so valid payloads pass the time check. We verify the + // payload is NOT treated as corrupted. + let rt = tokio::runtime::Runtime::new().expect("runtime"); + let result = rt.block_on(target.send_raw_from_store(stored_key.clone(), body, meta)); + + assert!(!matches!(result, Err(TargetError::Dropped(_))), "valid payload should not return Dropped"); + + // Verify entry is NOT deleted on non-Dropped errors + assert!(target.store().unwrap().get_raw(&stored_key).is_ok(), "valid entry should remain in store"); + } + + #[test] + fn validate_rejects_unpaired_tls_client_fields() { + let args = MySqlArgs { + enable: true, + dsn_string: "rustfs:password@tcp(127.0.0.1:3306)/db".to_string(), + table: "events".to_string(), + format: "access".to_string(), + tls_ca: String::new(), + tls_client_cert: "/etc/ssl/mysql/client.pem".to_string(), + tls_client_key: String::new(), + queue_dir: "/tmp".to_string(), + queue_limit: 100, + max_open_connections: 2, + target_type: TargetType::NotifyEvent, + }; + + let err = args.validate().expect_err("unpaired tls client fields should fail"); + assert!(err.to_string().contains("must be specified together")); + } + + #[test] + fn validate_rejects_relative_tls_paths() { + let args = MySqlArgs { + enable: true, + dsn_string: "rustfs:password@tcp(127.0.0.1:3306)/db".to_string(), + table: "events".to_string(), + format: "access".to_string(), + tls_ca: "ca.pem".to_string(), + tls_client_cert: String::new(), + tls_client_key: String::new(), + queue_dir: "/tmp".to_string(), + queue_limit: 100, + max_open_connections: 2, + target_type: TargetType::NotifyEvent, + }; + + let err = args.validate().expect_err("relative tls_ca should fail"); + assert!(err.to_string().contains("absolute path")); + } + + #[test] + fn validate_accepts_absolute_tls_paths() { + let args = MySqlArgs { + enable: true, + dsn_string: "rustfs:password@tcp(127.0.0.1:3306)/db".to_string(), + table: "events".to_string(), + format: "access".to_string(), + tls_ca: "/etc/ssl/mysql/ca.pem".to_string(), + tls_client_cert: "/etc/ssl/mysql/client.pem".to_string(), + tls_client_key: "/etc/ssl/mysql/client.key".to_string(), + queue_dir: "/tmp".to_string(), + queue_limit: 100, + max_open_connections: 2, + target_type: TargetType::NotifyEvent, + }; + + args.validate().expect("absolute tls paths should pass"); + } +} diff --git a/crates/targets/src/target/nats.rs b/crates/targets/src/target/nats.rs index 5076ad01c..fdde8e3f4 100644 --- a/crates/targets/src/target/nats.rs +++ b/crates/targets/src/target/nats.rs @@ -19,7 +19,7 @@ use crate::{ store::{Key, QueueStore, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, - TargetType, + TargetType, queue_store_subdir_name, }, }; use async_trait::async_trait; @@ -197,7 +197,7 @@ where let target_id = TargetID::new(id, ChannelTargetType::Nats.as_str().to_string()); let queue_store = if !args.queue_dir.is_empty() { let base_path = PathBuf::from(&args.queue_dir); - let specific_queue_path = base_path.join(format!("rustfs-{}-{}", ChannelTargetType::Nats.as_str(), target_id.id)); + let specific_queue_path = base_path.join(queue_store_subdir_name(ChannelTargetType::Nats.as_str(), &target_id.id)); let extension = match args.target_type { TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION, TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION, diff --git a/crates/targets/src/target/postgres.rs b/crates/targets/src/target/postgres.rs new file mode 100644 index 000000000..fcd489095 --- /dev/null +++ b/crates/targets/src/target/postgres.rs @@ -0,0 +1,1095 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! PostgreSQL event notification target. +//! +//! Persists S3 events into a user-provided PostgreSQL table using the +//! `Target` trait. Two output formats are supported: +//! +//! - `namespace` (default): single row per object key, UPSERT on each event. +//! - `access`: append-only audit log with one row per delivered event. +//! +//! TLS is provided via `tokio-postgres-rustls` with rustls + aws-lc-rs. +//! When `tls_ca` is empty the connector loads native OS trust roots. +//! Connection pooling is delegated to `deadpool-postgres`; the pool itself +//! is `Clone`, so no `Mutex` is required around it. + +use crate::{ + StoreError, Target, + arn::TargetID, + error::TargetError, + store::{Key, QueueStore, Store}, + target::{ + ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, + TargetType, build_queued_payload, queue_store_subdir_name, + }, +}; +use async_trait::async_trait; +use deadpool_postgres::{Manager, ManagerConfig, Pool, RecyclingMethod}; +use rustfs_config::{POSTGRES_DSN_STRING, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY}; +use rustls_pki_types::pem::PemObject; +use rustls_pki_types::{CertificateDer, PrivateKeyDer}; +use serde::Serialize; +use serde::de::DeserializeOwned; +use std::fmt; +use std::io::BufReader; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use tokio_postgres::Config; +use tokio_postgres_rustls::MakeRustlsConnect; +use tracing::{debug, error, info, instrument, warn}; +use url::Url; +use uuid::Uuid; + +const TARGET_LOG_KEY_FIELD: &str = "Key"; + +/// Output format selection for the PostgreSQL target. +/// +/// - `Namespace`: single-row UPSERT per object key (MinIO `namespace` style). +/// - `Access`: append-only insert per event (audit/compliance use case). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PostgresFormat { + Namespace, + Access, +} + +impl PostgresFormat { + pub fn as_str(&self) -> &'static str { + match self { + PostgresFormat::Namespace => "namespace", + PostgresFormat::Access => "access", + } + } +} + +/// Parses the `format` configuration value. +/// +/// Accepts case-insensitive `"namespace"` or `"access"`. Defaults to +/// `Namespace` when the value is missing or empty. +pub fn parse_postgres_format(value: Option<&str>) -> Result { + let raw = value.unwrap_or("").trim(); + if raw.is_empty() { + return Ok(PostgresFormat::Namespace); + } + match raw.to_ascii_lowercase().as_str() { + "namespace" => Ok(PostgresFormat::Namespace), + "access" => Ok(PostgresFormat::Access), + other => Err(TargetError::Configuration(format!( + "PostgreSQL format must be 'namespace' or 'access', got: {other}" + ))), + } +} + +/// Parsed representation of a PostgreSQL DSN string. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PostgresDsn { + pub host: String, + pub port: u16, + pub user: String, + pub password: Option, + pub database: String, + pub schema: String, +} + +impl PostgresDsn { + /// Parses and validates PostgreSQL DSN string. + /// + /// Supports canonical URL format like: + /// `postgres://user:password@host:5432/database?search_path=public` + pub fn parse(dsn_string: &str) -> Result { + let input = dsn_string.trim(); + if input.is_empty() { + return Err(TargetError::Configuration(format!("PostgreSQL {POSTGRES_DSN_STRING} cannot be empty"))); + } + + let url = Url::parse(input).map_err(|e| TargetError::Configuration(format!("invalid PostgreSQL dsn_string: {e}")))?; + let scheme = url.scheme().to_ascii_lowercase(); + if scheme != "postgres" && scheme != "postgresql" { + return Err(TargetError::Configuration( + "invalid PostgreSQL dsn_string: URL scheme must be postgres or postgresql".to_string(), + )); + } + + if url.host_str().is_none() { + return Err(TargetError::Configuration( + "invalid PostgreSQL dsn_string: host cannot be empty".to_string(), + )); + } + + let user = url.username().trim(); + if user.is_empty() { + return Err(TargetError::Configuration( + "invalid PostgreSQL dsn_string: user cannot be empty".to_string(), + )); + } + + let host = url.host_str().unwrap_or_default().trim(); + if host.is_empty() { + return Err(TargetError::Configuration( + "invalid PostgreSQL dsn_string: host cannot be empty".to_string(), + )); + } + let port = url.port().unwrap_or(5432); + + let database = url.path().trim_start_matches('/').trim(); + if database.is_empty() { + return Err(TargetError::Configuration( + "invalid PostgreSQL dsn_string: database cannot be empty".to_string(), + )); + } + + let mut schema = "public".to_string(); + for (key, value) in url.query_pairs() { + if !key.eq_ignore_ascii_case("search_path") { + return Err(TargetError::Configuration(format!( + "invalid PostgreSQL dsn_string: unsupported query parameter '{key}'" + ))); + } + let value = value.trim(); + if value.is_empty() { + return Err(TargetError::Configuration( + "invalid PostgreSQL dsn_string: search_path cannot be empty".to_string(), + )); + } + let first_schema = value + .split(',') + .next() + .map(str::trim) + .filter(|segment| !segment.is_empty()) + .ok_or_else(|| { + TargetError::Configuration( + "invalid PostgreSQL dsn_string: search_path must contain at least one schema".to_string(), + ) + })?; + validate_pg_identifier(first_schema, "schema")?; + schema = first_schema.to_string(); + } + + Ok(PostgresDsn { + host: host.to_string(), + port, + user: user.to_string(), + password: url.password().map(ToOwned::to_owned), + database: database.to_string(), + schema, + }) + } +} + +/// Returns a redacted version of the DSN string with the password replaced by +/// `***` while preserving non-secret connection details for diagnostics. +pub(crate) fn redact_postgres_dsn(dsn_string: &str) -> String { + let input = dsn_string.trim(); + if input.is_empty() { + return String::new(); + } + + let mut url = match Url::parse(input) { + Ok(url) => url, + Err(_) => return "***".to_string(), + }; + + let scheme = url.scheme().to_ascii_lowercase(); + if scheme != "postgres" && scheme != "postgresql" { + return "***".to_string(); + } + + if url.password().is_some() { + let _ = url.set_password(Some("***")); + } + + let mut query_pairs: Vec<(String, String)> = Vec::new(); + let mut has_password_param = false; + for (key, value) in url.query_pairs() { + if key.eq_ignore_ascii_case("password") { + has_password_param = true; + query_pairs.push((key.into_owned(), "***".to_string())); + } else { + query_pairs.push((key.into_owned(), value.into_owned())); + } + } + if has_password_param { + url.set_query(None); + let mut serializer = url.query_pairs_mut(); + for (key, value) in query_pairs { + serializer.append_pair(&key, &value); + } + } + + url.to_string() +} + +/// Validates a PostgreSQL identifier (schema or table name). +/// +/// Accepts only `^[A-Za-z_][A-Za-z0-9_]*$`. Quoted identifiers, dots, and +/// special characters are intentionally rejected to keep SQL string +/// construction safe without runtime escaping. +pub fn validate_pg_identifier(name: &str, kind: &str) -> Result<(), TargetError> { + if name.is_empty() { + return Err(TargetError::Configuration(format!("PostgreSQL {kind} cannot be empty"))); + } + let mut chars = name.chars(); + let Some(first) = chars.next() else { + return Err(TargetError::Configuration(format!("PostgreSQL {kind} cannot be empty"))); + }; + if !(first.is_ascii_alphabetic() || first == '_') { + return Err(TargetError::Configuration(format!( + "PostgreSQL {kind} must start with a letter or underscore" + ))); + } + for c in chars { + if !(c.is_ascii_alphanumeric() || c == '_') { + return Err(TargetError::Configuration(format!( + "PostgreSQL {kind} must match ^[A-Za-z_][A-Za-z0-9_]*$" + ))); + } + } + Ok(()) +} + +/// PostgreSQL target configuration. +/// +/// Implements a manual `Debug` that redacts the DSN password to prevent secret +/// leakage through logging or `tracing::instrument` capture. +#[derive(Clone)] +pub struct PostgresArgs { + pub enable: bool, + + // Connection + pub dsn_string: String, + + // Schema/Table/Format + pub schema: String, + pub table: String, + pub format: PostgresFormat, + + // TLS + pub tls_required: bool, + pub tls_ca: String, + pub tls_client_cert: String, + pub tls_client_key: String, + + // Queue + pub queue_dir: String, + pub queue_limit: u64, + + pub target_type: TargetType, +} + +impl fmt::Debug for PostgresArgs { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PostgresArgs") + .field("enable", &self.enable) + .field("dsn_string", &redact_postgres_dsn(&self.dsn_string)) + .field("schema", &self.schema) + .field("table", &self.table) + .field("format", &self.format) + .field("tls_required", &self.tls_required) + .field("tls_ca", &self.tls_ca) + .field("tls_client_cert", &self.tls_client_cert) + .field( + "tls_client_key", + if self.tls_client_key.is_empty() { + &"" + } else { + &"***REDACTED***" + }, + ) + .field("queue_dir", &self.queue_dir) + .field("queue_limit", &self.queue_limit) + .field("target_type", &self.target_type) + .finish() + } +} + +impl PostgresArgs { + pub fn validate(&self) -> Result<(), TargetError> { + if !self.enable { + return Ok(()); + } + + let parsed = PostgresDsn::parse(&self.dsn_string)?; + + if self.schema.trim().is_empty() { + return Err(TargetError::Configuration("PostgreSQL schema cannot be empty".to_string())); + } + validate_pg_identifier(&self.schema, "schema")?; + if self.schema != parsed.schema { + return Err(TargetError::Configuration(format!( + "PostgreSQL schema must match DSN search_path first schema ('{}')", + parsed.schema + ))); + } + validate_pg_identifier(&self.table, "table")?; + + // TLS pair must be both empty or both set + if self.tls_client_cert.is_empty() != self.tls_client_key.is_empty() { + return Err(TargetError::Configuration(format!( + "PostgreSQL {POSTGRES_TLS_CLIENT_CERT} and {POSTGRES_TLS_CLIENT_KEY} must be specified together" + ))); + } + + // Optional TLS path values must be absolute when present + if !self.tls_ca.is_empty() && !Path::new(&self.tls_ca).is_absolute() { + return Err(TargetError::Configuration(format!("{POSTGRES_TLS_CA} must be an absolute path"))); + } + if !self.tls_client_cert.is_empty() && !Path::new(&self.tls_client_cert).is_absolute() { + return Err(TargetError::Configuration(format!("{POSTGRES_TLS_CLIENT_CERT} must be an absolute path"))); + } + if !self.tls_client_key.is_empty() && !Path::new(&self.tls_client_key).is_absolute() { + return Err(TargetError::Configuration(format!("{POSTGRES_TLS_CLIENT_KEY} must be an absolute path"))); + } + + if !self.queue_dir.is_empty() && !Path::new(&self.queue_dir).is_absolute() { + return Err(TargetError::Configuration( + "PostgreSQL queue directory must be an absolute path".to_string(), + )); + } + + Ok(()) + } +} + +/// Returns the qualified `"schema"."table"` SQL identifier for `args`. +/// +/// Both schema and table are pre-validated in `PostgresArgs::validate()` so the +/// values cannot contain quote, dot, or whitespace characters; double-quoting +/// preserves case-sensitivity for users who created their tables with quoted +/// identifiers. +pub fn qualified_table(schema: &str, table: &str) -> String { + format!(r#""{schema}"."{table}""#) +} + +/// SQL for the `namespace` format. Performs UPSERT keyed on the object key. +pub fn namespace_upsert_sql(schema: &str, table: &str) -> String { + format!( + "INSERT INTO {} (key, value) VALUES ($1, $2::jsonb) \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + qualified_table(schema, table) + ) +} + +/// SQL for the `access` format. Append-only with `event_id` as PK so that +/// store-replay scenarios silently skip duplicates while distinct events still +/// land as separate rows. +pub fn access_insert_sql(schema: &str, table: &str) -> String { + format!( + "INSERT INTO {} (event_id, event_name, key, value, queued_at_ms) \ + VALUES ($1, $2, $3, $4::jsonb, $5) \ + ON CONFLICT (event_id) DO NOTHING", + qualified_table(schema, table) + ) +} + +/// SQL used by both `init()` and the connectivity probe to verify the table +/// exists and is readable without producing rows or triggering side effects. +pub fn table_probe_sql(schema: &str, table: &str) -> String { + format!("SELECT 1 FROM {} LIMIT 0", qualified_table(schema, table)) +} + +fn ensure_rustls_provider_installed() { + if rustls::crypto::CryptoProvider::get_default().is_none() + && rustls::crypto::aws_lc_rs::default_provider().install_default().is_err() + { + debug!("rustls crypto provider was installed concurrently, skipping aws-lc-rs install"); + } +} + +/// Builds a rustls `ClientConfig` for the PostgreSQL connection. +/// +/// When `tls_ca` is empty the OS native trust store is used via +/// `rustls-native-certs` (0.8 API: `CertificateResult { certs, errors }`). +/// When `tls_client_cert` and `tls_client_key` are both set the connection +/// uses mTLS authentication; otherwise no client cert is sent. +pub fn build_tls_config(args: &PostgresArgs) -> Result { + ensure_rustls_provider_installed(); + + let mut root_store = rustls::RootCertStore::empty(); + + if args.tls_ca.is_empty() { + let result = rustls_native_certs::load_native_certs(); + if !result.errors.is_empty() { + warn!(error_count = result.errors.len(), "some native CA certs failed to load"); + } + if result.certs.is_empty() { + return Err(TargetError::Configuration( + "no native CA certs available; specify tls_ca explicitly".to_string(), + )); + } + for cert in result.certs { + // Skip individual add failures; corrupted certs in the system store + // shouldn't block the rest from loading. + let _ = root_store.add(cert); + } + } else { + let pem = std::fs::read(&args.tls_ca) + .map_err(|e| TargetError::Configuration(format!("failed to read {POSTGRES_TLS_CA}: {e}")))?; + let mut reader = BufReader::new(pem.as_slice()); + for cert in CertificateDer::pem_reader_iter(&mut reader) { + let cert = cert.map_err(|e| TargetError::Configuration(format!("invalid {POSTGRES_TLS_CA}: {e}")))?; + root_store + .add(cert) + .map_err(|e| TargetError::Configuration(format!("failed to add CA cert: {e}")))?; + } + } + + let builder = rustls::ClientConfig::builder().with_root_certificates(root_store); + + let client_config = if !args.tls_client_cert.is_empty() && !args.tls_client_key.is_empty() { + let cert_pem = std::fs::read(&args.tls_client_cert) + .map_err(|e| TargetError::Configuration(format!("failed to read {POSTGRES_TLS_CLIENT_CERT}: {e}")))?; + let key_pem = std::fs::read(&args.tls_client_key) + .map_err(|e| TargetError::Configuration(format!("failed to read {POSTGRES_TLS_CLIENT_KEY}: {e}")))?; + + let certs: Vec<_> = CertificateDer::pem_reader_iter(&mut BufReader::new(cert_pem.as_slice())) + .collect::>() + .map_err(|e| TargetError::Configuration(format!("invalid {POSTGRES_TLS_CLIENT_CERT}: {e}")))?; + + let key = PrivateKeyDer::from_pem_reader(&mut BufReader::new(key_pem.as_slice())) + .map_err(|e| TargetError::Configuration(format!("invalid {POSTGRES_TLS_CLIENT_KEY}: {e}")))?; + + builder + .with_client_auth_cert(certs, key) + .map_err(|e| TargetError::Configuration(format!("invalid mTLS pair: {e}")))? + } else { + builder.with_no_client_auth() + }; + + Ok(client_config) +} + +/// Builds the deadpool-postgres `Pool` used by the target. +/// +/// `args.tls_required` decides whether the connection is plain TCP or wrapped +/// in rustls. The pool is `Clone` and cheap to share across `clone_box`. +pub fn build_pool(args: &PostgresArgs) -> Result { + let parsed = PostgresDsn::parse(&args.dsn_string)?; + let mut pg_config = Config::new(); + pg_config + .host(&parsed.host) + .port(parsed.port) + .user(&parsed.user) + .dbname(&parsed.database) + .options(format!("-c search_path={}", parsed.schema)); + if let Some(password) = parsed.password.as_deref() + && !password.is_empty() + { + pg_config.password(password); + } + + let manager_config = ManagerConfig { + recycling_method: RecyclingMethod::Fast, + }; + + let manager = if args.tls_required { + let tls_config = build_tls_config(args)?; + let connector = MakeRustlsConnect::new(tls_config); + Manager::from_config(pg_config, connector, manager_config) + } else { + Manager::from_config(pg_config, tokio_postgres::NoTls, manager_config) + }; + + Pool::builder(manager) + .build() + .map_err(|e| TargetError::Configuration(format!("failed to build PostgreSQL pool: {e}"))) +} + +/// Maps a `tokio_postgres::Error` to the proper `TargetError` variant. +/// +/// Connection-class errors (SQLSTATE 08, closed connection, IO) become +/// `NotConnected` so the queue store retains the payload for replay. +/// Schema and constraint problems (SQLSTATE 23, 42) become `Configuration` +/// so they are surfaced to the operator without endless retry. +pub fn map_pg_error(err: &tokio_postgres::Error, context: &str) -> TargetError { + if err.is_closed() { + return TargetError::NotConnected; + } + if let Some(db_err) = err.as_db_error() { + let class = db_err.code().code().get(..2).unwrap_or(""); + return match class { + "08" => TargetError::NotConnected, + "28" => TargetError::Authentication(format!("{context}: {db_err}")), + "23" | "42" => TargetError::Configuration(format!("{context}: {db_err}")), + "40" => TargetError::Request(format!("{context}: {db_err}")), + _ => TargetError::Request(format!("{context}: {db_err}")), + }; + } + TargetError::NotConnected +} + +/// Maps a `deadpool_postgres::PoolError` to the proper `TargetError` variant. +pub fn map_pool_error(err: deadpool_postgres::PoolError, context: &str) -> TargetError { + match err { + deadpool_postgres::PoolError::Timeout(_) => TargetError::Timeout(format!("{context}: pool timeout")), + deadpool_postgres::PoolError::Backend(pg_err) => map_pg_error(&pg_err, context), + deadpool_postgres::PoolError::Closed => TargetError::NotConnected, + other => TargetError::Request(format!("{context}: {other}")), + } +} + +fn resolve_payload_key(payload: &serde_json::Value, meta: &QueuedPayloadMeta) -> String { + payload + .get(TARGET_LOG_KEY_FIELD) + .and_then(serde_json::Value::as_str) + .map(ToOwned::to_owned) + .unwrap_or_else(|| { + let decoded_object = + crate::target::decode_object_name(&meta.object_name).unwrap_or_else(|_| meta.object_name.clone()); + format!("{}/{}", meta.bucket_name, decoded_object) + }) +} + +/// PostgreSQL notification target. +/// +/// Holds a cloneable `deadpool_postgres::Pool` rather than a `Mutex>` +/// so that `clone_box` does not duplicate connection state. The optional +/// `QueueStore` provides at-least-once delivery semantics consistent with the +/// other built-in targets. +pub struct PostgresTarget +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + id: TargetID, + args: PostgresArgs, + pool: Pool, + namespace_sql: String, + access_sql: String, + store: Option + Send + Sync>>, + delivery_counters: Arc, + _phantom: std::marker::PhantomData, +} + +impl PostgresTarget +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + pub fn clone_box(&self) -> Box + Send + Sync> { + Box::new(PostgresTarget:: { + id: self.id.clone(), + args: self.args.clone(), + pool: self.pool.clone(), + namespace_sql: self.namespace_sql.clone(), + access_sql: self.access_sql.clone(), + store: self.store.as_ref().map(|s| s.boxed_clone()), + delivery_counters: Arc::clone(&self.delivery_counters), + _phantom: std::marker::PhantomData, + }) + } + + #[instrument(skip(args), fields(target_id_as_string = %id))] + pub fn new(id: String, args: PostgresArgs) -> Result { + args.validate()?; + let target_id = TargetID::new(id, ChannelTargetType::Postgres.as_str().to_string()); + let pool = build_pool(&args)?; + + let queue_store = if !args.queue_dir.is_empty() { + let base_path = PathBuf::from(&args.queue_dir); + let specific_queue_path = + base_path.join(queue_store_subdir_name(ChannelTargetType::Postgres.as_str(), &target_id.id)); + let extension = match args.target_type { + TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION, + TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION, + }; + let store = QueueStore::::new(specific_queue_path, args.queue_limit, extension); + if let Err(e) = store.open() { + error!(target_id = %target_id, error = %e, "Failed to open store for PostgreSQL target"); + return Err(TargetError::Storage(format!("{e}"))); + } + Some(Box::new(store) as Box + Send + Sync>) + } else { + None + }; + + Ok(Self { + id: target_id, + namespace_sql: namespace_upsert_sql(&args.schema, &args.table), + access_sql: access_insert_sql(&args.schema, &args.table), + args, + pool, + store: queue_store, + delivery_counters: Arc::new(TargetDeliveryCounters::default()), + _phantom: std::marker::PhantomData, + }) + } + + /// Sends a serialized event body to PostgreSQL using the configured format. + /// + /// Identifier validation has already happened in `PostgresArgs::validate()`, + /// so `qualified_table` cannot produce a malformed SQL string here. + async fn send_body(&self, body: &[u8], event_id: &str, meta: &QueuedPayloadMeta) -> Result<(), TargetError> { + let client = self + .pool + .get() + .await + .map_err(|e| map_pool_error(e, "PostgreSQL pool checkout failed"))?; + + let payload: serde_json::Value = + serde_json::from_slice(body).map_err(|e| TargetError::Serialization(format!("Failed to parse JSON payload: {e}")))?; + + let key = resolve_payload_key(&payload, meta); + + let result = match self.args.format { + PostgresFormat::Namespace => client.execute(&self.namespace_sql, &[&key, &payload]).await, + PostgresFormat::Access => { + let event_name_str = meta.event_name.to_string(); + let queued_at_ms = meta.queued_at_unix_ms as i64; + client + .execute(&self.access_sql, &[&event_id, &event_name_str, &key, &payload, &queued_at_ms]) + .await + } + }; + + match result { + Ok(_) => { + self.delivery_counters.record_success(); + Ok(()) + } + Err(err) => Err(map_pg_error(&err, "PostgreSQL insert failed")), + } + } + + /// Probes the table from `init()`. Failure is non-fatal when a queue is + /// configured: events buffer in the store until the schema is fixed. + async fn probe_table(&self) -> Result<(), TargetError> { + let client = self + .pool + .get() + .await + .map_err(|e| map_pool_error(e, "PostgreSQL pool checkout failed during init probe"))?; + let sql = table_probe_sql(&self.args.schema, &self.args.table); + client + .execute(sql.as_str(), &[]) + .await + .map_err(|e| map_pg_error(&e, "PostgreSQL table probe failed"))?; + Ok(()) + } +} + +#[async_trait] +impl Target for PostgresTarget +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + fn id(&self) -> TargetID { + self.id.clone() + } + + async fn is_active(&self) -> Result { + if !self.is_enabled() { + return Ok(false); + } + + match tokio::time::timeout(std::time::Duration::from_secs(10), async { + let client = self + .pool + .get() + .await + .map_err(|e| map_pool_error(e, "PostgreSQL pool checkout failed"))?; + client + .execute("SELECT 1", &[]) + .await + .map_err(|e| map_pg_error(&e, "PostgreSQL liveness probe failed"))?; + Ok::<(), TargetError>(()) + }) + .await + { + Ok(Ok(())) => Ok(true), + Ok(Err(err)) => Err(err), + Err(_) => Err(TargetError::Timeout("PostgreSQL liveness probe timed out after 10s".to_string())), + } + } + + async fn save(&self, event: Arc>) -> Result<(), TargetError> { + let queued = match build_queued_payload(event.as_ref()) { + Ok(queued) => queued, + Err(err) => { + self.delivery_counters.record_final_failure(); + return Err(err); + } + }; + + if let Some(store) = &self.store { + let encoded = match queued.encode() { + Ok(encoded) => encoded, + Err(err) => { + self.delivery_counters.record_final_failure(); + return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}"))); + } + }; + if let Err(e) = store.put_raw(&encoded) { + self.delivery_counters.record_final_failure(); + return Err(TargetError::Storage(format!("Failed to save event to store: {e}"))); + } + Ok(()) + } else { + // No queue: deliver immediately. Fresh UUID acts as the access-format + // event_id so retries from the caller produce distinct rows. + let event_id = Uuid::new_v4().to_string(); + if let Err(err) = self.send_body(&queued.body, &event_id, &queued.meta).await { + self.delivery_counters.record_final_failure(); + return Err(err); + } + Ok(()) + } + } + + async fn send_raw_from_store(&self, key: Key, body: Vec, meta: QueuedPayloadMeta) -> Result<(), TargetError> { + // Use the store key as a stable event_id so replays of the same physical + // event are idempotent under the access-format composite PK. + let event_id = key.to_string(); + self.send_body(&body, &event_id, &meta).await + } + + async fn close(&self) -> Result<(), TargetError> { + self.pool.close(); + info!(target_id = %self.id, "PostgreSQL target closed"); + Ok(()) + } + + fn store(&self) -> Option<&(dyn Store + Send + Sync)> { + self.store.as_deref() + } + + fn clone_dyn(&self) -> Box + Send + Sync> { + self.clone_box() + } + + async fn init(&self) -> Result<(), TargetError> { + if !self.is_enabled() { + return Ok(()); + } + match self.probe_table().await { + Ok(()) => Ok(()), + Err(err) if self.store.is_some() => { + warn!(target_id = %self.id, error = %err, "PostgreSQL init probe failed; events will buffer in store"); + Ok(()) + } + Err(err) => Err(err), + } + } + + fn is_enabled(&self) -> bool { + self.args.enable + } + + fn delivery_snapshot(&self) -> TargetDeliverySnapshot { + self.delivery_counters + .snapshot(self.store.as_deref().map_or(0, |store| store.len() as u64)) + } + + fn record_final_failure(&self) { + self.delivery_counters.record_final_failure(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn base_args() -> PostgresArgs { + PostgresArgs { + enable: true, + dsn_string: "postgres://postgres:secret@localhost:5432/rustfs_events?search_path=public".to_string(), + schema: "public".to_string(), + table: "rustfs_events_namespace".to_string(), + format: PostgresFormat::Namespace, + tls_required: false, + tls_ca: String::new(), + tls_client_cert: String::new(), + tls_client_key: String::new(), + queue_dir: String::new(), + queue_limit: 100_000, + target_type: TargetType::NotifyEvent, + } + } + + #[test] + fn validate_disabled_skips_all_checks() { + let args = PostgresArgs { + enable: false, + dsn_string: String::new(), + schema: String::new(), + table: String::new(), + ..base_args() + }; + assert!(args.validate().is_ok()); + } + + #[test] + fn validate_accepts_base_args() { + assert!(base_args().validate().is_ok()); + } + + #[tokio::test] + async fn is_active_returns_false_when_disabled() { + let target = PostgresTarget::::new( + "postgres:test".to_string(), + PostgresArgs { + enable: false, + dsn_string: "postgres://postgres:secret@localhost:5432/rustfs_events?search_path=public".to_string(), + ..base_args() + }, + ) + .expect("disabled target should still construct"); + + assert!(!target.is_active().await.expect("disabled target should not probe")); + } + + #[test] + fn validate_rejects_empty_dsn_string() { + let args = PostgresArgs { + dsn_string: String::new(), + ..base_args() + }; + let err = args.validate().expect_err("empty dsn string should fail"); + assert!(err.to_string().contains("dsn_string cannot be empty")); + } + + #[test] + fn validate_rejects_invalid_dsn_string() { + let args = PostgresArgs { + dsn_string: "postgres://".to_string(), + ..base_args() + }; + let err = args.validate().expect_err("invalid dsn should fail"); + assert!(err.to_string().contains("invalid PostgreSQL dsn_string")); + } + + #[test] + fn validate_rejects_invalid_schema_identifier() { + let args = PostgresArgs { + schema: "public; DROP TABLE".to_string(), + ..base_args() + }; + let err = args.validate().expect_err("invalid schema should fail"); + assert!(err.to_string().contains("schema")); + } + + #[test] + fn validate_rejects_invalid_table_identifier() { + let args = PostgresArgs { + table: "events;".to_string(), + ..base_args() + }; + let err = args.validate().expect_err("invalid table should fail"); + assert!(err.to_string().contains("table")); + } + + #[test] + fn validate_rejects_table_starting_with_digit() { + let args = PostgresArgs { + table: "1events".to_string(), + ..base_args() + }; + let err = args.validate().expect_err("digit-leading table should fail"); + assert!(err.to_string().contains("table")); + } + + #[test] + fn validate_rejects_mtls_without_key() { + let args = PostgresArgs { + tls_client_cert: "/etc/ssl/client.pem".to_string(), + tls_client_key: String::new(), + ..base_args() + }; + let err = args.validate().expect_err("missing key should fail"); + assert!(err.to_string().contains("must be specified together")); + } + + #[test] + fn validate_rejects_relative_queue_dir() { + let args = PostgresArgs { + queue_dir: "relative/path".to_string(), + ..base_args() + }; + let err = args.validate().expect_err("relative queue_dir should fail"); + assert!(err.to_string().contains("absolute path")); + } + + #[test] + fn validate_rejects_relative_tls_ca() { + let args = PostgresArgs { + tls_ca: "ca.pem".to_string(), + ..base_args() + }; + let err = args.validate().expect_err("relative tls_ca should fail"); + assert!(err.to_string().contains("absolute path")); + } + + #[test] + fn parse_format_defaults_to_namespace() { + assert_eq!(parse_postgres_format(None).expect("ok"), PostgresFormat::Namespace); + assert_eq!(parse_postgres_format(Some("")).expect("ok"), PostgresFormat::Namespace); + assert_eq!(parse_postgres_format(Some(" ")).expect("ok"), PostgresFormat::Namespace); + } + + #[test] + fn parse_format_accepts_variants() { + assert_eq!(parse_postgres_format(Some("namespace")).expect("ok"), PostgresFormat::Namespace); + assert_eq!(parse_postgres_format(Some("ACCESS")).expect("ok"), PostgresFormat::Access); + assert_eq!(parse_postgres_format(Some("Access")).expect("ok"), PostgresFormat::Access); + } + + #[test] + fn parse_format_rejects_unknown() { + let err = parse_postgres_format(Some("structured")).expect_err("unknown format should fail"); + assert!(err.to_string().contains("must be 'namespace' or 'access'")); + } + + #[test] + fn parse_dsn_extracts_search_path_schema() { + let parsed = PostgresDsn::parse("postgres://postgres:secret@localhost:5432/rustfs_events?search_path=audit,public") + .expect("dsn should parse"); + assert_eq!(parsed.host, "localhost"); + assert_eq!(parsed.port, 5432); + assert_eq!(parsed.user, "postgres"); + assert_eq!(parsed.password.as_deref(), Some("secret")); + assert_eq!(parsed.database, "rustfs_events"); + assert_eq!(parsed.schema, "audit"); + } + + #[test] + fn parse_dsn_defaults_schema_to_public() { + let parsed = PostgresDsn::parse("postgres://postgres:secret@localhost:5432/rustfs_events").expect("dsn should parse"); + assert_eq!(parsed.schema, "public"); + } + + #[test] + fn parse_dsn_rejects_invalid_scheme() { + let err = PostgresDsn::parse("mysql://user:pass@localhost:5432/db").expect_err("scheme should fail"); + assert!(err.to_string().contains("scheme must be postgres or postgresql")); + } + + #[test] + fn parse_dsn_rejects_invalid_search_path_identifier() { + let err = PostgresDsn::parse("postgres://postgres:secret@localhost:5432/rustfs_events?search_path=public;drop") + .expect_err("invalid search_path should fail"); + assert!(err.to_string().contains("schema")); + } + + #[test] + fn validate_rejects_schema_mismatch_with_dsn_search_path() { + let args = PostgresArgs { + schema: "public".to_string(), + dsn_string: "postgres://postgres:secret@localhost:5432/rustfs_events?search_path=audit".to_string(), + ..base_args() + }; + let err = args.validate().expect_err("schema mismatch should fail"); + assert!(err.to_string().contains("schema must match DSN search_path")); + } + + #[test] + fn debug_masks_password() { + let args = base_args(); + let rendered = format!("{args:?}"); + assert!(!rendered.contains("secret"), "password leaked: {rendered}"); + assert!(rendered.contains("postgres:***@")); + } + + #[test] + fn debug_masks_password_when_empty_shows_blank() { + let args = PostgresArgs { + dsn_string: "postgres://postgres@localhost:5432/rustfs_events?search_path=public".to_string(), + ..base_args() + }; + let rendered = format!("{args:?}"); + assert!(!rendered.contains(":***@")); + } + + #[test] + fn redact_postgres_dsn_masks_password_query_parameter() { + let redacted = redact_postgres_dsn("postgres://postgres@localhost:5432/db?search_path=public&password=secret"); + assert!(!redacted.contains("secret")); + assert!(redacted.contains("password=%2A%2A%2A") || redacted.contains("password=***")); + } + + #[test] + fn qualified_table_double_quotes_both_parts() { + assert_eq!(qualified_table("public", "events"), r#""public"."events""#); + assert_eq!(qualified_table("audit", "rustfs_events"), r#""audit"."rustfs_events""#); + } + + #[test] + fn namespace_upsert_uses_on_conflict_update() { + let sql = namespace_upsert_sql("public", "events"); + assert!(sql.contains("ON CONFLICT (key) DO UPDATE")); + assert!(sql.contains(r#""public"."events""#)); + assert!(sql.contains("$2::jsonb")); + } + + #[test] + fn access_insert_uses_event_id_pk_with_on_conflict_do_nothing() { + let sql = access_insert_sql("public", "events_access"); + assert!(sql.contains("event_id")); + assert!(sql.contains("ON CONFLICT (event_id) DO NOTHING")); + assert!(sql.contains(r#""public"."events_access""#)); + assert!(sql.contains("$4::jsonb")); + } + + #[test] + fn table_probe_does_not_select_rows() { + let sql = table_probe_sql("public", "events"); + assert!(sql.contains("LIMIT 0")); + assert!(sql.contains(r#""public"."events""#)); + } + + #[test] + fn validate_pg_identifier_accepts_alphanumerics() { + assert!(validate_pg_identifier("events", "table").is_ok()); + assert!(validate_pg_identifier("rustfs_events_v2", "table").is_ok()); + assert!(validate_pg_identifier("_underscored", "table").is_ok()); + } + + #[test] + fn validate_pg_identifier_rejects_dot_and_quote() { + assert!(validate_pg_identifier("public.events", "table").is_err()); + assert!(validate_pg_identifier("events\"DROP", "table").is_err()); + assert!(validate_pg_identifier("a b", "table").is_err()); + } + + #[test] + fn resolve_payload_key_prefers_serialized_key_field() { + let payload = serde_json::json!({ + "EventName": "s3:ObjectCreated:Put", + "Key": "bucket-a/folder/object.txt", + "Records": [] + }); + let meta = QueuedPayloadMeta::new( + rustfs_s3_common::EventName::ObjectCreatedPut, + "bucket-a".to_string(), + "fallback%2Fvalue.txt".to_string(), + "application/json", + 0, + ); + + assert_eq!(resolve_payload_key(&payload, &meta), "bucket-a/folder/object.txt"); + } + + #[test] + fn resolve_payload_key_falls_back_to_decoded_meta_key() { + let payload = serde_json::json!({ + "EventName": "s3:ObjectCreated:Put", + "Records": [] + }); + let meta = QueuedPayloadMeta::new( + rustfs_s3_common::EventName::ObjectCreatedPut, + "bucket-a".to_string(), + "hello+world%2Ftest.txt".to_string(), + "application/json", + 0, + ); + + assert_eq!(resolve_payload_key(&payload, &meta), "bucket-a/hello world/test.txt"); + } +} diff --git a/crates/targets/src/target/pulsar.rs b/crates/targets/src/target/pulsar.rs index 1c6090f87..07a816a4a 100644 --- a/crates/targets/src/target/pulsar.rs +++ b/crates/targets/src/target/pulsar.rs @@ -19,7 +19,7 @@ use crate::{ store::{Key, QueueStore, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, - TargetType, + TargetType, queue_store_subdir_name, }, }; use async_trait::async_trait; @@ -188,7 +188,7 @@ where let target_id = TargetID::new(id, ChannelTargetType::Pulsar.as_str().to_string()); let queue_store = if !args.queue_dir.is_empty() { let base_path = PathBuf::from(&args.queue_dir); - let specific_queue_path = base_path.join(format!("rustfs-{}-{}", ChannelTargetType::Pulsar.as_str(), target_id.id)); + let specific_queue_path = base_path.join(queue_store_subdir_name(ChannelTargetType::Pulsar.as_str(), &target_id.id)); let extension = match args.target_type { TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION, TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION, diff --git a/crates/targets/src/target/redis.rs b/crates/targets/src/target/redis.rs new file mode 100644 index 000000000..bcc96468a --- /dev/null +++ b/crates/targets/src/target/redis.rs @@ -0,0 +1,1205 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + StoreError, Target, + arn::TargetID, + error::TargetError, + store::{Key, QueueStore, Store}, + target::{ + ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, + TargetType, build_queued_payload, queue_store_subdir_name, + }, +}; +use async_trait::async_trait; +use redis::{ + AsyncCommands, Client, ClientTlsConfig, ConnectionInfo, IntoConnectionInfo, RedisError, TlsCertificates, + aio::{ConnectionManager, ConnectionManagerConfig}, + cmd, + io::tcp::{TcpSettings, socket2}, +}; +use rustfs_config::{REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY}; +use serde::Serialize; +use serde::de::DeserializeOwned; +use std::fmt; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; +use tokio::sync::Mutex; +use tracing::{debug, error, info, instrument, warn}; +use url::Url; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RedisTlsPolicy { + SystemCa, + CustomCa, +} + +impl RedisTlsPolicy { + fn parse(value: &str) -> Result { + match value.trim() { + value if value.eq_ignore_ascii_case("system_ca") => Ok(Self::SystemCa), + value if value.eq_ignore_ascii_case("custom_ca") => Ok(Self::CustomCa), + _ => Err(TargetError::Configuration( + "Redis tls_policy must be one of: system_ca, custom_ca".to_string(), + )), + } + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct RedisTlsConfig { + pub policy: Option, + pub ca_path: String, + pub client_cert_path: String, + pub client_key_path: String, + pub allow_insecure: bool, +} + +impl RedisTlsConfig { + pub fn from_values( + policy: Option<&str>, + ca_path: Option<&str>, + client_cert_path: Option<&str>, + client_key_path: Option<&str>, + allow_insecure: Option<&str>, + ) -> Result { + let policy = match policy.map(str::trim).filter(|value| !value.is_empty()) { + Some(value) => Some(RedisTlsPolicy::parse(value)?), + None => None, + }; + let allow_insecure = allow_insecure + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(|value| { + value + .parse::() + .map(rustfs_config::EnableState::is_enabled) + .or_else(|_| value.parse::()) + .map_err(|_| TargetError::Configuration("Redis tls_allow_insecure must be a boolean value".to_string())) + }) + .transpose()? + .unwrap_or(false); + + Ok(Self { + policy, + ca_path: ca_path.unwrap_or_default().trim().to_string(), + client_cert_path: client_cert_path.unwrap_or_default().trim().to_string(), + client_key_path: client_key_path.unwrap_or_default().trim().to_string(), + allow_insecure, + }) + } +} + +#[derive(Clone)] +pub struct RedisArgs { + /// Whether the target is enabled + pub enable: bool, + /// The Redis server URL in format: `{redis|rediss|valkey|valkeys}://[][:@][:port][/]` + pub url: Url, + /// The Redis pub/sub channel to publish to + pub channel: String, + /// The username for the Redis connection (leave it empty if you parse with url) + pub username: Option, + /// The password for the Redis connection (leave it empty if you parse with url) + pub password: Option, + /// TLS configuration + pub tls: RedisTlsConfig, + /// The keep alive interval + pub keep_alive: Duration, + /// The directory to store events in case of failure + pub queue_dir: String, + /// The maximum number of events to store + pub queue_limit: u64, + /// Maximum number of synchronous publish retries per payload + pub max_retry_attempts: usize, + /// Maximum number of reconnect retries in the underlying connection manager (6 if not provided) + pub reconnect_retry_attempts: Option, + /// Minimum retry delay between publish retry attempts (100ms if not provided) + pub min_retry_delay: Option, + /// Maximum retry delay between publish retry attempts (2s if not provided) + pub max_retry_delay: Option, + /// Timeout for establishing a Redis connection (5s if not provided) + pub connection_timeout: Option, + /// Timeout for command responses (5s if not provided) + pub response_timeout: Option, + /// Internal command buffer size for the multiplexed connection (50 if not provided) + pub pipeline_buffer_size: Option, + /// the target type + pub target_type: TargetType, +} + +fn redact_redis_url(url: &Url) -> String { + let mut redacted = url.clone(); + if redacted.password().is_some() { + let _ = redacted.set_password(Some("***")); + } + redacted.to_string() +} + +impl fmt::Debug for RedisArgs { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RedisArgs") + .field("enable", &self.enable) + .field("url", &redact_redis_url(&self.url)) + .field("channel", &self.channel) + .field("username", &self.username) + .field( + "password", + if self.password.as_deref().unwrap_or_default().is_empty() { + &"" + } else { + &"***REDACTED***" + }, + ) + .field("tls", &self.tls) + .field("keep_alive", &self.keep_alive) + .field("queue_dir", &self.queue_dir) + .field("queue_limit", &self.queue_limit) + .field("max_retry_attempts", &self.max_retry_attempts) + .field("reconnect_retry_attempts", &self.reconnect_retry_attempts) + .field("min_retry_delay", &self.min_retry_delay) + .field("max_retry_delay", &self.max_retry_delay) + .field("connection_timeout", &self.connection_timeout) + .field("response_timeout", &self.response_timeout) + .field("pipeline_buffer_size", &self.pipeline_buffer_size) + .field("target_type", &self.target_type) + .finish() + } +} + +impl RedisArgs { + pub fn validate(&self) -> Result<(), TargetError> { + if !self.enable { + return Ok(()); + } + + validate_redis_url(&self.url)?; + validate_redis_tls_config(&self.url, &self.tls)?; + + if self.channel.trim().is_empty() { + return Err(TargetError::Configuration("Redis channel cannot be empty".to_string())); + } + + if self.username.as_deref().unwrap_or_default().is_empty() != self.password.as_deref().unwrap_or_default().is_empty() + && !(self.username.is_none() && self.password.is_none()) + { + return Err(TargetError::Configuration( + "Redis username and password must be specified together when provided explicitly".to_string(), + )); + } + + if self.max_retry_attempts == 0 { + return Err(TargetError::Configuration( + "Redis max_retry_attempts must be greater than zero".to_string(), + )); + } + + if self.pipeline_buffer_size == Some(0) { + return Err(TargetError::Configuration( + "Redis pipeline_buffer_size must be greater than zero".to_string(), + )); + } + + if let (Some(min_retry_delay), Some(max_retry_delay)) = (self.min_retry_delay, self.max_retry_delay) + && max_retry_delay < min_retry_delay + { + return Err(TargetError::Configuration( + "Redis max_retry_delay must be greater than or equal to min_retry_delay".to_string(), + )); + } + + if !self.queue_dir.is_empty() && !Path::new(&self.queue_dir).is_absolute() { + return Err(TargetError::Configuration("Redis queue_dir path should be absolute".to_string())); + } + + Ok(()) + } +} + +pub fn validate_redis_url(url: &Url) -> Result<(), TargetError> { + let _: ConnectionInfo = url.clone().into_connection_info().map_err(map_redis_error)?; + Ok(()) +} + +fn validate_redis_tls_config(url: &Url, tls: &RedisTlsConfig) -> Result<(), TargetError> { + let secure_scheme = matches!(url.scheme(), "rediss" | "valkeys"); + + if !tls.client_cert_path.is_empty() && !Path::new(&tls.client_cert_path).is_absolute() { + return Err(TargetError::Configuration(format!("{REDIS_TLS_CLIENT_CERT} must be an absolute path"))); + } + if !tls.client_key_path.is_empty() && !Path::new(&tls.client_key_path).is_absolute() { + return Err(TargetError::Configuration(format!("{REDIS_TLS_CLIENT_KEY} must be an absolute path"))); + } + if tls.client_cert_path.is_empty() != tls.client_key_path.is_empty() { + return Err(TargetError::Configuration( + "Redis tls_client_cert and tls_client_key must be specified together".to_string(), + )); + } + + if !secure_scheme { + if tls.policy.is_some() + || !tls.ca_path.is_empty() + || !tls.client_cert_path.is_empty() + || !tls.client_key_path.is_empty() + || tls.allow_insecure + { + return Err(TargetError::Configuration( + "TLS settings are only allowed for rediss/valkeys schemes".to_string(), + )); + } + return Ok(()); + } + + if let Some(policy) = tls.policy { + match policy { + RedisTlsPolicy::SystemCa => { + if !tls.ca_path.is_empty() { + return Err(TargetError::Configuration(format!( + "{REDIS_TLS_CA} is not allowed when {REDIS_TLS_POLICY}=system_ca" + ))); + } + } + RedisTlsPolicy::CustomCa => { + if tls.ca_path.is_empty() { + return Err(TargetError::Configuration(format!( + "{REDIS_TLS_CA} is required when {REDIS_TLS_POLICY}=custom_ca" + ))); + } + if !Path::new(&tls.ca_path).is_absolute() { + return Err(TargetError::Configuration(format!("{REDIS_TLS_CA} must be an absolute path"))); + } + } + } + } else if !tls.ca_path.is_empty() && !Path::new(&tls.ca_path).is_absolute() { + return Err(TargetError::Configuration(format!("{REDIS_TLS_CA} must be an absolute path"))); + } + + Ok(()) +} + +fn ensure_rustls_provider_installed() { + if rustls::crypto::CryptoProvider::get_default().is_none() + && rustls::crypto::aws_lc_rs::default_provider().install_default().is_err() + { + debug!("rustls crypto provider was installed concurrently, skipping aws-lc-rs install"); + } +} + +pub struct RedisTarget +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + id: TargetID, + args: RedisArgs, + publisher_client: Client, + publisher: Arc>>, + store: Option + Send + Sync>>, + /// Business-level liveness flag. + /// + /// We only flip this to `false` on final/terminal failure paths (for example: init failed, + /// publish exhausted retries, or the target was explicitly closed). Temporary reconnectable + /// errors only invalidate the cached publisher so that a later request can lazily rebuild it. + connected: Arc, + delivery_counters: Arc, + _phantom: std::marker::PhantomData, +} + +impl RedisTarget +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + #[instrument(skip(args), fields(target_id_as_string = %id))] + pub fn new(id: String, args: RedisArgs) -> Result { + args.validate()?; + + let target_id = TargetID::new(id, ChannelTargetType::Redis.as_str().to_string()); + let publisher_client = build_redis_client(&args)?; + + let queue_store = if !args.queue_dir.is_empty() { + let base_path = PathBuf::from(&args.queue_dir); + let specific_queue_path = base_path.join(queue_store_subdir_name(ChannelTargetType::Redis.as_str(), &target_id.id)); + let extension = match args.target_type { + TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION, + TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION, + }; + let store = QueueStore::::new(specific_queue_path, args.queue_limit, extension); + if let Err(e) = store.open() { + error!(target_id = %target_id, error = %e, "Failed to open store for Redis target"); + return Err(TargetError::Storage(format!("{e}"))); + } + Some(Box::new(store) as Box + Send + Sync>) + } else { + None + }; + + info!(target_id = %target_id, "Redis target created"); + Ok(Self { + id: target_id, + args, + publisher_client, + publisher: Arc::new(Mutex::new(None)), + store: queue_store, + connected: Arc::new(AtomicBool::new(false)), + delivery_counters: Arc::new(TargetDeliveryCounters::default()), + _phantom: std::marker::PhantomData, + }) + } + + pub fn clone_box(&self) -> Box + Send + Sync> { + Box::new(Self { + id: self.id.clone(), + args: self.args.clone(), + publisher_client: self.publisher_client.clone(), + publisher: Arc::clone(&self.publisher), + store: self.store.as_ref().map(|s| s.boxed_clone()), + connected: Arc::clone(&self.connected), + delivery_counters: Arc::clone(&self.delivery_counters), + _phantom: std::marker::PhantomData, + }) + } + + async fn get_or_create_publisher(&self) -> Result { + let mut guard = self.publisher.lock().await; + if let Some(manager) = guard.clone() { + return Ok(manager); + } + + let manager = self + .publisher_client + .get_connection_manager_lazy(build_redis_connection_manager_config(&self.args)) + .map_err(map_redis_error)?; + + *guard = Some(manager.clone()); + Ok(manager) + } + + async fn invalidate_cached_publisher(&self) { + // Intentionally does not touch `connected`: invalidating the current manager only means + // "recreate the publisher on the next attempt", not "this target is now definitively + // inactive". That distinction preserves the business semantics of `is_active()`. + *self.publisher.lock().await = None; + } + + async fn ensure_publisher_ready(&self) -> Result<(), TargetError> { + let mut publisher = self.get_or_create_publisher().await?; + match cmd("PING").query_async::(&mut publisher).await { + Ok(_) => Ok(()), + Err(err) => { + let mapped = map_redis_error(err); + if is_retryable_target_error(&mapped) { + self.invalidate_cached_publisher().await; + } + Err(mapped) + } + } + } + + async fn init_inner(&self) -> Result<(), TargetError> { + if let Err(err) = self.ensure_publisher_ready().await { + self.connected.store(false, Ordering::SeqCst); + return Err(err); + } + self.connected.store(true, Ordering::SeqCst); + Ok(()) + } + + #[instrument(skip(self, body, meta), fields(target_id = %self.id))] + async fn send_body(&self, body: Vec, meta: &QueuedPayloadMeta) -> Result<(), TargetError> { + debug!( + target = %self.id, + bucket = %meta.bucket_name, + object = %meta.object_name, + event = %meta.event_name, + payload_len = body.len(), + channel = %self.args.channel, + "Sending Redis payload" + ); + + let mut attempt = 0usize; + let mut last_error = None; + while attempt < self.args.max_retry_attempts { + attempt += 1; + + let mut publisher = self.get_or_create_publisher().await?; + match publisher + .publish::<_, _, i64>(self.args.channel.as_str(), body.as_slice()) + .await + { + Ok(_) => { + debug!(target_id = %self.id, channel = %self.args.channel, attempt, "Event published to Redis channel"); + self.delivery_counters.record_success(); + return Ok(()); + } + Err(err) => { + let mapped = map_redis_error(err); + if is_retryable_target_error(&mapped) { + self.invalidate_cached_publisher().await; + } + + warn!( + target_id = %self.id, + channel = %self.args.channel, + attempt, + max_attempts = self.args.max_retry_attempts, + error = %mapped, + "Redis publish attempt failed" + ); + + if !is_retryable_target_error(&mapped) || attempt >= self.args.max_retry_attempts { + last_error = Some(mapped); + break; + } + + last_error = Some(mapped); + tokio::time::sleep(compute_retry_delay( + attempt, + self.args.min_retry_delay.unwrap_or(Duration::from_millis(100)), + self.args.max_retry_delay.unwrap_or(Duration::from_secs(2)), + )) + .await; + } + } + } + + self.connected.store(false, Ordering::SeqCst); + + Err(last_error.unwrap_or(TargetError::Unknown("Redis publish failed without a captured error".to_string()))) + } +} + +#[async_trait] +impl Target for RedisTarget +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + fn id(&self) -> TargetID { + self.id.clone() + } + + async fn is_active(&self) -> Result { + if !self.is_enabled() { + return Ok(false); + } + + match tokio::time::timeout(Duration::from_secs(5), ping_redis_server(&self.publisher_client, &self.args)).await { + Ok(Ok(())) => { + self.connected.store(true, Ordering::SeqCst); + Ok(true) + } + Ok(Err(err)) => { + self.invalidate_cached_publisher().await; + self.connected.store(false, Ordering::SeqCst); + Err(err) + } + Err(_) => { + self.invalidate_cached_publisher().await; + self.connected.store(false, Ordering::SeqCst); + Err(TargetError::Timeout("Redis connection timed out".to_string())) + } + } + } + + async fn save(&self, event: Arc>) -> Result<(), TargetError> { + let queued = match build_queued_payload(event.as_ref()) { + Ok(queued) => queued, + Err(err) => { + self.delivery_counters.record_final_failure(); + return Err(err); + } + }; + + if let Some(store) = &self.store { + let encoded = match queued.encode() { + Ok(encoded) => encoded, + Err(err) => { + self.delivery_counters.record_final_failure(); + return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}"))); + } + }; + + if let Err(e) = store.put_raw(&encoded) { + self.delivery_counters.record_final_failure(); + return Err(TargetError::Storage(format!("Failed to save event to store: {e}"))); + } + + debug!(target_id = %self.id, "Event saved to store for Redis target"); + Ok(()) + } else { + if !self.is_enabled() { + return Err(TargetError::Disabled); + } + + if let Err(err) = self.init_inner().await { + self.delivery_counters.record_final_failure(); + return Err(err); + } + + if let Err(err) = self.send_body(queued.body, &queued.meta).await { + self.delivery_counters.record_final_failure(); + return Err(err); + } + + Ok(()) + } + } + + async fn send_raw_from_store(&self, key: Key, body: Vec, meta: QueuedPayloadMeta) -> Result<(), TargetError> { + debug!(target_id = %self.id, ?key, "Attempting to send queued payload from Redis store"); + + if !self.is_enabled() { + return Err(TargetError::Disabled); + } + + if let Err(err) = self.init_inner().await { + if matches!(err, TargetError::NotConnected | TargetError::Timeout(_) | TargetError::Network(_)) { + warn!(target_id = %self.id, error = %err, "Redis target not ready; queued event remains in store"); + } + return Err(err); + } + + if let Err(err) = self.send_body(body, &meta).await { + if matches!(err, TargetError::NotConnected | TargetError::Timeout(_) | TargetError::Network(_)) { + warn!(target_id = %self.id, error = %err, "Failed to send Redis event from store: target not connected. Event remains queued."); + } + return Err(err); + } + + debug!(target_id = %self.id, ?key, "Queued Redis payload sent successfully"); + Ok(()) + } + + async fn close(&self) -> Result<(), TargetError> { + self.invalidate_cached_publisher().await; + self.connected.store(false, Ordering::SeqCst); + info!(target_id = %self.id, "Redis target closed"); + Ok(()) + } + + fn store(&self) -> Option<&(dyn Store + Send + Sync)> { + self.store.as_deref() + } + + fn clone_dyn(&self) -> Box + Send + Sync> { + self.clone_box() + } + + async fn init(&self) -> Result<(), TargetError> { + if !self.is_enabled() { + return Ok(()); + } + self.init_inner().await + } + + fn is_enabled(&self) -> bool { + self.args.enable + } + + fn delivery_snapshot(&self) -> TargetDeliverySnapshot { + self.delivery_counters + .snapshot(self.store.as_deref().map_or(0, |store| store.len() as u64)) + } + + fn record_final_failure(&self) { + self.delivery_counters.record_final_failure(); + } +} + +pub(crate) fn build_redis_client(args: &RedisArgs) -> Result { + let mut url = args.url.clone(); + if args.tls.allow_insecure { + url.set_fragment(Some("insecure")); + } + + let mut connection_info: ConnectionInfo = url.into_connection_info().map_err(map_redis_error)?; + + let base_redis = connection_info.redis_settings().clone(); + + let mut redis_settings = base_redis.clone().set_lib_name("rustfs-targets", env!("CARGO_PKG_VERSION")); + + if let Some(username) = args.username.as_deref().filter(|value| !value.is_empty()) { + if base_redis.username().is_some_and(|base| base != username) { + warn!(url_username = ?base_redis.username(), arg_username = %username, "Redis target protocol username from URL is being overridden"); + } + redis_settings = redis_settings.set_username(username); + } + if let Some(password) = args.password.as_deref().filter(|value| !value.is_empty()) { + if base_redis.password().is_some() { + warn!("RedisArgs.password overrides password from Redis URL"); + } + redis_settings = redis_settings.set_password(password); + } + + let mut tcp_settings = TcpSettings::default().set_nodelay(true); + #[cfg(not(target_family = "wasm"))] + { + if !args.keep_alive.is_zero() { + tcp_settings = tcp_settings.set_keepalive(socket2::TcpKeepalive::new().with_time(args.keep_alive)); + } + } + + connection_info = connection_info + .set_redis_settings(redis_settings) + .set_tcp_settings(tcp_settings); + + let secure_scheme = matches!(args.url.scheme(), "rediss" | "valkeys"); + if secure_scheme { + ensure_rustls_provider_installed(); + let tls_certs = TlsCertificates { + client_tls: read_client_tls(&args.tls)?, + root_cert: read_root_cert(&args.tls)?, + }; + Client::build_with_tls(connection_info, tls_certs).map_err(map_redis_error) + } else { + Client::open(connection_info).map_err(map_redis_error) + } +} + +pub(crate) fn build_redis_connection_manager_config(args: &RedisArgs) -> ConnectionManagerConfig { + let mut config = ConnectionManagerConfig::new(); + + if let Some(reconnect_retry_attempts) = args.reconnect_retry_attempts { + config = config.set_number_of_retries(reconnect_retry_attempts); + } + if let Some(min_retry_delay) = args.min_retry_delay { + config = config.set_min_delay(min_retry_delay); + } + if let Some(max_retry_delay) = args.max_retry_delay { + config = config.set_max_delay(max_retry_delay); + } + if let Some(connection_timeout) = args.connection_timeout { + config = config.set_connection_timeout(Some(connection_timeout)); + } + if let Some(response_timeout) = args.response_timeout { + config = config.set_response_timeout(Some(response_timeout)); + } + if let Some(pipeline_buffer_size) = args.pipeline_buffer_size { + config = config.set_pipeline_buffer_size(pipeline_buffer_size); + } + + config +} + +pub(crate) async fn ping_redis_server(client: &Client, args: &RedisArgs) -> Result<(), TargetError> { + let config = build_redis_connection_manager_config(args); + let mut conn = client + .get_connection_manager_with_config(config) + .await + .map_err(map_redis_error)?; + + cmd("PING").query_async::(&mut conn).await.map_err(map_redis_error)?; + + Ok(()) +} + +fn read_client_tls(tls: &RedisTlsConfig) -> Result, TargetError> { + if tls.client_cert_path.is_empty() { + return Ok(None); + } + + let client_cert = std::fs::read(&tls.client_cert_path) + .map_err(|e| TargetError::Configuration(format!("Failed to read Redis client cert: {e}")))?; + let client_key = std::fs::read(&tls.client_key_path) + .map_err(|e| TargetError::Configuration(format!("Failed to read Redis client key: {e}")))?; + + Ok(Some(ClientTlsConfig { client_cert, client_key })) +} + +fn read_root_cert(tls: &RedisTlsConfig) -> Result>, TargetError> { + if tls.ca_path.is_empty() { + return Ok(None); + } + + std::fs::read(&tls.ca_path) + .map(Some) + .map_err(|e| TargetError::Configuration(format!("Failed to read Redis root CA cert: {e}"))) +} + +fn map_redis_error(err: RedisError) -> TargetError { + use redis::ErrorKind; + + match err.kind() { + ErrorKind::AuthenticationFailed => TargetError::Authentication(err.to_string()), + ErrorKind::RESP3NotSupported => TargetError::Initialization(err.to_string()), + ErrorKind::InvalidClientConfig => TargetError::Configuration(err.to_string()), + ErrorKind::Io if err.is_timeout() => TargetError::Timeout(err.to_string()), + ErrorKind::Io if err.is_connection_dropped() || err.is_connection_refusal() => TargetError::NotConnected, + ErrorKind::Io => TargetError::Network(err.to_string()), + _ if err.is_unrecoverable_error() => TargetError::NotConnected, + _ => TargetError::Request(err.to_string()), + } +} + +fn is_retryable_target_error(err: &TargetError) -> bool { + matches!(err, TargetError::NotConnected | TargetError::Timeout(_) | TargetError::Network(_)) +} + +fn compute_retry_delay(attempt: usize, min_delay: Duration, max_delay: Duration) -> Duration { + let shift = attempt.saturating_sub(1).min(16) as u32; + let factor = 1u32 << shift; + min_delay.saturating_mul(factor).min(max_delay) +} + +#[cfg(test)] +mod tests { + use super::*; + use redis::ProtocolVersion; + use std::sync::atomic::Ordering; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + + fn base_args() -> RedisArgs { + RedisArgs { + enable: true, + url: Url::parse("redis://127.0.0.1:6379").unwrap(), + channel: "rustfs-events".to_string(), + username: None, + password: None, + tls: RedisTlsConfig::default(), + keep_alive: Duration::from_secs(15), + queue_dir: String::new(), + queue_limit: 0, + max_retry_attempts: 3, + reconnect_retry_attempts: None, + min_retry_delay: None, + max_retry_delay: None, + connection_timeout: None, + response_timeout: None, + pipeline_buffer_size: None, + target_type: TargetType::NotifyEvent, + } + } + + #[test] + fn validate_rejects_empty_channel() { + let args = RedisArgs { + channel: String::new(), + ..base_args() + }; + assert!(args.validate().is_err()); + } + + #[test] + fn validate_accepts_embedded_credentials_in_url() { + let url = Url::parse("redis://user:pass@127.0.0.1:6379").unwrap(); + assert!(validate_redis_url(&url).is_ok()); + } + + #[test] + fn validate_rejects_relative_queue_dir() { + let args = RedisArgs { + queue_dir: "relative/path".to_string(), + ..base_args() + }; + assert!(args.validate().is_err()); + } + + #[test] + fn validate_accepts_custom_ca_tls_policy() { + let args = RedisArgs { + url: Url::parse("rediss://127.0.0.1:6379").unwrap(), + tls: RedisTlsConfig { + policy: Some(RedisTlsPolicy::CustomCa), + ca_path: "/tmp/ca.pem".to_string(), + ..RedisTlsConfig::default() + }, + ..base_args() + }; + assert!(args.validate().is_ok()); + } + + #[test] + fn debug_redacts_passwords_from_url_and_args() { + let args = RedisArgs { + url: Url::parse("redis://user:secret@127.0.0.1:6379/0").unwrap(), + password: Some("override-secret".to_string()), + ..base_args() + }; + + let rendered = format!("{args:?}"); + assert!(!rendered.contains("secret"), "url password leaked: {rendered}"); + assert!(!rendered.contains("override-secret"), "args password leaked: {rendered}"); + assert!(rendered.contains("redis://user:***@127.0.0.1:6379/0")); + assert!(rendered.contains("\"***REDACTED***\"")); + } + + #[test] + fn validate_rejects_insecure_tls_for_non_secure_scheme() { + let args = RedisArgs { + tls: RedisTlsConfig { + allow_insecure: true, + ..RedisTlsConfig::default() + }, + ..base_args() + }; + assert!(args.validate().is_err()); + } + + #[test] + fn build_redis_client_preserves_url_auth_when_args_are_none() { + let args = RedisArgs { + url: Url::parse("redis://user:pass@127.0.0.1:6379/2").unwrap(), + ..base_args() + }; + + let client = build_redis_client(&args).expect("client should build"); + let info = client.get_connection_info(); + let redis = info.redis_settings(); + + assert_eq!(redis.username(), Some("user")); + assert_eq!(redis.password(), Some("pass")); + assert_eq!(redis.db(), 2); + } + + #[test] + fn build_redis_client_overrides_url_auth_when_args_are_set() { + let args = RedisArgs { + url: Url::parse("redis://user:pass@127.0.0.1:6379/2").unwrap(), + username: Some("override-user".to_string()), + password: Some("override-pass".to_string()), + ..base_args() + }; + + let client = build_redis_client(&args).expect("client should build"); + let redis = client.get_connection_info().redis_settings(); + + assert_eq!(redis.username(), Some("override-user")); + assert_eq!(redis.password(), Some("override-pass")); + assert_eq!(redis.db(), 2); + } + + #[test] + fn build_redis_client_preserves_url_protocol_when_args_do_not_override_it() { + let args = RedisArgs { + url: Url::parse("redis://127.0.0.1:6379/?protocol=resp3").unwrap(), + ..base_args() + }; + + let client = build_redis_client(&args).expect("client should build"); + let redis = client.get_connection_info().redis_settings(); + + assert_eq!(redis.protocol(), ProtocolVersion::RESP3); + } + + #[test] + fn build_redis_client_enables_insecure_tls_when_requested() { + let args = RedisArgs { + url: Url::parse("rediss://127.0.0.1:6379").unwrap(), + tls: RedisTlsConfig { + allow_insecure: true, + ..RedisTlsConfig::default() + }, + ..base_args() + }; + + let client = build_redis_client(&args).expect("client should build"); + match client.get_connection_info().addr() { + redis::ConnectionAddr::TcpTls { insecure, .. } => assert!(*insecure), + other => panic!("expected TLS address, got {other:?}"), + } + } + + #[tokio::test] + async fn invalidate_cached_publisher_keeps_connected_state() { + let target = RedisTarget::::new("redis:test".to_string(), base_args()).expect("target should build"); + target.connected.store(true, Ordering::SeqCst); + + target.invalidate_cached_publisher().await; + + assert!(target.connected.load(Ordering::SeqCst)); + assert!(target.publisher.lock().await.is_none()); + } + + #[tokio::test] + async fn is_active_succeeds_when_ping_returns_pong() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind fake redis"); + let addr = listener.local_addr().expect("listener addr"); + tokio::spawn(run_fake_redis_server(listener, false)); + + let mut args = base_args(); + args.url = Url::parse(&format!("redis://{}:{}/0", addr.ip(), addr.port())).unwrap(); + + let target = RedisTarget::::new("redis:test".to_string(), args).expect("target should build"); + target.connected.store(false, Ordering::SeqCst); + + assert!(target.is_active().await.expect("ping should succeed")); + assert!(target.connected.load(Ordering::SeqCst)); + } + + #[tokio::test] + async fn is_active_returns_error_when_ping_fails() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind fake redis"); + let addr = listener.local_addr().expect("listener addr"); + tokio::spawn(async move { + loop { + let Ok((socket, _)) = listener.accept().await else { + return; + }; + drop(socket); + } + }); + + let mut args = base_args(); + args.url = Url::parse(&format!("redis://{}:{}/0", addr.ip(), addr.port())).unwrap(); + + let target = RedisTarget::::new("redis:test".to_string(), args).expect("target should build"); + target.connected.store(true, Ordering::SeqCst); + + let err = target.is_active().await.expect_err("ping should fail"); + assert!(matches!( + err, + TargetError::NotConnected | TargetError::Network(_) | TargetError::Timeout(_) + )); + assert!(!target.connected.load(Ordering::SeqCst)); + assert!(target.publisher.lock().await.is_none()); + } + + #[tokio::test] + async fn is_active_returns_false_when_disabled() { + let target = RedisTarget::::new( + "redis:test".to_string(), + RedisArgs { + enable: false, + ..base_args() + }, + ) + .expect("target should build"); + + assert!(!target.is_active().await.expect("disabled target should not probe")); + } + + #[test] + fn compute_retry_delay_is_bounded() { + let min = Duration::from_millis(100); + let max = Duration::from_secs(2); + + assert_eq!(compute_retry_delay(1, min, max), min); + assert!(compute_retry_delay(5, min, max) <= max); + assert_eq!(compute_retry_delay(50, min, max), max); + } + + #[test] + fn queued_payload_uses_event_data_in_records() { + let payload = build_queued_payload(&EntityTarget { + object_name: "greeting+file+%282%29.csv".to_string(), + bucket_name: "bucket".to_string(), + event_name: rustfs_s3_common::EventName::ObjectCreatedPut, + data: "payload-data".to_string(), + }) + .expect("payload should build"); + + let value: serde_json::Value = serde_json::from_slice(&payload.body).expect("payload JSON"); + assert_eq!(value["Key"], "bucket/greeting file (2).csv"); + assert_eq!(value["Records"][0], "payload-data"); + } + + fn parse_resp_array(input: &[u8]) -> Option<(Vec, usize)> { + if input.first()? != &b'*' { + return None; + } + + let mut index = 1; + let len_end = input[index..].windows(2).position(|w| w == b"\r\n")? + index; + let items: usize = std::str::from_utf8(&input[index..len_end]).ok()?.parse().ok()?; + index = len_end + 2; + + let mut out = Vec::with_capacity(items); + for _ in 0..items { + if input.get(index)? != &b'$' { + return None; + } + index += 1; + let bulk_end = input[index..].windows(2).position(|w| w == b"\r\n")? + index; + let bulk_len: usize = std::str::from_utf8(&input[index..bulk_end]).ok()?.parse().ok()?; + index = bulk_end + 2; + + let data_end = index.checked_add(bulk_len)?; + let data = std::str::from_utf8(input.get(index..data_end)?).ok()?.to_string(); + out.push(data); + index = data_end + 2; + } + + Some((out, index)) + } + + async fn run_fake_redis_server(listener: TcpListener, close_first_connection: bool) { + let mut first = close_first_connection; + loop { + let Ok((mut socket, _)) = listener.accept().await else { + return; + }; + + if first { + first = false; + drop(socket); + continue; + } + + tokio::spawn(async move { + let mut buf = vec![0_u8; 4096]; + let mut pending = Vec::new(); + + loop { + let Ok(read) = socket.read(&mut buf).await else { + return; + }; + if read == 0 { + return; + } + + pending.extend_from_slice(&buf[..read]); + + while let Some((command, consumed)) = parse_resp_array(&pending) { + pending.drain(..consumed); + let response = match command.first().map(|s| s.as_str()) { + Some("PING") => b"+PONG\r\n".as_slice(), + Some("PUBLISH") => b":1\r\n".as_slice(), + Some("CLIENT") => b"+OK\r\n".as_slice(), + Some("AUTH") => b"+OK\r\n".as_slice(), + Some("SELECT") => b"+OK\r\n".as_slice(), + Some("HELLO") => b"%1\r\n+server\r\n+redis\r\n".as_slice(), + _ => b"+OK\r\n".as_slice(), + }; + + if socket.write_all(response).await.is_err() { + return; + } + } + } + }); + } + } + + #[tokio::test] + async fn send_body_keeps_connected_true_when_retryable_error_eventually_recovers() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind fake redis"); + let addr = listener.local_addr().expect("listener addr"); + tokio::spawn(run_fake_redis_server(listener, true)); + + let mut args = base_args(); + args.url = Url::parse(&format!("redis://{}:{}/0", addr.ip(), addr.port())).unwrap(); + args.max_retry_attempts = 3; + args.reconnect_retry_attempts = Some(0); + args.min_retry_delay = Some(Duration::from_millis(10)); + args.max_retry_delay = Some(Duration::from_millis(20)); + + let target = RedisTarget::::new("redis:test".to_string(), args).expect("target should build"); + let meta = QueuedPayloadMeta::new( + rustfs_s3_common::EventName::ObjectCreatedPut, + "bucket".to_string(), + "object".to_string(), + "application/json", + 2, + ); + + target.connected.store(true, Ordering::SeqCst); + target + .send_body(b"{}".to_vec(), &meta) + .await + .expect("eventual retry should succeed"); + + assert!(target.connected.load(Ordering::SeqCst)); + assert_eq!(target.delivery_snapshot().total_messages, 1); + } + + #[tokio::test] + async fn send_body_sets_connected_false_after_retry_exhaustion() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind fake redis"); + let addr = listener.local_addr().expect("listener addr"); + tokio::spawn(async move { + loop { + let Ok((socket, _)) = listener.accept().await else { + return; + }; + drop(socket); + } + }); + + let mut args = base_args(); + args.url = Url::parse(&format!("redis://{}:{}/0", addr.ip(), addr.port())).unwrap(); + args.max_retry_attempts = 2; + args.reconnect_retry_attempts = Some(0); + args.min_retry_delay = Some(Duration::from_millis(10)); + args.max_retry_delay = Some(Duration::from_millis(20)); + + let target = RedisTarget::::new("redis:test".to_string(), args).expect("target should build"); + let meta = QueuedPayloadMeta::new( + rustfs_s3_common::EventName::ObjectCreatedPut, + "bucket".to_string(), + "object".to_string(), + "application/json", + 2, + ); + + let err = target + .send_body(b"{}".to_vec(), &meta) + .await + .expect_err("all retries should fail"); + assert!(matches!( + err, + TargetError::NotConnected | TargetError::Network(_) | TargetError::Timeout(_) + )); + assert!(!target.connected.load(Ordering::SeqCst)); + assert_eq!(target.delivery_snapshot().total_messages, 0); + } + + #[tokio::test] + async fn send_raw_from_store_failure_does_not_count_as_success() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind fake redis"); + let addr = listener.local_addr().expect("listener addr"); + tokio::spawn(async move { + loop { + let Ok((socket, _)) = listener.accept().await else { + return; + }; + drop(socket); + } + }); + + let mut args = base_args(); + args.url = Url::parse(&format!("redis://{}:{}/0", addr.ip(), addr.port())).unwrap(); + args.max_retry_attempts = 1; + args.reconnect_retry_attempts = Some(0); + + let target = RedisTarget::::new("redis:test".to_string(), args).expect("target should build"); + let meta = QueuedPayloadMeta::new( + rustfs_s3_common::EventName::ObjectCreatedPut, + "bucket".to_string(), + "object".to_string(), + "application/json", + 2, + ); + + let err = target + .send_raw_from_store( + Key { + name: "key".to_string(), + extension: String::new(), + item_count: 1, + compress: false, + }, + b"{}".to_vec(), + meta, + ) + .await + .expect_err("send from store should fail"); + + assert!(matches!( + err, + TargetError::NotConnected | TargetError::Network(_) | TargetError::Timeout(_) + )); + assert_eq!(target.delivery_snapshot().total_messages, 0); + } +} diff --git a/crates/targets/src/target/webhook.rs b/crates/targets/src/target/webhook.rs index 7372ead04..e0437359f 100644 --- a/crates/targets/src/target/webhook.rs +++ b/crates/targets/src/target/webhook.rs @@ -19,7 +19,7 @@ use crate::{ store::{Key, QueueStore, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, - TargetType, + TargetType, queue_store_subdir_name, }, }; use async_trait::async_trait; @@ -147,7 +147,7 @@ where // Build storage let queue_store = if !args.queue_dir.is_empty() { let queue_dir = - PathBuf::from(&args.queue_dir).join(format!("rustfs-{}-{}", ChannelTargetType::Webhook.as_str(), target_id.id)); + PathBuf::from(&args.queue_dir).join(queue_store_subdir_name(ChannelTargetType::Webhook.as_str(), &target_id.id)); let extension = match args.target_type { TargetType::AuditLog => AUDIT_STORE_EXTENSION, diff --git a/crates/targets/tests/mysql_integration.rs b/crates/targets/tests/mysql_integration.rs new file mode 100644 index 000000000..430b70f4f --- /dev/null +++ b/crates/targets/tests/mysql_integration.rs @@ -0,0 +1,252 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! MySQL notification target integration tests. +//! +//! These tests require a running MySQL 8.0+ or TiDB 8.5+ instance. +//! Set `RUSTFS_MYSQL_TEST_DSN` to enable them: +//! +//! ```bash +//! RUSTFS_MYSQL_TEST_DSN="user:pass@tcp(127.0.0.1:3306)/testdb" \ +//! cargo test -p rustfs-targets -- --ignored +//! ``` + +use mysql_async::{Opts, OptsBuilder, Pool, SslOpts, prelude::Queryable}; +use rustfs_targets::{Target, TargetError, target::mysql::*, target::*}; +use std::env; +use std::sync::Arc; +use tempfile::TempDir; +use uuid::Uuid; + +fn test_dsn() -> String { + env::var("RUSTFS_MYSQL_TEST_DSN").expect("RUSTFS_MYSQL_TEST_DSN must be set") +} + +fn table_name(prefix: &str) -> String { + let suffix = Uuid::new_v4().simple().to_string(); + format!("{prefix}_{}", &suffix[..16]) +} + +fn make_args(dsn: &str, table: &str, queue_dir: &str) -> MySqlArgs { + MySqlArgs { + enable: true, + dsn_string: dsn.to_string(), + table: table.to_string(), + format: "access".to_string(), + tls_ca: String::new(), + tls_client_cert: String::new(), + tls_client_key: String::new(), + queue_dir: queue_dir.to_string(), + queue_limit: 100, + max_open_connections: 2, + target_type: TargetType::NotifyEvent, + } +} + +fn make_entity(bucket: &str, object: &str, event_name: rustfs_s3_common::EventName) -> EntityTarget { + EntityTarget { + object_name: object.to_string(), + bucket_name: bucket.to_string(), + event_name, + data: serde_json::json!({"eventTime": "2026-05-03T10:00:00Z"}), + } +} + +fn build_test_pool(dsn_string: &str) -> Pool { + let parsed = MySqlDsn::parse(dsn_string).expect("parse test DSN"); + + let mut builder = OptsBuilder::default() + .user(Some(parsed.user)) + .pass(Some(parsed.password)) + .ip_or_hostname(parsed.host) + .tcp_port(parsed.port) + .db_name(Some(parsed.database)); + + if parsed.tls { + if rustls::crypto::CryptoProvider::get_default().is_none() { + rustls::crypto::aws_lc_rs::default_provider().install_default().ok(); + } + builder = builder.ssl_opts(Some(SslOpts::default())); + } + + Pool::new(Opts::from(builder)) +} + +async fn drop_table(dsn: &str, table: &str) { + let pool = build_test_pool(dsn); + let mut conn = pool.get_conn().await.expect("get conn for drop table"); + let _ = conn + .query_drop(format!("DROP TABLE IF EXISTS `{}`", table.replace('.', "`.`"))) + .await; +} + +#[ignore] +#[tokio::test] +async fn direct_write_and_read() { + let dsn = test_dsn(); + let table = table_name("test_direct"); + let target: MySqlTarget = + MySqlTarget::new("direct".to_string(), make_args(&dsn, &table, "")).expect("create target"); + + target.init().await.expect("init"); + + let entity = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectCreatedPut); + target.save(Arc::new(entity)).await.expect("save"); + + let pool = build_test_pool(&dsn); + let mut conn = pool.get_conn().await.expect("get conn"); + let rows: Vec = conn.query(format!("SELECT * FROM `{table}`")).await.expect("select"); + assert_eq!(rows.len(), 1); + + let data: String = mysql_async::from_value(rows[0].get(1).unwrap()); + assert!(data.contains("mybucket"), "event_data should contain bucket name, got: {data}"); + + drop_table(&dsn, &table).await; +} + +#[ignore] +#[tokio::test] +async fn delete_appends_row_does_not_remove_old() { + let dsn = test_dsn(); + let table = table_name("test_delete"); + let target: MySqlTarget = + MySqlTarget::new("delete".to_string(), make_args(&dsn, &table, "")).expect("create target"); + + target.init().await.expect("init"); + + let put = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectCreatedPut); + target.save(Arc::new(put)).await.expect("save put"); + + let delete = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectRemovedDelete); + target.save(Arc::new(delete)).await.expect("save delete"); + + let pool = build_test_pool(&dsn); + let mut conn = pool.get_conn().await.expect("get conn"); + let rows: Vec = conn.query(format!("SELECT * FROM `{table}`")).await.expect("select"); + assert_eq!(rows.len(), 2, "both PUT and DELETE should produce rows"); + + drop_table(&dsn, &table).await; +} + +#[ignore] +#[tokio::test] +async fn queue_store_saves_entry_and_replays() { + let dsn = test_dsn(); + let table = table_name("test_queue"); + let tmpdir = TempDir::new().expect("temp dir"); + let queue_dir = tmpdir.path().to_str().expect("valid path"); + + let target: MySqlTarget = + MySqlTarget::new("queue".to_string(), make_args(&dsn, &table, queue_dir)).expect("create target"); + + let entity = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectCreatedPut); + target.save(Arc::new(entity)).await.expect("save to queue"); + + let store = target.store().expect("store should exist"); + assert_eq!(store.len(), 1, "one entry should be in queue"); + + // Init creates the table; no rows should exist yet + target.init().await.expect("init"); + + { + let pool = build_test_pool(&dsn); + let mut conn = pool.get_conn().await.expect("get conn"); + let rows: Vec = conn.query(format!("SELECT * FROM `{table}`")).await.expect("select"); + assert_eq!(rows.len(), 0, "no row should exist before replay"); + } + + for key in store.list() { + target.send_from_store(key).await.expect("replay should succeed"); + } + + let pool = build_test_pool(&dsn); + let mut conn = pool.get_conn().await.expect("get conn"); + let rows: Vec = conn.query(format!("SELECT * FROM `{table}`")).await.expect("select"); + assert_eq!(rows.len(), 1, "one row should exist after replay"); + assert_eq!(store.len(), 0, "queue should be empty after replay"); + + drop_table(&dsn, &table).await; +} + +#[ignore] +#[tokio::test] +async fn duplicate_replay_produces_duplicate_rows() { + let dsn = test_dsn(); + let table = table_name("test_dupe"); + let tmpdir = TempDir::new().expect("temp dir"); + let queue_dir = tmpdir.path().to_str().expect("valid path"); + + let target: MySqlTarget = + MySqlTarget::new("dupe".to_string(), make_args(&dsn, &table, queue_dir)).expect("create target"); + + let entity = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectCreatedPut); + target.save(Arc::new(entity)).await.expect("save to queue"); + + target.init().await.expect("init"); + + let store = target.store().expect("store should exist"); + let keys: Vec<_> = store.list(); + + for key in &keys { + let raw = store.get_raw(key).expect("get raw"); + let queued = QueuedPayload::decode(&raw).expect("decode"); + + // Replay twice: duplicate rows are expected (at-least-once) + for _ in 0..2 { + target + .send_raw_from_store(key.clone(), queued.body.clone(), queued.meta.clone()) + .await + .expect("replay"); + } + let _ = store.del(key); + } + + let pool = build_test_pool(&dsn); + let mut conn = pool.get_conn().await.expect("get conn"); + let rows: Vec = conn.query(format!("SELECT * FROM `{table}`")).await.expect("select"); + assert_eq!(rows.len(), 2, "duplicate replay should produce 2 rows"); + + drop_table(&dsn, &table).await; +} + +#[ignore] +#[tokio::test] +async fn incompatible_schema_init_fails() { + let dsn = test_dsn(); + let table = table_name("test_schema"); + + { + let pool = build_test_pool(&dsn); + let mut conn = pool.get_conn().await.expect("get conn"); + conn.query_drop(format!("CREATE TABLE IF NOT EXISTS `{table}` (wrong_col INT NOT NULL)")) + .await + .expect("create incompatible table"); + } + + let target = MySqlTarget::::new("schema".to_string(), make_args(&dsn, &table, "")).expect("create target"); + + let result = target.init().await; + + match result { + Err(TargetError::Initialization(msg)) => { + assert!( + msg.contains("event_time") || msg.contains("event_data"), + "error should mention missing columns, got: {msg}" + ); + } + other => panic!("expected Initialization error, got {:?}", other), + } + + drop_table(&dsn, &table).await; +} diff --git a/crates/targets/tests/postgres_integration.rs b/crates/targets/tests/postgres_integration.rs new file mode 100644 index 000000000..33eae7fa4 --- /dev/null +++ b/crates/targets/tests/postgres_integration.rs @@ -0,0 +1,305 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Integration tests for the PostgreSQL notification target. +//! +//! All tests in this file are `#[ignore]` because they require a running +//! PostgreSQL server. CI does not run them by default. To run locally: +//! +//! ```bash +//! docker run -d --name rustfs-pg -p 5432:5432 \ +//! -e POSTGRES_PASSWORD=rustfs -e POSTGRES_DB=rustfs_events postgres:16 +//! export RUSTFS_TEST_PG_DSN="postgres://postgres:rustfs@localhost:5432/rustfs_events?search_path=public" +//! cargo test -p rustfs-targets --test postgres_integration -- --ignored +//! ``` +//! +//! Connection parameters can be overridden via environment variable: +//! `RUSTFS_TEST_PG_DSN`. + +use rustfs_s3_common::EventName; +use rustfs_targets::Target; +use rustfs_targets::check_postgres_server_available; +use rustfs_targets::target::EntityTarget; +use rustfs_targets::target::TargetType; +use rustfs_targets::target::postgres::{PostgresArgs, PostgresDsn, PostgresFormat, PostgresTarget}; +use serde_json::Value; +use std::sync::Arc; +use tokio_postgres::NoTls; +use url::Url; +use uuid::Uuid; + +fn env_or(key: &str, default: &str) -> String { + std::env::var(key).unwrap_or_else(|_| default.to_string()) +} + +fn test_args(table: &str, format: PostgresFormat) -> PostgresArgs { + let dsn = env_or( + "RUSTFS_TEST_PG_DSN", + "postgres://postgres:rustfs@localhost:5432/rustfs_events?search_path=public", + ); + let schema = PostgresDsn::parse(&dsn) + .expect("RUSTFS_TEST_PG_DSN must be a valid PostgreSQL DSN") + .schema; + PostgresArgs { + enable: true, + dsn_string: dsn, + schema, + table: table.to_string(), + format, + tls_required: false, + tls_ca: String::new(), + tls_client_cert: String::new(), + tls_client_key: String::new(), + queue_dir: String::new(), + queue_limit: 100_000, + target_type: TargetType::NotifyEvent, + } +} + +fn with_search_path(dsn: &str, schema: &str) -> String { + let mut url = Url::parse(dsn).expect("RUSTFS_TEST_PG_DSN must be a valid PostgreSQL DSN URL"); + url.query_pairs_mut().clear().append_pair("search_path", schema); + url.to_string() +} + +async fn raw_client(args: &PostgresArgs) -> tokio_postgres::Client { + let (client, connection) = tokio_postgres::connect(&args.dsn_string, NoTls) + .await + .expect("connect to postgres test server"); + tokio::spawn(async move { + let _ = connection.await; + }); + client +} + +fn unique_table(prefix: &str) -> String { + let suffix = Uuid::new_v4().simple().to_string(); + format!("{prefix}_{}", &suffix[..16]) +} + +fn entity_for(bucket: &str, object: &str) -> Arc> { + Arc::new(EntityTarget { + bucket_name: bucket.to_string(), + object_name: object.to_string(), + event_name: EventName::ObjectCreatedPut, + data: serde_json::json!({"bucket": bucket, "object": object}), + }) +} + +#[tokio::test] +#[ignore = "requires running PostgreSQL server"] +async fn test_check_postgres_server_available_with_existing_table() { + let args = test_args("pg_class", PostgresFormat::Namespace); + // Use a real existing table: pg_class always exists. + let mut args = args; + args.dsn_string = with_search_path(&args.dsn_string, "pg_catalog"); + args.schema = "pg_catalog".to_string(); + args.table = "pg_class".to_string(); + + check_postgres_server_available(&args) + .await + .expect("connectivity probe should succeed against pg_catalog.pg_class"); +} + +#[tokio::test] +#[ignore = "requires running PostgreSQL server"] +async fn test_check_postgres_server_available_missing_table_fails() { + let args = test_args("does_not_exist_table_xyz", PostgresFormat::Namespace); + let result = check_postgres_server_available(&args).await; + assert!(result.is_err(), "missing table should fail the probe"); +} + +#[tokio::test] +#[ignore = "requires running PostgreSQL server"] +async fn test_namespace_format_upsert_replaces_value() { + let table = unique_table("rustfs_test_namespace"); + let args = test_args(&table, PostgresFormat::Namespace); + + // Setup: create the namespace table. + let setup = raw_client(&args).await; + setup + .execute( + &format!("CREATE TABLE \"{}\" (key VARCHAR PRIMARY KEY, value JSONB NOT NULL)", table), + &[], + ) + .await + .expect("create namespace table"); + + // Build target and deliver two events for the same key. + let target: PostgresTarget = + PostgresTarget::new("test_namespace".to_string(), args.clone()).expect("construct target"); + + target + .save(entity_for("bucket1", "obj-A")) + .await + .expect("first save should succeed"); + target + .save(entity_for("bucket1", "obj-A")) + .await + .expect("second save should succeed"); + + // Verify only one row exists for the key (UPSERT collapsed). + let row = setup + .query_one(&format!("SELECT count(*)::bigint FROM \"{}\" WHERE key = $1", table), &[&"bucket1/obj-A"]) + .await + .expect("count rows"); + let count: i64 = row.get(0); + assert_eq!(count, 1, "namespace format should keep only one row per key"); + + // Cleanup. + setup + .execute(&format!("DROP TABLE \"{}\"", table), &[]) + .await + .expect("drop namespace table"); +} + +#[tokio::test] +#[ignore = "requires running PostgreSQL server"] +async fn test_access_format_appends_distinct_events() { + let table = unique_table("rustfs_test_access"); + let args = test_args(&table, PostgresFormat::Access); + + let setup = raw_client(&args).await; + setup + .execute( + &format!( + "CREATE TABLE \"{}\" (\ + event_id TEXT PRIMARY KEY, \ + event_time TIMESTAMPTZ NOT NULL DEFAULT NOW(), \ + event_name TEXT NOT NULL, \ + key TEXT NOT NULL, \ + value JSONB NOT NULL, \ + queued_at_ms BIGINT NOT NULL\ + )", + table + ), + &[], + ) + .await + .expect("create access table"); + + let target: PostgresTarget = + PostgresTarget::new("test_access".to_string(), args.clone()).expect("construct target"); + + // Two distinct events for different objects produce two rows. + target.save(entity_for("bucket1", "obj-A")).await.expect("save A"); + target.save(entity_for("bucket1", "obj-B")).await.expect("save B"); + + let row = setup + .query_one(&format!("SELECT count(*)::bigint FROM \"{}\"", table), &[]) + .await + .expect("count rows"); + let count: i64 = row.get(0); + assert_eq!(count, 2, "access format should append two distinct rows"); + + setup + .execute(&format!("DROP TABLE \"{}\"", table), &[]) + .await + .expect("drop access table"); +} + +#[tokio::test] +#[ignore = "requires running PostgreSQL server"] +async fn test_access_format_replay_is_idempotent() { + let table = unique_table("rustfs_test_access_replay"); + let args = test_args(&table, PostgresFormat::Access); + + let setup = raw_client(&args).await; + setup + .execute( + &format!( + "CREATE TABLE \"{}\" (\ + event_id TEXT PRIMARY KEY, \ + event_time TIMESTAMPTZ NOT NULL DEFAULT NOW(), \ + event_name TEXT NOT NULL, \ + key TEXT NOT NULL, \ + value JSONB NOT NULL, \ + queued_at_ms BIGINT NOT NULL\ + )", + table + ), + &[], + ) + .await + .expect("create access table"); + + // Insert the same row twice with the same event_id via direct SQL — this + // simulates store replay where send_raw_from_store is called twice with + // the same Key. + let event_id = Uuid::new_v4().to_string(); + let payload: Value = serde_json::json!({"EventName": "s3:ObjectCreated:Put", "Key": "bucket1/obj-A", "Records": []}); + let queued_at_ms: i64 = 1234567890; + + let sql = format!( + "INSERT INTO \"{}\" (event_id, event_name, key, value, queued_at_ms) \ + VALUES ($1, $2, $3, $4::jsonb, $5) ON CONFLICT (event_id) DO NOTHING", + table + ); + setup + .execute(&sql, &[&event_id, &"s3:ObjectCreated:Put", &"bucket1/obj-A", &payload, &queued_at_ms]) + .await + .expect("first insert"); + setup + .execute(&sql, &[&event_id, &"s3:ObjectCreated:Put", &"bucket1/obj-A", &payload, &queued_at_ms]) + .await + .expect("second insert (should be silent skip)"); + + let row = setup + .query_one(&format!("SELECT count(*)::bigint FROM \"{}\"", table), &[]) + .await + .expect("count rows"); + let count: i64 = row.get(0); + assert_eq!(count, 1, "ON CONFLICT (event_id) DO NOTHING should make replay idempotent"); + + setup + .execute(&format!("DROP TABLE \"{}\"", table), &[]) + .await + .expect("drop access table"); +} + +#[tokio::test] +#[ignore = "requires running PostgreSQL server"] +async fn test_init_succeeds_against_existing_table() { + let table = unique_table("rustfs_test_init"); + let args = test_args(&table, PostgresFormat::Namespace); + + let setup = raw_client(&args).await; + setup + .execute( + &format!("CREATE TABLE \"{}\" (key VARCHAR PRIMARY KEY, value JSONB NOT NULL)", table), + &[], + ) + .await + .expect("create table"); + + let target: PostgresTarget = + PostgresTarget::new("test_init".to_string(), args.clone()).expect("construct target"); + target.init().await.expect("init should succeed against existing table"); + target.close().await.expect("close should succeed"); + + setup + .execute(&format!("DROP TABLE \"{}\"", table), &[]) + .await + .expect("drop table"); +} + +#[tokio::test] +async fn test_invalid_identifier_rejected_at_construction() { + // No #[ignore] — pure validation, no DB needed. + let args = test_args("malicious; DROP TABLE users", PostgresFormat::Namespace); + match PostgresTarget::::new("bad_id".to_string(), args) { + Ok(_) => panic!("malicious table identifier must fail at construction"), + Err(e) => assert!(e.to_string().contains("table"), "unexpected error: {e}"), + } +} diff --git a/rustfs/src/admin/handlers/audit.rs b/rustfs/src/admin/handlers/audit.rs index b7b5b2bab..c6ed8abe1 100644 --- a/rustfs/src/admin/handlers/audit.rs +++ b/rustfs/src/admin/handlers/audit.rs @@ -16,7 +16,7 @@ use crate::admin::{ auth::validate_admin_request, handlers::target_descriptor::{ AdminTargetSpec, AdminTargetValidator, EndpointKey, TargetDomain, TargetEndpointSource, allowed_target_keys, - collect_validated_key_values as shared_collect_validated_key_values, + build_json_response, collect_validated_key_values as shared_collect_validated_key_values, merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason, target_mutation_block_reason as shared_target_mutation_block_reason, target_service_name, target_spec, validate_target_request, @@ -28,18 +28,20 @@ use crate::server::{ ADMIN_PREFIX, RemoteAddr, is_audit_module_enabled, refresh_audit_module_enabled, refresh_persisted_module_switches_from_store, }; use futures::stream::{FuturesUnordered, StreamExt}; -use http::{HeaderMap, StatusCode}; +use http::StatusCode; use hyper::Method; use matchit::Params; use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState}; use rustfs_config::audit::{ - AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS, - AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS, AUDIT_ROUTE_PREFIX, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS, + AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_MYSQL_KEYS, AUDIT_MYSQL_SUB_SYS, + AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS, AUDIT_POSTGRES_KEYS, AUDIT_POSTGRES_SUB_SYS, AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS, + AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_REDIS_KEYS, AUDIT_REDIS_SUB_SYS, AUDIT_ROUTE_PREFIX, AUDIT_WEBHOOK_KEYS, + AUDIT_WEBHOOK_SUB_SYS, }; use rustfs_config::{AUDIT_DEFAULT_DIR, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE}; use rustfs_ecstore::config::Config; use rustfs_policy::policy::action::{Action, AdminAction}; -use s3s::{Body, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error}; +use s3s::{Body, S3Request, S3Response, S3Result, s3_error}; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -93,7 +95,7 @@ struct AuditEndpointsResponse { audit_endpoints: Vec, } -fn audit_target_specs() -> [AdminTargetSpec; 5] { +fn audit_target_specs() -> [AdminTargetSpec; 8] { [ AdminTargetSpec { subsystem: AUDIT_WEBHOOK_SUB_SYS, @@ -113,18 +115,36 @@ fn audit_target_specs() -> [AdminTargetSpec; 5] { valid_keys: AUDIT_MQTT_KEYS, validator: AdminTargetValidator::Mqtt, }, + AdminTargetSpec { + subsystem: AUDIT_MYSQL_SUB_SYS, + service: "mysql", + valid_keys: AUDIT_MYSQL_KEYS, + validator: AdminTargetValidator::MySql, + }, AdminTargetSpec { subsystem: AUDIT_NATS_SUB_SYS, service: "nats", valid_keys: AUDIT_NATS_KEYS, validator: AdminTargetValidator::Nats(TargetDomain::Audit), }, + AdminTargetSpec { + subsystem: AUDIT_POSTGRES_SUB_SYS, + service: "postgres", + valid_keys: AUDIT_POSTGRES_KEYS, + validator: AdminTargetValidator::Postgres(TargetDomain::Audit), + }, AdminTargetSpec { subsystem: AUDIT_PULSAR_SUB_SYS, service: "pulsar", valid_keys: AUDIT_PULSAR_KEYS, validator: AdminTargetValidator::Pulsar(TargetDomain::Audit), }, + AdminTargetSpec { + subsystem: AUDIT_REDIS_SUB_SYS, + service: "redis", + valid_keys: AUDIT_REDIS_KEYS, + validator: AdminTargetValidator::Redis(TargetDomain::Audit, AUDIT_REDIS_DEFAULT_CHANNEL), + }, ] } @@ -138,15 +158,6 @@ async fn authorize_audit_admin_request(req: &S3Request, action: AdminActio validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await } -fn build_response(status: StatusCode, body: Body, request_id: Option<&http::HeaderValue>) -> S3Response<(StatusCode, Body)> { - let mut header = HeaderMap::new(); - header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); - if let Some(v) = request_id { - header.insert("x-request-id", v.clone()); - } - S3Response::with_headers((status, body), header) -} - fn has_any_audit_targets(config: &Config) -> bool { for spec in audit_target_specs() { let Some(targets) = config.0.get(spec.subsystem) else { @@ -334,7 +345,7 @@ impl Operation for AuditTargetConfig { }) .await?; - Ok(build_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id"))) + Ok(build_json_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id"))) } } @@ -375,7 +386,7 @@ impl Operation for ListAuditTargets { let data = serde_json::to_vec(&AuditEndpointsResponse { audit_endpoints }) .map_err(|e| s3_error!(InternalError, "failed to serialize audit targets: {}", e))?; - Ok(build_response(StatusCode::OK, Body::from(data), req.headers.get("x-request-id"))) + Ok(build_json_response(StatusCode::OK, Body::from(data), req.headers.get("x-request-id"))) } } @@ -411,7 +422,7 @@ impl Operation for RemoveAuditTarget { }) .await?; - Ok(build_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id"))) + Ok(build_json_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id"))) } } diff --git a/rustfs/src/admin/handlers/event.rs b/rustfs/src/admin/handlers/event.rs index c363b2af4..bd58e567c 100644 --- a/rustfs/src/admin/handlers/event.rs +++ b/rustfs/src/admin/handlers/event.rs @@ -16,7 +16,7 @@ use crate::admin::{ auth::validate_admin_request, handlers::target_descriptor::{ AdminTargetSpec, AdminTargetValidator, EndpointKey, TargetDomain, TargetEndpointSource, allowed_target_keys, - collect_validated_key_values as shared_collect_validated_key_values, + build_json_response, collect_validated_key_values as shared_collect_validated_key_values, merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason, target_mutation_block_reason as shared_target_mutation_block_reason, target_service_name, target_spec, validate_target_request, @@ -29,17 +29,19 @@ use crate::server::{ refresh_persisted_module_switches_from_store, }; use futures::stream::{FuturesUnordered, StreamExt}; -use http::{HeaderMap, StatusCode}; +use http::StatusCode; use hyper::Method; use matchit::Params; use rustfs_config::notify::{ - NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, - NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_ROUTE_PREFIX, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS, + NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_KEYS, NOTIFY_MYSQL_SUB_SYS, + NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_KEYS, NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_KEYS, + NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_DEFAULT_CHANNEL, NOTIFY_REDIS_KEYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_ROUTE_PREFIX, + NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS, }; use rustfs_config::{ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE}; use rustfs_ecstore::config::Config; use rustfs_policy::policy::action::{Action, AdminAction}; -use s3s::{Body, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error}; +use s3s::{Body, S3Request, S3Response, S3Result, s3_error}; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -99,7 +101,7 @@ struct NotificationEndpointsResponse { notification_endpoints: Vec, } -fn notification_target_specs() -> [AdminTargetSpec; 5] { +fn notification_target_specs() -> [AdminTargetSpec; 8] { [ AdminTargetSpec { subsystem: NOTIFY_WEBHOOK_SUB_SYS, @@ -119,12 +121,30 @@ fn notification_target_specs() -> [AdminTargetSpec; 5] { valid_keys: NOTIFY_MQTT_KEYS, validator: AdminTargetValidator::Mqtt, }, + AdminTargetSpec { + subsystem: NOTIFY_MYSQL_SUB_SYS, + service: "mysql", + valid_keys: NOTIFY_MYSQL_KEYS, + validator: AdminTargetValidator::MySql, + }, AdminTargetSpec { subsystem: NOTIFY_NATS_SUB_SYS, service: "nats", valid_keys: NOTIFY_NATS_KEYS, validator: AdminTargetValidator::Nats(TargetDomain::Notify), }, + AdminTargetSpec { + subsystem: NOTIFY_POSTGRES_SUB_SYS, + service: "postgres", + valid_keys: NOTIFY_POSTGRES_KEYS, + validator: AdminTargetValidator::Postgres(TargetDomain::Notify), + }, + AdminTargetSpec { + subsystem: NOTIFY_REDIS_SUB_SYS, + service: "redis", + valid_keys: NOTIFY_REDIS_KEYS, + validator: AdminTargetValidator::Redis(TargetDomain::Notify, NOTIFY_REDIS_DEFAULT_CHANNEL), + }, AdminTargetSpec { subsystem: NOTIFY_PULSAR_SUB_SYS, service: "pulsar", @@ -150,15 +170,6 @@ fn get_notification_system() -> S3Result> rustfs_notify::notification_system().ok_or_else(|| s3_error!(InternalError, "notification system not initialized")) } -fn build_response(status: StatusCode, body: Body, request_id: Option<&http::HeaderValue>) -> S3Response<(StatusCode, Body)> { - let mut header = HeaderMap::new(); - header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); - if let Some(v) = request_id { - header.insert("x-request-id", v.clone()); - } - S3Response::with_headers((status, body), header) -} - fn target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option { shared_target_mutation_block_reason( ¬ification_target_specs(), @@ -258,7 +269,7 @@ impl Operation for NotificationTarget { .await .map_err(|e| s3_error!(InternalError, "failed to set target config: {}", e))?; - Ok(build_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id"))) + Ok(build_json_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id"))) } } @@ -297,7 +308,7 @@ impl Operation for ListNotificationTargets { let data = serde_json::to_vec(&NotificationEndpointsResponse { notification_endpoints }) .map_err(|e| s3_error!(InternalError, "failed to serialize targets: {}", e))?; - Ok(build_response(StatusCode::OK, Body::from(data), req.headers.get("x-request-id"))) + Ok(build_json_response(StatusCode::OK, Body::from(data), req.headers.get("x-request-id"))) } } @@ -347,7 +358,7 @@ impl Operation for ListTargetsArns { let data = serde_json::to_vec(&data_target_arn_list) .map_err(|e| s3_error!(InternalError, "failed to serialize targets: {}", e))?; - Ok(build_response(StatusCode::OK, Body::from(data), req.headers.get("x-request-id"))) + Ok(build_json_response(StatusCode::OK, Body::from(data), req.headers.get("x-request-id"))) } } @@ -374,7 +385,7 @@ impl Operation for RemoveNotificationTarget { .await .map_err(|e| s3_error!(InternalError, "failed to remove target config: {}", e))?; - Ok(build_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id"))) + Ok(build_json_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id"))) } } diff --git a/rustfs/src/admin/handlers/target_descriptor.rs b/rustfs/src/admin/handlers/target_descriptor.rs index 686953fad..31a641bfc 100644 --- a/rustfs/src/admin/handlers/target_descriptor.rs +++ b/rustfs/src/admin/handlers/target_descriptor.rs @@ -13,19 +13,23 @@ // limitations under the License. use hashbrown::HashSet as HbHashSet; +use http::{HeaderMap, HeaderValue, StatusCode}; use rustfs_config::{ ENABLE_KEY, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_TOPIC, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, - MQTT_WS_PATH_ALLOWLIST, + MQTT_WS_PATH_ALLOWLIST, MYSQL_QUEUE_DIR, POSTGRES_QUEUE_DIR, REDIS_QUEUE_DIR, }; use rustfs_ecstore::config::Config; use rustfs_targets::{ TargetError, check_kafka_broker_available, check_mqtt_broker_available_with_tls, check_nats_server_available, - check_pulsar_broker_available, - config::{build_kafka_args, build_nats_args, build_pulsar_args, collect_env_target_instance_ids}, + check_postgres_server_available, check_pulsar_broker_available, check_redis_server_available, + config::{ + build_kafka_args, build_nats_args, build_postgres_args, build_pulsar_args, build_redis_args, + collect_env_target_instance_ids, validate_mysql_config, validate_redis_config, + }, target::{TargetType, mqtt::MQTTTlsConfig}, }; -use s3s::{S3Result, s3_error}; +use s3s::{Body, S3Response, S3Result, header::CONTENT_TYPE, s3_error}; use serde::Serialize; use std::collections::{HashMap, HashSet}; use std::io::{Error, ErrorKind}; @@ -71,8 +75,11 @@ pub(crate) enum AdminTargetValidator { Webhook, Mqtt, Kafka(TargetDomain), + MySql, Nats(TargetDomain), + Postgres(TargetDomain), Pulsar(TargetDomain), + Redis(TargetDomain, &'static str), } #[derive(Clone, Copy)] @@ -201,6 +208,19 @@ pub(crate) fn target_module_disabled_reason(module_name: &str, env_key: &str, en }) } +pub(crate) fn build_json_response( + status: StatusCode, + body: Body, + request_id: Option<&HeaderValue>, +) -> S3Response<(StatusCode, Body)> { + let mut header = HeaderMap::new(); + header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + if let Some(v) = request_id { + header.insert("x-request-id", v.clone()); + } + S3Response::with_headers((status, body), header) +} + pub(crate) fn merge_target_endpoints( specs: &[AdminTargetSpec], route_prefix: &str, @@ -336,8 +356,13 @@ pub(crate) async fn validate_target_request( AdminTargetValidator::Webhook => validate_webhook_request(kv_map).await, AdminTargetValidator::Mqtt => validate_mqtt_request(kv_map).await, AdminTargetValidator::Kafka(domain) => validate_kafka_request(kv_map, default_queue_dir, domain).await, + AdminTargetValidator::MySql => validate_mysql_request(kv_map, default_queue_dir).await, AdminTargetValidator::Nats(domain) => validate_nats_request(kv_map, default_queue_dir, domain).await, AdminTargetValidator::Pulsar(domain) => validate_pulsar_request(kv_map, default_queue_dir, domain).await, + AdminTargetValidator::Postgres(domain) => validate_postgres_request(kv_map, default_queue_dir, domain).await, + AdminTargetValidator::Redis(domain, default_channel) => { + validate_redis_request(kv_map, default_queue_dir, domain, default_channel).await + } } } @@ -487,6 +512,53 @@ async fn validate_pulsar_request( }) } +async fn validate_mysql_request(kv_map: &HashMap, default_queue_dir: &str) -> S3Result<()> { + if let Some(queue_dir) = kv_map.get(MYSQL_QUEUE_DIR) { + validate_queue_dir(queue_dir.as_str()).await?; + } + + validate_mysql_config(&to_kvs(kv_map), default_queue_dir).map_err(|e| s3_error!(InvalidArgument, "{}", e))?; + + Ok(()) +} + +async fn validate_postgres_request( + kv_map: &HashMap, + default_queue_dir: &str, + domain: TargetDomain, +) -> S3Result<()> { + if let Some(queue_dir) = kv_map.get(POSTGRES_QUEUE_DIR) { + validate_queue_dir(queue_dir.as_str()).await?; + } + let args = build_postgres_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type()) + .map_err(|e| s3_error!(InvalidArgument, "{}", e))?; + check_postgres_server_available(&args).await.map_err(|e| match e { + TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e), + _ => s3_error!(InvalidArgument, "PostgreSQL server check failed: {}", e), + }) +} + +async fn validate_redis_request( + kv_map: &HashMap, + default_queue_dir: &str, + domain: TargetDomain, + default_channel: &str, +) -> S3Result<()> { + if let Some(queue_dir) = kv_map.get(REDIS_QUEUE_DIR) { + validate_queue_dir(queue_dir.as_str()).await?; + } + + validate_redis_config(&to_kvs(kv_map), default_queue_dir, default_channel) + .map_err(|e| s3_error!(InvalidArgument, "{}", e))?; + + let args = build_redis_args(&to_kvs(kv_map), default_queue_dir, default_channel, domain.runtime_target_type()) + .map_err(|e| s3_error!(InvalidArgument, "{}", e))?; + check_redis_server_available(&args).await.map_err(|e| match e { + TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e), + _ => s3_error!(InvalidArgument, "Redis server check failed: {}", e), + }) +} + fn to_kvs(kv_map: &HashMap) -> rustfs_ecstore::config::KVS { let mut kvs = rustfs_ecstore::config::KVS::new(); for (key, value) in kv_map {