From 81ad48dac2067df374d7491b908a8eb0d7b5636f Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 9 May 2026 09:56:26 +0800 Subject: [PATCH] feat(targets): add AMQP support for notify and audit (#2879) Co-authored-by: Hyesook Yun <74169420+suk13574@users.noreply.github.com> --- Cargo.lock | 636 +++++++++++++--- Cargo.toml | 3 +- crates/audit/src/factory.rs | 343 ++++----- crates/audit/src/registry.rs | 106 +-- crates/config/src/audit/amqp.rs | 60 ++ crates/config/src/audit/mod.rs | 6 +- crates/config/src/constants/targets.rs | 13 + crates/config/src/notify/amqp.rs | 60 ++ crates/config/src/notify/mod.rs | 4 +- crates/ecstore/src/config/audit.rs | 93 ++- crates/ecstore/src/config/com.rs | 212 +++++- crates/ecstore/src/config/mod.rs | 8 +- crates/ecstore/src/config/notify.rs | 93 ++- crates/notify/src/factory.rs | 351 ++++----- crates/notify/src/integration.rs | 12 +- crates/notify/src/registry.rs | 91 +-- crates/targets/Cargo.toml | 5 +- crates/targets/src/check.rs | 20 + crates/targets/src/config/mod.rs | 7 +- crates/targets/src/config/target_args.rs | 223 +++++- crates/targets/src/lib.rs | 6 +- crates/targets/src/plugin.rs | 231 ++++++ crates/targets/src/target/amqp.rs | 706 ++++++++++++++++++ crates/targets/src/target/mod.rs | 15 +- crates/targets/tests/amqp_integration.rs | 221 ++++++ rustfs/src/admin/handlers/audit.rs | 138 ++-- rustfs/src/admin/handlers/event.rs | 168 +++-- .../src/admin/handlers/target_descriptor.rs | 270 ++++++- rustfs/src/server/audit.rs | 9 +- 29 files changed, 3280 insertions(+), 830 deletions(-) create mode 100644 crates/config/src/audit/amqp.rs create mode 100644 crates/config/src/notify/amqp.rs create mode 100644 crates/targets/src/plugin.rs create mode 100644 crates/targets/src/target/amqp.rs create mode 100644 crates/targets/tests/amqp_integration.rs diff --git a/Cargo.lock b/Cargo.lock index 69d1be16e..dda1fe56f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,7 +40,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b657e772794c6b04730ea897b66a058ccd866c16d1967da05eeeecec39043fe" dependencies = [ "crypto-common 0.2.1", - "inout", + "inout 0.2.2", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher 0.4.4", + "cpufeatures 0.2.17", ] [[package]] @@ -49,7 +60,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66bd29a732b644c0431c6140f370d097879203d79b80c94a6747ba0872adaef8" dependencies = [ - "cipher", + "cipher 0.5.1", "cpubits", "cpufeatures 0.3.0", ] @@ -61,8 +72,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e22c0c90bbe8d4f77c3ca9ddabe41a1f8382d6fc1f7cea89459d0f320371f972" dependencies = [ "aead", - "aes", - "cipher", + "aes 0.9.0", + "cipher 0.5.1", "ctr", "ghash", "subtle", @@ -141,6 +152,56 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "amq-protocol" +version = "10.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5b2e8843d88d935a75cbdb1c5b9ad80987da6e6472c2703914e41dc14560990" +dependencies = [ + "amq-protocol-tcp", + "amq-protocol-types", + "amq-protocol-uri", + "cookie-factory", + "nom 8.0.0", + "serde", +] + +[[package]] +name = "amq-protocol-tcp" +version = "10.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "998fb81655e11de5a336bb609042c11633678fc01f90b0772fb9a7886b6cc4c2" +dependencies = [ + "amq-protocol-uri", + "async-rs", + "cfg-if", + "tcp-stream", + "tracing", +] + +[[package]] +name = "amq-protocol-types" +version = "10.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee5b3a9e458bd2e452536995c8cf861b01c17283f55c4bbe0a1b9626b8253add" +dependencies = [ + "cookie-factory", + "nom 8.0.0", + "serde", + "serde_json", +] + +[[package]] +name = "amq-protocol-uri" +version = "10.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca3316970d20cdcca9123f4e8feb7a2c1c8fdca572a9692fc10002db35407aa" +dependencies = [ + "amq-protocol-types", + "percent-encoding", + "url", +] + [[package]] name = "android_system_properties" version = "0.1.5" @@ -551,6 +612,19 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-compat" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ba85bc55464dcbf728b56d97e119d673f4cf9062be330a9a26f3acf504a590" +dependencies = [ + "futures-core", + "futures-io", + "once_cell", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-compression" version = "0.4.42" @@ -563,6 +637,34 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-executor" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "pin-project-lite", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13f937e26114b93193065fd44f507aa2e9169ad0cdabbb996920b1fe1ddea7ba" +dependencies = [ + "async-channel", + "async-executor", + "async-lock", + "blocking", + "futures-lite", + "tokio", +] + [[package]] name = "async-lock" version = "3.4.2" @@ -621,6 +723,28 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "async-rs" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e32bd31386d41d0c06bd79b0397ec96e544d69d9dbd6db0236c7ceefe1ad61b" +dependencies = [ + "async-compat", + "async-global-executor", + "async-trait", + "futures-core", + "futures-io", + "hickory-resolver", + "tokio", + "tokio-stream", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + [[package]] name = "async-trait" version = "0.1.89" @@ -1331,6 +1455,28 @@ dependencies = [ "zeroize", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + +[[package]] +name = "blocking" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + [[package]] name = "bon" version = "3.9.1" @@ -1487,10 +1633,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] -name = "cc" -version = "1.2.61" +name = "cbc" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher 0.4.4", +] + +[[package]] +name = "cc" +version = "1.2.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" dependencies = [ "find-msvc-tools", "jobserver", @@ -1517,7 +1672,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ "cfg-if", - "cipher", + "cipher 0.5.1", "cpufeatures 0.3.0", "rand_core 0.10.1", ] @@ -1530,7 +1685,7 @@ checksum = "1c9ed179664f12fd6f155f6dd632edf5f3806d48c228c67ff78366f2a0eb6b5e" dependencies = [ "aead", "chacha20", - "cipher", + "cipher 0.5.1", "poly1305", ] @@ -1585,6 +1740,16 @@ dependencies = [ "half", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout 0.1.4", +] + [[package]] name = "cipher" version = "0.5.1" @@ -1593,7 +1758,7 @@ checksum = "e34d8227fe1ba289043aeb13792056ff80fd6de1a9f49137a5f499de8e8c78ea" dependencies = [ "block-buffer 0.12.0", "crypto-common 0.2.1", - "inout", + "inout 0.2.2", ] [[package]] @@ -1662,6 +1827,18 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746" +[[package]] +name = "cms" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b77c319abfd5219629c45c34c89ba945ed3c5e49fcde9d16b6c3885f118a730" +dependencies = [ + "const-oid 0.9.6", + "der 0.7.10", + "spki 0.7.3", + "x509-cert", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -1820,6 +1997,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "cookie-factory" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2" + [[package]] name = "core-foundation" version = "0.9.4" @@ -1959,6 +2142,12 @@ dependencies = [ "itertools 0.13.0", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -2048,9 +2237,9 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", "typenum", @@ -2105,7 +2294,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17469f8eb9bdbfad10f71f4cfddfd38b01143520c0e717d8796ccb4d44d44e42" dependencies = [ - "cipher", + "cipher 0.5.1", ] [[package]] @@ -3159,6 +3348,15 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "des" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdd80ce8ce993de27e9f063a444a4d53ce8e8db4c1f00cc03af5ad5a9867a1e" +dependencies = [ + "cipher 0.4.4", +] + [[package]] name = "dial9-macro" version = "0.3.6" @@ -3232,7 +3430,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid 0.9.6", - "crypto-common 0.1.6", + "crypto-common 0.1.7", "subtle", ] @@ -3826,6 +4024,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "futures-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" +dependencies = [ + "futures-io", + "rustls", + "rustls-pki-types", +] + [[package]] name = "futures-sink" version = "0.3.32" @@ -3857,9 +4066,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.9" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -4350,6 +4559,76 @@ dependencies = [ "vsimd", ] +[[package]] +name = "hickory-net" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "futures-channel", + "futures-io", + "futures-util", + "hickory-proto", + "idna", + "ipnet", + "jni", + "rand 0.10.1", + "thiserror 2.0.18", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "prefix-trie", + "rand 0.10.1", + "ring", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-net", + "hickory-proto", + "ipconfig", + "ipnet", + "jni", + "moka", + "ndk-context", + "once_cell", + "parking_lot 0.12.5", + "rand 0.10.1", + "resolv-conf", + "smallvec", + "system-configuration", + "thiserror 2.0.18", + "tokio", + "tracing", +] + [[package]] name = "highway" version = "1.3.0" @@ -4771,6 +5050,16 @@ dependencies = [ "str_stack", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "inout" version = "0.2.2" @@ -4806,11 +5095,27 @@ dependencies = [ "libc", ] +[[package]] +name = "ipconfig" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" +dependencies = [ + "socket2", + "widestring", + "windows-registry", + "windows-result", + "windows-sys 0.61.2", +] + [[package]] name = "ipnet" version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +dependencies = [ + "serde", +] [[package]] name = "ipnetwork" @@ -5062,6 +5367,24 @@ version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" +[[package]] +name = "lapin" +version = "4.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b2e572c434103775d7598b09b2385e02f1ea8ff18cc452335591b5f1cbff12a" +dependencies = [ + "amq-protocol", + "async-rs", + "async-trait", + "atomic-waker", + "backon", + "cfg-if", + "flume", + "futures-core", + "futures-io", + "tracing", +] + [[package]] name = "lazy-regex" version = "3.6.0" @@ -5825,6 +6148,12 @@ dependencies = [ "rand 0.8.6", ] +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "neli" version = "0.7.4" @@ -6277,6 +6606,10 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -6463,6 +6796,29 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" +[[package]] +name = "p12-keystore" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb9bf5222606eb712d3bb30e01bc9420545b00859970897e70c682353a034f2" +dependencies = [ + "base64 0.22.1", + "cbc", + "cms", + "der 0.7.10", + "des", + "hex", + "hmac 0.12.1", + "pkcs12", + "pkcs5", + "rand 0.10.1", + "rc2", + "sha1 0.10.6", + "sha2 0.10.9", + "thiserror 2.0.18", + "x509-parser", +] + [[package]] name = "p256" version = "0.11.1" @@ -6638,6 +6994,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest 0.10.7", + "hmac 0.12.1", +] + [[package]] name = "pbkdf2" version = "0.13.0" @@ -6797,6 +7163,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + [[package]] name = "pkcs1" version = "0.7.5" @@ -6818,6 +7195,36 @@ dependencies = [ "spki 0.8.0", ] +[[package]] +name = "pkcs12" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "695b3df3d3cc1015f12d70235e35b6b79befc5fa7a9b95b951eab1dd07c9efc2" +dependencies = [ + "cms", + "const-oid 0.9.6", + "der 0.7.10", + "digest 0.10.7", + "spki 0.7.3", + "x509-cert", + "zeroize", +] + +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes 0.8.4", + "cbc", + "der 0.7.10", + "pbkdf2 0.12.2", + "scrypt", + "sha2 0.10.9", + "spki 0.7.3", +] + [[package]] name = "pkcs8" version = "0.9.0" @@ -7033,6 +7440,17 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prefix-trie" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f561214012d3fc240a1f9c817cc4d57f5310910d066069c1b093f766bb5966" +dependencies = [ + "either", + "ipnet", + "num-traits", +] + [[package]] name = "pretty_assertions" version = "1.4.1" @@ -7461,7 +7879,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -7630,6 +8048,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rc2" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62c64daa8e9438b84aaae55010a93f396f8e60e3911590fcba770d04643fc1dd" +dependencies = [ + "cipher 0.4.4", +] + [[package]] name = "rcgen" version = "0.14.7" @@ -7919,6 +8346,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "resolv-conf" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" + [[package]] name = "rfc6979" version = "0.3.1" @@ -8357,7 +8790,7 @@ dependencies = [ "argon2", "chacha20poly1305", "jsonwebtoken", - "pbkdf2", + "pbkdf2 0.13.0", "rand 0.10.1", "serde_json", "sha2 0.11.0", @@ -8588,7 +9021,7 @@ dependencies = [ "indexmap 2.14.0", "kafka-protocol", "metrics", - "pbkdf2", + "pbkdf2 0.13.0", "rand 0.10.1", "rustls", "rustls-native-certs", @@ -8611,7 +9044,7 @@ dependencies = [ "hmac 0.13.0", "kafka-protocol", "metrics", - "pbkdf2", + "pbkdf2 0.13.0", "rand 0.10.1", "rustfs-kafka", "rustls", @@ -9042,8 +9475,11 @@ dependencies = [ "chrono", "criterion", "deadpool-postgres", + "hashbrown 0.17.0", "hyper-rustls", + "lapin", "mysql_async", + "parking_lot 0.12.5", "pulsar", "redis", "reqwest 0.13.3", @@ -9242,6 +9678,21 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-connector" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26bcb6901a3319d57589047c0da93a0f3228f13abf8dd949deef024749cb5e2" +dependencies = [ + "futures-io", + "futures-rustls", + "log", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "rustls-webpki", +] + [[package]] name = "rustls-native-certs" version = "0.8.3" @@ -9375,6 +9826,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher 0.4.4", +] + [[package]] name = "same-file" version = "1.0.6" @@ -9438,6 +9898,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2 0.12.2", + "salsa20", + "sha2 0.10.9", +] + [[package]] name = "sdd" version = "3.0.10" @@ -10312,6 +10783,19 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" +[[package]] +name = "tcp-stream" +version = "0.34.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993be82e36e4545bdf988701c9fbbde2a452f9e85bc2b91729c7df4e43a80ad3" +dependencies = [ + "async-rs", + "cfg-if", + "futures-io", + "p12-keystore", + "rustls-connector", +] + [[package]] name = "temp-env" version = "0.3.6" @@ -10569,9 +11053,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.2" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "110a78583f19d5cdb2c5ccf321d1290344e71313c6c37d43520d386027d18386" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "io-uring", @@ -11444,6 +11928,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + [[package]] name = "wildmatch" version = "2.6.1" @@ -11602,7 +12092,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -11611,16 +12101,7 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", + "windows-targets", ] [[package]] @@ -11638,31 +12119,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -11680,96 +12144,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" version = "1.0.2" @@ -11933,9 +12349,9 @@ dependencies = [ [[package]] name = "xml" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8aa498d22c9bbaf482329839bc5620c46be275a19a812e9a22a2b07529a642a" +checksum = "636f85e5ca6488e96401b61eb7de54f4e44755c988af0f52cf90230c312a1a89" [[package]] name = "xml-rs" @@ -12105,7 +12521,7 @@ version = "8.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" dependencies = [ - "aes", + "aes 0.9.0", "bzip2", "constant_time_eq", "crc32fast", @@ -12116,7 +12532,7 @@ dependencies = [ "indexmap 2.14.0", "lzma-rust2", "memchr", - "pbkdf2", + "pbkdf2 0.13.0", "ppmd-rust", "sha1 0.11.0", "time", diff --git a/Cargo.toml b/Cargo.toml index 66816b0d5..ad5fb598a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -126,6 +126,7 @@ futures-core = "0.3.32" futures-util = "0.3.32" pollster = "0.4.0" pulsar = { version = "6.7.2", default-features = false, features = ["tokio-rustls-runtime"] } +lapin = { version = "4.7.2", default-features = false, features = ["tokio", "rustls", "rustls--aws_lc_rs"] } hyper = { version = "1.9.0", features = ["http2", "http1", "server"] } hyper-rustls = { version = "0.27.9", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] } hyper-util = { version = "0.1.20", features = ["tokio", "server-auto", "server-graceful", "tracing"] } @@ -135,7 +136,7 @@ http-body-util = "0.1.3" reqwest = { version = "0.13.3", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] } rustfs-kafka-async = { version = "1.2.0" } socket2 = { version = "0.6.3", features = ["all"] } -tokio = { version = "1.52.2", features = ["fs", "rt-multi-thread"] } +tokio = { version = "1.52.3", features = ["fs", "rt-multi-thread"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] } tokio-stream = { version = "0.1.18" } tokio-test = "0.4.5" diff --git a/crates/audit/src/factory.rs b/crates/audit/src/factory.rs index ad6570459..9ee36731e 100644 --- a/crates/audit/src/factory.rs +++ b/crates/audit/src/factory.rs @@ -13,189 +13,192 @@ // limitations under the License. use crate::AuditEntry; -use async_trait::async_trait; use rustfs_config::AUDIT_DEFAULT_DIR; use rustfs_config::audit::{ - AUDIT_KAFKA_KEYS, AUDIT_MQTT_KEYS, AUDIT_MYSQL_KEYS, AUDIT_NATS_KEYS, AUDIT_POSTGRES_KEYS, AUDIT_PULSAR_KEYS, - AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_REDIS_KEYS, AUDIT_WEBHOOK_KEYS, + AUDIT_AMQP_KEYS, AUDIT_KAFKA_KEYS, AUDIT_MQTT_KEYS, AUDIT_MYSQL_KEYS, AUDIT_NATS_KEYS, AUDIT_POSTGRES_KEYS, + AUDIT_PULSAR_KEYS, AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_REDIS_KEYS, AUDIT_WEBHOOK_KEYS, }; -use rustfs_ecstore::config::KVS; -use rustfs_targets::{ - Target, - config::{ - build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args, build_pulsar_args, - build_redis_args, build_webhook_args, validate_kafka_config, validate_mqtt_config, validate_mysql_config, - validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, validate_webhook_config, - }, - error::TargetError, - target::TargetType, +use rustfs_targets::config::{ + build_amqp_args, build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args, + build_pulsar_args, build_redis_args, build_webhook_args, validate_amqp_config, validate_kafka_config, validate_mqtt_config, + validate_mysql_config, validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, + validate_webhook_config, }; -use std::collections::HashSet; +use rustfs_targets::target::{ChannelTargetType, TargetType}; +use rustfs_targets::{BuiltinTargetDescriptor, TargetPluginDescriptor, TargetRequestValidator, boxed_target}; -/// Trait for creating targets from configuration -#[async_trait] -pub trait TargetFactory: Send + Sync { - /// Creates a target from configuration - async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError>; - - /// Validates target configuration - fn validate_config(&self, id: &str, config: &KVS) -> Result<(), TargetError>; - - /// Returns a set of valid configuration field names for this target type. - /// This is used to filter environment variables. - fn get_valid_fields(&self) -> HashSet; +pub fn builtin_target_descriptors() -> Vec> { + vec![ + BuiltinTargetDescriptor::new( + rustfs_config::audit::AUDIT_AMQP_SUB_SYS, + TargetRequestValidator::Amqp(TargetType::AuditLog), + TargetPluginDescriptor::new( + ChannelTargetType::Amqp.as_str(), + AUDIT_AMQP_KEYS, + |config| validate_amqp_config(config, AUDIT_DEFAULT_DIR), + |id, config| { + let args = build_amqp_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; + Ok(boxed_target(rustfs_targets::target::amqp::AMQPTarget::new(id, args)?)) + }, + ), + ), + BuiltinTargetDescriptor::new( + rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS, + TargetRequestValidator::Webhook, + TargetPluginDescriptor::new( + ChannelTargetType::Webhook.as_str(), + AUDIT_WEBHOOK_KEYS, + |config| validate_webhook_config(config, AUDIT_DEFAULT_DIR), + |id, config| { + let args = build_webhook_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; + Ok(boxed_target(rustfs_targets::target::webhook::WebhookTarget::new(id, args)?)) + }, + ), + ), + BuiltinTargetDescriptor::new( + rustfs_config::audit::AUDIT_MQTT_SUB_SYS, + TargetRequestValidator::Mqtt, + TargetPluginDescriptor::new(ChannelTargetType::Mqtt.as_str(), AUDIT_MQTT_KEYS, validate_mqtt_config, |id, config| { + let args = build_mqtt_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; + Ok(boxed_target(rustfs_targets::target::mqtt::MQTTTarget::new(id, args)?)) + }), + ), + BuiltinTargetDescriptor::new( + rustfs_config::audit::AUDIT_NATS_SUB_SYS, + TargetRequestValidator::Nats(TargetType::AuditLog), + TargetPluginDescriptor::new( + ChannelTargetType::Nats.as_str(), + AUDIT_NATS_KEYS, + |config| validate_nats_config(config, AUDIT_DEFAULT_DIR), + |id, config| { + let args = build_nats_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; + Ok(boxed_target(rustfs_targets::target::nats::NATSTarget::new(id, args)?)) + }, + ), + ), + BuiltinTargetDescriptor::new( + rustfs_config::audit::AUDIT_PULSAR_SUB_SYS, + TargetRequestValidator::Pulsar(TargetType::AuditLog), + TargetPluginDescriptor::new( + ChannelTargetType::Pulsar.as_str(), + AUDIT_PULSAR_KEYS, + |config| validate_pulsar_config(config, AUDIT_DEFAULT_DIR), + |id, config| { + let args = build_pulsar_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; + Ok(boxed_target(rustfs_targets::target::pulsar::PulsarTarget::new(id, args)?)) + }, + ), + ), + BuiltinTargetDescriptor::new( + rustfs_config::audit::AUDIT_KAFKA_SUB_SYS, + TargetRequestValidator::Kafka(TargetType::AuditLog), + TargetPluginDescriptor::new( + ChannelTargetType::Kafka.as_str(), + AUDIT_KAFKA_KEYS, + |config| validate_kafka_config(config, AUDIT_DEFAULT_DIR), + |id, config| { + let args = build_kafka_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; + Ok(boxed_target(rustfs_targets::target::kafka::KafkaTarget::new(id, args)?)) + }, + ), + ), + BuiltinTargetDescriptor::new( + rustfs_config::audit::AUDIT_REDIS_SUB_SYS, + TargetRequestValidator::Redis { + default_channel: AUDIT_REDIS_DEFAULT_CHANNEL, + target_type: TargetType::AuditLog, + }, + TargetPluginDescriptor::new( + ChannelTargetType::Redis.as_str(), + AUDIT_REDIS_KEYS, + |config| validate_redis_config(config, AUDIT_DEFAULT_DIR, AUDIT_REDIS_DEFAULT_CHANNEL), + |id, config| { + let args = build_redis_args(config, AUDIT_DEFAULT_DIR, AUDIT_REDIS_DEFAULT_CHANNEL, TargetType::AuditLog)?; + Ok(boxed_target(rustfs_targets::target::redis::RedisTarget::new(id, args)?)) + }, + ), + ), + BuiltinTargetDescriptor::new( + rustfs_config::audit::AUDIT_MYSQL_SUB_SYS, + TargetRequestValidator::MySql, + TargetPluginDescriptor::new( + ChannelTargetType::MySql.as_str(), + AUDIT_MYSQL_KEYS, + |config| validate_mysql_config(config, AUDIT_DEFAULT_DIR), + |id, config| { + let args = build_mysql_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; + Ok(boxed_target(rustfs_targets::target::mysql::MySqlTarget::new(id, args)?)) + }, + ), + ), + BuiltinTargetDescriptor::new( + rustfs_config::audit::AUDIT_POSTGRES_SUB_SYS, + TargetRequestValidator::Postgres(TargetType::AuditLog), + TargetPluginDescriptor::new( + ChannelTargetType::Postgres.as_str(), + AUDIT_POSTGRES_KEYS, + |config| validate_postgres_config(config, AUDIT_DEFAULT_DIR), + |id, config| { + let args = build_postgres_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; + Ok(boxed_target(rustfs_targets::target::postgres::PostgresTarget::new(id, args)?)) + }, + ), + ), + ] } -/// Factory for creating Webhook targets -pub struct WebhookTargetFactory; - -#[async_trait] -impl TargetFactory for WebhookTargetFactory { - async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError> { - let args = build_webhook_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; - let target = rustfs_targets::target::webhook::WebhookTarget::new(id, args)?; - Ok(Box::new(target)) - } - - fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> { - validate_webhook_config(config, AUDIT_DEFAULT_DIR) - } - - fn get_valid_fields(&self) -> HashSet { - AUDIT_WEBHOOK_KEYS.iter().map(|s| s.to_string()).collect() - } +pub fn builtin_target_plugins() -> Vec> { + builtin_target_descriptors() + .into_iter() + .map(|descriptor| descriptor.plugin().clone()) + .collect() } -/// Factory for creating MQTT targets -pub struct MQTTTargetFactory; +#[cfg(test)] +mod tests { + use super::builtin_target_descriptors; + use rustfs_config::audit::AUDIT_AMQP_KEYS; + use rustfs_config::{AMQP_EXCHANGE, AMQP_QUEUE_DIR, AMQP_ROUTING_KEY, AMQP_URL}; + use rustfs_ecstore::config::KVS; + use rustfs_targets::target::ChannelTargetType; -#[async_trait] -impl TargetFactory for MQTTTargetFactory { - async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError> { - let args = build_mqtt_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; - let target = rustfs_targets::target::mqtt::MQTTTarget::new(id, args)?; - Ok(Box::new(target)) + fn amqp_base_config() -> KVS { + let mut config = KVS::new(); + config.insert(AMQP_URL.to_string(), "amqp://127.0.0.1:5672/%2f".to_string()); + config.insert(AMQP_EXCHANGE.to_string(), "rustfs.audit".to_string()); + config.insert(AMQP_ROUTING_KEY.to_string(), "audit".to_string()); + config.insert(AMQP_QUEUE_DIR.to_string(), String::new()); + config } - fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> { - validate_mqtt_config(config) + #[test] + fn builtin_plugins_include_amqp_descriptor() { + let plugin = builtin_target_descriptors() + .into_iter() + .find(|plugin| plugin.plugin().target_type() == ChannelTargetType::Amqp.as_str()) + .expect("amqp plugin should exist"); + + assert!(plugin.plugin().valid_fields().contains(&AMQP_URL)); + assert!(plugin.plugin().valid_fields().contains(&AMQP_EXCHANGE)); + assert!(plugin.plugin().valid_fields().contains(&AMQP_ROUTING_KEY)); + assert_eq!(plugin.plugin().valid_fields().len(), AUDIT_AMQP_KEYS.len()); } - fn get_valid_fields(&self) -> HashSet { - AUDIT_MQTT_KEYS.iter().map(|s| s.to_string()).collect() - } -} - -pub struct NATSTargetFactory; - -#[async_trait] -impl TargetFactory for NATSTargetFactory { - async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError> { - let args = build_nats_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; - let target = rustfs_targets::target::nats::NATSTarget::new(id, args)?; - Ok(Box::new(target)) - } - - fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> { - validate_nats_config(config, AUDIT_DEFAULT_DIR) - } - - fn get_valid_fields(&self) -> HashSet { - AUDIT_NATS_KEYS.iter().map(|s| s.to_string()).collect() - } -} - -pub struct PulsarTargetFactory; - -#[async_trait] -impl TargetFactory for PulsarTargetFactory { - async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError> { - let args = build_pulsar_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; - let target = rustfs_targets::target::pulsar::PulsarTarget::new(id, args)?; - Ok(Box::new(target)) - } - - fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> { - validate_pulsar_config(config, AUDIT_DEFAULT_DIR) - } - - fn get_valid_fields(&self) -> HashSet { - AUDIT_PULSAR_KEYS.iter().map(|s| s.to_string()).collect() - } -} - -pub struct KafkaTargetFactory; - -#[async_trait] -impl TargetFactory for KafkaTargetFactory { - async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError> { - let args = build_kafka_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; - let target = rustfs_targets::target::kafka::KafkaTarget::new(id, args)?; - Ok(Box::new(target)) - } - - fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> { - validate_kafka_config(config, AUDIT_DEFAULT_DIR) - } - - fn get_valid_fields(&self) -> HashSet { - AUDIT_KAFKA_KEYS.iter().map(|s| s.to_string()).collect() - } -} - -pub struct RedisTargetFactory; - -#[async_trait] -impl TargetFactory for RedisTargetFactory { - async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError> { - let args = build_redis_args(config, AUDIT_DEFAULT_DIR, AUDIT_REDIS_DEFAULT_CHANNEL, TargetType::AuditLog)?; - let target = rustfs_targets::target::redis::RedisTarget::new(id, args)?; - Ok(Box::new(target)) - } - - fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> { - validate_redis_config(config, AUDIT_DEFAULT_DIR, AUDIT_REDIS_DEFAULT_CHANNEL) - } - - fn get_valid_fields(&self) -> HashSet { - AUDIT_REDIS_KEYS.iter().map(|s| s.to_string()).collect() - } -} - -pub struct MySqlTargetFactory; - -#[async_trait] -impl TargetFactory for MySqlTargetFactory { - async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError> { - let args = build_mysql_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; - let target = rustfs_targets::target::mysql::MySqlTarget::new(id, args)?; - Ok(Box::new(target)) - } - - fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> { - validate_mysql_config(config, AUDIT_DEFAULT_DIR) - } - - fn get_valid_fields(&self) -> HashSet { - AUDIT_MYSQL_KEYS.iter().map(|s| s.to_string()).collect() - } -} - -pub struct PostgresTargetFactory; - -#[async_trait] -impl TargetFactory for PostgresTargetFactory { - async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError> { - let args = build_postgres_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; - let target = rustfs_targets::target::postgres::PostgresTarget::new(id, args)?; - Ok(Box::new(target)) - } - - fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> { - validate_postgres_config(config, AUDIT_DEFAULT_DIR) - } - - fn get_valid_fields(&self) -> HashSet { - AUDIT_POSTGRES_KEYS.iter().map(|s| s.to_string()).collect() + #[test] + fn builtin_plugins_create_audit_amqp_target() { + let plugin = builtin_target_descriptors() + .into_iter() + .find(|plugin| plugin.plugin().target_type() == ChannelTargetType::Amqp.as_str()) + .expect("amqp plugin should exist"); + + let target = plugin + .plugin() + .create_target("primary".to_string(), &amqp_base_config()) + .expect("AMQP audit target should be created"); + + let target_id = target.id(); + assert_eq!(target_id.id, "primary"); + assert_eq!(target_id.name, "amqp"); + assert!(target.store().is_none()); } } diff --git a/crates/audit/src/registry.rs b/crates/audit/src/registry.rs index 82c3921b6..b038eb783 100644 --- a/crates/audit/src/registry.rs +++ b/crates/audit/src/registry.rs @@ -12,29 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{ - AuditEntry, AuditError, AuditResult, - factory::{ - KafkaTargetFactory, MQTTTargetFactory, MySqlTargetFactory, NATSTargetFactory, PostgresTargetFactory, PulsarTargetFactory, - RedisTargetFactory, TargetFactory, WebhookTargetFactory, - }, -}; -use futures::StreamExt; -use futures::stream::FuturesUnordered; +use crate::{AuditEntry, AuditError, AuditResult, factory::builtin_target_plugins}; use hashbrown::HashMap; use rustfs_config::audit::AUDIT_ROUTE_PREFIX; use rustfs_ecstore::config::{Config, KVS}; use rustfs_targets::arn::TargetID; -use rustfs_targets::{Target, TargetError, config::collect_target_configs, target::ChannelTargetType}; -use std::sync::Arc; +use rustfs_targets::{Target, TargetError, TargetPluginRegistry}; use tracing::{error, info}; /// Registry for managing audit targets pub struct AuditRegistry { /// Storage for created targets targets: HashMap + Send + Sync>>, - /// Factories for creating targets - factories: HashMap>, + /// Registered plugins for creating targets + plugins: TargetPluginRegistry, } impl Default for AuditRegistry { @@ -46,31 +37,17 @@ impl Default for AuditRegistry { impl AuditRegistry { /// Creates a new AuditRegistry pub fn new() -> Self { - let mut registry = AuditRegistry { - factories: HashMap::new(), + let mut plugins = TargetPluginRegistry::new(); + plugins.register_all(builtin_target_plugins()); + + AuditRegistry { targets: HashMap::new(), - }; - - // Register built-in factories - registry.register(ChannelTargetType::Webhook.as_str(), Box::new(WebhookTargetFactory)); - registry.register(ChannelTargetType::Mqtt.as_str(), Box::new(MQTTTargetFactory)); - registry.register(ChannelTargetType::Nats.as_str(), Box::new(NATSTargetFactory)); - registry.register(ChannelTargetType::Pulsar.as_str(), Box::new(PulsarTargetFactory)); - registry.register(ChannelTargetType::Kafka.as_str(), Box::new(KafkaTargetFactory)); - registry.register(ChannelTargetType::Redis.as_str(), Box::new(RedisTargetFactory)); - registry.register(ChannelTargetType::MySql.as_str(), Box::new(MySqlTargetFactory)); - registry.register(ChannelTargetType::Postgres.as_str(), Box::new(PostgresTargetFactory)); - - registry + plugins, + } } - /// Registers a new factory for a target type - /// - /// # Arguments - /// * `target_type` - The type of the target (e.g., "webhook", "mqtt"). - /// * `factory` - The factory instance to create targets of this type. - pub fn register(&mut self, target_type: &str, factory: Box) { - self.factories.insert(target_type.to_string(), factory); + pub fn supports_target_type(&self, target_type: &str) -> bool { + self.plugins.supports_target_type(target_type) } /// Creates a target of the specified type with the given ID and configuration @@ -88,16 +65,7 @@ impl AuditRegistry { id: String, config: &KVS, ) -> Result + Send + Sync>, TargetError> { - let factory = self - .factories - .get(target_type) - .ok_or_else(|| TargetError::Configuration(format!("Unknown target type: {target_type}")))?; - - // Validate configuration before creating target - factory.validate_config(&id, config)?; - - // Create target - factory.create_target(id, config).await + self.plugins.create_target(target_type, id, config) } /// Creates all targets from a configuration @@ -113,37 +81,10 @@ impl AuditRegistry { &self, config: &Config, ) -> AuditResult + Send + Sync>>> { - let mut tasks = FuturesUnordered::new(); - for (target_type, factory) in &self.factories { - tracing::Span::current().record("target_type", target_type.as_str()); - info!("Start working on target types..."); - let valid_fields = factory.get_valid_fields(); - for (id, merged_config) in collect_target_configs(config, AUDIT_ROUTE_PREFIX, target_type, &valid_fields) { - info!(instance_id = %id, "Target is enabled, ready to create a task"); - let tid = id.clone(); - let merged_config_arc = Arc::new(merged_config); - tasks.push(async move { - let result = factory.create_target(tid.clone(), &merged_config_arc).await; - (tid, result) - }); - } - } - - let mut successful_targets = Vec::new(); - while let Some((id, result)) = tasks.next().await { - match result { - Ok(target) => { - info!(target_type = %target.id().name, instance_id = %id, "Create a target successfully"); - successful_targets.push(target); - } - Err(e) => { - error!(instance_id = %id, error = %e, "Failed to create a target"); - } - } - } - - info!(count = successful_targets.len(), "All target processing completed"); - Ok(successful_targets) + self.plugins + .create_targets_from_config(config, AUDIT_ROUTE_PREFIX) + .await + .map_err(AuditError::from) } /// Adds a target to the registry @@ -287,3 +228,16 @@ impl AuditRegistry { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::AuditRegistry; + use rustfs_targets::target::ChannelTargetType; + + #[test] + fn registry_registers_amqp_factory() { + let registry = AuditRegistry::new(); + + assert!(registry.supports_target_type(ChannelTargetType::Amqp.as_str())); + } +} diff --git a/crates/config/src/audit/amqp.rs b/crates/config/src/audit/amqp.rs new file mode 100644 index 000000000..8011343ad --- /dev/null +++ b/crates/config/src/audit/amqp.rs @@ -0,0 +1,60 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub const AUDIT_AMQP_KEYS: &[&str] = &[ + crate::ENABLE_KEY, + crate::AMQP_URL, + crate::AMQP_EXCHANGE, + crate::AMQP_ROUTING_KEY, + crate::AMQP_MANDATORY, + crate::AMQP_PERSISTENT, + crate::AMQP_USERNAME, + crate::AMQP_PASSWORD, + crate::AMQP_TLS_CA, + crate::AMQP_TLS_CLIENT_CERT, + crate::AMQP_TLS_CLIENT_KEY, + crate::AMQP_QUEUE_DIR, + crate::AMQP_QUEUE_LIMIT, + crate::COMMENT_KEY, +]; + +pub const ENV_AUDIT_AMQP_ENABLE: &str = "RUSTFS_AUDIT_AMQP_ENABLE"; +pub const ENV_AUDIT_AMQP_URL: &str = "RUSTFS_AUDIT_AMQP_URL"; +pub const ENV_AUDIT_AMQP_EXCHANGE: &str = "RUSTFS_AUDIT_AMQP_EXCHANGE"; +pub const ENV_AUDIT_AMQP_ROUTING_KEY: &str = "RUSTFS_AUDIT_AMQP_ROUTING_KEY"; +pub const ENV_AUDIT_AMQP_MANDATORY: &str = "RUSTFS_AUDIT_AMQP_MANDATORY"; +pub const ENV_AUDIT_AMQP_PERSISTENT: &str = "RUSTFS_AUDIT_AMQP_PERSISTENT"; +pub const ENV_AUDIT_AMQP_USERNAME: &str = "RUSTFS_AUDIT_AMQP_USERNAME"; +pub const ENV_AUDIT_AMQP_PASSWORD: &str = "RUSTFS_AUDIT_AMQP_PASSWORD"; +pub const ENV_AUDIT_AMQP_TLS_CA: &str = "RUSTFS_AUDIT_AMQP_TLS_CA"; +pub const ENV_AUDIT_AMQP_TLS_CLIENT_CERT: &str = "RUSTFS_AUDIT_AMQP_TLS_CLIENT_CERT"; +pub const ENV_AUDIT_AMQP_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_AMQP_TLS_CLIENT_KEY"; +pub const ENV_AUDIT_AMQP_QUEUE_DIR: &str = "RUSTFS_AUDIT_AMQP_QUEUE_DIR"; +pub const ENV_AUDIT_AMQP_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_AMQP_QUEUE_LIMIT"; + +pub const ENV_AUDIT_AMQP_KEYS: &[&str; 13] = &[ + ENV_AUDIT_AMQP_ENABLE, + ENV_AUDIT_AMQP_URL, + ENV_AUDIT_AMQP_EXCHANGE, + ENV_AUDIT_AMQP_ROUTING_KEY, + ENV_AUDIT_AMQP_MANDATORY, + ENV_AUDIT_AMQP_PERSISTENT, + ENV_AUDIT_AMQP_USERNAME, + ENV_AUDIT_AMQP_PASSWORD, + ENV_AUDIT_AMQP_TLS_CA, + ENV_AUDIT_AMQP_TLS_CLIENT_CERT, + ENV_AUDIT_AMQP_TLS_CLIENT_KEY, + ENV_AUDIT_AMQP_QUEUE_DIR, + ENV_AUDIT_AMQP_QUEUE_LIMIT, +]; diff --git a/crates/config/src/audit/mod.rs b/crates/config/src/audit/mod.rs index 03e0834b3..62de9f097 100644 --- a/crates/config/src/audit/mod.rs +++ b/crates/config/src/audit/mod.rs @@ -16,6 +16,7 @@ //! This module defines the configuration for audit systems, including //! webhook and MQTT audit-related settings. +mod amqp; mod kafka; mod mqtt; mod mysql; @@ -25,6 +26,7 @@ mod pulsar; mod redis; mod webhook; +pub use amqp::*; pub use kafka::*; pub use mqtt::*; pub use mysql::*; @@ -41,6 +43,7 @@ pub const AUDIT_PREFIX: &str = "audit"; pub const AUDIT_ROUTE_PREFIX: &str = const_str::concat!(AUDIT_PREFIX, DEFAULT_DELIMITER); pub const AUDIT_WEBHOOK_SUB_SYS: &str = "audit_webhook"; +pub const AUDIT_AMQP_SUB_SYS: &str = "audit_amqp"; pub const AUDIT_KAFKA_SUB_SYS: &str = "audit_kafka"; pub const AUDIT_MQTT_SUB_SYS: &str = "audit_mqtt"; pub const AUDIT_MYSQL_SUB_SYS: &str = "audit_mysql"; @@ -49,10 +52,9 @@ pub const AUDIT_POSTGRES_SUB_SYS: &str = "audit_postgres"; pub const AUDIT_PULSAR_SUB_SYS: &str = "audit_pulsar"; pub const AUDIT_REDIS_SUB_SYS: &str = "audit_redis"; pub const AUDIT_REDIS_DEFAULT_CHANNEL: &str = "rustfs_audit_channel"; - pub const AUDIT_STORE_EXTENSION: &str = ".audit"; -#[allow(dead_code)] pub const AUDIT_SUB_SYSTEMS: &[&str] = &[ + AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_MYSQL_SUB_SYS, diff --git a/crates/config/src/constants/targets.rs b/crates/config/src/constants/targets.rs index 9c9800b0a..40344589d 100644 --- a/crates/config/src/constants/targets.rs +++ b/crates/config/src/constants/targets.rs @@ -50,6 +50,19 @@ pub const KAFKA_TLS_CA: &str = "tls_ca"; pub const KAFKA_TLS_CLIENT_CERT: &str = "tls_client_cert"; pub const KAFKA_TLS_CLIENT_KEY: &str = "tls_client_key"; +pub const AMQP_URL: &str = "url"; +pub const AMQP_EXCHANGE: &str = "exchange"; +pub const AMQP_ROUTING_KEY: &str = "routing_key"; +pub const AMQP_MANDATORY: &str = "mandatory"; +pub const AMQP_PERSISTENT: &str = "persistent"; +pub const AMQP_USERNAME: &str = "username"; +pub const AMQP_PASSWORD: &str = "password"; +pub const AMQP_TLS_CA: &str = "tls_ca"; +pub const AMQP_TLS_CLIENT_CERT: &str = "tls_client_cert"; +pub const AMQP_TLS_CLIENT_KEY: &str = "tls_client_key"; +pub const AMQP_QUEUE_DIR: &str = "queue_dir"; +pub const AMQP_QUEUE_LIMIT: &str = "queue_limit"; + pub const NATS_ADDRESS: &str = "address"; pub const NATS_SUBJECT: &str = "subject"; pub const NATS_USERNAME: &str = "username"; diff --git a/crates/config/src/notify/amqp.rs b/crates/config/src/notify/amqp.rs new file mode 100644 index 000000000..16d7b2d20 --- /dev/null +++ b/crates/config/src/notify/amqp.rs @@ -0,0 +1,60 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub const NOTIFY_AMQP_KEYS: &[&str] = &[ + crate::ENABLE_KEY, + crate::AMQP_URL, + crate::AMQP_EXCHANGE, + crate::AMQP_ROUTING_KEY, + crate::AMQP_MANDATORY, + crate::AMQP_PERSISTENT, + crate::AMQP_USERNAME, + crate::AMQP_PASSWORD, + crate::AMQP_TLS_CA, + crate::AMQP_TLS_CLIENT_CERT, + crate::AMQP_TLS_CLIENT_KEY, + crate::AMQP_QUEUE_DIR, + crate::AMQP_QUEUE_LIMIT, + crate::COMMENT_KEY, +]; + +pub const ENV_NOTIFY_AMQP_ENABLE: &str = "RUSTFS_NOTIFY_AMQP_ENABLE"; +pub const ENV_NOTIFY_AMQP_URL: &str = "RUSTFS_NOTIFY_AMQP_URL"; +pub const ENV_NOTIFY_AMQP_EXCHANGE: &str = "RUSTFS_NOTIFY_AMQP_EXCHANGE"; +pub const ENV_NOTIFY_AMQP_ROUTING_KEY: &str = "RUSTFS_NOTIFY_AMQP_ROUTING_KEY"; +pub const ENV_NOTIFY_AMQP_MANDATORY: &str = "RUSTFS_NOTIFY_AMQP_MANDATORY"; +pub const ENV_NOTIFY_AMQP_PERSISTENT: &str = "RUSTFS_NOTIFY_AMQP_PERSISTENT"; +pub const ENV_NOTIFY_AMQP_USERNAME: &str = "RUSTFS_NOTIFY_AMQP_USERNAME"; +pub const ENV_NOTIFY_AMQP_PASSWORD: &str = "RUSTFS_NOTIFY_AMQP_PASSWORD"; +pub const ENV_NOTIFY_AMQP_TLS_CA: &str = "RUSTFS_NOTIFY_AMQP_TLS_CA"; +pub const ENV_NOTIFY_AMQP_TLS_CLIENT_CERT: &str = "RUSTFS_NOTIFY_AMQP_TLS_CLIENT_CERT"; +pub const ENV_NOTIFY_AMQP_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_AMQP_TLS_CLIENT_KEY"; +pub const ENV_NOTIFY_AMQP_QUEUE_DIR: &str = "RUSTFS_NOTIFY_AMQP_QUEUE_DIR"; +pub const ENV_NOTIFY_AMQP_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_AMQP_QUEUE_LIMIT"; + +pub const ENV_NOTIFY_AMQP_KEYS: &[&str; 13] = &[ + ENV_NOTIFY_AMQP_ENABLE, + ENV_NOTIFY_AMQP_URL, + ENV_NOTIFY_AMQP_EXCHANGE, + ENV_NOTIFY_AMQP_ROUTING_KEY, + ENV_NOTIFY_AMQP_MANDATORY, + ENV_NOTIFY_AMQP_PERSISTENT, + ENV_NOTIFY_AMQP_USERNAME, + ENV_NOTIFY_AMQP_PASSWORD, + ENV_NOTIFY_AMQP_TLS_CA, + ENV_NOTIFY_AMQP_TLS_CLIENT_CERT, + ENV_NOTIFY_AMQP_TLS_CLIENT_KEY, + ENV_NOTIFY_AMQP_QUEUE_DIR, + ENV_NOTIFY_AMQP_QUEUE_LIMIT, +]; diff --git a/crates/config/src/notify/mod.rs b/crates/config/src/notify/mod.rs index 5f7262674..96a1afd1e 100644 --- a/crates/config/src/notify/mod.rs +++ b/crates/config/src/notify/mod.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +mod amqp; mod arn; mod kafka; mod mqtt; @@ -23,6 +24,7 @@ mod redis; mod store; mod webhook; +pub use amqp::*; pub use arn::*; pub use kafka::*; pub use mqtt::*; @@ -76,6 +78,7 @@ pub const ENV_NOTIFY_SEND_CONCURRENCY: &str = "RUSTFS_NOTIFY_SEND_CONCURRENCY"; pub const DEFAULT_NOTIFY_SEND_CONCURRENCY: usize = 64; pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[ + NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, @@ -95,7 +98,6 @@ pub const NOTIFY_NATS_SUB_SYS: &str = "notify_nats"; pub const NOTIFY_NSQ_SUB_SYS: &str = "notify_nsq"; #[allow(dead_code)] pub const NOTIFY_ES_SUB_SYS: &str = "notify_elasticsearch"; -#[allow(dead_code)] pub const NOTIFY_AMQP_SUB_SYS: &str = "notify_amqp"; pub const NOTIFY_POSTGRES_SUB_SYS: &str = "notify_postgres"; #[allow(dead_code)] diff --git a/crates/ecstore/src/config/audit.rs b/crates/ecstore/src/config/audit.rs index bcb5c694b..508eae1ef 100644 --- a/crates/ecstore/src/config/audit.rs +++ b/crates/ecstore/src/config/audit.rs @@ -15,14 +15,16 @@ use crate::config::{KV, KVS}; use rustfs_config::audit::AUDIT_REDIS_DEFAULT_CHANNEL; use rustfs_config::{ - COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, - KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, - MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, - MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, - MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, - MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, - NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, - NATS_TOKEN, NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE, + AMQP_EXCHANGE, AMQP_MANDATORY, AMQP_PASSWORD, AMQP_PERSISTENT, AMQP_QUEUE_DIR, AMQP_QUEUE_LIMIT, AMQP_ROUTING_KEY, + AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY, AMQP_URL, AMQP_USERNAME, COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, + EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, + KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, + MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, + MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, + MYSQL_DSN_STRING, MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA, + MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, + NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, + NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT, @@ -206,6 +208,81 @@ pub static DEFAULT_AUDIT_MQTT_KVS: LazyLock = LazyLock::new(|| { ]) }); +pub static DEFAULT_AUDIT_AMQP_KVS: LazyLock = LazyLock::new(|| { + KVS(vec![ + KV { + key: ENABLE_KEY.to_owned(), + value: EnableState::Off.to_string(), + hidden_if_empty: false, + }, + KV { + key: AMQP_URL.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: AMQP_EXCHANGE.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: AMQP_ROUTING_KEY.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: AMQP_MANDATORY.to_owned(), + value: EnableState::Off.to_string(), + hidden_if_empty: false, + }, + KV { + key: AMQP_PERSISTENT.to_owned(), + value: EnableState::On.to_string(), + hidden_if_empty: false, + }, + KV { + key: AMQP_USERNAME.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: AMQP_PASSWORD.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: AMQP_TLS_CA.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: AMQP_TLS_CLIENT_CERT.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: AMQP_TLS_CLIENT_KEY.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: AMQP_QUEUE_DIR.to_owned(), + value: EVENT_DEFAULT_DIR.to_owned(), + hidden_if_empty: false, + }, + KV { + key: AMQP_QUEUE_LIMIT.to_owned(), + value: DEFAULT_LIMIT.to_string(), + hidden_if_empty: false, + }, + KV { + key: COMMENT_KEY.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + ]) +}); + pub static DEFAULT_AUDIT_NATS_KVS: LazyLock = LazyLock::new(|| { KVS(vec![ KV { diff --git a/crates/ecstore/src/config/com.rs b/crates/ecstore/src/config/com.rs index 32e25f5fa..bbef691e6 100644 --- a/crates/ecstore/src/config/com.rs +++ b/crates/ecstore/src/config/com.rs @@ -19,14 +19,15 @@ use crate::global::is_first_cluster_node_local; use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI}; use http::HeaderMap; use rustfs_config::audit::{ - AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_MYSQL_KEYS, AUDIT_MYSQL_SUB_SYS, - AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS, AUDIT_POSTGRES_KEYS, AUDIT_POSTGRES_SUB_SYS, AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS, - AUDIT_REDIS_KEYS, AUDIT_REDIS_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS, + AUDIT_AMQP_KEYS, AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, + AUDIT_MYSQL_KEYS, AUDIT_MYSQL_SUB_SYS, AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS, AUDIT_POSTGRES_KEYS, AUDIT_POSTGRES_SUB_SYS, + AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS, AUDIT_REDIS_KEYS, AUDIT_REDIS_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS, }; use rustfs_config::notify::{ - NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_KEYS, NOTIFY_MYSQL_SUB_SYS, - NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_KEYS, NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_KEYS, - NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_KEYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS, + NOTIFY_AMQP_KEYS, NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, + NOTIFY_MYSQL_KEYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_KEYS, + NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_KEYS, NOTIFY_REDIS_SUB_SYS, + NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS, }; use rustfs_config::oidc::{IDENTITY_OPENID_KEYS, IDENTITY_OPENID_SUB_SYS, OIDC_REDIRECT_URI_DYNAMIC}; use rustfs_config::{COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, RUSTFS_REGION}; @@ -60,7 +61,7 @@ struct TargetConfigDescriptor { valid_keys: &'static [&'static str], } -fn notify_target_descriptors() -> [TargetConfigDescriptor; 8] { +fn notify_target_descriptors() -> [TargetConfigDescriptor; 9] { [ TargetConfigDescriptor { external_key: "webhook", @@ -68,6 +69,12 @@ fn notify_target_descriptors() -> [TargetConfigDescriptor; 8] { default_kvs: ¬ify::DEFAULT_NOTIFY_WEBHOOK_KVS, valid_keys: NOTIFY_WEBHOOK_KEYS, }, + TargetConfigDescriptor { + external_key: "amqp", + subsystem_key: NOTIFY_AMQP_SUB_SYS, + default_kvs: ¬ify::DEFAULT_NOTIFY_AMQP_KVS, + valid_keys: NOTIFY_AMQP_KEYS, + }, TargetConfigDescriptor { external_key: "kafka", subsystem_key: NOTIFY_KAFKA_SUB_SYS, @@ -113,7 +120,7 @@ fn notify_target_descriptors() -> [TargetConfigDescriptor; 8] { ] } -fn audit_target_descriptors() -> [TargetConfigDescriptor; 8] { +fn audit_target_descriptors() -> [TargetConfigDescriptor; 9] { [ TargetConfigDescriptor { external_key: "webhook", @@ -121,6 +128,12 @@ fn audit_target_descriptors() -> [TargetConfigDescriptor; 8] { default_kvs: &audit::DEFAULT_AUDIT_WEBHOOK_KVS, valid_keys: AUDIT_WEBHOOK_KEYS, }, + TargetConfigDescriptor { + external_key: "amqp", + subsystem_key: AUDIT_AMQP_SUB_SYS, + default_kvs: &audit::DEFAULT_AUDIT_AMQP_KVS, + valid_keys: AUDIT_AMQP_KEYS, + }, TargetConfigDescriptor { external_key: "kafka", subsystem_key: AUDIT_KAFKA_SUB_SYS, @@ -717,6 +730,8 @@ fn is_target_bool_key(key: &str) -> bool { matches!( key, ENABLE_KEY + | rustfs_config::AMQP_MANDATORY + | rustfs_config::AMQP_PERSISTENT | rustfs_config::WEBHOOK_SKIP_TLS_VERIFY | rustfs_config::KAFKA_TLS_ENABLE | rustfs_config::MQTT_TLS_TRUST_LEAF_AS_CA @@ -726,14 +741,31 @@ fn is_target_bool_key(key: &str) -> bool { ) } +fn parse_target_bool_scalar(value: &str) -> Option { + if let Ok(state) = value.parse::() { + return Some(state.is_enabled()); + } + if let Ok(boolean) = value.parse::() { + return Some(boolean); + } + None +} + +fn target_scalar_values_equal(key: &str, lhs: &str, rhs: &str) -> bool { + if is_target_bool_key(key) + && let (Some(lhs), Some(rhs)) = (parse_target_bool_scalar(lhs), parse_target_bool_scalar(rhs)) + { + return lhs == rhs; + } + + lhs == rhs +} + fn encode_target_scalar_value(key: &str, value: &str) -> Value { - if is_target_bool_key(key) { - if let Ok(state) = value.parse::() { - return Value::Bool(state.is_enabled()); - } - if let Ok(boolean) = value.parse::() { - return Value::Bool(boolean); - } + if is_target_bool_key(key) + && let Some(boolean) = parse_target_bool_scalar(value) + { + return Value::Bool(boolean); } Value::String(value.to_string()) @@ -759,7 +791,7 @@ fn build_target_instance_diff_object(kvs: &KVS, baseline: &KVS, valid_keys: &[&s let baseline_value = baseline.lookup(key).unwrap_or_default(); let effective_value = kvs.lookup(key).unwrap_or_else(|| baseline_value.clone()); - if effective_value == baseline_value { + if target_scalar_values_equal(key, &effective_value, &baseline_value) { continue; } @@ -1169,8 +1201,10 @@ mod tests { ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, WalkOptions, }; use http::HeaderMap; - use rustfs_config::audit::{AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS}; - use rustfs_config::notify::{NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS}; + use rustfs_config::audit::{AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS}; + use rustfs_config::notify::{ + NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, + }; use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS; use rustfs_config::{ DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MYSQL_DSN_STRING, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_TABLE, @@ -1816,6 +1850,15 @@ mod tests { "tls_enable":true } }, + "amqp":{ + "primary":{ + "enable":true, + "url":"amqp://127.0.0.1:5672/%2f", + "exchange":"rustfs.events", + "routing_key":"objects", + "persistent":true + } + }, "mysql":{ "primary":{ "enable":true, @@ -1861,6 +1904,15 @@ mod tests { assert_eq!(kafka.get(rustfs_config::KAFKA_ACKS), "all"); assert_eq!(kafka.get(rustfs_config::KAFKA_TLS_ENABLE), "true"); + let amqp = cfg + .get_value(NOTIFY_AMQP_SUB_SYS, "primary") + .expect("amqp target should be decoded"); + assert_eq!(amqp.get(ENABLE_KEY), EnableState::On.to_string()); + assert_eq!(amqp.get(rustfs_config::AMQP_URL), "amqp://127.0.0.1:5672/%2f"); + assert_eq!(amqp.get(rustfs_config::AMQP_EXCHANGE), "rustfs.events"); + assert_eq!(amqp.get(rustfs_config::AMQP_ROUTING_KEY), "objects"); + assert_eq!(amqp.get(rustfs_config::AMQP_PERSISTENT), "true"); + let mysql = cfg .get_value(NOTIFY_MYSQL_SUB_SYS, "primary") .expect("mysql target should be decoded"); @@ -1928,6 +1980,15 @@ mod tests { "queue_dir":"/tmp/audit-queue" } }, + "amqp":{ + "primary":{ + "enable":true, + "url":"amqp://127.0.0.1:5672/%2f", + "exchange":"rustfs.audit", + "routing_key":"audit", + "persistent":true + } + }, "mqtt":{ "default":{ "enable":true, @@ -1959,6 +2020,15 @@ mod tests { assert_eq!(webhook.get(rustfs_config::WEBHOOK_ENDPOINT), "https://example.com/audit-hook"); assert_eq!(webhook.get(rustfs_config::WEBHOOK_QUEUE_DIR), "/tmp/audit-queue"); + let amqp = cfg + .get_value(AUDIT_AMQP_SUB_SYS, "primary") + .expect("audit amqp target should be decoded"); + assert_eq!(amqp.get(ENABLE_KEY), EnableState::On.to_string()); + assert_eq!(amqp.get(rustfs_config::AMQP_URL), "amqp://127.0.0.1:5672/%2f"); + assert_eq!(amqp.get(rustfs_config::AMQP_EXCHANGE), "rustfs.audit"); + assert_eq!(amqp.get(rustfs_config::AMQP_ROUTING_KEY), "audit"); + assert_eq!(amqp.get(rustfs_config::AMQP_PERSISTENT), "true"); + let mqtt_default = cfg .get_value(AUDIT_MQTT_SUB_SYS, DEFAULT_DELIMITER) .expect("audit mqtt default should be decoded"); @@ -2127,6 +2197,44 @@ mod tests { ); cfg.0.insert(NOTIFY_KAFKA_SUB_SYS.to_string(), kafka_section); + let mut amqp_section = std::collections::HashMap::new(); + amqp_section.insert( + "primary".to_string(), + crate::config::KVS(vec![ + crate::config::KV { + key: ENABLE_KEY.to_string(), + value: EnableState::On.to_string(), + hidden_if_empty: false, + }, + crate::config::KV { + key: rustfs_config::AMQP_URL.to_string(), + value: "amqp://127.0.0.1:5672/%2f".to_string(), + hidden_if_empty: false, + }, + crate::config::KV { + key: rustfs_config::AMQP_EXCHANGE.to_string(), + value: "rustfs.events".to_string(), + hidden_if_empty: false, + }, + crate::config::KV { + key: rustfs_config::AMQP_ROUTING_KEY.to_string(), + value: "objects".to_string(), + hidden_if_empty: false, + }, + crate::config::KV { + key: rustfs_config::AMQP_MANDATORY.to_string(), + value: "false".to_string(), + hidden_if_empty: false, + }, + crate::config::KV { + key: rustfs_config::AMQP_PERSISTENT.to_string(), + value: "false".to_string(), + hidden_if_empty: false, + }, + ]), + ); + cfg.0.insert(NOTIFY_AMQP_SUB_SYS.to_string(), amqp_section); + let out = encode_server_config_blob(&cfg, None).expect("encode should succeed"); let v: Value = serde_json::from_slice(&out).expect("output should be json"); let notify = v @@ -2175,6 +2283,21 @@ mod tests { ); assert_eq!(kafka.get(rustfs_config::KAFKA_ACKS).and_then(Value::as_str), Some("all")); assert_eq!(kafka.get(rustfs_config::KAFKA_TLS_ENABLE).and_then(Value::as_bool), Some(true)); + + let amqp = notify + .get("amqp") + .and_then(Value::as_object) + .and_then(|targets| targets.get("primary")) + .and_then(Value::as_object) + .expect("amqp target should be encoded"); + assert_eq!( + amqp.get(rustfs_config::AMQP_URL).and_then(Value::as_str), + Some("amqp://127.0.0.1:5672/%2f") + ); + assert_eq!(amqp.get(rustfs_config::AMQP_EXCHANGE).and_then(Value::as_str), Some("rustfs.events")); + assert_eq!(amqp.get(rustfs_config::AMQP_ROUTING_KEY).and_then(Value::as_str), Some("objects")); + assert!(!amqp.contains_key(rustfs_config::AMQP_MANDATORY)); + assert_eq!(amqp.get(rustfs_config::AMQP_PERSISTENT).and_then(Value::as_bool), Some(false)); } #[test] @@ -2204,6 +2327,44 @@ mod tests { ); cfg.0.insert(AUDIT_WEBHOOK_SUB_SYS.to_string(), webhook_section); + let mut amqp_section = std::collections::HashMap::new(); + amqp_section.insert( + "primary".to_string(), + crate::config::KVS(vec![ + crate::config::KV { + key: ENABLE_KEY.to_string(), + value: EnableState::On.to_string(), + hidden_if_empty: false, + }, + crate::config::KV { + key: rustfs_config::AMQP_URL.to_string(), + value: "amqp://127.0.0.1:5672/%2f".to_string(), + hidden_if_empty: false, + }, + crate::config::KV { + key: rustfs_config::AMQP_EXCHANGE.to_string(), + value: "rustfs.audit".to_string(), + hidden_if_empty: false, + }, + crate::config::KV { + key: rustfs_config::AMQP_ROUTING_KEY.to_string(), + value: "audit".to_string(), + hidden_if_empty: false, + }, + crate::config::KV { + key: rustfs_config::AMQP_MANDATORY.to_string(), + value: "false".to_string(), + hidden_if_empty: false, + }, + crate::config::KV { + key: rustfs_config::AMQP_PERSISTENT.to_string(), + value: "false".to_string(), + hidden_if_empty: false, + }, + ]), + ); + cfg.0.insert(AUDIT_AMQP_SUB_SYS.to_string(), amqp_section); + let mut mqtt_default = audit::DEFAULT_AUDIT_MQTT_KVS.clone(); mqtt_default.insert(ENABLE_KEY.to_string(), EnableState::On.to_string()); mqtt_default.insert(rustfs_config::MQTT_TOPIC.to_string(), "audit-events".to_string()); @@ -2266,6 +2427,21 @@ mod tests { ); assert_eq!(webhook.get(ENABLE_KEY).and_then(Value::as_bool), Some(true)); + let amqp = logger + .get("amqp") + .and_then(Value::as_object) + .and_then(|targets| targets.get("primary")) + .and_then(Value::as_object) + .expect("audit amqp target should be encoded"); + assert_eq!( + amqp.get(rustfs_config::AMQP_URL).and_then(Value::as_str), + Some("amqp://127.0.0.1:5672/%2f") + ); + assert_eq!(amqp.get(rustfs_config::AMQP_EXCHANGE).and_then(Value::as_str), Some("rustfs.audit")); + assert_eq!(amqp.get(rustfs_config::AMQP_ROUTING_KEY).and_then(Value::as_str), Some("audit")); + assert!(!amqp.contains_key(rustfs_config::AMQP_MANDATORY)); + assert_eq!(amqp.get(rustfs_config::AMQP_PERSISTENT).and_then(Value::as_bool), Some(false)); + let mqtt_default = logger .get("mqtt") .and_then(Value::as_object) diff --git a/crates/ecstore/src/config/mod.rs b/crates/ecstore/src/config/mod.rs index ddef5e638..0b2935381 100644 --- a/crates/ecstore/src/config/mod.rs +++ b/crates/ecstore/src/config/mod.rs @@ -26,12 +26,12 @@ use com::{STORAGE_CLASS_SUB_SYS, lookup_configs, read_config_without_migrate}; use rustfs_config::COMMENT_KEY; use rustfs_config::DEFAULT_DELIMITER; use rustfs_config::audit::{ - AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_MYSQL_SUB_SYS, AUDIT_NATS_SUB_SYS, AUDIT_POSTGRES_SUB_SYS, + AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_MYSQL_SUB_SYS, AUDIT_NATS_SUB_SYS, AUDIT_POSTGRES_SUB_SYS, AUDIT_PULSAR_SUB_SYS, AUDIT_REDIS_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS, }; use rustfs_config::notify::{ - NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_SUB_SYS, - NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, + NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS, + NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, }; use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS; use serde::{Deserialize, Serialize}; @@ -243,6 +243,8 @@ pub fn init() { kvs.insert(AUDIT_WEBHOOK_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_WEBHOOK_KVS.clone()); kvs.insert(NOTIFY_MQTT_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_MQTT_KVS.clone()); kvs.insert(AUDIT_MQTT_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_MQTT_KVS.clone()); + kvs.insert(NOTIFY_AMQP_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_AMQP_KVS.clone()); + kvs.insert(AUDIT_AMQP_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_AMQP_KVS.clone()); kvs.insert(NOTIFY_NATS_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_NATS_KVS.clone()); kvs.insert(AUDIT_NATS_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_NATS_KVS.clone()); kvs.insert(NOTIFY_REDIS_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_REDIS_KVS.clone()); diff --git a/crates/ecstore/src/config/notify.rs b/crates/ecstore/src/config/notify.rs index 2032bb4f3..516682ff6 100644 --- a/crates/ecstore/src/config/notify.rs +++ b/crates/ecstore/src/config/notify.rs @@ -15,14 +15,16 @@ use crate::config::{KV, KVS}; use rustfs_config::notify::NOTIFY_REDIS_DEFAULT_CHANNEL; use rustfs_config::{ - COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, - KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, - MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, - MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, - MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, - MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, - NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, - NATS_TOKEN, NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE, + AMQP_EXCHANGE, AMQP_MANDATORY, AMQP_PASSWORD, AMQP_PERSISTENT, AMQP_QUEUE_DIR, AMQP_QUEUE_LIMIT, AMQP_ROUTING_KEY, + AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY, AMQP_URL, AMQP_USERNAME, COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, + EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, + KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, + MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, + MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, + MYSQL_DSN_STRING, MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA, + MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, + NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, + NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT, @@ -184,6 +186,81 @@ pub static DEFAULT_NOTIFY_MQTT_KVS: LazyLock = LazyLock::new(|| { ]) }); +pub static DEFAULT_NOTIFY_AMQP_KVS: LazyLock = LazyLock::new(|| { + KVS(vec![ + KV { + key: ENABLE_KEY.to_owned(), + value: EnableState::Off.to_string(), + hidden_if_empty: false, + }, + KV { + key: AMQP_URL.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: AMQP_EXCHANGE.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: AMQP_ROUTING_KEY.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: AMQP_MANDATORY.to_owned(), + value: EnableState::Off.to_string(), + hidden_if_empty: false, + }, + KV { + key: AMQP_PERSISTENT.to_owned(), + value: EnableState::On.to_string(), + hidden_if_empty: false, + }, + KV { + key: AMQP_USERNAME.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: AMQP_PASSWORD.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: AMQP_TLS_CA.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: AMQP_TLS_CLIENT_CERT.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: AMQP_TLS_CLIENT_KEY.to_owned(), + value: "".to_owned(), + hidden_if_empty: true, + }, + KV { + key: AMQP_QUEUE_DIR.to_owned(), + value: EVENT_DEFAULT_DIR.to_owned(), + hidden_if_empty: false, + }, + KV { + key: AMQP_QUEUE_LIMIT.to_owned(), + value: DEFAULT_LIMIT.to_string(), + hidden_if_empty: false, + }, + KV { + key: COMMENT_KEY.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + ]) +}); + pub static DEFAULT_NOTIFY_NATS_KVS: LazyLock = LazyLock::new(|| { KVS(vec![ KV { diff --git a/crates/notify/src/factory.rs b/crates/notify/src/factory.rs index 5a2b6bade..6489f6053 100644 --- a/crates/notify/src/factory.rs +++ b/crates/notify/src/factory.rs @@ -13,189 +13,200 @@ // limitations under the License. use crate::Event; -use async_trait::async_trait; use rustfs_config::EVENT_DEFAULT_DIR; use rustfs_config::notify::{ - NOTIFY_KAFKA_KEYS, NOTIFY_MQTT_KEYS, NOTIFY_MYSQL_KEYS, NOTIFY_NATS_KEYS, NOTIFY_POSTGRES_KEYS, NOTIFY_PULSAR_KEYS, - NOTIFY_REDIS_DEFAULT_CHANNEL, NOTIFY_REDIS_KEYS, NOTIFY_WEBHOOK_KEYS, + NOTIFY_AMQP_KEYS, NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, + NOTIFY_MYSQL_KEYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_KEYS, + NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_DEFAULT_CHANNEL, NOTIFY_REDIS_KEYS, + NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS, }; -use rustfs_ecstore::config::KVS; -use rustfs_targets::{ - Target, - config::{ - build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args, build_pulsar_args, - build_redis_args, build_webhook_args, validate_kafka_config, validate_mqtt_config, validate_mysql_config, - validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, validate_webhook_config, - }, - error::TargetError, - target::TargetType, +use rustfs_targets::config::{ + build_amqp_args, build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args, + build_pulsar_args, build_redis_args, build_webhook_args, validate_amqp_config, validate_kafka_config, validate_mqtt_config, + validate_mysql_config, validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, + validate_webhook_config, }; -use std::collections::HashSet; +use rustfs_targets::target::{ChannelTargetType, TargetType}; +use rustfs_targets::{BuiltinTargetDescriptor, TargetPluginDescriptor, TargetRequestValidator, boxed_target}; -/// Trait for creating targets from configuration -#[async_trait] -pub trait TargetFactory: Send + Sync { - /// Creates a target from configuration - async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError>; - - /// Validates target configuration - fn validate_config(&self, id: &str, config: &KVS) -> Result<(), TargetError>; - - /// Returns a set of valid configuration field names for this target type. - /// This is used to filter environment variables. - fn get_valid_fields(&self) -> HashSet; +pub fn builtin_target_descriptors() -> Vec> { + vec![ + BuiltinTargetDescriptor::new( + NOTIFY_WEBHOOK_SUB_SYS, + TargetRequestValidator::Webhook, + TargetPluginDescriptor::new( + ChannelTargetType::Webhook.as_str(), + NOTIFY_WEBHOOK_KEYS, + |config| validate_webhook_config(config, EVENT_DEFAULT_DIR), + |id, config| { + let args = build_webhook_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; + Ok(boxed_target(rustfs_targets::target::webhook::WebhookTarget::new(id, args)?)) + }, + ), + ), + BuiltinTargetDescriptor::new( + NOTIFY_AMQP_SUB_SYS, + TargetRequestValidator::Amqp(TargetType::NotifyEvent), + TargetPluginDescriptor::new( + ChannelTargetType::Amqp.as_str(), + NOTIFY_AMQP_KEYS, + |config| validate_amqp_config(config, EVENT_DEFAULT_DIR), + |id, config| { + let args = build_amqp_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; + Ok(boxed_target(rustfs_targets::target::amqp::AMQPTarget::new(id, args)?)) + }, + ), + ), + BuiltinTargetDescriptor::new( + NOTIFY_KAFKA_SUB_SYS, + TargetRequestValidator::Kafka(TargetType::NotifyEvent), + TargetPluginDescriptor::new( + ChannelTargetType::Kafka.as_str(), + NOTIFY_KAFKA_KEYS, + |config| validate_kafka_config(config, EVENT_DEFAULT_DIR), + |id, config| { + let args = build_kafka_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; + Ok(boxed_target(rustfs_targets::target::kafka::KafkaTarget::new(id, args)?)) + }, + ), + ), + BuiltinTargetDescriptor::new( + NOTIFY_MQTT_SUB_SYS, + TargetRequestValidator::Mqtt, + TargetPluginDescriptor::new( + ChannelTargetType::Mqtt.as_str(), + NOTIFY_MQTT_KEYS, + validate_mqtt_config, + |id, config| { + let args = build_mqtt_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; + Ok(boxed_target(rustfs_targets::target::mqtt::MQTTTarget::new(id, args)?)) + }, + ), + ), + BuiltinTargetDescriptor::new( + NOTIFY_MYSQL_SUB_SYS, + TargetRequestValidator::MySql, + TargetPluginDescriptor::new( + ChannelTargetType::MySql.as_str(), + NOTIFY_MYSQL_KEYS, + |config| validate_mysql_config(config, EVENT_DEFAULT_DIR), + |id, config| { + let args = build_mysql_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; + Ok(boxed_target(rustfs_targets::target::mysql::MySqlTarget::new(id, args)?)) + }, + ), + ), + BuiltinTargetDescriptor::new( + NOTIFY_NATS_SUB_SYS, + TargetRequestValidator::Nats(TargetType::NotifyEvent), + TargetPluginDescriptor::new( + ChannelTargetType::Nats.as_str(), + NOTIFY_NATS_KEYS, + |config| validate_nats_config(config, EVENT_DEFAULT_DIR), + |id, config| { + let args = build_nats_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; + Ok(boxed_target(rustfs_targets::target::nats::NATSTarget::new(id, args)?)) + }, + ), + ), + BuiltinTargetDescriptor::new( + NOTIFY_POSTGRES_SUB_SYS, + TargetRequestValidator::Postgres(TargetType::NotifyEvent), + TargetPluginDescriptor::new( + ChannelTargetType::Postgres.as_str(), + NOTIFY_POSTGRES_KEYS, + |config| validate_postgres_config(config, EVENT_DEFAULT_DIR), + |id, config| { + let args = build_postgres_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; + Ok(boxed_target(rustfs_targets::target::postgres::PostgresTarget::new(id, args)?)) + }, + ), + ), + BuiltinTargetDescriptor::new( + NOTIFY_REDIS_SUB_SYS, + TargetRequestValidator::Redis { + default_channel: NOTIFY_REDIS_DEFAULT_CHANNEL, + target_type: TargetType::NotifyEvent, + }, + TargetPluginDescriptor::new( + ChannelTargetType::Redis.as_str(), + NOTIFY_REDIS_KEYS, + |config| validate_redis_config(config, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL), + |id, config| { + let args = + build_redis_args(config, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL, TargetType::NotifyEvent)?; + Ok(boxed_target(rustfs_targets::target::redis::RedisTarget::new(id, args)?)) + }, + ), + ), + BuiltinTargetDescriptor::new( + NOTIFY_PULSAR_SUB_SYS, + TargetRequestValidator::Pulsar(TargetType::NotifyEvent), + TargetPluginDescriptor::new( + ChannelTargetType::Pulsar.as_str(), + NOTIFY_PULSAR_KEYS, + |config| validate_pulsar_config(config, EVENT_DEFAULT_DIR), + |id, config| { + let args = build_pulsar_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; + Ok(boxed_target(rustfs_targets::target::pulsar::PulsarTarget::new(id, args)?)) + }, + ), + ), + ] } -/// Factory for creating Webhook targets -pub struct WebhookTargetFactory; - -#[async_trait] -impl TargetFactory for WebhookTargetFactory { - async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError> { - let args = build_webhook_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; - let target = rustfs_targets::target::webhook::WebhookTarget::new(id, args)?; - Ok(Box::new(target)) - } - - fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> { - validate_webhook_config(config, EVENT_DEFAULT_DIR) - } - - fn get_valid_fields(&self) -> HashSet { - NOTIFY_WEBHOOK_KEYS.iter().map(|s| s.to_string()).collect() - } +pub fn builtin_target_plugins() -> Vec> { + builtin_target_descriptors() + .into_iter() + .map(|descriptor| descriptor.plugin().clone()) + .collect() } -/// Factory for creating MQTT targets -pub struct MQTTTargetFactory; +#[cfg(test)] +mod tests { + use super::builtin_target_descriptors; + use rustfs_config::notify::NOTIFY_AMQP_KEYS; + use rustfs_config::{AMQP_EXCHANGE, AMQP_QUEUE_DIR, AMQP_ROUTING_KEY, AMQP_URL}; + use rustfs_ecstore::config::KVS; + use rustfs_targets::target::ChannelTargetType; -#[async_trait] -impl TargetFactory for MQTTTargetFactory { - async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError> { - let args = build_mqtt_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; - let target = rustfs_targets::target::mqtt::MQTTTarget::new(id, args)?; - Ok(Box::new(target)) + fn amqp_base_config() -> KVS { + let mut config = KVS::new(); + config.insert(AMQP_URL.to_string(), "amqp://127.0.0.1:5672/%2f".to_string()); + config.insert(AMQP_EXCHANGE.to_string(), "rustfs.events".to_string()); + config.insert(AMQP_ROUTING_KEY.to_string(), "objects".to_string()); + config.insert(AMQP_QUEUE_DIR.to_string(), String::new()); + config } - fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> { - validate_mqtt_config(config) + #[test] + fn builtin_plugins_include_amqp_descriptor() { + let plugin = builtin_target_descriptors() + .into_iter() + .find(|plugin| plugin.plugin().target_type() == ChannelTargetType::Amqp.as_str()) + .expect("amqp plugin should exist"); + + assert!(plugin.plugin().valid_fields().contains(&AMQP_URL)); + assert!(plugin.plugin().valid_fields().contains(&AMQP_EXCHANGE)); + assert!(plugin.plugin().valid_fields().contains(&AMQP_ROUTING_KEY)); + assert_eq!(plugin.plugin().valid_fields().len(), NOTIFY_AMQP_KEYS.len()); } - fn get_valid_fields(&self) -> HashSet { - NOTIFY_MQTT_KEYS.iter().map(|s| s.to_string()).collect() - } -} - -pub struct NATSTargetFactory; - -#[async_trait] -impl TargetFactory for NATSTargetFactory { - async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError> { - let args = build_nats_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; - let target = rustfs_targets::target::nats::NATSTarget::new(id, args)?; - Ok(Box::new(target)) - } - - fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> { - validate_nats_config(config, EVENT_DEFAULT_DIR) - } - - fn get_valid_fields(&self) -> HashSet { - NOTIFY_NATS_KEYS.iter().map(|s| s.to_string()).collect() - } -} - -pub struct PulsarTargetFactory; - -#[async_trait] -impl TargetFactory for PulsarTargetFactory { - async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError> { - let args = build_pulsar_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; - let target = rustfs_targets::target::pulsar::PulsarTarget::new(id, args)?; - Ok(Box::new(target)) - } - - fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> { - validate_pulsar_config(config, EVENT_DEFAULT_DIR) - } - - fn get_valid_fields(&self) -> HashSet { - NOTIFY_PULSAR_KEYS.iter().map(|s| s.to_string()).collect() - } -} - -pub struct PostgresTargetFactory; - -#[async_trait] -impl TargetFactory for PostgresTargetFactory { - async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError> { - let args = build_postgres_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; - let target = rustfs_targets::target::postgres::PostgresTarget::new(id, args)?; - Ok(Box::new(target)) - } - - fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> { - validate_postgres_config(config, EVENT_DEFAULT_DIR) - } - - fn get_valid_fields(&self) -> HashSet { - NOTIFY_POSTGRES_KEYS.iter().map(|s| s.to_string()).collect() - } -} - -pub struct KafkaTargetFactory; - -#[async_trait] -impl TargetFactory for KafkaTargetFactory { - async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError> { - let args = build_kafka_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; - let target = rustfs_targets::target::kafka::KafkaTarget::new(id, args)?; - Ok(Box::new(target)) - } - - fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> { - validate_kafka_config(config, EVENT_DEFAULT_DIR) - } - - fn get_valid_fields(&self) -> HashSet { - NOTIFY_KAFKA_KEYS.iter().map(|s| s.to_string()).collect() - } -} - -pub struct MySqlTargetFactory; - -#[async_trait] -impl TargetFactory for MySqlTargetFactory { - async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError> { - let args = build_mysql_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; - let target = rustfs_targets::target::mysql::MySqlTarget::new(id, args)?; - Ok(Box::new(target)) - } - - fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> { - validate_mysql_config(config, EVENT_DEFAULT_DIR) - } - - fn get_valid_fields(&self) -> HashSet { - NOTIFY_MYSQL_KEYS.iter().map(|s| s.to_string()).collect() - } -} - -pub struct RedisTargetFactory; - -#[async_trait] -impl TargetFactory for RedisTargetFactory { - async fn create_target(&self, id: String, config: &KVS) -> Result + Send + Sync>, TargetError> { - let args = build_redis_args(config, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL, TargetType::NotifyEvent)?; - let target = rustfs_targets::target::redis::RedisTarget::new(id, args)?; - Ok(Box::new(target)) - } - - fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> { - validate_redis_config(config, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL) - } - - fn get_valid_fields(&self) -> HashSet { - NOTIFY_REDIS_KEYS.iter().map(|s| s.to_string()).collect() + #[test] + fn builtin_plugins_create_notify_amqp_target() { + let plugin = builtin_target_descriptors() + .into_iter() + .find(|plugin| plugin.plugin().target_type() == ChannelTargetType::Amqp.as_str()) + .expect("amqp plugin should exist"); + + let target = plugin + .plugin() + .create_target("primary".to_string(), &amqp_base_config()) + .expect("AMQP target should be created"); + + let target_id = target.id(); + assert_eq!(target_id.id, "primary"); + assert_eq!(target_id.name, "amqp"); + assert!(target.store().is_none()); } } diff --git a/crates/notify/src/integration.rs b/crates/notify/src/integration.rs index b0f5b0e46..04b8fe557 100644 --- a/crates/notify/src/integration.rs +++ b/crates/notify/src/integration.rs @@ -25,8 +25,8 @@ use crate::{ use hashbrown::HashMap; use rustfs_config::notify::{ DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_WEBHOOK_ENABLE, - ENV_NOTIFY_WEBHOOK_ENDPOINT, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS, - NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, + ENV_NOTIFY_WEBHOOK_ENDPOINT, NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, + NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, }; use rustfs_config::{ENV_NOTIFY_ENABLE, EVENT_DEFAULT_DIR}; use rustfs_ecstore::config::{Config, KVS}; @@ -54,6 +54,7 @@ fn notify_configuration_hint() -> String { fn subsystem_target_type(target_type: &str) -> &str { match target_type { + NOTIFY_AMQP_SUB_SYS => "amqp", NOTIFY_WEBHOOK_SUB_SYS => "webhook", NOTIFY_KAFKA_SUB_SYS => "kafka", NOTIFY_MQTT_SUB_SYS => "mqtt", @@ -778,6 +779,13 @@ mod tests { assert_eq!(target_id.name, "webhook"); } + #[test] + fn runtime_target_id_for_subsystem_maps_notify_amqp_to_runtime_type() { + let target_id = runtime_target_id_for_subsystem(NOTIFY_AMQP_SUB_SYS, "Primary"); + assert_eq!(target_id.id, "primary"); + assert_eq!(target_id.name, "amqp"); + } + #[test] fn runtime_target_id_for_subsystem_maps_notify_mqtt_to_runtime_type() { let target_id = runtime_target_id_for_subsystem(NOTIFY_MQTT_SUB_SYS, "Analytics"); diff --git a/crates/notify/src/registry.rs b/crates/notify/src/registry.rs index 35efbb268..3ee881d09 100644 --- a/crates/notify/src/registry.rs +++ b/crates/notify/src/registry.rs @@ -13,21 +13,14 @@ // limitations under the License. use crate::Event; -use crate::factory::{ - KafkaTargetFactory, MQTTTargetFactory, MySqlTargetFactory, NATSTargetFactory, PostgresTargetFactory, PulsarTargetFactory, - RedisTargetFactory, TargetFactory, WebhookTargetFactory, -}; -use futures::stream::{FuturesUnordered, StreamExt}; -use hashbrown::HashMap; +use crate::factory::builtin_target_plugins; use rustfs_config::notify::NOTIFY_ROUTE_PREFIX; use rustfs_ecstore::config::{Config, KVS}; -use rustfs_targets::{Target, TargetError, config::collect_target_configs, target::ChannelTargetType}; -use std::sync::Arc; -use tracing::{error, info}; +use rustfs_targets::{Target, TargetError, TargetPluginRegistry}; /// Registry for managing target factories pub struct TargetRegistry { - factories: HashMap>, + plugins: TargetPluginRegistry, } impl Default for TargetRegistry { @@ -39,26 +32,14 @@ impl Default for TargetRegistry { impl TargetRegistry { /// Creates a new TargetRegistry with built-in factories pub fn new() -> Self { - let mut registry = TargetRegistry { - factories: HashMap::new(), - }; + let mut plugins = TargetPluginRegistry::new(); + plugins.register_all(builtin_target_plugins()); - // Register built-in factories - registry.register(ChannelTargetType::Webhook.as_str(), Box::new(WebhookTargetFactory)); - registry.register(ChannelTargetType::Mqtt.as_str(), Box::new(MQTTTargetFactory)); - registry.register(ChannelTargetType::Nats.as_str(), Box::new(NATSTargetFactory)); - registry.register(ChannelTargetType::Postgres.as_str(), Box::new(PostgresTargetFactory)); - registry.register(ChannelTargetType::Pulsar.as_str(), Box::new(PulsarTargetFactory)); - registry.register(ChannelTargetType::Kafka.as_str(), Box::new(KafkaTargetFactory)); - registry.register(ChannelTargetType::MySql.as_str(), Box::new(MySqlTargetFactory)); - registry.register(ChannelTargetType::Redis.as_str(), Box::new(RedisTargetFactory)); - - registry + TargetRegistry { plugins } } - /// Registers a new factory for a target type - pub fn register(&mut self, target_type: &str, factory: Box) { - self.factories.insert(target_type.to_string(), factory); + pub fn supports_target_type(&self, target_type: &str) -> bool { + self.plugins.supports_target_type(target_type) } /// Creates a target from configuration @@ -68,16 +49,7 @@ impl TargetRegistry { id: String, config: &KVS, ) -> Result + Send + Sync>, TargetError> { - let factory = self - .factories - .get(target_type) - .ok_or_else(|| TargetError::Configuration(format!("Unknown target type: {target_type}")))?; - - // Validate configuration before creating target - factory.validate_config(&id, config)?; - - // Create target - factory.create_target(id, config).await + self.plugins.create_target(target_type, id, config) } /// Creates all targets from a configuration @@ -93,36 +65,19 @@ impl TargetRegistry { &self, config: &Config, ) -> Result + Send + Sync>>, TargetError> { - let mut tasks = FuturesUnordered::new(); - for (target_type, factory) in &self.factories { - tracing::Span::current().record("target_type", target_type.as_str()); - info!("Start working on target types..."); - let valid_fields = factory.get_valid_fields(); - for (id, merged_config) in collect_target_configs(config, NOTIFY_ROUTE_PREFIX, target_type, &valid_fields) { - info!(instance_id = %id, "Target is enabled, ready to create a task"); - let tid = id.clone(); - let merged_config_arc = Arc::new(merged_config); - tasks.push(async move { - let result = factory.create_target(tid.clone(), &merged_config_arc).await; - (tid, result) - }); - } - } - - let mut successful_targets = Vec::new(); - while let Some((id, result)) = tasks.next().await { - match result { - Ok(target) => { - info!(instance_id = %id, "Create target successfully"); - successful_targets.push(target); - } - Err(e) => { - error!(instance_id = %id, error = %e, "Failed to create target"); - } - } - } - - info!(count = successful_targets.len(), "All target processing completed"); - Ok(successful_targets) + self.plugins.create_targets_from_config(config, NOTIFY_ROUTE_PREFIX).await + } +} + +#[cfg(test)] +mod tests { + use super::TargetRegistry; + use rustfs_targets::target::ChannelTargetType; + + #[test] + fn registry_registers_amqp_factory() { + let registry = TargetRegistry::new(); + + assert!(registry.supports_target_type(ChannelTargetType::Amqp.as_str())); } } diff --git a/crates/targets/Cargo.toml b/crates/targets/Cargo.toml index 313509d5f..daa7de6d7 100644 --- a/crates/targets/Cargo.toml +++ b/crates/targets/Cargo.toml @@ -20,6 +20,7 @@ async-trait = { workspace = true } async-nats = { workspace = true } deadpool-postgres = { workspace = true } hyper-rustls = { workspace = true } +lapin = { workspace = true } pulsar = { workspace = true } reqwest = { workspace = true } rumqttc = { workspace = true } @@ -31,7 +32,7 @@ serde = { workspace = true } serde_json = { workspace = true } snap = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time"] } +tokio = { workspace = true, features = ["fs", "rt-multi-thread", "sync", "time"] } tokio-postgres = { workspace = true } tokio-postgres-rustls = { workspace = true } tracing = { workspace = true } @@ -42,6 +43,8 @@ sysinfo = { workspace = true, features = ["multithread"] } rustfs-kafka-async = { workspace = true } mysql_async = { workspace = true } chrono = { workspace = true } +parking_lot = { workspace = true } +hashbrown = { workspace = true } [dev-dependencies] criterion = { workspace = true } diff --git a/crates/targets/src/check.rs b/crates/targets/src/check.rs index 730613905..7b486137f 100644 --- a/crates/targets/src/check.rs +++ b/crates/targets/src/check.rs @@ -220,3 +220,23 @@ pub async fn check_redis_server_available(args: &crate::target::redis::RedisArgs .await .unwrap_or_else(|_| Err(crate::TargetError::Timeout("Redis connection timed out".to_string()))) } + +pub async fn check_amqp_broker_available(args: &crate::target::amqp::AMQPArgs) -> Result<(), crate::TargetError> { + match tokio::time::timeout(std::time::Duration::from_secs(5), async { + let connection = crate::target::amqp::connect_amqp(args).await?; + if !connection.connection.status().connected() || !connection.channel.status().connected() { + return Err(crate::TargetError::NotConnected); + } + connection + .connection + .close(200, "OK".into()) + .await + .map_err(|e| crate::TargetError::Network(format!("Failed to close AMQP check connection: {e}")))?; + Ok(()) + }) + .await + { + Ok(result) => result, + Err(_) => Err(crate::TargetError::Timeout("AMQP connection timed out".to_string())), + } +} diff --git a/crates/targets/src/config/mod.rs b/crates/targets/src/config/mod.rs index bfd9a37c2..6386f7939 100644 --- a/crates/targets/src/config/mod.rs +++ b/crates/targets/src/config/mod.rs @@ -21,7 +21,8 @@ pub use loader::{ collect_target_configs_from_env, }; pub use target_args::{ - build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args, build_pulsar_args, - build_redis_args, build_webhook_args, validate_kafka_config, validate_mqtt_config, validate_mysql_config, - validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, validate_webhook_config, + build_amqp_args, build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args, + build_pulsar_args, build_redis_args, build_webhook_args, validate_amqp_config, validate_kafka_config, validate_mqtt_config, + validate_mysql_config, validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, + validate_webhook_config, }; diff --git a/crates/targets/src/config/target_args.rs b/crates/targets/src/config/target_args.rs index 4d86ad297..42226f93f 100644 --- a/crates/targets/src/config/target_args.rs +++ b/crates/targets/src/config/target_args.rs @@ -16,6 +16,7 @@ use super::common::{parse_target_bool, parse_url, validate_nats_server_config, v use crate::error::TargetError; use crate::target::{ TargetType, + amqp::AMQPArgs, kafka::KafkaArgs, mqtt::{MQTTArgs, MQTTTlsConfig, validate_mqtt_broker_url}, mysql::MySqlArgs, @@ -27,22 +28,23 @@ use crate::target::{ }; use rumqttc::QoS; use rustfs_config::{ - DEFAULT_LIMIT, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, - KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, - MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, - MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, - MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA, - MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, - NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, - NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE, - POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER, - PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, - PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT, - REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD, - REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT, - REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL, - REDIS_USERNAME, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, - WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY, + AMQP_EXCHANGE, AMQP_MANDATORY, AMQP_PASSWORD, AMQP_PERSISTENT, AMQP_QUEUE_DIR, AMQP_QUEUE_LIMIT, AMQP_ROUTING_KEY, + AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY, AMQP_URL, AMQP_USERNAME, DEFAULT_LIMIT, KAFKA_ACKS, KAFKA_BROKERS, + KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, + MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, + MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, + MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, + MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, + NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, + NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, + POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, + POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, + PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, + REDIS_CONNECTION_TIMEOUT, REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, + REDIS_PASSWORD, REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS, + REDIS_RESPONSE_TIMEOUT, REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, + REDIS_TLS_POLICY, REDIS_URL, REDIS_USERNAME, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, + WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY, }; use rustfs_ecstore::config::KVS; use std::path::Path; @@ -66,6 +68,55 @@ fn parse_kafka_acks_value(value: Option<&str>) -> Result { } } +fn parse_amqp_bool_value(field: &str, config: &KVS, default: bool) -> Result { + match config.lookup(field) { + Some(value) => parse_target_bool(Some(value.as_str())) + .ok_or_else(|| TargetError::Configuration(format!("Invalid AMQP {field} boolean value: {value}"))), + None => Ok(default), + } +} + +pub fn build_amqp_args(config: &KVS, default_queue_dir: &str, target_type: TargetType) -> Result { + let url = config + .lookup(AMQP_URL) + .ok_or_else(|| TargetError::Configuration("Missing AMQP url".to_string()))?; + let url = parse_url(url.trim(), "AMQP URL")?; + + let exchange = config + .lookup(AMQP_EXCHANGE) + .ok_or_else(|| TargetError::Configuration("Missing AMQP exchange".to_string()))?; + let routing_key = config + .lookup(AMQP_ROUTING_KEY) + .ok_or_else(|| TargetError::Configuration("Missing AMQP routing_key".to_string()))?; + + let args = AMQPArgs { + enable: true, + url, + exchange, + routing_key, + mandatory: parse_amqp_bool_value(AMQP_MANDATORY, config, false)?, + persistent: parse_amqp_bool_value(AMQP_PERSISTENT, config, true)?, + username: config.lookup(AMQP_USERNAME).unwrap_or_default(), + password: config.lookup(AMQP_PASSWORD).unwrap_or_default(), + tls_ca: config.lookup(AMQP_TLS_CA).unwrap_or_default(), + tls_client_cert: config.lookup(AMQP_TLS_CLIENT_CERT).unwrap_or_default(), + tls_client_key: config.lookup(AMQP_TLS_CLIENT_KEY).unwrap_or_default(), + queue_dir: config.lookup(AMQP_QUEUE_DIR).unwrap_or_else(|| default_queue_dir.to_string()), + queue_limit: config + .lookup(AMQP_QUEUE_LIMIT) + .and_then(|v| v.parse::().ok()) + .unwrap_or(DEFAULT_LIMIT), + target_type, + }; + args.validate()?; + Ok(args) +} + +pub fn validate_amqp_config(config: &KVS, default_queue_dir: &str) -> Result<(), TargetError> { + let _ = build_amqp_args(config, default_queue_dir, TargetType::NotifyEvent)?; + Ok(()) +} + pub fn build_webhook_args(config: &KVS, default_queue_dir: &str, target_type: TargetType) -> Result { let endpoint = config .lookup(WEBHOOK_ENDPOINT) @@ -540,19 +591,28 @@ pub fn validate_mysql_config(config: &KVS, default_queue_dir: &str) -> Result<() #[cfg(test)] mod tests { use super::{ - build_kafka_args, build_mysql_args, build_postgres_args, build_redis_args, validate_kafka_config, validate_mysql_config, - validate_postgres_config, validate_redis_config, + build_amqp_args, build_kafka_args, build_mysql_args, build_postgres_args, build_redis_args, validate_amqp_config, + validate_kafka_config, validate_mysql_config, validate_postgres_config, validate_redis_config, }; use crate::target::{TargetType, postgres::PostgresFormat}; use rustfs_config::{ - KAFKA_ACKS, KAFKA_BROKERS, KAFKA_TOPIC, MYSQL_DSN_STRING, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_TABLE, - MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, - POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, REDIS_CHANNEL, - REDIS_CONNECTION_TIMEOUT, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PIPELINE_BUFFER_SIZE, - REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT, REDIS_TLS_ALLOW_INSECURE, REDIS_URL, + AMQP_EXCHANGE, AMQP_MANDATORY, AMQP_PASSWORD, AMQP_PERSISTENT, AMQP_QUEUE_DIR, AMQP_ROUTING_KEY, AMQP_TLS_CLIENT_CERT, + AMQP_TLS_CLIENT_KEY, AMQP_URL, AMQP_USERNAME, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_TOPIC, MYSQL_DSN_STRING, + MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, + POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, + POSTGRES_TLS_CLIENT_KEY, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, + REDIS_PIPELINE_BUFFER_SIZE, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT, REDIS_TLS_ALLOW_INSECURE, REDIS_URL, }; use rustfs_ecstore::config::KVS; + fn amqp_base_config() -> KVS { + let mut config = KVS::new(); + config.insert(AMQP_URL.to_string(), "amqp://127.0.0.1:5672/%2f".to_string()); + config.insert(AMQP_EXCHANGE.to_string(), "rustfs.events".to_string()); + config.insert(AMQP_ROUTING_KEY.to_string(), "objects".to_string()); + config + } + fn kafka_base_config() -> KVS { let mut config = KVS::new(); config.insert(KAFKA_BROKERS.to_string(), "127.0.0.1:9092".to_string()); @@ -570,6 +630,123 @@ mod tests { config } + #[test] + fn build_amqp_args_accepts_valid_config() { + let args = build_amqp_args(&amqp_base_config(), "", TargetType::NotifyEvent).expect("valid AMQP args"); + + assert_eq!(args.url.as_str(), "amqp://127.0.0.1:5672/%2f"); + assert_eq!(args.exchange, "rustfs.events"); + assert_eq!(args.routing_key, "objects"); + assert!(!args.mandatory); + assert!(args.persistent); + } + + #[test] + fn build_amqp_args_accepts_bool_aliases() { + let mut config = amqp_base_config(); + config.insert(AMQP_MANDATORY.to_string(), "on".to_string()); + config.insert(AMQP_PERSISTENT.to_string(), "no".to_string()); + + let args = build_amqp_args(&config, "", TargetType::NotifyEvent).expect("valid AMQP bool aliases"); + + assert!(args.mandatory); + assert!(!args.persistent); + } + + #[test] + fn validate_amqp_config_rejects_invalid_bool() { + let mut config = amqp_base_config(); + config.insert(AMQP_MANDATORY.to_string(), "sometimes".to_string()); + + let err = validate_amqp_config(&config, "").expect_err("invalid AMQP bool should fail"); + + assert!(err.to_string().contains("Invalid AMQP mandatory boolean")); + } + + #[test] + fn validate_amqp_config_rejects_invalid_scheme() { + let mut config = amqp_base_config(); + config.insert(AMQP_URL.to_string(), "http://127.0.0.1:5672".to_string()); + + let err = validate_amqp_config(&config, "").expect_err("invalid AMQP scheme should fail"); + + assert!(err.to_string().contains("only amqp and amqps")); + } + + #[test] + fn validate_amqp_config_rejects_missing_url_host() { + let mut config = amqp_base_config(); + config.insert(AMQP_URL.to_string(), "amqp:///objects".to_string()); + + let err = validate_amqp_config(&config, "").expect_err("missing AMQP host should fail"); + + assert!(err.to_string().contains("missing host")); + } + + #[test] + fn validate_amqp_config_rejects_missing_exchange() { + let mut config = amqp_base_config(); + config.0.retain(|kv| kv.key != AMQP_EXCHANGE); + + let err = validate_amqp_config(&config, "").expect_err("missing AMQP exchange should fail"); + + assert!(err.to_string().contains("Missing AMQP exchange")); + } + + #[test] + fn validate_amqp_config_rejects_missing_routing_key() { + let mut config = amqp_base_config(); + config.0.retain(|kv| kv.key != AMQP_ROUTING_KEY); + + let err = validate_amqp_config(&config, "").expect_err("missing AMQP routing_key should fail"); + + assert!(err.to_string().contains("Missing AMQP routing_key")); + } + + #[test] + fn validate_amqp_config_rejects_relative_queue_dir() { + let mut config = amqp_base_config(); + config.insert(AMQP_QUEUE_DIR.to_string(), "relative-queue".to_string()); + + let err = validate_amqp_config(&config, "").expect_err("relative queue_dir should fail"); + + assert!(err.to_string().contains("absolute path")); + } + + #[test] + fn validate_amqp_config_rejects_unpaired_tls_client_cert_key() { + let mut config = amqp_base_config(); + config.insert(AMQP_URL.to_string(), "amqps://127.0.0.1:5671/%2f".to_string()); + config.insert(AMQP_TLS_CLIENT_CERT.to_string(), "/tmp/client.crt".to_string()); + + let err = validate_amqp_config(&config, "").expect_err("unpaired TLS cert should fail"); + + assert!(err.to_string().contains("tls_client_cert and tls_client_key")); + } + + #[test] + fn validate_amqp_config_rejects_tls_paths_without_amqps() { + let mut config = amqp_base_config(); + config.insert(AMQP_TLS_CLIENT_CERT.to_string(), "/tmp/client.crt".to_string()); + config.insert(AMQP_TLS_CLIENT_KEY.to_string(), "/tmp/client.key".to_string()); + + let err = validate_amqp_config(&config, "").expect_err("TLS paths without amqps should fail"); + + assert!(err.to_string().contains("only allowed with amqps")); + } + + #[test] + fn validate_amqp_config_rejects_ambiguous_credentials() { + let mut config = amqp_base_config(); + config.insert(AMQP_URL.to_string(), "amqp://guest:guest@127.0.0.1:5672/%2f".to_string()); + config.insert(AMQP_USERNAME.to_string(), "user".to_string()); + config.insert(AMQP_PASSWORD.to_string(), "password".to_string()); + + let err = validate_amqp_config(&config, "").expect_err("ambiguous credentials should fail"); + + assert!(err.to_string().contains("either in url or username/password")); + } + #[test] fn build_kafka_args_accepts_all_ack_alias() { let mut config = kafka_base_config(); diff --git a/crates/targets/src/lib.rs b/crates/targets/src/lib.rs index 43b177f03..aeff41e37 100644 --- a/crates/targets/src/lib.rs +++ b/crates/targets/src/lib.rs @@ -16,15 +16,17 @@ pub mod arn; mod check; pub mod config; pub mod error; +pub mod plugin; pub mod store; pub mod sys; pub mod target; pub use check::{ - check_kafka_broker_available, check_mqtt_broker_available, check_mqtt_broker_available_with_tls, check_nats_server_available, - check_postgres_server_available, check_pulsar_broker_available, check_redis_server_available, + check_amqp_broker_available, check_kafka_broker_available, check_mqtt_broker_available, check_mqtt_broker_available_with_tls, + check_nats_server_available, check_postgres_server_available, check_pulsar_broker_available, check_redis_server_available, }; pub use error::{StoreError, TargetError}; +pub use plugin::{BuiltinTargetDescriptor, TargetPluginDescriptor, TargetPluginRegistry, TargetRequestValidator, boxed_target}; pub use rustfs_s3_common::EventName; use serde::{Deserialize, Serialize}; pub use sys::user_agent::*; diff --git a/crates/targets/src/plugin.rs b/crates/targets/src/plugin.rs new file mode 100644 index 000000000..24743e315 --- /dev/null +++ b/crates/targets/src/plugin.rs @@ -0,0 +1,231 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{Target, TargetError, config::collect_target_configs}; +use hashbrown::HashMap; +use rustfs_ecstore::config::{Config, KVS}; +use serde::Serialize; +use serde::de::DeserializeOwned; +use std::collections::HashSet; +use std::sync::Arc; +use tracing::{error, info}; + +type BoxedTarget = Box + Send + Sync>; +type TargetCreateFn = Arc Result, TargetError> + Send + Sync>; +type TargetValidateFn = Arc Result<(), TargetError> + Send + Sync>; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TargetRequestValidator { + Webhook, + Mqtt, + Amqp(crate::target::TargetType), + Kafka(crate::target::TargetType), + MySql, + Nats(crate::target::TargetType), + Postgres(crate::target::TargetType), + Pulsar(crate::target::TargetType), + Redis { + default_channel: &'static str, + target_type: crate::target::TargetType, + }, +} + +#[derive(Clone)] +pub struct TargetPluginDescriptor +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + create_target: TargetCreateFn, + target_type: &'static str, + valid_fields: &'static [&'static str], + valid_fields_set: Arc>, + validate_config: TargetValidateFn, +} + +impl TargetPluginDescriptor +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + pub fn new( + target_type: &'static str, + valid_fields: &'static [&'static str], + validate_config: Validate, + create_target: Create, + ) -> Self + where + Create: Fn(String, &KVS) -> Result, TargetError> + Send + Sync + 'static, + Validate: Fn(&KVS) -> Result<(), TargetError> + Send + Sync + 'static, + { + Self { + create_target: Arc::new(create_target), + target_type, + valid_fields, + valid_fields_set: Arc::new(valid_fields.iter().map(|field| (*field).to_string()).collect()), + validate_config: Arc::new(validate_config), + } + } + + #[inline] + pub fn target_type(&self) -> &'static str { + self.target_type + } + + #[inline] + pub fn valid_fields(&self) -> &'static [&'static str] { + self.valid_fields + } + + #[inline] + pub fn valid_fields_set(&self) -> &HashSet { + self.valid_fields_set.as_ref() + } + + #[inline] + pub fn validate_config(&self, config: &KVS) -> Result<(), TargetError> { + (self.validate_config)(config) + } + + #[inline] + pub fn create_target(&self, id: String, config: &KVS) -> Result, TargetError> { + (self.create_target)(id, config) + } +} + +#[derive(Clone)] +pub struct BuiltinTargetDescriptor +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + plugin: TargetPluginDescriptor, + request_validator: TargetRequestValidator, + subsystem: &'static str, +} + +impl BuiltinTargetDescriptor +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + pub fn new(subsystem: &'static str, request_validator: TargetRequestValidator, plugin: TargetPluginDescriptor) -> Self { + Self { + plugin, + request_validator, + subsystem, + } + } + + #[inline] + pub fn plugin(&self) -> &TargetPluginDescriptor { + &self.plugin + } + + #[inline] + pub fn request_validator(&self) -> TargetRequestValidator { + self.request_validator + } + + #[inline] + pub fn subsystem(&self) -> &'static str { + self.subsystem + } +} + +pub struct TargetPluginRegistry +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + plugins: HashMap>, +} + +impl Default for TargetPluginRegistry +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + fn default() -> Self { + Self::new() + } +} + +impl TargetPluginRegistry +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + pub fn new() -> Self { + Self { plugins: HashMap::new() } + } + + pub fn register(&mut self, plugin: TargetPluginDescriptor) -> Option> { + self.plugins.insert(plugin.target_type().to_string(), plugin) + } + + pub fn register_all(&mut self, plugins: I) + where + I: IntoIterator>, + { + for plugin in plugins { + self.register(plugin); + } + } + + pub fn supports_target_type(&self, target_type: &str) -> bool { + self.plugins.contains_key(target_type) + } + + pub fn registered_target_types(&self) -> Vec { + self.plugins.keys().cloned().collect() + } + + pub fn create_target(&self, target_type: &str, id: String, config: &KVS) -> Result, TargetError> { + let plugin = self + .plugins + .get(target_type) + .ok_or_else(|| TargetError::Configuration(format!("Unknown target type: {target_type}")))?; + plugin.validate_config(config)?; + plugin.create_target(id, config) + } + + pub async fn create_targets_from_config( + &self, + config: &Config, + route_prefix: &str, + ) -> Result>, TargetError> { + let mut successful_targets = Vec::new(); + + for (target_type, plugin) in &self.plugins { + info!(target_type = %target_type, "Start working on target type"); + for (id, merged_config) in collect_target_configs(config, route_prefix, target_type, plugin.valid_fields_set()) { + info!(target_type = %target_type, instance_id = %id, "Target is enabled, ready to create"); + match self.create_target(target_type, id.clone(), &merged_config) { + Ok(target) => { + info!(target_type = %target.id().name, instance_id = %id, "Create target successfully"); + successful_targets.push(target); + } + Err(err) => { + error!(target_type = %target_type, instance_id = %id, error = %err, "Failed to create target"); + } + } + } + } + + info!(count = successful_targets.len(), "All target processing completed"); + Ok(successful_targets) + } +} + +pub fn boxed_target(target: T) -> BoxedTarget +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, + T: Target + Send + Sync + 'static, +{ + Box::new(target) +} diff --git a/crates/targets/src/target/amqp.rs b/crates/targets/src/target/amqp.rs new file mode 100644 index 000000000..58408c916 --- /dev/null +++ b/crates/targets/src/target/amqp.rs @@ -0,0 +1,706 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! AMQP 0-9-1 event notification target. +//! +//! Publishes S3 events to RabbitMQ-compatible AMQP 0-9-1 brokers via `lapin`. +//! Queue-store mode uses the shared target store and replays the same raw JSON +//! body through `send_raw_from_store`. + +use crate::{ + StoreError, Target, TargetLog, + arn::TargetID, + error::TargetError, + store::{Key, QueueStore, Store}, + target::{ + ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, + TargetType, queue_store_subdir_name, + }, +}; +use async_trait::async_trait; +use lapin::{ + BasicProperties, Channel, Confirmation, Connection, ConnectionProperties, ErrorKind as LapinErrorKind, + options::{BasicPublishOptions, ConfirmSelectOptions}, + tcp::{OwnedIdentity, OwnedTLSConfig}, +}; +use parking_lot::Mutex; +use rustfs_config::{AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY}; +use serde::Serialize; +use serde::de::DeserializeOwned; +use std::fmt; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::Mutex as AsyncMutex; +use tracing::{error, info, instrument, warn}; +use url::Url; + +#[derive(Clone)] +pub struct AMQPArgs { + pub enable: bool, + pub url: Url, + pub exchange: String, + pub routing_key: String, + pub mandatory: bool, + pub persistent: bool, + pub username: String, + pub password: String, + pub tls_ca: String, + pub tls_client_cert: String, + pub tls_client_key: String, + pub queue_dir: String, + pub queue_limit: u64, + pub target_type: TargetType, +} + +impl fmt::Debug for AMQPArgs { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AMQPArgs") + .field("enable", &self.enable) + .field("url", &redacted_amqp_url(&self.url)) + .field("exchange", &self.exchange) + .field("routing_key", &self.routing_key) + .field("mandatory", &self.mandatory) + .field("persistent", &self.persistent) + .field("username", &self.username) + .field("password", if self.password.is_empty() { &"" } else { &"***REDACTED***" }) + .field("tls_ca", &self.tls_ca) + .field("tls_client_cert", &self.tls_client_cert) + .field( + "tls_client_key", + if self.tls_client_key.is_empty() { + &"" + } else { + &"***REDACTED***" + }, + ) + .field("queue_dir", &self.queue_dir) + .field("queue_limit", &self.queue_limit) + .field("target_type", &self.target_type) + .finish() + } +} + +impl AMQPArgs { + pub fn validate(&self) -> Result<(), TargetError> { + if !self.enable { + return Ok(()); + } + + validate_amqp_url(&self.url)?; + + if self.exchange.trim().is_empty() { + return Err(TargetError::Configuration("AMQP exchange cannot be empty".to_string())); + } + if self.routing_key.trim().is_empty() { + return Err(TargetError::Configuration("AMQP routing_key cannot be empty".to_string())); + } + + let url_has_credentials = !self.url.username().is_empty() || self.url.password().is_some(); + let config_has_credentials = !self.username.is_empty() || !self.password.is_empty(); + if self.username.is_empty() != self.password.is_empty() { + return Err(TargetError::Configuration( + "AMQP username and password must be specified together".to_string(), + )); + } + if url_has_credentials && config_has_credentials { + return Err(TargetError::Configuration( + "AMQP credentials must be specified either in url or username/password, not both".to_string(), + )); + } + + validate_amqp_tls_paths(self)?; + + if !self.queue_dir.is_empty() && !Path::new(&self.queue_dir).is_absolute() { + return Err(TargetError::Configuration("AMQP queue directory must be an absolute path".to_string())); + } + + Ok(()) + } +} + +fn redacted_amqp_url(url: &Url) -> String { + if url.password().is_none() { + return url.to_string(); + } + let mut redacted = url.clone(); + let _ = redacted.set_password(Some("***REDACTED***")); + redacted.to_string() +} + +pub fn validate_amqp_url(url: &Url) -> Result<(), TargetError> { + match url.scheme() { + "amqp" | "amqps" => { + if url.host_str().is_none() { + return Err(TargetError::Configuration("AMQP URL is missing host".to_string())); + } + Ok(()) + } + scheme => Err(TargetError::Configuration(format!( + "Unsupported AMQP URL scheme: {scheme} (only amqp and amqps are allowed)" + ))), + } +} + +fn validate_amqp_tls_paths(args: &AMQPArgs) -> Result<(), TargetError> { + let has_tls_settings = !args.tls_ca.is_empty() || !args.tls_client_cert.is_empty() || !args.tls_client_key.is_empty(); + if has_tls_settings && args.url.scheme() != "amqps" { + return Err(TargetError::Configuration( + "AMQP TLS settings are only allowed with amqps URLs".to_string(), + )); + } + + if args.tls_client_cert.is_empty() != args.tls_client_key.is_empty() { + return Err(TargetError::Configuration( + "AMQP tls_client_cert and tls_client_key must be specified together".to_string(), + )); + } + + if !args.tls_ca.is_empty() && !Path::new(&args.tls_ca).is_absolute() { + return Err(TargetError::Configuration(format!("{AMQP_TLS_CA} must be an absolute path"))); + } + if !args.tls_client_cert.is_empty() && !Path::new(&args.tls_client_cert).is_absolute() { + return Err(TargetError::Configuration(format!("{AMQP_TLS_CLIENT_CERT} must be an absolute path"))); + } + if !args.tls_client_key.is_empty() && !Path::new(&args.tls_client_key).is_absolute() { + return Err(TargetError::Configuration(format!("{AMQP_TLS_CLIENT_KEY} must be an absolute path"))); + } + + Ok(()) +} + +fn connection_url(args: &AMQPArgs) -> Result { + let mut url = args.url.clone(); + if !args.username.is_empty() { + url.set_username(&args.username) + .map_err(|_| TargetError::Configuration("AMQP username cannot be set on URL".to_string()))?; + url.set_password(Some(&args.password)) + .map_err(|_| TargetError::Configuration("AMQP password cannot be set on URL".to_string()))?; + } + Ok(url.to_string()) +} + +async fn build_tls_config(args: &AMQPArgs) -> Result { + let cert_chain = if args.tls_ca.is_empty() { + None + } else { + Some( + tokio::fs::read_to_string(&args.tls_ca) + .await + .map_err(|e| TargetError::Configuration(format!("Failed to read {AMQP_TLS_CA}: {e}")))?, + ) + }; + + let identity = if args.tls_client_cert.is_empty() { + None + } else { + let pem = tokio::fs::read(&args.tls_client_cert) + .await + .map_err(|e| TargetError::Configuration(format!("Failed to read {AMQP_TLS_CLIENT_CERT}: {e}")))?; + let key = tokio::fs::read(&args.tls_client_key) + .await + .map_err(|e| TargetError::Configuration(format!("Failed to read {AMQP_TLS_CLIENT_KEY}: {e}")))?; + Some(OwnedIdentity::PKCS8 { pem, key }) + }; + + Ok(OwnedTLSConfig { identity, cert_chain }) +} + +fn build_publish_properties(args: &AMQPArgs) -> BasicProperties { + let mut properties = BasicProperties::default().with_content_type("application/json".into()); + if args.persistent { + properties = properties.with_delivery_mode(2); + } + properties +} + +fn map_lapin_error(err: lapin::Error, context: &str) -> TargetError { + let message = format!("{context}: {err}"); + match err.kind() { + LapinErrorKind::IOError(io_err) if io_err.kind() == std::io::ErrorKind::TimedOut => TargetError::Timeout(message), + LapinErrorKind::IOError(_) + | LapinErrorKind::InvalidConnectionState(_) + | LapinErrorKind::InvalidChannelState(..) + | LapinErrorKind::MissingHeartbeatError + | LapinErrorKind::ProtocolError(_) + if err.can_be_recovered() => + { + TargetError::NotConnected + } + _ => TargetError::Network(message), + } +} + +pub async fn connect_amqp(args: &AMQPArgs) -> Result { + args.validate()?; + tokio::time::timeout(Duration::from_secs(5), async { + let url = connection_url(args)?; + // Reconnect explicitly so every new channel enables publisher confirms below. + let properties = ConnectionProperties::default(); + let connection = if args.url.scheme() == "amqps" && (!args.tls_ca.is_empty() || !args.tls_client_cert.is_empty()) { + Connection::connect_with_config( + &url, + properties, + build_tls_config(args).await?, + lapin::runtime::default_runtime() + .map_err(|e| TargetError::Initialization(format!("Failed to create AMQP runtime: {e}")))?, + ) + .await + } else { + Connection::connect(&url, properties).await + } + .map_err(|e| map_lapin_error(e, "Failed to connect to AMQP broker"))?; + + let channel = connection + .create_channel() + .await + .map_err(|e| map_lapin_error(e, "Failed to create AMQP channel"))?; + channel + .confirm_select(ConfirmSelectOptions::default()) + .await + .map_err(|e| map_lapin_error(e, "Failed to enable AMQP publisher confirms"))?; + + Ok(AMQPConnection { connection, channel }) + }) + .await + .unwrap_or_else(|_| Err(TargetError::Timeout("AMQP connection timed out".to_string()))) +} + +pub struct AMQPConnection { + pub(crate) connection: Connection, + pub(crate) channel: Channel, +} + +pub struct AMQPTarget +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + id: TargetID, + args: AMQPArgs, + connection: Arc>>>, + connect_lock: Arc>, + store: Option + Send + Sync>>, + delivery_counters: Arc, + _phantom: std::marker::PhantomData, +} + +impl AMQPTarget +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + pub fn clone_box(&self) -> Box + Send + Sync> { + Box::new(AMQPTarget:: { + id: self.id.clone(), + args: self.args.clone(), + connection: Arc::clone(&self.connection), + connect_lock: Arc::clone(&self.connect_lock), + store: self.store.as_ref().map(|s| s.boxed_clone()), + delivery_counters: Arc::clone(&self.delivery_counters), + _phantom: std::marker::PhantomData, + }) + } + + #[instrument(skip(args), fields(target_id_as_string = %id))] + pub fn new(id: String, args: AMQPArgs) -> Result { + args.validate()?; + let target_id = TargetID::new(id, ChannelTargetType::Amqp.as_str().to_string()); + let queue_store = if !args.queue_dir.is_empty() { + let base_path = PathBuf::from(&args.queue_dir); + let specific_queue_path = base_path.join(queue_store_subdir_name(ChannelTargetType::Amqp.as_str(), &target_id.id)); + let extension = match args.target_type { + TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION, + TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION, + }; + let store = QueueStore::::new(specific_queue_path, args.queue_limit, extension); + if let Err(e) = store.open() { + error!(target_id = %target_id, error = %e, "Failed to open store for AMQP target"); + return Err(TargetError::Storage(format!("{e}"))); + } + Some(Box::new(store) as Box + Send + Sync>) + } else { + None + }; + + Ok(Self { + id: target_id, + args, + connection: Arc::new(Mutex::new(None)), + connect_lock: Arc::new(AsyncMutex::new(())), + store: queue_store, + delivery_counters: Arc::new(TargetDeliveryCounters::default()), + _phantom: std::marker::PhantomData, + }) + } + + fn build_queued_payload(&self, event: &EntityTarget) -> Result { + let object_name = crate::target::decode_object_name(&event.object_name)?; + let key = format!("{}/{}", event.bucket_name, object_name); + let log = TargetLog { + event_name: event.event_name, + key, + records: vec![event.clone()], + }; + let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?; + let meta = QueuedPayloadMeta::new( + event.event_name, + event.bucket_name.clone(), + event.object_name.clone(), + "application/json", + body.len(), + ); + Ok(QueuedPayload::new(meta, body)) + } + + async fn get_or_connect(&self) -> Result, TargetError> { + if let Some(connection) = self.connection.lock().clone() + && connection.connection.status().connected() + && connection.channel.status().connected() + { + return Ok(connection); + } + + let _guard = self.connect_lock.lock().await; + if let Some(connection) = self.connection.lock().clone() + && connection.connection.status().connected() + && connection.channel.status().connected() + { + return Ok(connection); + } + + let connection = Arc::new(connect_amqp(&self.args).await?); + let mut guard = self.connection.lock(); + *guard = Some(Arc::clone(&connection)); + Ok(connection) + } + + fn clear_connection(&self) { + *self.connection.lock() = None; + } + + async fn send_body(&self, body: &[u8]) -> Result<(), TargetError> { + let connection = self.get_or_connect().await?; + let publish = connection + .channel + .basic_publish( + self.args.exchange.clone().into(), + self.args.routing_key.clone().into(), + BasicPublishOptions { + mandatory: self.args.mandatory, + ..BasicPublishOptions::default() + }, + body, + build_publish_properties(&self.args), + ) + .await; + + let confirm = match publish { + Ok(confirm) => confirm.await, + Err(err) => { + self.clear_connection(); + return Err(map_lapin_error(err, "Failed to publish AMQP message")); + } + }; + + match confirm { + Ok(Confirmation::Ack(None) | Confirmation::NotRequested) => { + self.delivery_counters.record_success(); + Ok(()) + } + Ok(Confirmation::Ack(Some(returned)) | Confirmation::Nack(Some(returned))) => { + Err(TargetError::Request(format!("AMQP broker returned message: {}", returned.reply_text))) + } + Ok(Confirmation::Nack(None)) => Err(TargetError::Request("AMQP broker negatively acknowledged message".to_string())), + Err(err) => { + self.clear_connection(); + Err(map_lapin_error(err, "Failed to confirm AMQP publish")) + } + } + } +} + +#[async_trait] +impl Target for AMQPTarget +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + fn id(&self) -> TargetID { + self.id.clone() + } + + async fn is_active(&self) -> Result { + let connection = self.get_or_connect().await?; + Ok(connection.connection.status().connected() && connection.channel.status().connected()) + } + + async fn save(&self, event: Arc>) -> Result<(), TargetError> { + let queued = match self.build_queued_payload(&event) { + Ok(queued) => queued, + Err(err) => { + self.delivery_counters.record_final_failure(); + return Err(err); + } + }; + + if let Some(store) = &self.store { + let encoded = match queued.encode() { + Ok(encoded) => encoded, + Err(err) => { + self.delivery_counters.record_final_failure(); + return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}"))); + } + }; + if let Err(e) = store.put_raw(&encoded) { + self.delivery_counters.record_final_failure(); + return Err(TargetError::Storage(format!("Failed to save event to store: {e}"))); + } + Ok(()) + } else { + if let Err(err) = self.send_body(&queued.body).await { + self.delivery_counters.record_final_failure(); + return Err(err); + } + Ok(()) + } + } + + async fn send_raw_from_store(&self, _key: Key, body: Vec, _meta: QueuedPayloadMeta) -> Result<(), TargetError> { + self.send_body(&body).await + } + + async fn close(&self) -> Result<(), TargetError> { + let connection = self.connection.lock().take(); + if let Some(connection) = connection { + connection + .connection + .close(200, "OK".into()) + .await + .map_err(|e| map_lapin_error(e, "Failed to close AMQP connection"))?; + } + info!(target_id = %self.id, "AMQP target closed"); + Ok(()) + } + + fn store(&self) -> Option<&(dyn Store + Send + Sync)> { + self.store.as_deref() + } + + fn clone_dyn(&self) -> Box + Send + Sync> { + self.clone_box() + } + + async fn init(&self) -> Result<(), TargetError> { + if !self.is_enabled() { + return Ok(()); + } + match self.get_or_connect().await { + Ok(_) => Ok(()), + Err(err) + if self.store.is_some() + && matches!(err, TargetError::Network(_) | TargetError::Timeout(_) | TargetError::NotConnected) => + { + warn!(target_id = %self.id, error = %err, "AMQP init failed; events will buffer in store"); + Ok(()) + } + Err(err) => Err(err), + } + } + + fn is_enabled(&self) -> bool { + self.args.enable + } + + fn delivery_snapshot(&self) -> TargetDeliverySnapshot { + self.delivery_counters + .snapshot(self.store.as_deref().map_or(0, |store| store.len() as u64)) + } + + fn record_final_failure(&self) { + self.delivery_counters.record_final_failure(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rustfs_s3_common::EventName; + use serde_json::json; + use std::sync::Arc; + use uuid::Uuid; + + fn valid_args() -> AMQPArgs { + AMQPArgs { + enable: true, + url: Url::parse("amqp://127.0.0.1:5672/%2f").unwrap(), + exchange: "rustfs.events".to_string(), + routing_key: "objects".to_string(), + mandatory: false, + persistent: true, + username: String::new(), + password: String::new(), + tls_ca: String::new(), + tls_client_cert: String::new(), + tls_client_key: String::new(), + queue_dir: String::new(), + queue_limit: 10, + target_type: TargetType::NotifyEvent, + } + } + + fn unreachable_args() -> AMQPArgs { + AMQPArgs { + url: Url::parse("amqp://127.0.0.1:1/%2f").unwrap(), + ..valid_args() + } + } + + fn test_event() -> Arc> { + Arc::new(EntityTarget { + object_name: "object.txt".to_string(), + bucket_name: "bucket".to_string(), + event_name: EventName::ObjectCreatedPut, + data: json!({"ok": true}), + }) + } + + fn temp_store_dir(name: &str) -> PathBuf { + std::env::temp_dir().join(format!("rustfs-amqp-target-{name}-{}", Uuid::new_v4())) + } + + fn assert_connect_failure(err: &TargetError) { + assert!( + matches!(err, TargetError::NotConnected | TargetError::Timeout(_)), + "unexpected error: {err}" + ); + } + + #[test] + fn new_rejects_invalid_args() { + let mut args = valid_args(); + args.exchange.clear(); + + let err = match AMQPTarget::::new("primary".to_string(), args) { + Ok(_) => panic!("invalid args should fail"), + Err(err) => err, + }; + + assert!(err.to_string().contains("exchange cannot be empty")); + } + + #[test] + fn new_accepts_queue_mode() { + let mut args = valid_args(); + args.queue_dir = temp_store_dir("queue-mode").to_string_lossy().to_string(); + + let target = + AMQPTarget::::new("primary".to_string(), args.clone()).expect("queue mode should be supported"); + + assert!(target.store().is_some()); + let _ = std::fs::remove_dir_all(args.queue_dir); + } + + #[tokio::test] + async fn save_with_store_queues_event_without_broker() { + let mut args = unreachable_args(); + args.queue_dir = temp_store_dir("save-store").to_string_lossy().to_string(); + let target = AMQPTarget::::new("primary".to_string(), args.clone()).expect("target should build"); + + target + .save(test_event()) + .await + .expect("store-backed save should queue without broker"); + + assert_eq!(target.delivery_snapshot().queue_length, 1); + assert_eq!(target.delivery_snapshot().failed_messages, 0); + let _ = std::fs::remove_dir_all(args.queue_dir); + } + + #[tokio::test] + async fn save_without_store_returns_connection_error() { + let target = + AMQPTarget::::new("primary".to_string(), unreachable_args()).expect("target should build"); + + let err = target + .save(test_event()) + .await + .expect_err("direct publish should fail without broker"); + + assert_connect_failure(&err); + assert_eq!(target.delivery_snapshot().failed_messages, 1); + } + + #[tokio::test] + async fn init_with_store_allows_broker_to_recover_later() { + let mut args = unreachable_args(); + args.queue_dir = temp_store_dir("init-store").to_string_lossy().to_string(); + let target = AMQPTarget::::new("primary".to_string(), args.clone()).expect("target should build"); + + target.init().await.expect("store-backed init should tolerate broker failure"); + let _ = std::fs::remove_dir_all(args.queue_dir); + } + + #[tokio::test] + async fn init_without_store_returns_connection_error() { + let target = + AMQPTarget::::new("primary".to_string(), unreachable_args()).expect("target should build"); + + let err = target + .init() + .await + .expect_err("init should fail without broker when no store exists"); + + assert_connect_failure(&err); + } + + #[tokio::test] + async fn send_raw_from_store_returns_connection_error() { + let target = + AMQPTarget::::new("primary".to_string(), unreachable_args()).expect("target should build"); + let key = Key { + name: "queued".to_string(), + extension: ".event".to_string(), + item_count: 1, + compress: false, + }; + let meta = QueuedPayloadMeta::new( + EventName::ObjectCreatedPut, + "bucket".to_string(), + "object.txt".to_string(), + "application/json", + 2, + ); + + let err = target + .send_raw_from_store(key, b"{}".to_vec(), meta) + .await + .expect_err("queue replay should fail without broker"); + + assert_connect_failure(&err); + } + + #[test] + fn debug_masks_secret_values() { + let args = AMQPArgs { + url: Url::parse("amqp://guest:secret@127.0.0.1:5672/%2f").unwrap(), + password: "secret".to_string(), + tls_client_key: "/tmp/client.key".to_string(), + ..valid_args() + }; + let rendered = format!("{args:?}"); + + assert!(!rendered.contains("guest:secret")); + assert!(!rendered.contains("password: \"secret\"")); + assert!(!rendered.contains("tls_client_key: \"/tmp/client.key\"")); + assert!(rendered.contains("***REDACTED***")); + } +} diff --git a/crates/targets/src/target/mod.rs b/crates/targets/src/target/mod.rs index 2e0ff4f95..066f4e23c 100644 --- a/crates/targets/src/target/mod.rs +++ b/crates/targets/src/target/mod.rs @@ -25,6 +25,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; use tracing::warn; +pub mod amqp; pub mod kafka; pub mod mqtt; pub mod mysql; @@ -268,9 +269,12 @@ impl QueuedPayload { /// used in the notification system. /// /// It includes: +/// - `Amqp`: Represents an AMQP 0-9-1 target for sending notifications to a broker. /// - `Webhook`: Represents a webhook target for sending notifications via HTTP requests. /// - `Kafka`: Represents a Kafka target for sending notifications to a Kafka topic. /// - `Mqtt`: Represents an MQTT target for sending notifications via MQTT protocol. +/// - `Nats`: Represents a NATS target for sending notifications to a subject. +/// - `Pulsar`: Represents a Pulsar target for sending notifications to a topic. /// /// Each variant has an associated string representation that can be used for serialization /// or logging purposes. @@ -289,6 +293,7 @@ impl QueuedPayload { /// example output: /// Target type: webhook pub enum ChannelTargetType { + Amqp, Webhook, Kafka, Mqtt, @@ -302,6 +307,7 @@ pub enum ChannelTargetType { impl ChannelTargetType { pub fn as_str(&self) -> &'static str { match self { + ChannelTargetType::Amqp => "amqp", ChannelTargetType::Webhook => "webhook", ChannelTargetType::Kafka => "kafka", ChannelTargetType::Mqtt => "mqtt", @@ -317,6 +323,7 @@ impl ChannelTargetType { impl std::fmt::Display for ChannelTargetType { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { + ChannelTargetType::Amqp => write!(f, "amqp"), ChannelTargetType::Webhook => write!(f, "webhook"), ChannelTargetType::Kafka => write!(f, "kafka"), ChannelTargetType::Mqtt => write!(f, "mqtt"), @@ -330,7 +337,7 @@ impl std::fmt::Display for ChannelTargetType { } /// `TargetType` enum represents the type of target in the notification system. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum TargetType { AuditLog, NotifyEvent, @@ -437,6 +444,12 @@ pub(crate) fn delete_stored_payload( mod tests { use super::*; + #[test] + fn channel_target_type_amqp_uses_runtime_name() { + assert_eq!(ChannelTargetType::Amqp.as_str(), "amqp"); + assert_eq!(ChannelTargetType::Amqp.to_string(), "amqp"); + } + #[test] fn queued_payload_round_trips_meta_and_body() { let meta = QueuedPayloadMeta::new( diff --git a/crates/targets/tests/amqp_integration.rs b/crates/targets/tests/amqp_integration.rs new file mode 100644 index 000000000..1f4de9146 --- /dev/null +++ b/crates/targets/tests/amqp_integration.rs @@ -0,0 +1,221 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Integration tests for the AMQP notification target. +//! +//! These tests are ignored because they require a running RabbitMQ-compatible +//! AMQP 0-9-1 broker. To run locally: +//! +//! ```bash +//! docker run -d --name rustfs-rabbitmq -p 5672:5672 rabbitmq:3 +//! cargo test -p rustfs-targets --test amqp_integration -- --ignored +//! ``` +//! +//! Override the broker URL with `RUSTFS_TEST_AMQP_URL`. + +use lapin::{ + BasicProperties, Connection, ConnectionProperties, + options::{BasicAckOptions, BasicGetOptions, QueueBindOptions, QueueDeclareOptions, QueueDeleteOptions}, + types::FieldTable, +}; +use rustfs_s3_common::EventName; +use rustfs_targets::Target; +use rustfs_targets::check_amqp_broker_available; +use rustfs_targets::target::EntityTarget; +use rustfs_targets::target::TargetType; +use rustfs_targets::target::amqp::{AMQPArgs, AMQPTarget}; +use serde_json::Value; +use std::sync::Arc; +use uuid::Uuid; + +fn broker_url() -> String { + std::env::var("RUSTFS_TEST_AMQP_URL").unwrap_or_else(|_| "amqp://guest:guest@127.0.0.1:5672/%2f".to_string()) +} + +fn test_args(routing_key: &str) -> AMQPArgs { + AMQPArgs { + enable: true, + url: broker_url().parse().expect("valid AMQP URL"), + exchange: "amq.topic".to_string(), + routing_key: routing_key.to_string(), + mandatory: true, + persistent: true, + username: String::new(), + password: String::new(), + tls_ca: String::new(), + tls_client_cert: String::new(), + tls_client_key: String::new(), + queue_dir: String::new(), + queue_limit: 100_000, + target_type: TargetType::NotifyEvent, + } +} + +fn entity_for(bucket: &str, object: &str) -> Arc> { + Arc::new(EntityTarget { + bucket_name: bucket.to_string(), + object_name: object.to_string(), + event_name: EventName::ObjectCreatedPut, + data: serde_json::json!({"bucket": bucket, "object": object}), + }) +} + +async fn bind_queue(queue: &str, routing_key: &str) -> lapin::Channel { + let conn = Connection::connect(&broker_url(), ConnectionProperties::default()) + .await + .expect("connect to AMQP broker"); + let channel = conn.create_channel().await.expect("create channel"); + channel + .queue_declare( + queue.into(), + QueueDeclareOptions { + durable: false, + exclusive: true, + auto_delete: true, + ..QueueDeclareOptions::default() + }, + FieldTable::default(), + ) + .await + .expect("declare queue"); + channel + .queue_bind( + queue.into(), + "amq.topic".into(), + routing_key.into(), + QueueBindOptions::default(), + FieldTable::default(), + ) + .await + .expect("bind queue"); + channel +} + +async fn read_one(channel: &lapin::Channel, queue: &str) -> (Value, BasicProperties) { + let msg = tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + if let Some(msg) = channel + .basic_get(queue.into(), BasicGetOptions::default()) + .await + .expect("basic_get") + { + msg.ack(BasicAckOptions::default()).await.expect("ack message"); + break msg; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + }) + .await + .expect("message should arrive"); + + let properties = msg.properties.clone(); + let payload = serde_json::from_slice(&msg.data).expect("message payload should be JSON"); + (payload, properties) +} + +#[tokio::test] +#[ignore = "requires running RabbitMQ-compatible AMQP broker"] +async fn test_check_amqp_broker_available() { + check_amqp_broker_available(&test_args("rustfs.check")) + .await + .expect("broker check should succeed"); +} + +#[tokio::test] +#[ignore = "requires running RabbitMQ-compatible AMQP broker"] +async fn test_direct_publish_delivers_json_payload() { + let routing_key = format!("rustfs.test.{}", Uuid::new_v4().simple()); + let queue = format!("rustfs-test-{}", Uuid::new_v4().simple()); + let channel = bind_queue(&queue, &routing_key).await; + let target = AMQPTarget::new("direct".to_string(), test_args(&routing_key)).expect("construct AMQP target"); + + target + .save(entity_for("bucket1", "object-A")) + .await + .expect("publish should succeed"); + + let (payload, properties) = read_one(&channel, &queue).await; + assert_eq!(payload["Key"], "bucket1/object-A"); + assert_eq!(payload["Records"][0]["data"]["bucket"], "bucket1"); + assert_eq!(properties.content_type().as_ref().map(|s| s.as_str()), Some("application/json")); + assert_eq!(*properties.delivery_mode(), Some(2)); + + channel + .queue_delete(queue.into(), QueueDeleteOptions::default()) + .await + .expect("delete queue"); +} + +#[tokio::test] +#[ignore = "requires running RabbitMQ-compatible AMQP broker"] +async fn test_publish_reconnects_after_close() { + let routing_key = format!("rustfs.reconnect.{}", Uuid::new_v4().simple()); + let queue = format!("rustfs-test-{}", Uuid::new_v4().simple()); + let channel = bind_queue(&queue, &routing_key).await; + let target = AMQPTarget::new("reconnect".to_string(), test_args(&routing_key)).expect("construct AMQP target"); + + target + .save(entity_for("bucket1", "object-before-close")) + .await + .expect("initial publish should succeed"); + let (payload, _) = read_one(&channel, &queue).await; + assert_eq!(payload["Key"], "bucket1/object-before-close"); + + target.close().await.expect("close cached AMQP connection"); + + target + .save(entity_for("bucket1", "object-after-close")) + .await + .expect("publish should reconnect after close"); + let (payload, _) = read_one(&channel, &queue).await; + assert_eq!(payload["Key"], "bucket1/object-after-close"); + + channel + .queue_delete(queue.into(), QueueDeleteOptions::default()) + .await + .expect("delete queue"); +} + +#[tokio::test] +#[ignore = "requires running RabbitMQ-compatible AMQP broker"] +async fn test_queue_replay_delivers_and_removes_stored_payload() { + let routing_key = format!("rustfs.replay.{}", Uuid::new_v4().simple()); + let queue = format!("rustfs-test-{}", Uuid::new_v4().simple()); + let channel = bind_queue(&queue, &routing_key).await; + let queue_dir = std::env::temp_dir().join(format!("rustfs-amqp-integration-{}", Uuid::new_v4())); + let mut args = test_args(&routing_key); + args.queue_dir = queue_dir.to_string_lossy().to_string(); + let target = AMQPTarget::new("queued".to_string(), args.clone()).expect("construct AMQP target"); + + target + .save(entity_for("bucket1", "object-B")) + .await + .expect("store-backed save should queue"); + assert_eq!(target.delivery_snapshot().queue_length, 1); + + let key = target.store().expect("store configured").list()[0].clone(); + target.send_from_store(key).await.expect("replay should publish and delete"); + + let (payload, properties) = read_one(&channel, &queue).await; + assert_eq!(payload["Key"], "bucket1/object-B"); + assert_eq!(properties.content_type().as_ref().map(|s| s.as_str()), Some("application/json")); + assert_eq!(*properties.delivery_mode(), Some(2)); + assert_eq!(target.delivery_snapshot().queue_length, 0); + + channel + .queue_delete(queue.into(), QueueDeleteOptions::default()) + .await + .expect("delete queue"); + let _ = std::fs::remove_dir_all(args.queue_dir); +} diff --git a/rustfs/src/admin/handlers/audit.rs b/rustfs/src/admin/handlers/audit.rs index c6ed8abe1..33efc11de 100644 --- a/rustfs/src/admin/handlers/audit.rs +++ b/rustfs/src/admin/handlers/audit.rs @@ -15,7 +15,7 @@ use crate::admin::{ auth::validate_admin_request, handlers::target_descriptor::{ - AdminTargetSpec, AdminTargetValidator, EndpointKey, TargetDomain, TargetEndpointSource, allowed_target_keys, + AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, allowed_target_keys, build_json_response, collect_validated_key_values as shared_collect_validated_key_values, merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason, target_mutation_block_reason as shared_target_mutation_block_reason, target_service_name, target_spec, @@ -31,13 +31,9 @@ use futures::stream::{FuturesUnordered, StreamExt}; use http::StatusCode; use hyper::Method; use matchit::Params; +use rustfs_audit::factory::builtin_target_descriptors as builtin_audit_target_descriptors; use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState}; -use rustfs_config::audit::{ - AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_MYSQL_KEYS, AUDIT_MYSQL_SUB_SYS, - AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS, AUDIT_POSTGRES_KEYS, AUDIT_POSTGRES_SUB_SYS, AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS, - AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_REDIS_KEYS, AUDIT_REDIS_SUB_SYS, AUDIT_ROUTE_PREFIX, AUDIT_WEBHOOK_KEYS, - AUDIT_WEBHOOK_SUB_SYS, -}; +use rustfs_config::audit::AUDIT_ROUTE_PREFIX; use rustfs_config::{AUDIT_DEFAULT_DIR, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE}; use rustfs_ecstore::config::Config; use rustfs_policy::policy::action::{Action, AdminAction}; @@ -45,6 +41,7 @@ use s3s::{Body, S3Request, S3Response, S3Result, s3_error}; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::sync::Arc; +use std::sync::LazyLock; use tokio::sync::Semaphore; use tokio::time::{Duration, timeout}; use tracing::{Span, warn}; @@ -95,57 +92,15 @@ struct AuditEndpointsResponse { audit_endpoints: Vec, } -fn audit_target_specs() -> [AdminTargetSpec; 8] { - [ - AdminTargetSpec { - subsystem: AUDIT_WEBHOOK_SUB_SYS, - service: "webhook", - valid_keys: AUDIT_WEBHOOK_KEYS, - validator: AdminTargetValidator::Webhook, - }, - AdminTargetSpec { - subsystem: AUDIT_KAFKA_SUB_SYS, - service: "kafka", - valid_keys: AUDIT_KAFKA_KEYS, - validator: AdminTargetValidator::Kafka(TargetDomain::Audit), - }, - AdminTargetSpec { - subsystem: AUDIT_MQTT_SUB_SYS, - service: "mqtt", - valid_keys: AUDIT_MQTT_KEYS, - validator: AdminTargetValidator::Mqtt, - }, - AdminTargetSpec { - subsystem: AUDIT_MYSQL_SUB_SYS, - service: "mysql", - valid_keys: AUDIT_MYSQL_KEYS, - validator: AdminTargetValidator::MySql, - }, - AdminTargetSpec { - subsystem: AUDIT_NATS_SUB_SYS, - service: "nats", - valid_keys: AUDIT_NATS_KEYS, - validator: AdminTargetValidator::Nats(TargetDomain::Audit), - }, - AdminTargetSpec { - subsystem: AUDIT_POSTGRES_SUB_SYS, - service: "postgres", - valid_keys: AUDIT_POSTGRES_KEYS, - validator: AdminTargetValidator::Postgres(TargetDomain::Audit), - }, - AdminTargetSpec { - subsystem: AUDIT_PULSAR_SUB_SYS, - service: "pulsar", - valid_keys: AUDIT_PULSAR_KEYS, - validator: AdminTargetValidator::Pulsar(TargetDomain::Audit), - }, - AdminTargetSpec { - subsystem: AUDIT_REDIS_SUB_SYS, - service: "redis", - valid_keys: AUDIT_REDIS_KEYS, - validator: AdminTargetValidator::Redis(TargetDomain::Audit, AUDIT_REDIS_DEFAULT_CHANNEL), - }, - ] +static AUDIT_TARGET_SPECS: LazyLock> = LazyLock::new(|| { + builtin_audit_target_descriptors() + .into_iter() + .map(|descriptor| admin_target_spec_from_builtin(&descriptor)) + .collect() +}); + +fn audit_target_specs() -> &'static [AdminTargetSpec] { + &AUDIT_TARGET_SPECS } async fn authorize_audit_admin_request(req: &S3Request, action: AdminAction) -> S3Result<()> { @@ -172,7 +127,7 @@ fn has_any_audit_targets(config: &Config) -> bool { fn audit_target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option { shared_target_mutation_block_reason( - &audit_target_specs(), + audit_target_specs(), AUDIT_ROUTE_PREFIX, config, target_type, @@ -193,7 +148,7 @@ async fn audit_target_operation_block_reason(action: &str) -> Option { } fn merge_audit_endpoints(config: &Config, runtime_statuses: HashMap) -> Vec { - shared_merge_target_endpoints(&audit_target_specs(), AUDIT_ROUTE_PREFIX, config, runtime_statuses) + shared_merge_target_endpoints(audit_target_specs(), AUDIT_ROUTE_PREFIX, config, runtime_statuses) .into_iter() .map(|endpoint| AuditEndpoint { account_id: endpoint.account_id, @@ -208,7 +163,7 @@ fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, & let target_type = params .get("target_type") .ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'target_type'"))?; - if target_service_name(&audit_target_specs(), target_type).is_none() { + if target_service_name(audit_target_specs(), target_type).is_none() { return Err(s3_error!(InvalidArgument, "unsupported audit target type: '{}'", target_type)); } let target_name = params @@ -314,7 +269,7 @@ impl Operation for AuditTargetConfig { .map_err(|e| s3_error!(InvalidArgument, "invalid json body for audit target config: {}", e))?; let specs = audit_target_specs(); - let allowed_keys: HashSet<&str> = allowed_target_keys(&specs, target_type); + let allowed_keys: HashSet<&str> = allowed_target_keys(specs, target_type); let kv_map = shared_collect_validated_key_values( audit_body.key_values.iter().map(|kv| (kv.key.as_str(), kv.value.as_str())), @@ -323,7 +278,7 @@ impl Operation for AuditTargetConfig { "audit target", )?; - let spec = target_spec(&specs, target_type) + let spec = target_spec(specs, target_type) .ok_or_else(|| s3_error!(InvalidArgument, "unsupported audit target type: '{}'", target_type))?; timeout(Duration::from_secs(10), validate_target_request(spec, &kv_map, AUDIT_DEFAULT_DIR)) .await @@ -431,7 +386,9 @@ mod tests { use super::*; use matchit::Router; use rustfs_config::ENV_PREFIX; + use rustfs_config::audit::{AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS}; use rustfs_ecstore::config::{KV, KVS}; + use serial_test::serial; use std::collections::{HashMap, HashSet}; use temp_env::{with_var, with_vars, with_vars_unset}; @@ -469,6 +426,7 @@ mod tests { } #[test] + #[serial] fn merge_audit_endpoints_marks_config_env_and_mixed_sources() { let config = Config(HashMap::from([( AUDIT_WEBHOOK_SUB_SYS.to_string(), @@ -513,6 +471,7 @@ mod tests { } #[test] + #[serial] fn merge_audit_endpoints_marks_kafka_env_and_mixed_sources() { let config = Config(HashMap::from([( AUDIT_KAFKA_SUB_SYS.to_string(), @@ -549,6 +508,44 @@ mod tests { } #[test] + #[serial] + fn merge_audit_endpoints_marks_amqp_env_and_mixed_sources() { + let config = Config(HashMap::from([( + AUDIT_AMQP_SUB_SYS.to_string(), + HashMap::from([("mixed-amqp".to_string(), enabled_kvs("on"))]), + )])); + + with_vars( + [ + ("RUSTFS_AUDIT_AMQP_ENABLE_MIXED-AMQP", Some("on")), + ("RUSTFS_AUDIT_AMQP_URL_MIXED-AMQP", Some("amqp://127.0.0.1:5672/%2f")), + ("RUSTFS_AUDIT_AMQP_ENABLE_ENV-AMQP", Some("on")), + ("RUSTFS_AUDIT_AMQP_URL_ENV-AMQP", Some("amqp://127.0.0.1:5672/%2f")), + ], + || { + let runtime = HashMap::from([ + (("mixed-amqp".to_string(), "amqp".to_string()), "online".to_string()), + (("env-amqp".to_string(), "amqp".to_string()), "online".to_string()), + ]); + let merged = merge_audit_endpoints(&config, runtime); + + let mixed = merged + .iter() + .find(|entry| entry.account_id == "mixed-amqp" && entry.service == "amqp") + .expect("mixed amqp target should be present"); + assert_eq!(mixed.source, TargetEndpointSource::Mixed); + + let env_only = merged + .iter() + .find(|entry| entry.account_id == "env-amqp" && entry.service == "amqp") + .expect("env amqp target should be present"); + assert_eq!(env_only.source, TargetEndpointSource::Env); + }, + ); + } + + #[test] + #[serial] fn audit_target_mutation_block_reason_rejects_env_managed_target() { with_vars( [ @@ -565,6 +562,7 @@ mod tests { } #[test] + #[serial] fn audit_target_operation_block_reason_requires_audit_module_enable() { with_var(rustfs_config::ENV_AUDIT_ENABLE, Some("false"), || { let reason = @@ -575,6 +573,7 @@ mod tests { } #[test] + #[serial] fn audit_target_operation_block_reason_allows_when_audit_module_enabled() { with_var(rustfs_config::ENV_AUDIT_ENABLE, Some("true"), || { assert!( @@ -585,6 +584,7 @@ mod tests { } #[test] + #[serial] fn audit_target_mutation_block_reason_rejects_mixed_target() { with_var("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_PRIMARY", Some("https://example.com/hook"), || { let config = Config(HashMap::from([( @@ -598,6 +598,7 @@ mod tests { } #[test] + #[serial] fn merge_audit_endpoints_marks_disabled_config_with_env_override_as_mixed() { let config = Config(HashMap::from([( AUDIT_WEBHOOK_SUB_SYS.to_string(), @@ -622,6 +623,7 @@ mod tests { } #[test] + #[serial] fn merge_audit_endpoints_includes_env_only_target_without_runtime_status() { let config = Config(HashMap::new()); @@ -710,6 +712,14 @@ mod tests { assert_eq!(target_type, AUDIT_KAFKA_SUB_SYS); assert_eq!(target_name, "primary"); + let supported_amqp_params = full_router + .at("/v3/audit/target/audit_amqp/primary") + .expect("route should match"); + let (target_type, target_name) = + extract_target_params(&supported_amqp_params.params).expect("audit amqp target should be supported"); + assert_eq!(target_type, AUDIT_AMQP_SUB_SYS); + assert_eq!(target_name, "primary"); + let mut partial_router = Router::new(); partial_router .insert("/v3/audit/target/{target_type}", ()) @@ -722,6 +732,7 @@ mod tests { } #[test] + #[serial] fn merge_audit_endpoints_marks_mixed_with_case_insensitive_instance_id() { let config = Config(HashMap::from([( AUDIT_WEBHOOK_SUB_SYS.to_string(), @@ -746,6 +757,7 @@ mod tests { } #[test] + #[serial] fn audit_target_mutation_block_reason_allows_case_insensitive_config_target_lookup() { let config = Config(HashMap::from([( AUDIT_WEBHOOK_SUB_SYS.to_string(), diff --git a/rustfs/src/admin/handlers/event.rs b/rustfs/src/admin/handlers/event.rs index bd58e567c..b2efde467 100644 --- a/rustfs/src/admin/handlers/event.rs +++ b/rustfs/src/admin/handlers/event.rs @@ -15,7 +15,7 @@ use crate::admin::{ auth::validate_admin_request, handlers::target_descriptor::{ - AdminTargetSpec, AdminTargetValidator, EndpointKey, TargetDomain, TargetEndpointSource, allowed_target_keys, + AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, allowed_target_keys, build_json_response, collect_validated_key_values as shared_collect_validated_key_values, merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason, target_mutation_block_reason as shared_target_mutation_block_reason, target_service_name, target_spec, @@ -32,19 +32,16 @@ use futures::stream::{FuturesUnordered, StreamExt}; use http::StatusCode; use hyper::Method; use matchit::Params; -use rustfs_config::notify::{ - NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_KEYS, NOTIFY_MYSQL_SUB_SYS, - NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_KEYS, NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_KEYS, - NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_DEFAULT_CHANNEL, NOTIFY_REDIS_KEYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_ROUTE_PREFIX, - NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS, -}; +use rustfs_config::notify::NOTIFY_ROUTE_PREFIX; use rustfs_config::{ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE}; use rustfs_ecstore::config::Config; +use rustfs_notify::factory::builtin_target_descriptors as builtin_notification_target_descriptors; use rustfs_policy::policy::action::{Action, AdminAction}; use s3s::{Body, S3Request, S3Response, S3Result, s3_error}; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::sync::Arc; +use std::sync::LazyLock; use tokio::sync::Semaphore; use tokio::time::{Duration, timeout}; use tracing::{Span, info, warn}; @@ -101,57 +98,15 @@ struct NotificationEndpointsResponse { notification_endpoints: Vec, } -fn notification_target_specs() -> [AdminTargetSpec; 8] { - [ - AdminTargetSpec { - subsystem: NOTIFY_WEBHOOK_SUB_SYS, - service: "webhook", - valid_keys: NOTIFY_WEBHOOK_KEYS, - validator: AdminTargetValidator::Webhook, - }, - AdminTargetSpec { - subsystem: NOTIFY_KAFKA_SUB_SYS, - service: "kafka", - valid_keys: NOTIFY_KAFKA_KEYS, - validator: AdminTargetValidator::Kafka(TargetDomain::Notify), - }, - AdminTargetSpec { - subsystem: NOTIFY_MQTT_SUB_SYS, - service: "mqtt", - valid_keys: NOTIFY_MQTT_KEYS, - validator: AdminTargetValidator::Mqtt, - }, - AdminTargetSpec { - subsystem: NOTIFY_MYSQL_SUB_SYS, - service: "mysql", - valid_keys: NOTIFY_MYSQL_KEYS, - validator: AdminTargetValidator::MySql, - }, - AdminTargetSpec { - subsystem: NOTIFY_NATS_SUB_SYS, - service: "nats", - valid_keys: NOTIFY_NATS_KEYS, - validator: AdminTargetValidator::Nats(TargetDomain::Notify), - }, - AdminTargetSpec { - subsystem: NOTIFY_POSTGRES_SUB_SYS, - service: "postgres", - valid_keys: NOTIFY_POSTGRES_KEYS, - validator: AdminTargetValidator::Postgres(TargetDomain::Notify), - }, - AdminTargetSpec { - subsystem: NOTIFY_REDIS_SUB_SYS, - service: "redis", - valid_keys: NOTIFY_REDIS_KEYS, - validator: AdminTargetValidator::Redis(TargetDomain::Notify, NOTIFY_REDIS_DEFAULT_CHANNEL), - }, - AdminTargetSpec { - subsystem: NOTIFY_PULSAR_SUB_SYS, - service: "pulsar", - valid_keys: NOTIFY_PULSAR_KEYS, - validator: AdminTargetValidator::Pulsar(TargetDomain::Notify), - }, - ] +static NOTIFICATION_TARGET_SPECS: LazyLock> = LazyLock::new(|| { + builtin_notification_target_descriptors() + .into_iter() + .map(|descriptor| admin_target_spec_from_builtin(&descriptor)) + .collect() +}); + +fn notification_target_specs() -> &'static [AdminTargetSpec] { + &NOTIFICATION_TARGET_SPECS } // --- Helper Functions --- @@ -172,7 +127,7 @@ fn get_notification_system() -> S3Result> fn target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option { shared_target_mutation_block_reason( - ¬ification_target_specs(), + notification_target_specs(), NOTIFY_ROUTE_PREFIX, config, target_type, @@ -193,7 +148,7 @@ async fn notification_target_operation_block_reason(action: &str) -> Option) -> Vec { - shared_merge_target_endpoints(¬ification_target_specs(), NOTIFY_ROUTE_PREFIX, config, runtime_statuses) + shared_merge_target_endpoints(notification_target_specs(), NOTIFY_ROUTE_PREFIX, config, runtime_statuses) .into_iter() .map(|endpoint| NotificationEndpoint { account_id: endpoint.account_id, @@ -241,7 +196,7 @@ impl Operation for NotificationTarget { .map_err(|e| s3_error!(InvalidArgument, "invalid json body for target config: {}", e))?; let specs = notification_target_specs(); - let allowed_keys: HashSet<&str> = allowed_target_keys(&specs, target_type); + let allowed_keys: HashSet<&str> = allowed_target_keys(specs, target_type); let kv_map = shared_collect_validated_key_values( notification_body @@ -252,7 +207,7 @@ impl Operation for NotificationTarget { target_type, "target", )?; - let spec = target_spec(&specs, target_type) + let spec = target_spec(specs, target_type) .ok_or_else(|| s3_error!(InvalidArgument, "unsupported target type: '{}'", target_type))?; timeout(Duration::from_secs(10), validate_target_request(spec, &kv_map, EVENT_DEFAULT_DIR)) .await @@ -397,7 +352,7 @@ fn extract_param<'a>(params: &'a Params<'_, '_>, key: &str) -> S3Result<&'a str> fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, &'a str)> { let target_type = extract_param(params, "target_type")?; - if target_service_name(¬ification_target_specs(), target_type).is_none() { + if target_service_name(notification_target_specs(), target_type).is_none() { return Err(s3_error!(InvalidArgument, "unsupported target type: '{}'", target_type)); } let target_name = extract_param(params, "target_name")?; @@ -409,8 +364,10 @@ mod tests { use super::*; use matchit::Router; use rustfs_config::DEFAULT_DELIMITER; + use rustfs_config::notify::{NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS}; use rustfs_ecstore::config::{KV, KVS}; use rustfs_targets::arn::TargetID; + use serial_test::serial; use std::collections::{HashMap, HashSet}; use temp_env::{with_var, with_vars}; @@ -483,6 +440,7 @@ mod tests { } #[test] + #[serial] fn merge_notification_endpoints_marks_env_and_mixed_sources() { let config = Config(HashMap::from([ ( @@ -530,6 +488,7 @@ mod tests { } #[test] + #[serial] fn merge_notification_endpoints_marks_kafka_env_and_mixed_sources() { let config = Config(HashMap::from([( NOTIFY_KAFKA_SUB_SYS.to_string(), @@ -566,6 +525,44 @@ mod tests { } #[test] + #[serial] + fn merge_notification_endpoints_marks_amqp_env_and_mixed_sources() { + let config = Config(HashMap::from([( + NOTIFY_AMQP_SUB_SYS.to_string(), + HashMap::from([("mixed-amqp".to_string(), enabled_kvs("on"))]), + )])); + + with_vars( + [ + ("RUSTFS_NOTIFY_AMQP_ENABLE_MIXED-AMQP", Some("on")), + ("RUSTFS_NOTIFY_AMQP_URL_MIXED-AMQP", Some("amqp://127.0.0.1:5672/%2f")), + ("RUSTFS_NOTIFY_AMQP_ENABLE_ENV-AMQP", Some("on")), + ("RUSTFS_NOTIFY_AMQP_URL_ENV-AMQP", Some("amqp://127.0.0.1:5672/%2f")), + ], + || { + let runtime = HashMap::from([ + (("mixed-amqp".to_string(), "amqp".to_string()), "online".to_string()), + (("env-amqp".to_string(), "amqp".to_string()), "online".to_string()), + ]); + let merged = merge_notification_endpoints(&config, runtime); + + let mixed = merged + .iter() + .find(|entry| entry.account_id == "mixed-amqp" && entry.service == "amqp") + .expect("mixed amqp target should be present"); + assert_eq!(mixed.source, TargetEndpointSource::Mixed); + + let env_only = merged + .iter() + .find(|entry| entry.account_id == "env-amqp" && entry.service == "amqp") + .expect("env amqp target should be present"); + assert_eq!(env_only.source, TargetEndpointSource::Env); + }, + ); + } + + #[test] + #[serial] fn target_mutation_block_reason_rejects_env_managed_target() { with_vars( [ @@ -582,6 +579,7 @@ mod tests { } #[test] + #[serial] fn notification_target_operation_block_reason_requires_notify_module_enable() { with_var(rustfs_config::ENV_NOTIFY_ENABLE, Some("false"), || { let reason = futures::executor::block_on(notification_target_operation_block_reason( @@ -593,6 +591,7 @@ mod tests { } #[test] + #[serial] fn notification_target_operation_block_reason_allows_when_notify_module_enabled() { with_var(rustfs_config::ENV_NOTIFY_ENABLE, Some("true"), || { assert!( @@ -605,6 +604,7 @@ mod tests { } #[test] + #[serial] fn target_mutation_block_reason_rejects_mixed_target() { with_var("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("https://example.com/hook"), || { let config = Config(HashMap::from([( @@ -628,6 +628,7 @@ mod tests { } #[test] + #[serial] fn merge_notification_endpoints_marks_disabled_config_with_env_override_as_mixed() { let config = Config(HashMap::from([( NOTIFY_WEBHOOK_SUB_SYS.to_string(), @@ -652,6 +653,7 @@ mod tests { } #[test] + #[serial] fn merge_notification_endpoints_includes_env_only_target_without_runtime_status() { let config = Config(HashMap::new()); @@ -697,6 +699,7 @@ mod tests { } #[test] + #[serial] fn merge_notification_endpoints_marks_mixed_with_case_insensitive_instance_id() { let config = Config(HashMap::from([( NOTIFY_WEBHOOK_SUB_SYS.to_string(), @@ -734,6 +737,7 @@ mod tests { } #[test] + #[serial] fn target_mutation_block_reason_allows_case_insensitive_config_target_lookup() { let config = Config(HashMap::from([( NOTIFY_WEBHOOK_SUB_SYS.to_string(), @@ -811,6 +815,44 @@ mod tests { assert_eq!(target_name, "streaming"); } + #[test] + fn extract_target_params_accepts_amqp_target_type() { + let mut router = Router::new(); + router + .insert("/v3/target/{target_type}/{target_name}", ()) + .expect("route should insert"); + + let params = router + .at("/v3/target/notify_amqp/rabbitmq") + .expect("route should match") + .params; + let (target_type, target_name) = extract_target_params(¶ms).expect("amqp target type should be accepted"); + assert_eq!(target_type, NOTIFY_AMQP_SUB_SYS); + assert_eq!(target_name, "rabbitmq"); + } + + #[test] + fn collect_validated_key_values_accepts_amqp_keys() { + let specs = notification_target_specs(); + let allowed_keys = allowed_target_keys(specs, NOTIFY_AMQP_SUB_SYS); + + let kv_map = shared_collect_validated_key_values( + [ + (rustfs_config::AMQP_URL, "amqp://127.0.0.1:5672/%2f"), + (rustfs_config::AMQP_EXCHANGE, "rustfs.events"), + (rustfs_config::AMQP_ROUTING_KEY, "objects"), + ], + &allowed_keys, + NOTIFY_AMQP_SUB_SYS, + "target", + ) + .expect("amqp keys should be accepted"); + + assert_eq!(kv_map.get(rustfs_config::AMQP_URL).map(String::as_str), Some("amqp://127.0.0.1:5672/%2f")); + assert!(allowed_keys.contains(rustfs_config::AMQP_MANDATORY)); + assert!(allowed_keys.contains(rustfs_config::AMQP_PERSISTENT)); + } + fn extract_block_between_markers<'a>(src: &'a str, start_marker: &str, end_marker: &str) -> &'a str { let start = src .find(start_marker) diff --git a/rustfs/src/admin/handlers/target_descriptor.rs b/rustfs/src/admin/handlers/target_descriptor.rs index 31a641bfc..62b0aa38f 100644 --- a/rustfs/src/admin/handlers/target_descriptor.rs +++ b/rustfs/src/admin/handlers/target_descriptor.rs @@ -15,16 +15,17 @@ use hashbrown::HashSet as HbHashSet; use http::{HeaderMap, HeaderValue, StatusCode}; use rustfs_config::{ - ENABLE_KEY, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_TOPIC, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_TLS_CA, + AMQP_QUEUE_DIR, ENABLE_KEY, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_TOPIC, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_QUEUE_DIR, POSTGRES_QUEUE_DIR, REDIS_QUEUE_DIR, }; use rustfs_ecstore::config::Config; use rustfs_targets::{ - TargetError, check_kafka_broker_available, check_mqtt_broker_available_with_tls, check_nats_server_available, - check_postgres_server_available, check_pulsar_broker_available, check_redis_server_available, + BuiltinTargetDescriptor, TargetError, TargetRequestValidator, check_amqp_broker_available, check_kafka_broker_available, + check_mqtt_broker_available_with_tls, check_nats_server_available, check_postgres_server_available, + check_pulsar_broker_available, check_redis_server_available, config::{ - build_kafka_args, build_nats_args, build_postgres_args, build_pulsar_args, build_redis_args, + build_amqp_args, build_kafka_args, build_nats_args, build_postgres_args, build_pulsar_args, build_redis_args, collect_env_target_instance_ids, validate_mysql_config, validate_redis_config, }, target::{TargetType, mqtt::MQTTTlsConfig}, @@ -34,10 +35,13 @@ use serde::Serialize; use std::collections::{HashMap, HashSet}; use std::io::{Error, ErrorKind}; use std::path::Path; +use std::sync::Arc; use tokio::time::{Duration, sleep}; use url::Url; pub(crate) type EndpointKey = (String, String); +type AdminRequestValidatorFn = + Arc, &str) -> futures::future::BoxFuture<'static, S3Result<()>> + Send + Sync>; #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] #[serde(rename_all = "lowercase")] @@ -55,14 +59,14 @@ pub(crate) struct MergedTargetEndpoint { pub source: TargetEndpointSource, } -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug)] pub(crate) enum TargetDomain { Notify, Audit, } impl TargetDomain { - fn runtime_target_type(self) -> TargetType { + pub(crate) fn runtime_target_type(self) -> TargetType { match self { TargetDomain::Notify => TargetType::NotifyEvent, TargetDomain::Audit => TargetType::AuditLog, @@ -70,24 +74,98 @@ impl TargetDomain { } } -#[derive(Clone, Copy)] -pub(crate) enum AdminTargetValidator { - Webhook, - Mqtt, - Kafka(TargetDomain), - MySql, - Nats(TargetDomain), - Postgres(TargetDomain), - Pulsar(TargetDomain), - Redis(TargetDomain, &'static str), +impl From for TargetDomain { + fn from(value: TargetType) -> Self { + match value { + TargetType::NotifyEvent => TargetDomain::Notify, + TargetType::AuditLog => TargetDomain::Audit, + } + } } -#[derive(Clone, Copy)] +#[derive(Clone)] pub(crate) struct AdminTargetSpec { pub subsystem: &'static str, pub service: &'static str, pub valid_keys: &'static [&'static str], - pub validator: AdminTargetValidator, + validator: AdminRequestValidatorFn, +} + +pub(crate) fn admin_target_spec_from_builtin(descriptor: &BuiltinTargetDescriptor) -> AdminTargetSpec +where + E: Send + Sync + 'static + Clone + serde::Serialize + serde::de::DeserializeOwned, +{ + AdminTargetSpec { + subsystem: descriptor.subsystem(), + service: descriptor.plugin().target_type(), + valid_keys: descriptor.plugin().valid_fields(), + validator: match descriptor.request_validator() { + TargetRequestValidator::Webhook => Arc::new(validate_webhook_request_entry), + TargetRequestValidator::Mqtt => Arc::new(validate_mqtt_request_entry), + TargetRequestValidator::Amqp(target_type) => { + if matches!(TargetDomain::from(target_type), TargetDomain::Audit) { + Arc::new(validate_audit_amqp_request_entry) + } else { + Arc::new(validate_notify_amqp_request_entry) + } + } + TargetRequestValidator::Kafka(target_type) => { + if matches!(TargetDomain::from(target_type), TargetDomain::Audit) { + Arc::new(validate_audit_kafka_request_entry) + } else { + Arc::new(validate_notify_kafka_request_entry) + } + } + TargetRequestValidator::MySql => Arc::new(validate_mysql_request_entry), + TargetRequestValidator::Nats(target_type) => { + if matches!(TargetDomain::from(target_type), TargetDomain::Audit) { + Arc::new(validate_audit_nats_request_entry) + } else { + Arc::new(validate_notify_nats_request_entry) + } + } + TargetRequestValidator::Postgres(target_type) => { + if matches!(TargetDomain::from(target_type), TargetDomain::Audit) { + Arc::new(validate_audit_postgres_request_entry) + } else { + Arc::new(validate_notify_postgres_request_entry) + } + } + TargetRequestValidator::Pulsar(target_type) => { + if matches!(TargetDomain::from(target_type), TargetDomain::Audit) { + Arc::new(validate_audit_pulsar_request_entry) + } else { + Arc::new(validate_notify_pulsar_request_entry) + } + } + TargetRequestValidator::Redis { + default_channel, + target_type, + } => { + if matches!(TargetDomain::from(target_type), TargetDomain::Audit) { + validate_audit_redis_request_entry(default_channel) + } else { + validate_notify_redis_request_entry(default_channel) + } + } + }, + } +} + +impl std::fmt::Debug for AdminTargetSpec { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AdminTargetSpec") + .field("subsystem", &self.subsystem) + .field("service", &self.service) + .field("valid_keys", &self.valid_keys) + .finish_non_exhaustive() + } +} + +impl AdminTargetSpec { + pub(crate) async fn validate_request(&self, kv_map: &HashMap, default_queue_dir: &str) -> S3Result<()> { + (self.validator)(kv_map, default_queue_dir).await + } } pub(crate) fn normalized_endpoint_key(account_id: &str, service: &str) -> EndpointKey { @@ -352,18 +430,7 @@ pub(crate) async fn validate_target_request( kv_map: &HashMap, default_queue_dir: &str, ) -> S3Result<()> { - match spec.validator { - AdminTargetValidator::Webhook => validate_webhook_request(kv_map).await, - AdminTargetValidator::Mqtt => validate_mqtt_request(kv_map).await, - AdminTargetValidator::Kafka(domain) => validate_kafka_request(kv_map, default_queue_dir, domain).await, - AdminTargetValidator::MySql => validate_mysql_request(kv_map, default_queue_dir).await, - AdminTargetValidator::Nats(domain) => validate_nats_request(kv_map, default_queue_dir, domain).await, - AdminTargetValidator::Pulsar(domain) => validate_pulsar_request(kv_map, default_queue_dir, domain).await, - AdminTargetValidator::Postgres(domain) => validate_postgres_request(kv_map, default_queue_dir, domain).await, - AdminTargetValidator::Redis(domain, default_channel) => { - validate_redis_request(kv_map, default_queue_dir, domain, default_channel).await - } - } + spec.validate_request(kv_map, default_queue_dir).await } fn config_enable_is_on(value: &str) -> bool { @@ -420,6 +487,14 @@ async fn validate_webhook_request(kv_map: &HashMap) -> S3Result< Ok(()) } +fn validate_webhook_request_entry( + kv_map: &HashMap, + _default_queue_dir: &str, +) -> futures::future::BoxFuture<'static, S3Result<()>> { + let kv_map = kv_map.clone(); + Box::pin(async move { validate_webhook_request(&kv_map).await }) +} + async fn validate_mqtt_request(kv_map: &HashMap) -> S3Result<()> { let endpoint = kv_map .get(MQTT_BROKER) @@ -464,6 +539,129 @@ async fn validate_mqtt_request(kv_map: &HashMap) -> S3Result<()> Ok(()) } +fn validate_mqtt_request_entry( + kv_map: &HashMap, + _default_queue_dir: &str, +) -> futures::future::BoxFuture<'static, S3Result<()>> { + let kv_map = kv_map.clone(); + Box::pin(async move { validate_mqtt_request(&kv_map).await }) +} + +fn validate_notify_nats_request_entry( + kv_map: &HashMap, + default_queue_dir: &str, +) -> futures::future::BoxFuture<'static, S3Result<()>> { + let kv_map = kv_map.clone(); + let default_queue_dir = default_queue_dir.to_string(); + Box::pin(async move { validate_nats_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await }) +} + +fn validate_audit_nats_request_entry( + kv_map: &HashMap, + default_queue_dir: &str, +) -> futures::future::BoxFuture<'static, S3Result<()>> { + let kv_map = kv_map.clone(); + let default_queue_dir = default_queue_dir.to_string(); + Box::pin(async move { validate_nats_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await }) +} + +fn validate_notify_kafka_request_entry( + kv_map: &HashMap, + default_queue_dir: &str, +) -> futures::future::BoxFuture<'static, S3Result<()>> { + let kv_map = kv_map.clone(); + let default_queue_dir = default_queue_dir.to_string(); + Box::pin(async move { validate_kafka_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await }) +} + +fn validate_audit_kafka_request_entry( + kv_map: &HashMap, + default_queue_dir: &str, +) -> futures::future::BoxFuture<'static, S3Result<()>> { + let kv_map = kv_map.clone(); + let default_queue_dir = default_queue_dir.to_string(); + Box::pin(async move { validate_kafka_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await }) +} + +fn validate_notify_amqp_request_entry( + kv_map: &HashMap, + default_queue_dir: &str, +) -> futures::future::BoxFuture<'static, S3Result<()>> { + let kv_map = kv_map.clone(); + let default_queue_dir = default_queue_dir.to_string(); + Box::pin(async move { validate_amqp_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await }) +} + +fn validate_audit_amqp_request_entry( + kv_map: &HashMap, + default_queue_dir: &str, +) -> futures::future::BoxFuture<'static, S3Result<()>> { + let kv_map = kv_map.clone(); + let default_queue_dir = default_queue_dir.to_string(); + Box::pin(async move { validate_amqp_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await }) +} + +fn validate_notify_pulsar_request_entry( + kv_map: &HashMap, + default_queue_dir: &str, +) -> futures::future::BoxFuture<'static, S3Result<()>> { + let kv_map = kv_map.clone(); + let default_queue_dir = default_queue_dir.to_string(); + Box::pin(async move { validate_pulsar_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await }) +} + +fn validate_audit_pulsar_request_entry( + kv_map: &HashMap, + default_queue_dir: &str, +) -> futures::future::BoxFuture<'static, S3Result<()>> { + let kv_map = kv_map.clone(); + let default_queue_dir = default_queue_dir.to_string(); + Box::pin(async move { validate_pulsar_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await }) +} + +fn validate_mysql_request_entry( + kv_map: &HashMap, + default_queue_dir: &str, +) -> futures::future::BoxFuture<'static, S3Result<()>> { + let kv_map = kv_map.clone(); + let default_queue_dir = default_queue_dir.to_string(); + Box::pin(async move { validate_mysql_request(&kv_map, &default_queue_dir).await }) +} + +fn validate_notify_postgres_request_entry( + kv_map: &HashMap, + default_queue_dir: &str, +) -> futures::future::BoxFuture<'static, S3Result<()>> { + let kv_map = kv_map.clone(); + let default_queue_dir = default_queue_dir.to_string(); + Box::pin(async move { validate_postgres_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await }) +} + +fn validate_audit_postgres_request_entry( + kv_map: &HashMap, + default_queue_dir: &str, +) -> futures::future::BoxFuture<'static, S3Result<()>> { + let kv_map = kv_map.clone(); + let default_queue_dir = default_queue_dir.to_string(); + Box::pin(async move { validate_postgres_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await }) +} + +fn validate_notify_redis_request_entry(default_channel: &'static str) -> AdminRequestValidatorFn { + Arc::new(move |kv_map: &HashMap, default_queue_dir: &str| { + let kv_map = kv_map.clone(); + let default_queue_dir = default_queue_dir.to_string(); + Box::pin(async move { validate_redis_request(&kv_map, &default_queue_dir, TargetDomain::Notify, default_channel).await }) + }) +} + +fn validate_audit_redis_request_entry(default_channel: &'static str) -> AdminRequestValidatorFn { + Arc::new(move |kv_map: &HashMap, default_queue_dir: &str| { + let kv_map = kv_map.clone(); + let default_queue_dir = default_queue_dir.to_string(); + Box::pin(async move { validate_redis_request(&kv_map, &default_queue_dir, TargetDomain::Audit, default_channel).await }) + }) +} + async fn validate_nats_request(kv_map: &HashMap, default_queue_dir: &str, domain: TargetDomain) -> S3Result<()> { if let Some(queue_dir) = kv_map.get("queue_dir") { validate_queue_dir(queue_dir.as_str()).await?; @@ -496,6 +694,18 @@ async fn validate_kafka_request(kv_map: &HashMap, default_queue_ }) } +async fn validate_amqp_request(kv_map: &HashMap, default_queue_dir: &str, domain: TargetDomain) -> S3Result<()> { + if let Some(queue_dir) = kv_map.get(AMQP_QUEUE_DIR) { + validate_queue_dir(queue_dir.as_str()).await?; + } + let args = build_amqp_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type()) + .map_err(|e| s3_error!(InvalidArgument, "{}", e))?; + check_amqp_broker_available(&args).await.map_err(|e| match e { + TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e), + _ => s3_error!(InvalidArgument, "AMQP broker check failed: {}", e), + }) +} + async fn validate_pulsar_request( kv_map: &HashMap, default_queue_dir: &str, diff --git a/rustfs/src/server/audit.rs b/rustfs/src/server/audit.rs index 2ebe93e80..75f8a89a1 100644 --- a/rustfs/src/server/audit.rs +++ b/rustfs/src/server/audit.rs @@ -35,12 +35,7 @@ pub fn is_audit_module_enabled() -> bool { } fn has_any_audit_targets(config: &rustfs_ecstore::config::Config) -> bool { - for subsystem in [ - rustfs_config::audit::AUDIT_MQTT_SUB_SYS, - rustfs_config::audit::AUDIT_NATS_SUB_SYS, - rustfs_config::audit::AUDIT_PULSAR_SUB_SYS, - rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS, - ] { + for &subsystem in rustfs_config::audit::AUDIT_SUB_SYSTEMS { let Some(targets) = config.0.get(subsystem) else { continue; }; @@ -102,7 +97,7 @@ pub async fn start_audit_system() -> AuditResult<()> { if !has_targets { info!( target: "rustfs::main::start_audit_system", - "Audit subsystem (Webhook/MQTT/NATS/Pulsar) is not configured, and audit system initialization is skipped." + "Audit subsystem targets are not configured, and audit system initialization is skipped." ); return Ok(()); }