feat(targets): add NATS and Pulsar target support (#2618)

crates/config/src/audit/mod.rs

@@ -17,9 +17,13 @@
 //! webhook and MQTT audit-related settings.
 
 mod mqtt;
+mod nats;
+mod pulsar;
 mod webhook;
 
 pub use mqtt::*;
+pub use nats::*;
+pub use pulsar::*;
 pub use webhook::*;
 
 use crate::DEFAULT_DELIMITER;
@@ -30,7 +34,14 @@ pub const AUDIT_ROUTE_PREFIX: &str = const_str::concat!(AUDIT_PREFIX, DEFAULT_DE
 
 pub const AUDIT_WEBHOOK_SUB_SYS: &str = "audit_webhook";
 pub const AUDIT_MQTT_SUB_SYS: &str = "audit_mqtt";
+pub const AUDIT_NATS_SUB_SYS: &str = "audit_nats";
+pub const AUDIT_PULSAR_SUB_SYS: &str = "audit_pulsar";
 
 pub const AUDIT_STORE_EXTENSION: &str = ".audit";
 #[allow(dead_code)]
-pub const AUDIT_SUB_SYSTEMS: &[&str] = &[AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS];
+pub const AUDIT_SUB_SYSTEMS: &[&str] = &[
+    AUDIT_MQTT_SUB_SYS,
+    AUDIT_NATS_SUB_SYS,
+    AUDIT_PULSAR_SUB_SYS,
+    AUDIT_WEBHOOK_SUB_SYS,
+];

crates/config/src/audit/nats.rs

@@ -0,0 +1,60 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub const ENV_AUDIT_NATS_ENABLE: &str = "RUSTFS_AUDIT_NATS_ENABLE";
+pub const ENV_AUDIT_NATS_ADDRESS: &str = "RUSTFS_AUDIT_NATS_ADDRESS";
+pub const ENV_AUDIT_NATS_SUBJECT: &str = "RUSTFS_AUDIT_NATS_SUBJECT";
+pub const ENV_AUDIT_NATS_USERNAME: &str = "RUSTFS_AUDIT_NATS_USERNAME";
+pub const ENV_AUDIT_NATS_PASSWORD: &str = "RUSTFS_AUDIT_NATS_PASSWORD";
+pub const ENV_AUDIT_NATS_TOKEN: &str = "RUSTFS_AUDIT_NATS_TOKEN";
+pub const ENV_AUDIT_NATS_CREDENTIALS_FILE: &str = "RUSTFS_AUDIT_NATS_CREDENTIALS_FILE";
+pub const ENV_AUDIT_NATS_TLS_CA: &str = "RUSTFS_AUDIT_NATS_TLS_CA";
+pub const ENV_AUDIT_NATS_TLS_CLIENT_CERT: &str = "RUSTFS_AUDIT_NATS_TLS_CLIENT_CERT";
+pub const ENV_AUDIT_NATS_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_NATS_TLS_CLIENT_KEY";
+pub const ENV_AUDIT_NATS_TLS_REQUIRED: &str = "RUSTFS_AUDIT_NATS_TLS_REQUIRED";
+pub const ENV_AUDIT_NATS_QUEUE_DIR: &str = "RUSTFS_AUDIT_NATS_QUEUE_DIR";
+pub const ENV_AUDIT_NATS_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_NATS_QUEUE_LIMIT";
+
+pub const ENV_AUDIT_NATS_KEYS: &[&str; 13] = &[
+    ENV_AUDIT_NATS_ENABLE,
+    ENV_AUDIT_NATS_ADDRESS,
+    ENV_AUDIT_NATS_SUBJECT,
+    ENV_AUDIT_NATS_USERNAME,
+    ENV_AUDIT_NATS_PASSWORD,
+    ENV_AUDIT_NATS_TOKEN,
+    ENV_AUDIT_NATS_CREDENTIALS_FILE,
+    ENV_AUDIT_NATS_TLS_CA,
+    ENV_AUDIT_NATS_TLS_CLIENT_CERT,
+    ENV_AUDIT_NATS_TLS_CLIENT_KEY,
+    ENV_AUDIT_NATS_TLS_REQUIRED,
+    ENV_AUDIT_NATS_QUEUE_DIR,
+    ENV_AUDIT_NATS_QUEUE_LIMIT,
+];
+
+pub const AUDIT_NATS_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::NATS_ADDRESS,
+    crate::NATS_SUBJECT,
+    crate::NATS_USERNAME,
+    crate::NATS_PASSWORD,
+    crate::NATS_TOKEN,
+    crate::NATS_CREDENTIALS_FILE,
+    crate::NATS_TLS_CA,
+    crate::NATS_TLS_CLIENT_CERT,
+    crate::NATS_TLS_CLIENT_KEY,
+    crate::NATS_TLS_REQUIRED,
+    crate::NATS_QUEUE_DIR,
+    crate::NATS_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];

crates/config/src/audit/pulsar.rs

@@ -0,0 +1,54 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub const ENV_AUDIT_PULSAR_ENABLE: &str = "RUSTFS_AUDIT_PULSAR_ENABLE";
+pub const ENV_AUDIT_PULSAR_BROKER: &str = "RUSTFS_AUDIT_PULSAR_BROKER";
+pub const ENV_AUDIT_PULSAR_TOPIC: &str = "RUSTFS_AUDIT_PULSAR_TOPIC";
+pub const ENV_AUDIT_PULSAR_AUTH_TOKEN: &str = "RUSTFS_AUDIT_PULSAR_AUTH_TOKEN";
+pub const ENV_AUDIT_PULSAR_USERNAME: &str = "RUSTFS_AUDIT_PULSAR_USERNAME";
+pub const ENV_AUDIT_PULSAR_PASSWORD: &str = "RUSTFS_AUDIT_PULSAR_PASSWORD";
+pub const ENV_AUDIT_PULSAR_TLS_CA: &str = "RUSTFS_AUDIT_PULSAR_TLS_CA";
+pub const ENV_AUDIT_PULSAR_TLS_ALLOW_INSECURE: &str = "RUSTFS_AUDIT_PULSAR_TLS_ALLOW_INSECURE";
+pub const ENV_AUDIT_PULSAR_TLS_HOSTNAME_VERIFICATION: &str = "RUSTFS_AUDIT_PULSAR_TLS_HOSTNAME_VERIFICATION";
+pub const ENV_AUDIT_PULSAR_QUEUE_DIR: &str = "RUSTFS_AUDIT_PULSAR_QUEUE_DIR";
+pub const ENV_AUDIT_PULSAR_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_PULSAR_QUEUE_LIMIT";
+
+pub const ENV_AUDIT_PULSAR_KEYS: &[&str; 11] = &[
+    ENV_AUDIT_PULSAR_ENABLE,
+    ENV_AUDIT_PULSAR_BROKER,
+    ENV_AUDIT_PULSAR_TOPIC,
+    ENV_AUDIT_PULSAR_AUTH_TOKEN,
+    ENV_AUDIT_PULSAR_USERNAME,
+    ENV_AUDIT_PULSAR_PASSWORD,
+    ENV_AUDIT_PULSAR_TLS_CA,
+    ENV_AUDIT_PULSAR_TLS_ALLOW_INSECURE,
+    ENV_AUDIT_PULSAR_TLS_HOSTNAME_VERIFICATION,
+    ENV_AUDIT_PULSAR_QUEUE_DIR,
+    ENV_AUDIT_PULSAR_QUEUE_LIMIT,
+];
+
+pub const AUDIT_PULSAR_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::PULSAR_BROKER,
+    crate::PULSAR_TOPIC,
+    crate::PULSAR_AUTH_TOKEN,
+    crate::PULSAR_USERNAME,
+    crate::PULSAR_PASSWORD,
+    crate::PULSAR_TLS_CA,
+    crate::PULSAR_TLS_ALLOW_INSECURE,
+    crate::PULSAR_TLS_HOSTNAME_VERIFICATION,
+    crate::PULSAR_QUEUE_DIR,
+    crate::PULSAR_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];

crates/config/src/constants/targets.rs

@@ -41,6 +41,30 @@ pub const MQTT_TLS_CLIENT_KEY: &str = "tls_client_key";
 pub const MQTT_TLS_TRUST_LEAF_AS_CA: &str = "tls_trust_leaf_as_ca";
 pub const MQTT_WS_PATH_ALLOWLIST: &str = "ws_path_allowlist";
 
+pub const NATS_ADDRESS: &str = "address";
+pub const NATS_SUBJECT: &str = "subject";
+pub const NATS_USERNAME: &str = "username";
+pub const NATS_PASSWORD: &str = "password";
+pub const NATS_TOKEN: &str = "token";
+pub const NATS_CREDENTIALS_FILE: &str = "credentials_file";
+pub const NATS_TLS_CA: &str = "tls_ca";
+pub const NATS_TLS_CLIENT_CERT: &str = "tls_client_cert";
+pub const NATS_TLS_CLIENT_KEY: &str = "tls_client_key";
+pub const NATS_TLS_REQUIRED: &str = "tls_required";
+pub const NATS_QUEUE_DIR: &str = "queue_dir";
+pub const NATS_QUEUE_LIMIT: &str = "queue_limit";
+
+pub const PULSAR_BROKER: &str = "broker";
+pub const PULSAR_TOPIC: &str = "topic";
+pub const PULSAR_AUTH_TOKEN: &str = "auth_token";
+pub const PULSAR_USERNAME: &str = "username";
+pub const PULSAR_PASSWORD: &str = "password";
+pub const PULSAR_TLS_CA: &str = "tls_ca";
+pub const PULSAR_TLS_ALLOW_INSECURE: &str = "tls_allow_insecure";
+pub const PULSAR_TLS_HOSTNAME_VERIFICATION: &str = "tls_hostname_verification";
+pub const PULSAR_QUEUE_DIR: &str = "queue_dir";
+pub const PULSAR_QUEUE_LIMIT: &str = "queue_limit";
+
 /// Environment variable controlling whether target queue files are Snappy-compressed.
 /// Applies to both notify and audit target queue stores.
 pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";

crates/config/src/notify/mod.rs

@@ -14,11 +14,15 @@
 
 mod arn;
 mod mqtt;
+mod nats;
+mod pulsar;
 mod store;
 mod webhook;
 
 pub use arn::*;
 pub use mqtt::*;
+pub use nats::*;
+pub use pulsar::*;
 pub use store::*;
 pub use webhook::*;
 
@@ -64,7 +68,12 @@ pub const ENV_NOTIFY_SEND_CONCURRENCY: &str = "RUSTFS_NOTIFY_SEND_CONCURRENCY";
 pub const DEFAULT_NOTIFY_SEND_CONCURRENCY: usize = 64;
 
 #[allow(dead_code)]
-pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS];
+pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[
+    NOTIFY_MQTT_SUB_SYS,
+    NOTIFY_NATS_SUB_SYS,
+    NOTIFY_PULSAR_SUB_SYS,
+    NOTIFY_WEBHOOK_SUB_SYS,
+];
 
 #[allow(dead_code)]
 pub const NOTIFY_KAFKA_SUB_SYS: &str = "notify_kafka";
@@ -83,4 +92,5 @@ pub const NOTIFY_AMQP_SUB_SYS: &str = "notify_amqp";
 pub const NOTIFY_POSTGRES_SUB_SYS: &str = "notify_postgres";
 #[allow(dead_code)]
 pub const NOTIFY_REDIS_SUB_SYS: &str = "notify_redis";
+pub const NOTIFY_PULSAR_SUB_SYS: &str = "notify_pulsar";
 pub const NOTIFY_WEBHOOK_SUB_SYS: &str = "notify_webhook";

crates/config/src/notify/nats.rs

@@ -0,0 +1,60 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub const NOTIFY_NATS_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::NATS_ADDRESS,
+    crate::NATS_SUBJECT,
+    crate::NATS_USERNAME,
+    crate::NATS_PASSWORD,
+    crate::NATS_TOKEN,
+    crate::NATS_CREDENTIALS_FILE,
+    crate::NATS_TLS_CA,
+    crate::NATS_TLS_CLIENT_CERT,
+    crate::NATS_TLS_CLIENT_KEY,
+    crate::NATS_TLS_REQUIRED,
+    crate::NATS_QUEUE_DIR,
+    crate::NATS_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];
+
+pub const ENV_NOTIFY_NATS_ENABLE: &str = "RUSTFS_NOTIFY_NATS_ENABLE";
+pub const ENV_NOTIFY_NATS_ADDRESS: &str = "RUSTFS_NOTIFY_NATS_ADDRESS";
+pub const ENV_NOTIFY_NATS_SUBJECT: &str = "RUSTFS_NOTIFY_NATS_SUBJECT";
+pub const ENV_NOTIFY_NATS_USERNAME: &str = "RUSTFS_NOTIFY_NATS_USERNAME";
+pub const ENV_NOTIFY_NATS_PASSWORD: &str = "RUSTFS_NOTIFY_NATS_PASSWORD";
+pub const ENV_NOTIFY_NATS_TOKEN: &str = "RUSTFS_NOTIFY_NATS_TOKEN";
+pub const ENV_NOTIFY_NATS_CREDENTIALS_FILE: &str = "RUSTFS_NOTIFY_NATS_CREDENTIALS_FILE";
+pub const ENV_NOTIFY_NATS_TLS_CA: &str = "RUSTFS_NOTIFY_NATS_TLS_CA";
+pub const ENV_NOTIFY_NATS_TLS_CLIENT_CERT: &str = "RUSTFS_NOTIFY_NATS_TLS_CLIENT_CERT";
+pub const ENV_NOTIFY_NATS_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_NATS_TLS_CLIENT_KEY";
+pub const ENV_NOTIFY_NATS_TLS_REQUIRED: &str = "RUSTFS_NOTIFY_NATS_TLS_REQUIRED";
+pub const ENV_NOTIFY_NATS_QUEUE_DIR: &str = "RUSTFS_NOTIFY_NATS_QUEUE_DIR";
+pub const ENV_NOTIFY_NATS_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_NATS_QUEUE_LIMIT";
+
+pub const ENV_NOTIFY_NATS_KEYS: &[&str; 13] = &[
+    ENV_NOTIFY_NATS_ENABLE,
+    ENV_NOTIFY_NATS_ADDRESS,
+    ENV_NOTIFY_NATS_SUBJECT,
+    ENV_NOTIFY_NATS_USERNAME,
+    ENV_NOTIFY_NATS_PASSWORD,
+    ENV_NOTIFY_NATS_TOKEN,
+    ENV_NOTIFY_NATS_CREDENTIALS_FILE,
+    ENV_NOTIFY_NATS_TLS_CA,
+    ENV_NOTIFY_NATS_TLS_CLIENT_CERT,
+    ENV_NOTIFY_NATS_TLS_CLIENT_KEY,
+    ENV_NOTIFY_NATS_TLS_REQUIRED,
+    ENV_NOTIFY_NATS_QUEUE_DIR,
+    ENV_NOTIFY_NATS_QUEUE_LIMIT,
+];

crates/config/src/notify/pulsar.rs

@@ -0,0 +1,54 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub const NOTIFY_PULSAR_KEYS: &[&str] = &[
+    crate::ENABLE_KEY,
+    crate::PULSAR_BROKER,
+    crate::PULSAR_TOPIC,
+    crate::PULSAR_AUTH_TOKEN,
+    crate::PULSAR_USERNAME,
+    crate::PULSAR_PASSWORD,
+    crate::PULSAR_TLS_CA,
+    crate::PULSAR_TLS_ALLOW_INSECURE,
+    crate::PULSAR_TLS_HOSTNAME_VERIFICATION,
+    crate::PULSAR_QUEUE_DIR,
+    crate::PULSAR_QUEUE_LIMIT,
+    crate::COMMENT_KEY,
+];
+
+pub const ENV_NOTIFY_PULSAR_ENABLE: &str = "RUSTFS_NOTIFY_PULSAR_ENABLE";
+pub const ENV_NOTIFY_PULSAR_BROKER: &str = "RUSTFS_NOTIFY_PULSAR_BROKER";
+pub const ENV_NOTIFY_PULSAR_TOPIC: &str = "RUSTFS_NOTIFY_PULSAR_TOPIC";
+pub const ENV_NOTIFY_PULSAR_AUTH_TOKEN: &str = "RUSTFS_NOTIFY_PULSAR_AUTH_TOKEN";
+pub const ENV_NOTIFY_PULSAR_USERNAME: &str = "RUSTFS_NOTIFY_PULSAR_USERNAME";
+pub const ENV_NOTIFY_PULSAR_PASSWORD: &str = "RUSTFS_NOTIFY_PULSAR_PASSWORD";
+pub const ENV_NOTIFY_PULSAR_TLS_CA: &str = "RUSTFS_NOTIFY_PULSAR_TLS_CA";
+pub const ENV_NOTIFY_PULSAR_TLS_ALLOW_INSECURE: &str = "RUSTFS_NOTIFY_PULSAR_TLS_ALLOW_INSECURE";
+pub const ENV_NOTIFY_PULSAR_TLS_HOSTNAME_VERIFICATION: &str = "RUSTFS_NOTIFY_PULSAR_TLS_HOSTNAME_VERIFICATION";
+pub const ENV_NOTIFY_PULSAR_QUEUE_DIR: &str = "RUSTFS_NOTIFY_PULSAR_QUEUE_DIR";
+pub const ENV_NOTIFY_PULSAR_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_PULSAR_QUEUE_LIMIT";
+
+pub const ENV_NOTIFY_PULSAR_KEYS: &[&str; 11] = &[
+    ENV_NOTIFY_PULSAR_ENABLE,
+    ENV_NOTIFY_PULSAR_BROKER,
+    ENV_NOTIFY_PULSAR_TOPIC,
+    ENV_NOTIFY_PULSAR_AUTH_TOKEN,
+    ENV_NOTIFY_PULSAR_USERNAME,
+    ENV_NOTIFY_PULSAR_PASSWORD,
+    ENV_NOTIFY_PULSAR_TLS_CA,
+    ENV_NOTIFY_PULSAR_TLS_ALLOW_INSECURE,
+    ENV_NOTIFY_PULSAR_TLS_HOSTNAME_VERIFICATION,
+    ENV_NOTIFY_PULSAR_QUEUE_DIR,
+    ENV_NOTIFY_PULSAR_QUEUE_LIMIT,
+];