diff --git a/Cargo.lock b/Cargo.lock index cd44c85f1..8b1df0b9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8390,11 +8390,11 @@ dependencies = [ "chrono", "const-str", "ecstore", + "form_urlencoded", "once_cell", "quick-xml", "reqwest", "rumqttc", - "rustfs-config", "rustfs-utils", "serde", "serde_json", @@ -8524,6 +8524,7 @@ dependencies = [ "sha2 0.10.9", "siphasher 1.0.1", "snap", + "sysinfo", "tempfile", "tokio", "tracing", diff --git a/Cargo.toml b/Cargo.toml index a888d171e..79e32b823 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -94,6 +94,7 @@ dirs = "6.0.0" dotenvy = "0.15.7" flatbuffers = "25.2.10" flexi_logger = { version = "0.30.2", features = ["trc", "dont_minimize_extra_stacks"] } +form_urlencoded = "1.2.1" futures = "0.3.31" futures-core = "0.3.31" futures-util = "0.3.31" diff --git a/crates/config/Cargo.toml b/crates/config/Cargo.toml index 3069ba941..0256f30e8 100644 --- a/crates/config/Cargo.toml +++ b/crates/config/Cargo.toml @@ -18,6 +18,5 @@ workspace = true [features] default = [] constants = ["dep:const-str"] -notify = [] observability = [] diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index 4584271bb..da496971f 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -3,8 +3,5 @@ pub mod constants; #[cfg(feature = "constants")] pub use constants::app::*; -#[cfg(feature = "notify")] -pub mod notify; - #[cfg(feature = "observability")] pub mod observability; diff --git a/crates/config/src/notify/config.rs b/crates/config/src/notify/config.rs deleted file mode 100644 index a33948031..000000000 --- a/crates/config/src/notify/config.rs +++ /dev/null @@ -1,53 +0,0 @@ -use crate::notify::mqtt::MQTTArgs; -use crate::notify::webhook::WebhookArgs; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; - -/// Config - notification target configuration structure, holds -/// information about various notification targets. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct NotifyConfig { - pub mqtt: HashMap, - pub webhook: HashMap, -} - -impl NotifyConfig { - /// Create a new configuration with default values. - pub fn new() -> Self { - let mut config = NotifyConfig { - webhook: HashMap::new(), - mqtt: HashMap::new(), - }; - // Insert default target for each backend - config.webhook.insert("1".to_string(), WebhookArgs::new()); - config.mqtt.insert("1".to_string(), MQTTArgs::new()); - config - } -} - -impl Default for NotifyConfig { - fn default() -> Self { - Self::new() - } -} - -#[cfg(test)] -mod tests { - use crate::notify::config::NotifyConfig; - - #[test] - fn test_notify_config_new() { - let config = NotifyConfig::new(); - assert_eq!(config.webhook.len(), 1); - assert_eq!(config.mqtt.len(), 1); - assert!(config.webhook.contains_key("1")); - assert!(config.mqtt.contains_key("1")); - } - - #[test] - fn test_notify_config_default() { - let config = NotifyConfig::default(); - assert_eq!(config.webhook.len(), 1); - assert_eq!(config.mqtt.len(), 1); - } -} diff --git a/crates/config/src/notify/help.rs b/crates/config/src/notify/help.rs deleted file mode 100644 index 8bbe58602..000000000 --- a/crates/config/src/notify/help.rs +++ /dev/null @@ -1,26 +0,0 @@ -/// Help text for Webhook configuration. -pub const HELP_WEBHOOK: &str = r#" -Webhook configuration: -- enable: Enable or disable the webhook target (true/false) -- endpoint: Webhook server endpoint (e.g., http://localhost:8080/rustfs/events) -- auth_token: Opaque string or JWT authorization token (optional) -- queue_dir: Absolute path for persistent event queue (optional) -- queue_limit: Maximum number of events to queue (optional, default: 0) -- client_cert: Path to client certificate file (optional) -- client_key: Path to client private key file (optional) -"#; - -/// Help text for MQTT configuration. -pub const HELP_MQTT: &str = r#" -MQTT configuration: -- enable: Enable or disable the MQTT target (true/false) -- broker: MQTT broker address (e.g., tcp://localhost:1883) -- topic: MQTT topic (e.g., rustfs/events) -- qos: Quality of Service level (0, 1, or 2) -- username: Username for MQTT authentication (optional) -- password: Password for MQTT authentication (optional) -- reconnect_interval: Reconnect interval in milliseconds (optional) -- keep_alive_interval: Keep alive interval in milliseconds (optional) -- queue_dir: Absolute path for persistent event queue (optional) -- queue_limit: Maximum number of events to queue (optional, default: 0) -"#; \ No newline at end of file diff --git a/crates/config/src/notify/legacy.rs b/crates/config/src/notify/legacy.rs deleted file mode 100644 index 75c0e19de..000000000 --- a/crates/config/src/notify/legacy.rs +++ /dev/null @@ -1,268 +0,0 @@ -use crate::notify::mqtt::MQTTArgs; -use crate::notify::webhook::WebhookArgs; -use std::collections::HashMap; - -/// Convert legacy webhook configuration to the new WebhookArgs struct. -pub fn convert_webhook_config(config: &HashMap) -> Result { - let mut args = WebhookArgs::new(); - args.enable = config.get("enable").map_or(false, |v| v == "true"); - args.endpoint = config.get("endpoint").unwrap_or(&"".to_string()).clone(); - args.auth_token = config.get("auth_token").unwrap_or(&"".to_string()).clone(); - args.queue_dir = config.get("queue_dir").unwrap_or(&"".to_string()).clone(); - args.queue_limit = config.get("queue_limit").map_or(0, |v| v.parse().unwrap_or(0)); - args.client_cert = config.get("client_cert").unwrap_or(&"".to_string()).clone(); - args.client_key = config.get("client_key").unwrap_or(&"".to_string()).clone(); - Ok(args) -} - -/// Convert legacy MQTT configuration to the new MQTTArgs struct. -pub fn convert_mqtt_config(config: &HashMap) -> Result { - let mut args = MQTTArgs::new(); - args.enable = config.get("enable").map_or(false, |v| v == "true"); - args.broker = config.get("broker").unwrap_or(&"".to_string()).clone(); - args.topic = config.get("topic").unwrap_or(&"".to_string()).clone(); - args.qos = config.get("qos").map_or(0, |v| v.parse().unwrap_or(0)); - args.username = config.get("username").unwrap_or(&"".to_string()).clone(); - args.password = config.get("password").unwrap_or(&"".to_string()).clone(); - args.reconnect_interval = config.get("reconnect_interval").map_or(0, |v| v.parse().unwrap_or(0)); - args.keep_alive_interval = config.get("keep_alive_interval").map_or(0, |v| v.parse().unwrap_or(0)); - args.queue_dir = config.get("queue_dir").unwrap_or(&"".to_string()).clone(); - args.queue_limit = config.get("queue_limit").map_or(0, |v| v.parse().unwrap_or(0)); - Ok(args) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_convert_webhook_config_invalid() { - let mut old_config = HashMap::new(); - old_config.insert("max_retries".to_string(), "invalid".to_string()); - let result = convert_webhook_config(&old_config); - assert!(result.is_err()); - } - - #[test] - fn test_convert_mqtt_config_invalid() { - let mut old_config = HashMap::new(); - old_config.insert("port".to_string(), "invalid".to_string()); - let result = convert_mqtt_config(&old_config); - assert!(result.is_err()); - } - - #[test] - fn test_convert_empty_config() { - let empty_config = HashMap::new(); - let webhook_result = convert_webhook_config(&empty_config); - assert!(webhook_result.is_ok()); - let mqtt_result = convert_mqtt_config(&empty_config); - assert!(mqtt_result.is_ok()); - } - - #[test] - fn test_convert_config_with_extra_fields() { - let mut extra_config = HashMap::new(); - extra_config.insert("endpoint".to_string(), "http://example.com".to_string()); - extra_config.insert("extra_field".to_string(), "extra_value".to_string()); - let webhook_result = convert_webhook_config(&extra_config); - assert!(webhook_result.is_ok()); - let args = webhook_result.unwrap(); - assert_eq!(args.endpoint, "http://example.com"); - - let mut extra_mqtt_config = HashMap::new(); - extra_mqtt_config.insert("broker".to_string(), "mqtt.example.com".to_string()); - extra_mqtt_config.insert("extra_field".to_string(), "extra_value".to_string()); - let mqtt_result = convert_mqtt_config(&extra_mqtt_config); - assert!(mqtt_result.is_ok()); - let args = mqtt_result.unwrap(); - assert_eq!(args.broker, "mqtt.example.com"); - } - - #[test] - fn test_convert_config_with_empty_values() { - let mut empty_values_config = HashMap::new(); - empty_values_config.insert("endpoint".to_string(), "".to_string()); - let webhook_result = convert_webhook_config(&empty_values_config); - assert!(webhook_result.is_ok()); - let args = webhook_result.unwrap(); - assert_eq!(args.endpoint, ""); - - let mut empty_mqtt_config = HashMap::new(); - empty_mqtt_config.insert("broker".to_string(), "".to_string()); - let mqtt_result = convert_mqtt_config(&empty_mqtt_config); - assert!(mqtt_result.is_ok()); - let args = mqtt_result.unwrap(); - assert_eq!(args.broker, ""); - } - - #[test] - fn test_convert_config_with_whitespace_values() { - let mut whitespace_config = HashMap::new(); - whitespace_config.insert("endpoint".to_string(), " http://example.com ".to_string()); - let webhook_result = convert_webhook_config(&whitespace_config); - assert!(webhook_result.is_ok()); - let args = webhook_result.unwrap(); - assert_eq!(args.endpoint, " http://example.com "); - - let mut whitespace_mqtt_config = HashMap::new(); - whitespace_mqtt_config.insert("broker".to_string(), " mqtt.example.com ".to_string()); - let mqtt_result = convert_mqtt_config(&whitespace_mqtt_config); - assert!(mqtt_result.is_ok()); - let args = mqtt_result.unwrap(); - assert_eq!(args.broker, " mqtt.example.com "); - } - - #[test] - fn test_convert_config_with_special_characters() { - let mut special_chars_config = HashMap::new(); - special_chars_config.insert("endpoint".to_string(), "http://example.com/path?param=value&other=123".to_string()); - let webhook_result = convert_webhook_config(&special_chars_config); - assert!(webhook_result.is_ok()); - let args = webhook_result.unwrap(); - assert_eq!(args.endpoint, "http://example.com/path?param=value&other=123"); - - let mut special_chars_mqtt_config = HashMap::new(); - special_chars_mqtt_config.insert("broker".to_string(), "mqtt.example.com:1883".to_string()); - let mqtt_result = convert_mqtt_config(&special_chars_mqtt_config); - assert!(mqtt_result.is_ok()); - let args = mqtt_result.unwrap(); - assert_eq!(args.broker, "mqtt.example.com:1883"); - } - - #[test] - fn test_convert_config_with_boolean_values() { - let mut boolean_config = HashMap::new(); - boolean_config.insert("enable".to_string(), "true".to_string()); - let webhook_result = convert_webhook_config(&boolean_config); - assert!(webhook_result.is_ok()); - let args = webhook_result.unwrap(); - assert_eq!(args.endpoint, ""); // default value - - let mut boolean_mqtt_config = HashMap::new(); - boolean_mqtt_config.insert("enable".to_string(), "false".to_string()); - let mqtt_result = convert_mqtt_config(&boolean_mqtt_config); - assert!(mqtt_result.is_ok()); - let args = mqtt_result.unwrap(); - assert_eq!(args.broker, "localhost"); // default value - } - - #[test] - fn test_convert_config_with_null_values() { - let mut null_config = HashMap::new(); - null_config.insert("endpoint".to_string(), "null".to_string()); - let webhook_result = convert_webhook_config(&null_config); - assert!(webhook_result.is_ok()); - let args = webhook_result.unwrap(); - assert_eq!(args.endpoint, "null"); - - let mut null_mqtt_config = HashMap::new(); - null_mqtt_config.insert("broker".to_string(), "null".to_string()); - let mqtt_result = convert_mqtt_config(&null_mqtt_config); - assert!(mqtt_result.is_ok()); - let args = mqtt_result.unwrap(); - assert_eq!(args.broker, "null"); - } - - #[test] - fn test_convert_config_with_duplicate_keys() { - let mut duplicate_config = HashMap::new(); - duplicate_config.insert("endpoint".to_string(), "http://example.org".to_string()); - let webhook_result = convert_webhook_config(&duplicate_config); - assert!(webhook_result.is_ok()); - let args = webhook_result.unwrap(); - assert_eq!(args.endpoint, "http://example.org"); // last value wins - - let mut duplicate_mqtt_config = HashMap::new(); - duplicate_mqtt_config.insert("broker".to_string(), "mqtt.example.org".to_string()); - let mqtt_result = convert_mqtt_config(&duplicate_mqtt_config); - assert!(mqtt_result.is_ok()); - let args = mqtt_result.unwrap(); - assert_eq!(args.broker, "mqtt.example.org"); // last value wins - } - - #[test] - fn test_convert_config_with_case_insensitive_keys() { - let mut case_insensitive_config = HashMap::new(); - case_insensitive_config.insert("ENDPOINT".to_string(), "http://example.com".to_string()); - let webhook_result = convert_webhook_config(&case_insensitive_config); - assert!(webhook_result.is_ok()); - let args = webhook_result.unwrap(); - assert_eq!(args.endpoint, "http://example.com"); - - let mut case_insensitive_mqtt_config = HashMap::new(); - case_insensitive_mqtt_config.insert("BROKER".to_string(), "mqtt.example.com".to_string()); - let mqtt_result = convert_mqtt_config(&case_insensitive_mqtt_config); - assert!(mqtt_result.is_ok()); - let args = mqtt_result.unwrap(); - assert_eq!(args.broker, "mqtt.example.com"); - } - - #[test] - fn test_convert_config_with_mixed_case_keys() { - let mut mixed_case_config = HashMap::new(); - mixed_case_config.insert("EndPoint".to_string(), "http://example.com".to_string()); - let webhook_result = convert_webhook_config(&mixed_case_config); - assert!(webhook_result.is_ok()); - let args = webhook_result.unwrap(); - assert_eq!(args.endpoint, "http://example.com"); - - let mut mixed_case_mqtt_config = HashMap::new(); - mixed_case_mqtt_config.insert("BroKer".to_string(), "mqtt.example.com".to_string()); - let mqtt_result = convert_mqtt_config(&mixed_case_mqtt_config); - assert!(mqtt_result.is_ok()); - let args = mqtt_result.unwrap(); - assert_eq!(args.broker, "mqtt.example.com"); - } - - #[test] - fn test_convert_config_with_snake_case_keys() { - let mut snake_case_config = HashMap::new(); - snake_case_config.insert("end_point".to_string(), "http://example.com".to_string()); - let webhook_result = convert_webhook_config(&snake_case_config); - assert!(webhook_result.is_ok()); - let args = webhook_result.unwrap(); - assert_eq!(args.endpoint, "http://example.com"); - - let mut snake_case_mqtt_config = HashMap::new(); - snake_case_mqtt_config.insert("bro_ker".to_string(), "mqtt.example.com".to_string()); - let mqtt_result = convert_mqtt_config(&snake_case_mqtt_config); - assert!(mqtt_result.is_ok()); - let args = mqtt_result.unwrap(); - assert_eq!(args.broker, "mqtt.example.com"); - } - - #[test] - fn test_convert_config_with_kebab_case_keys() { - let mut kebab_case_config = HashMap::new(); - kebab_case_config.insert("end-point".to_string(), "http://example.com".to_string()); - let webhook_result = convert_webhook_config(&kebab_case_config); - assert!(webhook_result.is_ok()); - let args = webhook_result.unwrap(); - assert_eq!(args.endpoint, "http://example.com"); - - let mut kebab_case_mqtt_config = HashMap::new(); - kebab_case_mqtt_config.insert("bro-ker".to_string(), "mqtt.example.com".to_string()); - let mqtt_result = convert_mqtt_config(&kebab_case_mqtt_config); - assert!(mqtt_result.is_ok()); - let args = mqtt_result.unwrap(); - assert_eq!(args.broker, "mqtt.example.com"); - } - - #[test] - fn test_convert_config_with_camel_case_keys() { - let mut camel_case_config = HashMap::new(); - camel_case_config.insert("endPoint".to_string(), "http://example.com".to_string()); - let webhook_result = convert_webhook_config(&camel_case_config); - assert!(webhook_result.is_ok()); - let args = webhook_result.unwrap(); - assert_eq!(args.endpoint, "http://example.com"); - - let mut camel_case_mqtt_config = HashMap::new(); - camel_case_mqtt_config.insert("broKer".to_string(), "mqtt.example.com".to_string()); - let mqtt_result = convert_mqtt_config(&camel_case_mqtt_config); - assert!(mqtt_result.is_ok()); - let args = mqtt_result.unwrap(); - assert_eq!(args.broker, "mqtt.example.com"); - } -} diff --git a/crates/config/src/notify/mod.rs b/crates/config/src/notify/mod.rs deleted file mode 100644 index 4c9c07113..000000000 --- a/crates/config/src/notify/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub mod config; -pub mod help; -pub mod legacy; -pub mod mqtt; -pub mod webhook; diff --git a/crates/config/src/notify/mqtt.rs b/crates/config/src/notify/mqtt.rs deleted file mode 100644 index 8b400f3f3..000000000 --- a/crates/config/src/notify/mqtt.rs +++ /dev/null @@ -1,114 +0,0 @@ -use serde::{Deserialize, Serialize}; - -/// MQTTArgs - MQTT target arguments. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MQTTArgs { - pub enable: bool, - pub broker: String, - pub topic: String, - pub qos: u8, - pub username: String, - pub password: String, - pub reconnect_interval: u64, - pub keep_alive_interval: u64, - #[serde(skip)] - pub root_cas: Option<()>, // Placeholder for *x509.CertPool - pub queue_dir: String, - pub queue_limit: u64, -} - -impl MQTTArgs { - /// Create a new configuration with default values. - pub fn new() -> Self { - Self { - enable: false, - broker: "".to_string(), - topic: "".to_string(), - qos: 0, - username: "".to_string(), - password: "".to_string(), - reconnect_interval: 0, - keep_alive_interval: 0, - root_cas: None, - queue_dir: "".to_string(), - queue_limit: 0, - } - } - - /// Validate MQTTArgs fields - pub fn validate(&self) -> Result<(), String> { - if !self.enable { - return Ok(()); - } - if self.broker.trim().is_empty() { - return Err("MQTT broker cannot be empty".to_string()); - } - if self.topic.trim().is_empty() { - return Err("MQTT topic cannot be empty".to_string()); - } - if self.queue_dir != "" && !self.queue_dir.starts_with('/') { - return Err("queueDir path should be absolute".to_string()); - } - if self.qos == 0 && self.queue_dir != "" { - return Err("qos should be set to 1 or 2 if queueDir is set".to_string()); - } - Ok(()) - } -} - -impl Default for MQTTArgs { - fn default() -> Self { - Self::new() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_mqtt_args_new() { - let args = MQTTArgs::new(); - assert_eq!(args.broker, ""); - assert_eq!(args.topic, ""); - assert_eq!(args.qos, 0); - assert_eq!(args.username, ""); - assert_eq!(args.password, ""); - assert_eq!(args.reconnect_interval, 0); - assert_eq!(args.keep_alive_interval, 0); - assert!(args.root_cas.is_none()); - assert_eq!(args.queue_dir, ""); - assert_eq!(args.queue_limit, 0); - assert!(!args.enable); - } - - #[test] - fn test_mqtt_args_validate() { - let mut args = MQTTArgs::new(); - assert!(args.validate().is_ok()); - args.broker = "".to_string(); - assert!(args.validate().is_err()); - args.broker = "localhost".to_string(); - args.topic = "".to_string(); - assert!(args.validate().is_err()); - args.topic = "mqtt_topic".to_string(); - args.reconnect_interval = 10001; - assert!(args.validate().is_err()); - args.reconnect_interval = 1000; - args.keep_alive_interval = 10001; - assert!(args.validate().is_err()); - args.keep_alive_interval = 1000; - args.queue_limit = 10001; - assert!(args.validate().is_err()); - args.queue_dir = "invalid_path".to_string(); - assert!(args.validate().is_err()); - args.queue_dir = "/valid_path".to_string(); - assert!(args.validate().is_ok()); - args.qos = 0; - assert!(args.validate().is_err()); - args.qos = 1; - assert!(args.validate().is_ok()); - args.qos = 2; - assert!(args.validate().is_ok()); - } -} diff --git a/crates/config/src/notify/webhook.rs b/crates/config/src/notify/webhook.rs deleted file mode 100644 index 80e0b38ff..000000000 --- a/crates/config/src/notify/webhook.rs +++ /dev/null @@ -1,81 +0,0 @@ -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; - -/// WebhookArgs - Webhook target arguments. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WebhookArgs { - pub enable: bool, - pub endpoint: String, - pub auth_token: String, - #[serde(skip)] - pub custom_headers: Option>, - pub queue_dir: String, - pub queue_limit: u64, - pub client_cert: String, - pub client_key: String, -} - -impl WebhookArgs { - /// Create a new configuration with default values. - pub fn new() -> Self { - Self { - enable: false, - endpoint: "".to_string(), - auth_token: "".to_string(), - custom_headers: None, - queue_dir: "".to_string(), - queue_limit: 0, - client_cert: "".to_string(), - client_key: "".to_string(), - } - } - - /// Validate WebhookArgs fields - pub fn validate(&self) -> Result<(), String> { - if !self.enable { - return Ok(()); - } - if self.endpoint.trim().is_empty() { - return Err("endpoint empty".to_string()); - } - if self.queue_dir != "" && !self.queue_dir.starts_with('/') { - return Err("queueDir path should be absolute".to_string()); - } - if (self.client_cert != "" && self.client_key == "") || (self.client_cert == "" && self.client_key != "") { - return Err("cert and key must be specified as a pair".to_string()); - } - Ok(()) - } -} - -impl Default for WebhookArgs { - fn default() -> Self { - Self::new() - } -} - -#[cfg(test)] -mod tests { - use crate::notify::webhook::WebhookArgs; - - #[test] - fn test_webhook_args_new() { - let args = WebhookArgs::new(); - assert_eq!(args.endpoint, ""); - assert_eq!(args.auth_token, ""); - assert!(args.custom_headers.is_none()); - assert_eq!(args.queue_dir, ""); - assert_eq!(args.queue_limit, 0); - assert_eq!(args.client_cert, ""); - assert_eq!(args.client_key, ""); - assert!(!args.enable); - } - - #[test] - fn test_webhook_args_validate() { - let mut args = WebhookArgs::new(); - assert!(args.validate().is_err()); - args.endpoint = "http://example.com".to_string(); - assert!(args.validate().is_ok()); - } -} diff --git a/crates/notify/Cargo.toml b/crates/notify/Cargo.toml index d9e07e0b0..37a3a5263 100644 --- a/crates/notify/Cargo.toml +++ b/crates/notify/Cargo.toml @@ -7,11 +7,12 @@ rust-version.workspace = true version.workspace = true [dependencies] -rustfs-config = { workspace = true, features = ["constants", "notify"] } +rustfs-utils = { workspace = true, features = ["path", "sys"] } async-trait = { workspace = true } chrono = { workspace = true, features = ["serde"] } const-str = { workspace = true } ecstore = { workspace = true } +form_urlencoded = { workspace = true } once_cell = { workspace = true } quick-xml = { workspace = true, features = ["serialize", "async-tokio"] } reqwest = { workspace = true } @@ -28,11 +29,12 @@ url = { workspace = true } urlencoding = { workspace = true } wildmatch = { workspace = true, features = ["serde"] } + + [dev-dependencies] tokio = { workspace = true, features = ["test-util"] } reqwest = { workspace = true, default-features = false, features = ["rustls-tls", "charset", "http2", "system-proxy", "stream", "json", "blocking"] } axum = { workspace = true } -rustfs-utils = { workspace = true, features = ["path"] } [lints] workspace = true diff --git a/crates/notify/examples/full_demo.rs b/crates/notify/examples/full_demo.rs index 448cd60c4..0cdf50f48 100644 --- a/crates/notify/examples/full_demo.rs +++ b/crates/notify/examples/full_demo.rs @@ -1,7 +1,7 @@ -use ecstore::config::{Config, KV, KVS}; +use ecstore::config::{Config, ENABLE_KEY, ENABLE_ON, KV, KVS}; use rustfs_notify::arn::TargetID; use rustfs_notify::factory::{ - DEFAULT_TARGET, ENABLE, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_TOPIC, MQTT_USERNAME, + DEFAULT_TARGET, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_TOPIC, MQTT_USERNAME, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, WEBHOOK_AUTH_TOKEN, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, }; use rustfs_notify::global::notification_system; @@ -9,12 +9,20 @@ use rustfs_notify::store::DEFAULT_LIMIT; use rustfs_notify::{init_logger, BucketNotificationConfig, Event, EventName, LogLevel, NotificationError}; use std::time::Duration; use tracing::info; +use tracing_subscriber::util::SubscriberInitExt; #[tokio::main] async fn main() -> Result<(), NotificationError> { init_logger(LogLevel::Debug); - let system = notification_system(); + let system = match notification_system() { + Some(sys) => sys, + None => { + let config = Config::new(); + notification_system::initialize(config).await?; + notification_system().expect("Failed to initialize notification system") + } + }; // --- Initial configuration (Webhook and MQTT) --- let mut config = Config::new(); @@ -23,8 +31,8 @@ async fn main() -> Result<(), NotificationError> { let webhook_kvs_vec = vec![ KV { - key: ENABLE.to_string(), - value: "on".to_string(), + key: ENABLE_KEY.to_string(), + value: ENABLE_ON.to_string(), hidden_if_empty: false, }, KV { @@ -62,8 +70,8 @@ async fn main() -> Result<(), NotificationError> { // MQTT target configuration let mqtt_kvs_vec = vec![ KV { - key: ENABLE.to_string(), - value: "on".to_string(), + key: ENABLE_KEY.to_string(), + value: ENABLE_ON.to_string(), hidden_if_empty: false, }, KV { diff --git a/crates/notify/examples/full_demo_one.rs b/crates/notify/examples/full_demo_one.rs index fb33f4bcd..2af55a588 100644 --- a/crates/notify/examples/full_demo_one.rs +++ b/crates/notify/examples/full_demo_one.rs @@ -1,8 +1,8 @@ -use ecstore::config::{Config, KV, KVS}; +use ecstore::config::{Config, ENABLE_KEY, ENABLE_ON, KV, KVS}; // Using Global Accessories use rustfs_notify::arn::TargetID; use rustfs_notify::factory::{ - DEFAULT_TARGET, ENABLE, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_TOPIC, MQTT_USERNAME, + DEFAULT_TARGET, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_TOPIC, MQTT_USERNAME, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, WEBHOOK_AUTH_TOKEN, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, }; use rustfs_notify::global::notification_system; @@ -10,13 +10,21 @@ use rustfs_notify::store::DEFAULT_LIMIT; use rustfs_notify::{init_logger, BucketNotificationConfig, Event, EventName, LogLevel, NotificationError}; use std::time::Duration; use tracing::info; +use tracing_subscriber::util::SubscriberInitExt; #[tokio::main] async fn main() -> Result<(), NotificationError> { init_logger(LogLevel::Debug); // Get global NotificationSystem instance - let system = notification_system(); + let system = match notification_system() { + Some(sys) => sys, + None => { + let config = Config::new(); + notification_system::initialize(config).await?; + notification_system().expect("Failed to initialize notification system") + } + }; // --- Initial configuration --- let mut config = Config::new(); @@ -24,8 +32,8 @@ async fn main() -> Result<(), NotificationError> { // Webhook target let webhook_kvs_vec = vec![ KV { - key: ENABLE.to_string(), - value: "on".to_string(), + key: ENABLE_KEY.to_string(), + value: ENABLE_ON.to_string(), hidden_if_empty: false, }, KV { @@ -72,8 +80,8 @@ async fn main() -> Result<(), NotificationError> { let mqtt_kvs_vec = vec![ KV { - key: ENABLE.to_string(), - value: "on".to_string(), + key: ENABLE_KEY.to_string(), + value: ENABLE_ON.to_string(), hidden_if_empty: false, }, KV { diff --git a/crates/notify/src/args.rs b/crates/notify/src/args.rs deleted file mode 100644 index 3eceb1eca..000000000 --- a/crates/notify/src/args.rs +++ /dev/null @@ -1,110 +0,0 @@ -use crate::{Event, EventName}; -use std::collections::HashMap; - -/// 事件参数 -#[derive(Debug, Clone)] -pub struct EventArgs { - pub event_name: EventName, - pub bucket_name: String, - pub object_name: String, - pub object_size: Option, - pub object_etag: Option, - pub object_version_id: Option, - pub object_content_type: Option, - pub object_user_metadata: Option>, - pub req_params: HashMap, - pub resp_elements: HashMap, - pub host: String, - pub user_agent: String, -} - -impl EventArgs { - /// 转换为通知事件 - pub fn to_event(&self) -> Event { - let event_time = chrono::Utc::now(); - let unique_id = format!("{:X}", event_time.timestamp_nanos_opt().unwrap_or(0)); - - let mut resp_elements = HashMap::new(); - if let Some(request_id) = self.resp_elements.get("requestId") { - resp_elements.insert("x-amz-request-id".to_string(), request_id.clone()); - } - if let Some(node_id) = self.resp_elements.get("nodeId") { - resp_elements.insert("x-amz-id-2".to_string(), node_id.clone()); - } - - // RustFS 特定的自定义元素 - // 注意:这里需要获取 endpoint 的逻辑在 Rust 中可能需要单独实现 - resp_elements.insert("x-rustfs-origin-endpoint".to_string(), "".to_string()); - - // 添加 deployment ID - resp_elements.insert("x-rustfs-deployment-id".to_string(), "".to_string()); - - if let Some(content_length) = self.resp_elements.get("content-length") { - resp_elements.insert("content-length".to_string(), content_length.clone()); - } - - let key_name = &self.object_name; - // 注意:这里可能需要根据 escape 参数进行 URL 编码 - - let mut event = Event { - event_version: "2.0".to_string(), - event_source: "rustfs:s3".to_string(), - aws_region: self.req_params.get("region").cloned().unwrap_or_default(), - event_time, - event_name: self.event_name, - user_identity: crate::event::Identity { - principal_id: self - .req_params - .get("principalId") - .cloned() - .unwrap_or_default(), - }, - request_parameters: self.req_params.clone(), - response_elements: resp_elements, - s3: crate::event::Metadata { - schema_version: "1.0".to_string(), - configuration_id: "Config".to_string(), - bucket: crate::event::Bucket { - name: self.bucket_name.clone(), - owner_identity: crate::event::Identity { - principal_id: self - .req_params - .get("principalId") - .cloned() - .unwrap_or_default(), - }, - arn: format!("arn:aws:s3:::{}", self.bucket_name), - }, - object: crate::event::Object { - key: key_name.clone(), - version_id: self.object_version_id.clone(), - sequencer: unique_id, - size: self.object_size, - etag: self.object_etag.clone(), - content_type: self.object_content_type.clone(), - user_metadata: Some(self.object_user_metadata.clone().unwrap_or_default()), - }, - }, - source: crate::event::Source { - host: self.host.clone(), - port: "".to_string(), - user_agent: self.user_agent.clone(), - }, - }; - - // 检查是否为删除事件,如果是删除事件,某些字段应当为空 - let is_removed_event = matches!( - self.event_name, - EventName::ObjectRemovedDelete | EventName::ObjectRemovedDeleteMarkerCreated - ); - - if is_removed_event { - event.s3.object.etag = None; - event.s3.object.size = None; - event.s3.object.content_type = None; - event.s3.object.user_metadata = None; - } - - event - } -} diff --git a/crates/notify/src/error.rs b/crates/notify/src/error.rs index 77e42a293..a7a38e9d4 100644 --- a/crates/notify/src/error.rs +++ b/crates/notify/src/error.rs @@ -92,6 +92,12 @@ pub enum NotificationError { #[error("System initialization error: {0}")] Initialization(String), + + #[error("Notification system has already been initialized")] + AlreadyInitialized, + + #[error("Io error: {0}")] + Io(std::io::Error), } impl From for TargetError { diff --git a/crates/notify/src/event.rs b/crates/notify/src/event.rs index db08bef76..22bb630fd 100644 --- a/crates/notify/src/event.rs +++ b/crates/notify/src/event.rs @@ -2,6 +2,7 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::fmt; +use url::form_urlencoded; /// Error returned when parsing event name string fails。 #[derive(Debug, Clone, PartialEq, Eq)] @@ -296,7 +297,7 @@ pub struct Bucket { } /// Represents the object that the event occurred on -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct Object { /// The key (name) of the object pub key: String, @@ -323,6 +324,7 @@ pub struct Object { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Metadata { /// The schema version of the event + #[serde(rename = "s3SchemaVersion")] pub schema_version: String, /// The ID of the configuration that triggered the event pub configuration_id: String, @@ -340,11 +342,13 @@ pub struct Source { /// The port on the host pub port: String, /// The user agent that caused the event + #[serde(rename = "userAgent")] pub user_agent: String, } /// Represents a storage event #[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] pub struct Event { /// The version of the event pub event_version: String, @@ -432,6 +436,85 @@ impl Event { pub fn mask(&self) -> u64 { self.event_name.mask() } + + pub fn new(args: EventArgs) -> Self { + let event_time = Utc::now().naive_local(); + let unique_id = match args.object.mod_time { + Some(t) => format!("{:X}", t.unix_timestamp_nanos()), + None => format!("{:X}", event_time.and_utc().timestamp_nanos_opt().unwrap_or(0)), + }; + + let mut resp_elements = args.resp_elements.clone(); + resp_elements + .entry("x-amz-request-id".to_string()) + .or_insert_with(|| "".to_string()); + resp_elements + .entry("x-amz-id-2".to_string()) + .or_insert_with(|| "".to_string()); + // ... Filling of other response elements + + // URL encoding of object keys + let key_name = form_urlencoded::byte_serialize(args.object.name.as_bytes()).collect::(); + + let principal_id = args.req_params.get("principalId").cloned().unwrap_or_default(); + let owner_identity = Identity { + principal_id: principal_id.clone(), + }; + let user_identity = Identity { principal_id }; + + let mut s3_metadata = Metadata { + schema_version: "1.0".to_string(), + configuration_id: "Config".to_string(), // or from args + bucket: Bucket { + name: args.bucket_name.clone(), + owner_identity, + arn: format!("arn:aws:s3:::{}", args.bucket_name), + }, + object: Object { + key: key_name, + version_id: Some(args.object.version_id.unwrap().to_string()), + sequencer: unique_id, + ..Default::default() + }, + }; + + let is_removed_event = matches!( + args.event_name, + EventName::ObjectRemovedDelete | EventName::ObjectRemovedDeleteMarkerCreated + ); + + if !is_removed_event { + s3_metadata.object.size = Some(args.object.size); + s3_metadata.object.etag = args.object.etag.clone(); + s3_metadata.object.content_type = args.object.content_type.clone(); + // Filter out internal reserved metadata + let user_metadata = args + .object + .user_defined + .iter() + .filter(|&(k, v)| !k.to_lowercase().starts_with("x-amz-meta-internal-")) + .map(|(k, v)| (k.clone(), v.clone())) + .collect::>(); + s3_metadata.object.user_metadata = Some(user_metadata); + } + + Self { + event_version: "2.1".to_string(), + event_source: "rustfs:s3".to_string(), + aws_region: args.req_params.get("region").cloned().unwrap_or_default(), + event_time: event_time.and_utc(), + event_name: args.event_name, + user_identity, + request_parameters: args.req_params, + response_elements: resp_elements, + s3: s3_metadata, + source: Source { + host: args.host, + port: "".to_string(), + user_agent: args.user_agent, + }, + } + } } /// Represents a log of events for sending to targets @@ -444,3 +527,21 @@ pub struct EventLog { /// The list of events pub records: Vec, } + +#[derive(Debug, Clone)] +pub struct EventArgs { + pub event_name: EventName, + pub bucket_name: String, + pub object: ecstore::store_api::ObjectInfo, + pub req_params: HashMap, + pub resp_elements: HashMap, + pub host: String, + pub user_agent: String, +} + +impl EventArgs { + // Helper function to check if it is a copy request + pub fn is_replication_request(&self) -> bool { + self.req_params.contains_key("x-rustfs-source-replication-request") + } +} diff --git a/crates/notify/src/factory.rs b/crates/notify/src/factory.rs index c76153f89..b40d85505 100644 --- a/crates/notify/src/factory.rs +++ b/crates/notify/src/factory.rs @@ -4,7 +4,7 @@ use crate::{ target::{mqtt::MQTTArgs, webhook::WebhookArgs, Target}, }; use async_trait::async_trait; -use ecstore::config::KVS; +use ecstore::config::{ENABLE_KEY, ENABLE_ON, KVS}; use rumqttc::QoS; use std::time::Duration; use tracing::warn; @@ -13,7 +13,6 @@ use url::Url; // --- Configuration Constants --- // General -pub const ENABLE: &str = "enable"; pub const DEFAULT_TARGET: &str = "1"; @@ -37,6 +36,9 @@ pub const NOTIFY_POSTGRES_SUB_SYS: &str = "notify_postgres"; pub const NOTIFY_REDIS_SUB_SYS: &str = "notify_redis"; pub const NOTIFY_WEBHOOK_SUB_SYS: &str = "notify_webhook"; +#[allow(dead_code)] +pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS]; + // Webhook Keys pub const WEBHOOK_ENDPOINT: &str = "endpoint"; pub const WEBHOOK_AUTH_TOKEN: &str = "auth_token"; @@ -111,8 +113,8 @@ impl TargetFactory for WebhookTargetFactory { async fn create_target(&self, id: String, config: &KVS) -> Result, TargetError> { let get = |base_env_key: &str, config_key: &str| get_config_value(&id, base_env_key, config_key, config); - let enable = get(ENV_WEBHOOK_ENABLE, ENABLE) - .map(|v| v.eq_ignore_ascii_case("on") || v.eq_ignore_ascii_case("true")) + let enable = get(ENV_WEBHOOK_ENABLE, ENABLE_KEY) + .map(|v| v.eq_ignore_ascii_case(ENABLE_ON) || v.eq_ignore_ascii_case("true")) .unwrap_or(false); if !enable { @@ -151,8 +153,8 @@ impl TargetFactory for WebhookTargetFactory { fn validate_config(&self, id: &str, config: &KVS) -> Result<(), TargetError> { let get = |base_env_key: &str, config_key: &str| get_config_value(id, base_env_key, config_key, config); - let enable = get(ENV_WEBHOOK_ENABLE, ENABLE) - .map(|v| v.eq_ignore_ascii_case("on") || v.eq_ignore_ascii_case("true")) + let enable = get(ENV_WEBHOOK_ENABLE, ENABLE_KEY) + .map(|v| v.eq_ignore_ascii_case(ENABLE_ON) || v.eq_ignore_ascii_case("true")) .unwrap_or(false); if !enable { @@ -189,8 +191,8 @@ impl TargetFactory for MQTTTargetFactory { async fn create_target(&self, id: String, config: &KVS) -> Result, TargetError> { let get = |base_env_key: &str, config_key: &str| get_config_value(&id, base_env_key, config_key, config); - let enable = get(ENV_MQTT_ENABLE, ENABLE) - .map(|v| v.eq_ignore_ascii_case("on") || v.eq_ignore_ascii_case("true")) + let enable = get(ENV_MQTT_ENABLE, ENABLE_KEY) + .map(|v| v.eq_ignore_ascii_case(ENABLE_ON) || v.eq_ignore_ascii_case("true")) .unwrap_or(false); if !enable { @@ -252,8 +254,8 @@ impl TargetFactory for MQTTTargetFactory { fn validate_config(&self, id: &str, config: &KVS) -> Result<(), TargetError> { let get = |base_env_key: &str, config_key: &str| get_config_value(id, base_env_key, config_key, config); - let enable = get(ENV_MQTT_ENABLE, ENABLE) - .map(|v| v.eq_ignore_ascii_case("on") || v.eq_ignore_ascii_case("true")) + let enable = get(ENV_MQTT_ENABLE, ENABLE_KEY) + .map(|v| v.eq_ignore_ascii_case(ENABLE_ON) || v.eq_ignore_ascii_case("true")) .unwrap_or(false); if !enable { diff --git a/crates/notify/src/global.rs b/crates/notify/src/global.rs index 6c93b8398..b2dffe5c1 100644 --- a/crates/notify/src/global.rs +++ b/crates/notify/src/global.rs @@ -1,12 +1,60 @@ -use crate::NotificationSystem; +use crate::{Event, EventArgs, NotificationError, NotificationSystem}; +use ecstore::config::Config; use once_cell::sync::Lazy; -use std::sync::Arc; +use std::sync::{Arc, OnceLock}; -static NOTIFICATION_SYSTEM: Lazy> = - Lazy::new(|| Arc::new(NotificationSystem::new())); +static NOTIFICATION_SYSTEM: OnceLock> = OnceLock::new(); +// Create a globally unique Notifier instance +pub static GLOBAL_NOTIFIER: Lazy = Lazy::new(|| Notifier {}); -/// Returns the handle to the global NotificationSystem instance. -/// This function can be called anywhere you need to interact with the notification system。 -pub fn notification_system() -> Arc { - NOTIFICATION_SYSTEM.clone() +/// Initialize the global notification system with the given configuration. +/// This function should only be called once throughout the application life cycle. +pub async fn initialize(config: Config) -> Result<(), NotificationError> { + // `new` is synchronous and responsible for creating instances + let system = NotificationSystem::new(config); + // `init` is asynchronous and responsible for performing I/O-intensive initialization + system.init().await?; + + match NOTIFICATION_SYSTEM.set(Arc::new(system)) { + Ok(_) => Ok(()), + Err(_) => Err(NotificationError::AlreadyInitialized), + } +} + +/// Returns a handle to the global NotificationSystem instance. +/// Return None if the system has not been initialized. +pub fn notification_system() -> Option> { + NOTIFICATION_SYSTEM.get().cloned() +} + +pub struct Notifier { + // Notifier can hold state, but in this design we make it stateless, + // Rely on getting an instance of NotificationSystem from the outside. +} + +impl crate::notifier::Notifier { + /// Notify an event asynchronously. + /// This is the only entry point for all event notifications in the system. + pub async fn notify(&self, args: EventArgs) { + // Dependency injection or service positioning mode obtain NotificationSystem instance + let notification_sys = match notification_system() { + // If the notification system itself cannot be retrieved, it will be returned directly + Some(sys) => sys, + None => { + tracing::error!("Notification system is not initialized."); + return; + } + }; + + // Avoid generating notifications for replica creation events + if args.is_replication_request() { + return; + } + + // Create an event and send it + let event = Event::new(args.clone()); + notification_sys + .send_event(&args.bucket_name, &args.event_name.as_str(), &args.object.name.clone(), event) + .await; + } } diff --git a/crates/notify/src/integration.rs b/crates/notify/src/integration.rs index 3e99ecb05..34bb9be65 100644 --- a/crates/notify/src/integration.rs +++ b/crates/notify/src/integration.rs @@ -40,7 +40,7 @@ impl NotificationMetrics { } } - // 提供公共方法增加计数 + // Provide public methods to increase count pub fn increment_processing(&self) { self.processing_events.fetch_add(1, Ordering::Relaxed); } @@ -55,7 +55,7 @@ impl NotificationMetrics { self.failed_events.fetch_add(1, Ordering::Relaxed); } - // 提供公共方法获取计数 + // Provide public methods to get count pub fn processing_count(&self) -> usize { self.processing_events.load(Ordering::Relaxed) } @@ -89,19 +89,13 @@ pub struct NotificationSystem { metrics: Arc, } -impl Default for NotificationSystem { - fn default() -> Self { - Self::new() - } -} - impl NotificationSystem { /// Creates a new NotificationSystem - pub fn new() -> Self { + pub fn new(config: Config) -> Self { NotificationSystem { notifier: Arc::new(EventNotifier::new()), registry: Arc::new(TargetRegistry::new()), - config: Arc::new(RwLock::new(Config::new())), + config: Arc::new(RwLock::new(config)), stream_cancellers: Arc::new(RwLock::new(HashMap::new())), concurrency_limiter: Arc::new(Semaphore::new( std::env::var("RUSTFS_TARGET_STREAM_CONCURRENCY") @@ -194,49 +188,43 @@ impl NotificationSystem { pub async fn remove_target(&self, target_id: &TargetID, target_type: &str) -> Result<(), NotificationError> { info!("Attempting to remove target: {}", target_id); - // Step 1: Stop the event stream (if present) - let mut cancellers_guard = self.stream_cancellers.write().await; - if let Some(cancel_tx) = cancellers_guard.remove(target_id) { - info!("Stopping event stream for target {}", target_id); - // Send a stop signal and continue execution even if it fails, because the receiver may have been closed - if let Err(e) = cancel_tx.send(()).await { - error!("Failed to send stop signal to target {} stream: {}", target_id, e); - } - } else { - info!("No active event stream found for target {}, skipping stop.", target_id); - } - drop(cancellers_guard); + let Some(store) = ecstore::global::new_object_layer_fn() else { + return Err(NotificationError::Io(std::io::Error::new( + std::io::ErrorKind::Other, + "errServerNotInitialized", + ))); + }; - // Step 2: Remove the Target instance from the activity list of Notifier - // TargetList::remove_target_only will call target.close() - let target_list = self.notifier.target_list(); - let mut target_list_guard = target_list.write().await; - if target_list_guard.remove_target_only(target_id).await.is_some() { - info!("Removed target {} from the active list.", target_id); - } else { - warn!("Target {} was not found in the active list.", target_id); - } - drop(target_list_guard); + let mut new_config = ecstore::config::com::read_config_without_migrate(store.clone()) + .await + .map_err(|e| NotificationError::Configuration(format!("Failed to read notification config: {}", e)))?; - // Step 3: Remove Target from persistent configuration - let mut config_guard = self.config.write().await; let mut changed = false; - if let Some(targets_of_type) = config_guard.0.get_mut(target_type) { + if let Some(targets_of_type) = new_config.0.get_mut(target_type) { if targets_of_type.remove(&target_id.name).is_some() { info!("Removed target {} from the configuration.", target_id); changed = true; } - // If there are no targets under this type, remove the entry for this type if targets_of_type.is_empty() { - config_guard.0.remove(target_type); + new_config.0.remove(target_type); } } if !changed { warn!("Target {} was not found in the configuration.", target_id); + return Ok(()); } - Ok(()) + if let Err(e) = ecstore::config::com::save_server_config(store, &new_config).await { + error!("Failed to save config for target removal: {}", e); + return Err(NotificationError::Configuration(format!("Failed to save config: {}", e))); + } + + info!( + "Configuration updated and persisted for target {} removal. Reloading system...", + target_id + ); + self.reload_config(new_config).await } /// Set or update a Target configuration. @@ -253,18 +241,49 @@ impl NotificationSystem { /// If the target configuration is invalid, it returns Err(NotificationError::Configuration). pub async fn set_target_config(&self, target_type: &str, target_name: &str, kvs: KVS) -> Result<(), NotificationError> { info!("Setting config for target {} of type {}", target_name, target_type); - let mut config_guard = self.config.write().await; - config_guard + // 1. Get the storage handle + let Some(store) = ecstore::global::new_object_layer_fn() else { + return Err(NotificationError::Io(std::io::Error::new( + std::io::ErrorKind::Other, + "errServerNotInitialized", + ))); + }; + + // 2. Read the latest configuration from storage + let mut new_config = ecstore::config::com::read_config_without_migrate(store.clone()) + .await + .map_err(|e| NotificationError::Configuration(format!("Failed to read notification config: {}", e)))?; + + // 3. Modify the configuration copy + new_config .0 .entry(target_type.to_string()) .or_default() .insert(target_name.to_string(), kvs); - let new_config = config_guard.clone(); - // Release the lock before calling reload_config - drop(config_guard); + // 4. Persist the new configuration + if let Err(e) = ecstore::config::com::save_server_config(store, &new_config).await { + error!("Failed to save notification config: {}", e); + return Err(NotificationError::Configuration(format!("Failed to save notification config: {}", e))); + } - self.reload_config(new_config).await + // 5. After the persistence is successful, the system will be reloaded to apply changes. + match self.reload_config(new_config).await { + Ok(_) => { + info!( + "Target {} of type {} configuration updated and reloaded successfully", + target_name, target_type + ); + Ok(()) + } + Err(e) => { + error!("Failed to reload config for target {} of type {}: {}", target_name, target_type, e); + Err(NotificationError::Configuration(format!( + "Configuration saved, but failed to reload: {}", + e + ))) + } + } } /// Removes all notification configurations for a bucket. @@ -286,27 +305,42 @@ impl NotificationSystem { /// If the target configuration does not exist, it returns Ok(()) without making any changes. pub async fn remove_target_config(&self, target_type: &str, target_name: &str) -> Result<(), NotificationError> { info!("Removing config for target {} of type {}", target_name, target_type); - let mut config_guard = self.config.write().await; - let mut changed = false; + let Some(store) = ecstore::global::new_object_layer_fn() else { + return Err(NotificationError::Io(std::io::Error::new( + std::io::ErrorKind::Other, + "errServerNotInitialized", + ))); + }; - if let Some(targets) = config_guard.0.get_mut(target_type) { + let mut new_config = ecstore::config::com::read_config_without_migrate(store.clone()) + .await + .map_err(|e| NotificationError::Configuration(format!("Failed to read notification config: {}", e)))?; + + let mut changed = false; + if let Some(targets) = new_config.0.get_mut(target_type) { if targets.remove(target_name).is_some() { changed = true; } if targets.is_empty() { - config_guard.0.remove(target_type); + new_config.0.remove(target_type); } } - if changed { - let new_config = config_guard.clone(); - // Release the lock before calling reload_config - drop(config_guard); - self.reload_config(new_config).await - } else { + if !changed { info!("Target {} of type {} not found, no changes made.", target_name, target_type); - Ok(()) + return Ok(()); } + + if let Err(e) = ecstore::config::com::save_server_config(store, &new_config).await { + error!("Failed to save config for target removal: {}", e); + return Err(NotificationError::Configuration(format!("Failed to save config: {}", e))); + } + + info!( + "Configuration updated and persisted for target {} removal. Reloading system...", + target_name + ); + self.reload_config(new_config).await } /// Enhanced event stream startup function, including monitoring and concurrency control diff --git a/crates/notify/src/lib.rs b/crates/notify/src/lib.rs index 66a0cbc82..9d0c5436e 100644 --- a/crates/notify/src/lib.rs +++ b/crates/notify/src/lib.rs @@ -4,7 +4,6 @@ //! similar to RustFS's notification system. It supports sending events to various targets //! (like Webhook and MQTT) and includes features like event persistence and retry on failure. -pub mod args; pub mod arn; pub mod error; pub mod event; @@ -17,11 +16,11 @@ pub mod rules; pub mod store; pub mod stream; pub mod target; -pub mod utils; // Re-exports pub use error::{NotificationError, StoreError, TargetError}; -pub use event::{Event, EventLog, EventName}; +pub use event::{Event, EventArgs, EventLog, EventName}; +pub use global::{initialize, notification_system}; pub use integration::NotificationSystem; pub use rules::BucketNotificationConfig; use std::io::IsTerminal; diff --git a/crates/notify/src/notifier.rs b/crates/notify/src/notifier.rs index f38e10f8a..a3827f68f 100644 --- a/crates/notify/src/notifier.rs +++ b/crates/notify/src/notifier.rs @@ -96,57 +96,42 @@ impl EventNotifier { let target_ids_len = target_ids.len(); let mut handles = vec![]; - // 使用作用域来限制 target_list 的借用范围 + // Use scope to limit the borrow scope of target_list { let target_list_guard = self.target_list.read().await; info!("Sending event to targets: {:?}", target_ids); for target_id in target_ids { // `get` now returns Option> if let Some(target_arc) = target_list_guard.get(&target_id) { - // 克隆 Arc> (target_list 存储的就是这个类型) 以便移入异步任务 + // Clone an Arc> (which is where target_list is stored) to move into an asynchronous task // target_arc is already Arc, clone it for the async task let cloned_target_for_task = target_arc.clone(); let event_clone = event.clone(); - let target_name_for_task = cloned_target_for_task.name(); // 在生成任务前获取名称 - debug!( - "Preparing to send event to target: {}", - target_name_for_task - ); - // 在闭包中使用克隆的数据,避免借用冲突 + let target_name_for_task = cloned_target_for_task.name(); // Get the name before generating the task + debug!("Preparing to send event to target: {}", target_name_for_task); + // Use cloned data in closures to avoid borrowing conflicts let handle = tokio::spawn(async move { if let Err(e) = cloned_target_for_task.save(event_clone).await { - error!( - "Failed to send event to target {}: {}", - target_name_for_task, e - ); + error!("Failed to send event to target {}: {}", target_name_for_task, e); } else { - debug!( - "Successfully saved event to target {}", - target_name_for_task - ); + debug!("Successfully saved event to target {}", target_name_for_task); } }); handles.push(handle); } else { - warn!( - "Target ID {:?} found in rules but not in target list.", - target_id - ); + warn!("Target ID {:?} found in rules but not in target list.", target_id); } } - // target_list 在这里自动释放 + // target_list is automatically released here } - // 等待所有任务完成 + // Wait for all tasks to be completed for handle in handles { if let Err(e) = handle.await { error!("Task for sending/saving event failed: {}", e); } } - info!( - "Event processing initiated for {} targets for bucket: {}", - target_ids_len, bucket_name - ); + info!("Event processing initiated for {} targets for bucket: {}", target_ids_len, bucket_name); } else { debug!("No rules found for bucket: {}", bucket_name); } @@ -158,22 +143,22 @@ impl EventNotifier { &self, targets_to_init: Vec>, ) -> Result<(), NotificationError> { - // 当前激活的、更简单的逻辑: - let mut target_list_guard = self.target_list.write().await; // 获取 TargetList 的写锁 + // Currently active, simpler logic + let mut target_list_guard = self.target_list.write().await; //Gets a write lock for the TargetList for target_boxed in targets_to_init { - // 遍历传入的 Box + // Traverse the incoming Box debug!("init bucket target: {}", target_boxed.name()); - // TargetList::add 方法期望 Arc - // 因此,需要将 Box 转换为 Arc + // TargetList::add method expectations Arc + // Therefore, you need to convert Box to Arc let target_arc: Arc = Arc::from(target_boxed); - target_list_guard.add(target_arc)?; // 将 Arc 添加到列表中 + target_list_guard.add(target_arc)?; // Add Arc to the list } info!( - "Initialized {} targets, list size: {}", // 更清晰的日志 + "Initialized {} targets, list size: {}", // Clearer logs target_list_guard.len(), target_list_guard.len() ); - Ok(()) // 确保返回 Result + Ok(()) // Make sure to return a Result } } @@ -191,9 +176,7 @@ impl Default for TargetList { impl TargetList { /// Creates a new TargetList pub fn new() -> Self { - TargetList { - targets: HashMap::new(), - } + TargetList { targets: HashMap::new() } } /// Adds a target to the list @@ -201,10 +184,7 @@ impl TargetList { let id = target.id(); if self.targets.contains_key(&id) { // Potentially update or log a warning/error if replacing an existing target. - warn!( - "Target with ID {} already exists in TargetList. It will be overwritten.", - id - ); + warn!("Target with ID {} already exists in TargetList. It will be overwritten.", id); } self.targets.insert(id, target); Ok(()) @@ -212,10 +192,7 @@ impl TargetList { /// Removes a target by ID. Note: This does not stop its associated event stream. /// Stream cancellation should be handled by EventNotifier. - pub async fn remove_target_only( - &mut self, - id: &TargetID, - ) -> Option> { + pub async fn remove_target_only(&mut self, id: &TargetID) -> Option> { if let Some(target_arc) = self.targets.remove(id) { if let Err(e) = target_arc.close().await { // Target's own close logic diff --git a/crates/notify/src/registry.rs b/crates/notify/src/registry.rs index e541f79d1..748b53563 100644 --- a/crates/notify/src/registry.rs +++ b/crates/notify/src/registry.rs @@ -4,7 +4,7 @@ use crate::{ factory::{MQTTTargetFactory, TargetFactory, WebhookTargetFactory}, target::Target, }; -use ecstore::config::{Config, KVS}; +use ecstore::config::{Config, ENABLE_KEY, ENABLE_OFF, ENABLE_ON, KVS}; use std::collections::HashMap; use tracing::{error, info}; @@ -74,7 +74,7 @@ impl TargetRegistry { // Iterate through subsections (each representing a target instance) for (target_id, target_config) in subsections { // Skip disabled targets - if target_config.lookup("enable").unwrap_or_else(|| "off".to_string()) != "on" { + if target_config.lookup(ENABLE_KEY).unwrap_or_else(|| ENABLE_OFF.to_string()) != ENABLE_ON { continue; } @@ -94,9 +94,3 @@ impl TargetRegistry { Ok(targets) } } - -#[cfg(test)] -mod tests { - #[tokio::test] - async fn test_target_registry() {} -} diff --git a/crates/notify/src/target/webhook.rs b/crates/notify/src/target/webhook.rs index 1086fec0f..9413067dd 100644 --- a/crates/notify/src/target/webhook.rs +++ b/crates/notify/src/target/webhook.rs @@ -4,7 +4,6 @@ use crate::{ arn::TargetID, error::TargetError, event::{Event, EventLog}, store::{Key, Store}, - utils, StoreError, Target, }; @@ -56,18 +55,14 @@ impl WebhookArgs { if !self.queue_dir.is_empty() { let path = std::path::Path::new(&self.queue_dir); if !path.is_absolute() { - return Err(TargetError::Configuration( - "webhook queueDir path should be absolute".to_string(), - )); + return Err(TargetError::Configuration("webhook queueDir path should be absolute".to_string())); } } if !self.client_cert.is_empty() && self.client_key.is_empty() || self.client_cert.is_empty() && !self.client_key.is_empty() { - return Err(TargetError::Configuration( - "cert and key must be specified as a pair".to_string(), - )); + return Err(TargetError::Configuration("cert and key must be specified as a pair".to_string())); } Ok(()) @@ -79,7 +74,7 @@ pub struct WebhookTarget { id: TargetID, args: WebhookArgs, http_client: Arc, - // 添加 Send + Sync 约束确保线程安全 + // Add Send + Sync constraints to ensure thread safety store: Option + Send + Sync>>, initialized: AtomicBool, addr: String, @@ -103,36 +98,35 @@ impl WebhookTarget { /// Creates a new WebhookTarget #[instrument(skip(args), fields(target_id = %id))] pub fn new(id: String, args: WebhookArgs) -> Result { - // 首先验证参数 + // First verify the parameters args.validate()?; - // 创建 TargetID + // Create a TargetID let target_id = TargetID::new(id, ChannelTargetType::Webhook.as_str().to_string()); - // 构建 HTTP client + // Build HTTP client let mut client_builder = Client::builder() .timeout(Duration::from_secs(30)) - .user_agent(utils::get_user_agent(utils::ServiceType::Basis)); + .user_agent(rustfs_utils::sys::get_user_agent(rustfs_utils::sys::ServiceType::Basis)); - // 补充证书处理逻辑 + // Supplementary certificate processing logic if !args.client_cert.is_empty() && !args.client_key.is_empty() { - // 添加客户端证书 - let cert = std::fs::read(&args.client_cert).map_err(|e| { - TargetError::Configuration(format!("Failed to read client cert: {}", e)) - })?; - let key = std::fs::read(&args.client_key).map_err(|e| { - TargetError::Configuration(format!("Failed to read client key: {}", e)) - })?; + // Add client certificate + let cert = std::fs::read(&args.client_cert) + .map_err(|e| TargetError::Configuration(format!("Failed to read client cert: {}", e)))?; + let key = std::fs::read(&args.client_key) + .map_err(|e| TargetError::Configuration(format!("Failed to read client key: {}", e)))?; - let identity = reqwest::Identity::from_pem(&[cert, key].concat()).map_err(|e| { - TargetError::Configuration(format!("Failed to create identity: {}", e)) - })?; + let identity = reqwest::Identity::from_pem(&[cert, key].concat()) + .map_err(|e| TargetError::Configuration(format!("Failed to create identity: {}", e)))?; client_builder = client_builder.identity(identity); } - let http_client = Arc::new(client_builder.build().map_err(|e| { - TargetError::Configuration(format!("Failed to build HTTP client: {}", e)) - })?); + let http_client = Arc::new( + client_builder + .build() + .map_err(|e| TargetError::Configuration(format!("Failed to build HTTP client: {}", e)))?, + ); - // 构建存储 + // Build storage let queue_store = if !args.queue_dir.is_empty() { let queue_dir = PathBuf::from(&args.queue_dir).join(format!( "rustfs-{}-{}-{}", @@ -140,43 +134,30 @@ impl WebhookTarget { target_id.name, target_id.id )); - let store = super::super::store::QueueStore::::new( - queue_dir, - args.queue_limit, - STORE_EXTENSION, - ); + let store = super::super::store::QueueStore::::new(queue_dir, args.queue_limit, STORE_EXTENSION); if let Err(e) = store.open() { - error!( - "Failed to open store for Webhook target {}: {}", - target_id.id, e - ); + error!("Failed to open store for Webhook target {}: {}", target_id.id, e); return Err(TargetError::Storage(format!("{}", e))); } - // 确保 QueueStore 实现的 Store trait 匹配预期的错误类型 - Some(Box::new(store) - as Box< - dyn Store + Send + Sync, - >) + // Make sure that the Store trait implemented by QueueStore matches the expected error type + Some(Box::new(store) as Box + Send + Sync>) } else { None }; - // 解析地址 + // resolved address let addr = { let host = args.endpoint.host_str().unwrap_or("localhost"); - let port = args.endpoint.port().unwrap_or_else(|| { - if args.endpoint.scheme() == "https" { - 443 - } else { - 80 - } - }); + let port = args + .endpoint + .port() + .unwrap_or_else(|| if args.endpoint.scheme() == "https" { 443 } else { 80 }); format!("{}:{}", host, port) }; - // 创建取消通道 + // Create a cancel channel let (cancel_sender, _) = mpsc::channel(1); info!(target_id = %target_id.id, "Webhook target created"); Ok(WebhookTarget { @@ -202,10 +183,7 @@ impl WebhookTarget { return Err(TargetError::NotConnected); } Err(e) => { - error!( - "Failed to check if Webhook target {} is active: {}", - self.id, e - ); + error!("Failed to check if Webhook target {} is active: {}", self.id, e); return Err(e); } } @@ -228,17 +206,13 @@ impl WebhookTarget { records: vec![event.clone()], }; - let data = serde_json::to_vec(&log) - .map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {}", e)))?; + let data = + serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {}", e)))?; // Vec 转换为 String - let data_string = String::from_utf8(data.clone()).map_err(|e| { - TargetError::Encoding(format!("Failed to convert event data to UTF-8: {}", e)) - })?; - debug!( - "Sending event to webhook target: {}, event log: {}", - self.id, data_string - ); + let data_string = String::from_utf8(data.clone()) + .map_err(|e| TargetError::Encoding(format!("Failed to convert event data to UTF-8: {}", e)))?; + debug!("Sending event to webhook target: {}, event log: {}", self.id, data_string); // 构建请求 let mut req_builder = self @@ -256,8 +230,7 @@ impl WebhookTarget { } 1 => { // 只有令牌,需要添加 "Bearer" 前缀 - req_builder = req_builder - .header("Authorization", format!("Bearer {}", self.args.auth_token)); + req_builder = req_builder.header("Authorization", format!("Bearer {}", self.args.auth_token)); } _ => { // 空字符串或其他情况,不添加认证头 @@ -305,16 +278,8 @@ impl Target for WebhookTarget { .map_err(|e| TargetError::Network(format!("Failed to resolve host: {}", e)))? .next() .ok_or_else(|| TargetError::Network("No address found".to_string()))?; - debug!( - "is_active socket addr: {},target id:{}", - socket_addr, self.id.id - ); - match tokio::time::timeout( - Duration::from_secs(5), - tokio::net::TcpStream::connect(socket_addr), - ) - .await - { + debug!("is_active socket addr: {},target id:{}", socket_addr, self.id.id); + match tokio::time::timeout(Duration::from_secs(5), tokio::net::TcpStream::connect(socket_addr)).await { Ok(Ok(_)) => { debug!("Connection to {} is active", self.addr); Ok(true) @@ -334,9 +299,9 @@ impl Target for WebhookTarget { async fn save(&self, event: Event) -> Result<(), TargetError> { if let Some(store) = &self.store { // Call the store method directly, no longer need to acquire the lock - store.put(event).map_err(|e| { - TargetError::Storage(format!("Failed to save event to store: {}", e)) - })?; + store + .put(event) + .map_err(|e| TargetError::Storage(format!("Failed to save event to store: {}", e)))?; debug!("Event saved to store for target: {}", self.id); Ok(()) } else { @@ -373,10 +338,7 @@ impl Target for WebhookTarget { Ok(event) => event, Err(StoreError::NotFound) => return Ok(()), Err(e) => { - return Err(TargetError::Storage(format!( - "Failed to get event from store: {}", - e - ))); + return Err(TargetError::Storage(format!("Failed to get event from store: {}", e))); } }; @@ -388,23 +350,12 @@ impl Target for WebhookTarget { } // Use the immutable reference of the store to delete the event content corresponding to the key - debug!( - "Deleting event from store for target: {}, key:{}, start", - self.id, - key.to_string() - ); + debug!("Deleting event from store for target: {}, key:{}, start", self.id, key.to_string()); match store.del(&key) { - Ok(_) => debug!( - "Event deleted from store for target: {}, key:{}, end", - self.id, - key.to_string() - ), + Ok(_) => debug!("Event deleted from store for target: {}, key:{}, end", self.id, key.to_string()), Err(e) => { error!("Failed to delete event from store: {}", e); - return Err(TargetError::Storage(format!( - "Failed to delete event from store: {}", - e - ))); + return Err(TargetError::Storage(format!("Failed to delete event from store: {}", e))); } } @@ -433,10 +384,7 @@ impl Target for WebhookTarget { async fn init(&self) -> Result<(), TargetError> { // If the target is disabled, return to success directly if !self.is_enabled() { - debug!( - "Webhook target {} is disabled, skipping initialization", - self.id - ); + debug!("Webhook target {} is disabled, skipping initialization", self.id); return Ok(()); } diff --git a/ecstore/src/config/com.rs b/ecstore/src/config/com.rs index 7642d691e..796f1d38a 100644 --- a/ecstore/src/config/com.rs +++ b/ecstore/src/config/com.rs @@ -115,59 +115,62 @@ async fn new_and_save_server_config(api: Arc) -> Result(api: Arc) -> Result { - let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, CONFIG_FILE); - let data = match read_config(api.clone(), config_file.as_str()).await { - Ok(res) => res, - Err(err) => { - return if err == Error::ConfigNotFound { - warn!("config not found, start to init"); - let cfg = new_and_save_server_config(api).await?; - warn!("config init done"); - Ok(cfg) - } else { - error!("read config err {:?}", &err); - Err(err) - }; - } - }; +fn get_config_file() -> String { + format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, CONFIG_FILE) +} - read_server_config(api, data.as_slice()).await +/// Handle the situation where the configuration file does not exist, create and save a new configuration +async fn handle_missing_config(api: Arc, context: &str) -> Result { + warn!("Configuration not found ({}): Start initializing new configuration", context); + let cfg = new_and_save_server_config(api).await?; + warn!("Configuration initialization complete ({})", context); + Ok(cfg) +} + +/// Handle configuration file read errors +fn handle_config_read_error(err: Error, file_path: &str) -> Result { + error!("Read configuration failed (path: '{}'): {:?}", file_path, err); + Err(err) +} + +pub async fn read_config_without_migrate(api: Arc) -> Result { + let config_file = get_config_file(); + + // Try to read the configuration file + match read_config(api.clone(), &config_file).await { + Ok(data) => read_server_config(api, &data).await, + Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration").await, + Err(err) => handle_config_read_error(err, &config_file), + } } async fn read_server_config(api: Arc, data: &[u8]) -> Result { - let cfg = { - if data.is_empty() { - let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, CONFIG_FILE); - let cfg_data = match read_config(api.clone(), config_file.as_str()).await { - Ok(res) => res, - Err(err) => { - return if err == Error::ConfigNotFound { - warn!("config not found init start"); - let cfg = new_and_save_server_config(api).await?; - warn!("config not found init done"); - Ok(cfg) - } else { - error!("read config err {:?}", &err); - Err(err) - }; - } - }; - // TODO: decrypt + // If the provided data is empty, try to read from the file again + if data.is_empty() { + let config_file = get_config_file(); + warn!("Received empty configuration data, try to reread from '{}'", config_file); - Config::unmarshal(cfg_data.as_slice())? - } else { - Config::unmarshal(data)? + // Try to read the configuration again + match read_config(api.clone(), &config_file).await { + Ok(cfg_data) => { + // TODO: decrypt + let cfg = Config::unmarshal(&cfg_data)?; + return Ok(cfg.merge()); + } + Err(Error::ConfigNotFound) => return handle_missing_config(api, "Read alternate configuration").await, + Err(err) => return handle_config_read_error(err, &config_file), } - }; + } + // Process non-empty configuration data + let cfg = Config::unmarshal(data)?; Ok(cfg.merge()) } -async fn save_server_config(api: Arc, cfg: &Config) -> Result<()> { +pub async fn save_server_config(api: Arc, cfg: &Config) -> Result<()> { let data = cfg.marshal()?; - let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, CONFIG_FILE); + let config_file = get_config_file(); save_config(api, &config_file, data).await } diff --git a/ecstore/src/config/mod.rs b/ecstore/src/config/mod.rs index 4e5a24528..5b06ea5dc 100644 --- a/ecstore/src/config/mod.rs +++ b/ecstore/src/config/mod.rs @@ -107,8 +107,8 @@ impl Config { cfg } - pub fn get_value(&self, subsys: &str, key: &str) -> Option { - if let Some(m) = self.0.get(subsys) { + pub fn get_value(&self, sub_sys: &str, key: &str) -> Option { + if let Some(m) = self.0.get(sub_sys) { m.get(key).cloned() } else { None diff --git a/rustfs/src/event.rs b/rustfs/src/event.rs index 52a008352..2a8552756 100644 --- a/rustfs/src/event.rs +++ b/rustfs/src/event.rs @@ -1,32 +1,34 @@ -// use rustfs_notify::EventNotifierConfig; -use tracing::{info, instrument}; +use ecstore::config::GLOBAL_ServerConfig; +use tracing::{error, info, instrument}; #[instrument] -pub(crate) async fn init_event_notifier(notifier_config: Option) { +pub(crate) async fn init_event_notifier() { info!("Initializing event notifier..."); - let notifier_config_present = notifier_config.is_some(); - let config = if notifier_config_present { - info!("event_config is not empty, path: {:?}", notifier_config); - // EventNotifierConfig::event_load_config(notifier_config) - } else { - info!("event_config is empty"); - // rustfs_notify::get_event_notifier_config().clone() - // EventNotifierConfig::default() + + // 1. Get the global configuration loaded by ecstore + let server_config = match GLOBAL_ServerConfig.get() { + Some(config) => config.clone(), // Clone the config to pass ownership + None => { + error!("Event notifier initialization failed: Global server config not loaded."); + return; + } }; - info!("using event_config: {:?}", config); + // 2. Check if the notify subsystem exists in the configuration, and skip initialization if it doesn't + if server_config.get_value("notify", "_").is_none() { + info!("'notify' subsystem not configured, skipping event notifier initialization."); + return; + } + + info!("Event notifier configuration found, proceeding with initialization."); + + // 3. Initialize the notification system asynchronously with a global configuration + // Put it into a separate task to avoid blocking the main initialization process tokio::spawn(async move { - // let result = rustfs_notify::initialize(&config).await; - // match result { - // Ok(_) => info!( - // "event notifier initialized successfully {}", - // if notifier_config_present { - // "by config file" - // } else { - // "by sys config" - // } - // ), - // Err(e) => error!("Failed to initialize event notifier: {}", e), - // } + if let Err(e) = rustfs_notify::initialize(server_config).await { + error!("Failed to initialize event notifier system: {}", e); + } else { + info!("Event notifier system initialized successfully."); + } }); } diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 936f868e3..066736f6f 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -16,8 +16,8 @@ use bytes::Bytes; use chrono::DateTime; use chrono::Utc; use datafusion::arrow::csv::WriterBuilder as CsvWriterBuilder; -use datafusion::arrow::json::WriterBuilder as JsonWriterBuilder; use datafusion::arrow::json::writer::JsonArray; +use datafusion::arrow::json::WriterBuilder as JsonWriterBuilder; use ecstore::bucket::metadata::BUCKET_LIFECYCLE_CONFIG; use ecstore::bucket::metadata::BUCKET_NOTIFICATION_CONFIG; use ecstore::bucket::metadata::BUCKET_POLICY_CONFIG; @@ -32,13 +32,13 @@ use ecstore::bucket::tagging::decode_tags; use ecstore::bucket::tagging::encode_tags; use ecstore::bucket::utils::serialize; use ecstore::bucket::versioning_sys::BucketVersioningSys; -use ecstore::cmd::bucket_replication::ReplicationStatusType; -use ecstore::cmd::bucket_replication::ReplicationType; use ecstore::cmd::bucket_replication::get_must_replicate_options; use ecstore::cmd::bucket_replication::must_replicate; use ecstore::cmd::bucket_replication::schedule_replication; -use ecstore::compress::MIN_COMPRESSIBLE_SIZE; +use ecstore::cmd::bucket_replication::ReplicationStatusType; +use ecstore::cmd::bucket_replication::ReplicationType; use ecstore::compress::is_compressible; +use ecstore::compress::MIN_COMPRESSIBLE_SIZE; use ecstore::error::StorageError; use ecstore::new_object_layer_fn; use ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE; @@ -52,40 +52,42 @@ use ecstore::store_api::ObjectIO; use ecstore::store_api::ObjectOptions; use ecstore::store_api::ObjectToDelete; use ecstore::store_api::PutObjReader; -use ecstore::store_api::StorageAPI; // use ecstore::store_api::RESERVED_METADATA_PREFIX; +use ecstore::store_api::StorageAPI; +// use ecstore::store_api::RESERVED_METADATA_PREFIX; use futures::StreamExt; use http::HeaderMap; use lazy_static::lazy_static; use policy::auth; +use policy::policy::action::Action; +use policy::policy::action::S3Action; use policy::policy::BucketPolicy; use policy::policy::BucketPolicyArgs; use policy::policy::Validator; -use policy::policy::action::Action; -use policy::policy::action::S3Action; use query::instance::make_rustfsms; use rustfs_filemeta::headers::RESERVED_METADATA_PREFIX_LOWER; use rustfs_filemeta::headers::{AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_TAGGING}; +use rustfs_notify::EventName; use rustfs_rio::CompressReader; use rustfs_rio::HashReader; use rustfs_rio::Reader; use rustfs_rio::WarpReader; -use rustfs_utils::CompressionAlgorithm; use rustfs_utils::path::path_join_buf; +use rustfs_utils::CompressionAlgorithm; use rustfs_zip::CompressionFormat; -use s3s::S3; +use s3s::dto::*; +use s3s::s3_error; use s3s::S3Error; use s3s::S3ErrorCode; use s3s::S3Result; -use s3s::dto::*; -use s3s::s3_error; +use s3s::S3; use s3s::{S3Request, S3Response}; use std::collections::HashMap; use std::fmt::Debug; use std::path::Path; use std::str::FromStr; use std::sync::Arc; -use time::OffsetDateTime; use time::format_description::well_known::Rfc3339; +use time::OffsetDateTime; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use tokio_tar::Archive; @@ -222,6 +224,21 @@ impl FS { // e_tag, // ..Default::default() // }; + + // let event_args = rustfs_notify::event::EventArgs { + // event_name: EventName::ObjectCreatedPut, // 或者其他相应的事件类型 + // bucket_name: bucket.clone(), + // object: _obj_info.clone(), // clone() 或传递所需字段 + // req_params: crate::storage::global::extract_req_params(&req), // 假设有一个辅助函数来提取请求参数 + // resp_elements: crate::storage::global::extract_resp_elements(&output), // 假设有一个辅助函数来提取响应元素 + // host: crate::storage::global::get_request_host(&req.headers), // 假设的辅助函数 + // user_agent: crate::storage::global::get_request_user_agent(&req.headers), // 假设的辅助函数 + // }; + // + // // 异步调用,不会阻塞当前请求的响应 + // tokio::spawn(async move { + // rustfs_notify::notifier::GLOBAL_NOTIFIER.notify(event_args).await; + // }); } } diff --git a/rustfs/src/storage/event.rs b/rustfs/src/storage/event.rs deleted file mode 100644 index 8b1378917..000000000 --- a/rustfs/src/storage/event.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/rustfs/src/storage/global.rs b/rustfs/src/storage/global.rs new file mode 100644 index 000000000..e55d79ab0 --- /dev/null +++ b/rustfs/src/storage/global.rs @@ -0,0 +1,43 @@ +use hyper::HeaderMap; +use s3s::{S3Request, S3Response}; +use std::collections::HashMap; + +/// Extract request parameters from S3Request, mainly header information. +pub fn extract_req_params(req: &S3Request) -> HashMap { + let mut params = HashMap::new(); + for (key, value) in req.headers.iter() { + if let Ok(val_str) = value.to_str() { + params.insert(key.as_str().to_string(), val_str.to_string()); + } + } + params +} + +/// Extract response elements from S3Response, mainly header information. +pub fn extract_resp_elements(resp: &S3Response) -> HashMap { + let mut params = HashMap::new(); + for (key, value) in resp.headers.iter() { + if let Ok(val_str) = value.to_str() { + params.insert(key.as_str().to_string(), val_str.to_string()); + } + } + params +} + +/// Get host from header information. +pub fn get_request_host(headers: &HeaderMap) -> String { + headers + .get("host") + .and_then(|v| v.to_str().ok()) + .unwrap_or_default() + .to_string() +} + +/// Get user-agent from header information. +pub fn get_request_user_agent(headers: &HeaderMap) -> String { + headers + .get("user-agent") + .and_then(|v| v.to_str().ok()) + .unwrap_or_default() + .to_string() +} diff --git a/rustfs/src/storage/mod.rs b/rustfs/src/storage/mod.rs index 5bb2e216e..21af29ec8 100644 --- a/rustfs/src/storage/mod.rs +++ b/rustfs/src/storage/mod.rs @@ -1,5 +1,5 @@ pub mod access; pub mod ecfs; // pub mod error; -mod event; +mod global; pub mod options;