From 21a829e7cf6e3aa6720c55cb71d515054c9486d6 Mon Sep 17 00:00:00 2001 From: houseme Date: Wed, 16 Apr 2025 16:15:53 +0800 Subject: [PATCH 01/16] init event notifer --- Cargo.lock | 21 ++ Cargo.toml | 4 + crates/event-notifier/Cargo.toml | 21 ++ crates/event-notifier/src/error.rs | 25 +++ crates/event-notifier/src/event.rs | 53 +++++ crates/event-notifier/src/event_name.rs | 79 ++++++++ crates/event-notifier/src/lib.rs | 80 ++++++++ crates/event-notifier/src/notifier.rs | 47 +++++ crates/event-notifier/src/rules.rs | 191 ++++++++++++++++++ crates/event-notifier/src/stats.rs | 81 ++++++++ crates/event-notifier/src/target.rs | 66 ++++++ crates/event-notifier/src/target_entry/mod.rs | 1 + .../src/target_entry/webhook.rs | 161 +++++++++++++++ rustfs/Cargo.toml | 1 + 14 files changed, 831 insertions(+) create mode 100644 crates/event-notifier/Cargo.toml create mode 100644 crates/event-notifier/src/error.rs create mode 100644 crates/event-notifier/src/event.rs create mode 100644 crates/event-notifier/src/event_name.rs create mode 100644 crates/event-notifier/src/lib.rs create mode 100644 crates/event-notifier/src/notifier.rs create mode 100644 crates/event-notifier/src/rules.rs create mode 100644 crates/event-notifier/src/stats.rs create mode 100644 crates/event-notifier/src/target.rs create mode 100644 crates/event-notifier/src/target_entry/mod.rs create mode 100644 crates/event-notifier/src/target_entry/webhook.rs diff --git a/Cargo.lock b/Cargo.lock index f69232ecc..c0e82fdbf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7200,6 +7200,7 @@ dependencies = [ "query", "rmp-serde", "rust-embed", + "rustfs-event-notifier", "rustfs-obs", "rustls", "rustls-pemfile", @@ -7224,6 +7225,20 @@ dependencies = [ "uuid", ] +[[package]] +name = "rustfs-event-notifier" +version = "0.0.1" +dependencies = [ + "async-trait", + "parking_lot 0.12.3", + "reqwest", + "serde", + "serde_json", + "thiserror 2.0.12", + "tokio", + "wildmatch", +] + [[package]] name = "rustfs-gui" version = "0.0.1" @@ -9317,6 +9332,12 @@ dependencies = [ "rustix 0.38.44", ] +[[package]] +name = "wildmatch" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ce1ab1f8c62655ebe1350f589c61e505cf94d385bc6a12899442d9081e71fd" + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 381305c48..99e5073f3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ "s3select/api", "s3select/query", "appauth", + "crates/event-notifier", ] resolver = "2" @@ -83,6 +84,7 @@ opentelemetry-stdout = { version = "0.29.0" } opentelemetry-otlp = { version = "0.29" } opentelemetry-prometheus = { version = "0.29.1" } opentelemetry-semantic-conventions = { version = "0.29.0", features = ["semconv_experimental"] } +parking_lot = "0.12.3" pin-project-lite = "0.2" prometheus = "0.14.0" # pin-utils = "0.1.0" @@ -98,6 +100,7 @@ rfd = { version = "0.15.3", default-features = false, features = ["xdg-portal", rmp = "0.8.14" rmp-serde = "1.3.0" rustfs-obs = { path = "crates/obs", version = "0.0.1" } +rustfs-event-notifier = { path = "crates/event-notifier", version = "0.0.1" } rust-embed = "8.7.0" rustls = { version = "0.23.26" } rustls-pki-types = "1.11.0" @@ -141,6 +144,7 @@ uuid = { version = "1.16.0", features = [ "fast-rng", "macro-diagnostics", ] } +wildmatch = "2.4.0" workers = { path = "./common/workers" } [profile.wasm-dev] diff --git a/crates/event-notifier/Cargo.toml b/crates/event-notifier/Cargo.toml new file mode 100644 index 000000000..2e4afc7bc --- /dev/null +++ b/crates/event-notifier/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "rustfs-event-notifier" +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[dependencies] +async-trait = { workspace = true } +parking_lot = { workspace = true } +reqwest = { workspace = true } +serde = { workspace = true } +tokio = { workspace = true, features = ["full"] } +thiserror = { workspace = true } +wildmatch = { workspace = true } +serde_json = { workspace = true } + + +[lints] +workspace = true diff --git a/crates/event-notifier/src/error.rs b/crates/event-notifier/src/error.rs new file mode 100644 index 000000000..1945c351c --- /dev/null +++ b/crates/event-notifier/src/error.rs @@ -0,0 +1,25 @@ +// src/error.rs +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum Error { + #[error("target not found: {0}")] + TargetNotFound(String), + + #[error("send failed: {0}")] + SendError(String), + + #[error("target error: {0}")] + TargetError(String), + + #[error("invalid configuration: {0}")] + ConfigError(String), + + #[error("store error: {0}")] + StoreError(String), + + #[error("invalid event name: {0}")] + InvalidEventName(String), // 添加此变体 +} + +pub type Result = std::result::Result; diff --git a/crates/event-notifier/src/event.rs b/crates/event-notifier/src/event.rs new file mode 100644 index 000000000..695f2c3d7 --- /dev/null +++ b/crates/event-notifier/src/event.rs @@ -0,0 +1,53 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Event { + pub event_version: String, + pub event_source: String, + pub aws_region: String, + pub event_time: String, + pub event_name: String, + pub user_identity: Identity, + pub request_parameters: HashMap, + pub response_elements: HashMap, + pub s3: Metadata, + pub source: Source, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Identity { + pub principal_id: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Metadata { + pub schema_version: String, + pub configuration_id: String, + pub bucket: Bucket, + pub object: Object, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Bucket { + pub name: String, + pub owner_identity: Identity, + pub arn: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Object { + pub key: String, + pub version_id: Option, + pub sequencer: String, + pub size: Option, + pub etag: Option, + pub content_type: Option, + pub user_metadata: HashMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Source { + pub host: String, + pub user_agent: String, +} diff --git a/crates/event-notifier/src/event_name.rs b/crates/event-notifier/src/event_name.rs new file mode 100644 index 000000000..c60cce253 --- /dev/null +++ b/crates/event-notifier/src/event_name.rs @@ -0,0 +1,79 @@ +use crate::error::Error; +use serde::{Deserialize, Serialize}; +use std::str::FromStr; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[repr(u64)] +pub enum EventName { + // 对象操作事件 + ObjectCreatedAll = 1 << 0, + ObjectCreatedPut = 1 << 1, + ObjectCreatedPost = 1 << 2, + ObjectCreatedCopy = 1 << 3, + ObjectCreatedCompleteMultipartUpload = 1 << 4, + + ObjectRemovedAll = 1 << 5, + ObjectRemovedDelete = 1 << 6, + ObjectRemovedDeleteMarkerCreated = 1 << 7, + + ObjectAccessedAll = 1 << 8, + ObjectAccessedGet = 1 << 9, + ObjectAccessedHead = 1 << 10, + + ObjectRestoredAll = 1 << 11, + ObjectRestoredPost = 1 << 12, + ObjectRestoredCompleted = 1 << 13, + + ReplicationAll = 1 << 14, + ReplicationFailed = 1 << 15, + ReplicationComplete = 1 << 16, +} + +impl EventName { + pub fn mask(&self) -> u64 { + *self as u64 + } + + pub fn expand(&self) -> Vec { + match self { + EventName::ObjectCreatedAll => vec![ + EventName::ObjectCreatedPut, + EventName::ObjectCreatedPost, + EventName::ObjectCreatedCopy, + EventName::ObjectCreatedCompleteMultipartUpload, + ], + EventName::ObjectRemovedAll => vec![EventName::ObjectRemovedDelete, EventName::ObjectRemovedDeleteMarkerCreated], + EventName::ObjectAccessedAll => vec![EventName::ObjectAccessedGet, EventName::ObjectAccessedHead], + EventName::ObjectRestoredAll => vec![EventName::ObjectRestoredPost, EventName::ObjectRestoredCompleted], + EventName::ReplicationAll => vec![EventName::ReplicationFailed, EventName::ReplicationComplete], + _ => vec![*self], + } + } +} + +impl FromStr for EventName { + type Err = Error; + + fn from_str(s: &str) -> Result { + match s { + "s3:ObjectCreated:*" => Ok(EventName::ObjectCreatedAll), + "s3:ObjectCreated:Put" => Ok(EventName::ObjectCreatedPut), + "s3:ObjectCreated:Post" => Ok(EventName::ObjectCreatedPost), + "s3:ObjectCreated:Copy" => Ok(EventName::ObjectCreatedCopy), + "s3:ObjectCreated:CompleteMultipartUpload" => Ok(EventName::ObjectCreatedCompleteMultipartUpload), + "s3:ObjectRemoved:*" => Ok(EventName::ObjectRemovedAll), + "s3:ObjectRemoved:Delete" => Ok(EventName::ObjectRemovedDelete), + "s3:ObjectRemoved:DeleteMarkerCreated" => Ok(EventName::ObjectRemovedDeleteMarkerCreated), + "s3:ObjectAccessed:*" => Ok(EventName::ObjectAccessedAll), + "s3:ObjectAccessed:Get" => Ok(EventName::ObjectAccessedGet), + "s3:ObjectAccessed:Head" => Ok(EventName::ObjectAccessedHead), + "s3:ObjectRestored:*" => Ok(EventName::ObjectRestoredAll), + "s3:ObjectRestored:Post" => Ok(EventName::ObjectRestoredPost), + "s3:ObjectRestored:Completed" => Ok(EventName::ObjectRestoredCompleted), + "s3:Replication:*" => Ok(EventName::ReplicationAll), + "s3:Replication:Failed" => Ok(EventName::ReplicationFailed), + "s3:Replication:Complete" => Ok(EventName::ReplicationComplete), + _ => Err(Error::InvalidEventName(format!("Unrecognized event name: {}", s))), + } + } +} diff --git a/crates/event-notifier/src/lib.rs b/crates/event-notifier/src/lib.rs new file mode 100644 index 000000000..fbfc9f863 --- /dev/null +++ b/crates/event-notifier/src/lib.rs @@ -0,0 +1,80 @@ +/// RustFS Event Notifier +/// This crate provides a simple event notification system for RustFS. +/// It allows for the registration of event handlers and the triggering of events. +/// +mod error; +mod event; +mod event_name; +mod notifier; +mod rules; +mod stats; +mod target; +mod target_entry; + +pub fn add(left: u64, right: u64) -> u64 { + left + right +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::event::{Bucket, Event, Identity, Metadata, Object, Source}; + use crate::target::{TargetID, TargetList}; + use std::collections::HashMap; + + #[test] + fn it_works() { + let result = add(2, 2); + assert_eq!(result, 4); + } + + #[tokio::main] + async fn main() { + let target_list = TargetList::new(); + let event = Event { + event_version: "1.0".to_string(), + event_source: "aws:s3".to_string(), + aws_region: "us-west-2".to_string(), + event_time: "2023-10-01T12:00:00Z".to_string(), + event_name: "PutObject".to_string(), + user_identity: Identity { + principal_id: "user123".to_string(), + }, + request_parameters: HashMap::new(), + response_elements: HashMap::new(), + s3: Metadata { + schema_version: "1.0".to_string(), + configuration_id: "config123".to_string(), + bucket: Bucket { + name: "my-bucket".to_string(), + owner_identity: Identity { + principal_id: "owner123".to_string(), + }, + arn: "arn:aws:s3:::my-bucket".to_string(), + }, + object: Object { + key: "my-object.txt".to_string(), + version_id: Some("version123".to_string()), + sequencer: "seq123".to_string(), + size: Some(1024), + etag: Some("etag123".to_string()), + content_type: Some("text/plain".to_string()), + user_metadata: HashMap::new(), + }, + }, + source: Source { + host: "localhost".to_string(), + user_agent: "RustFS/1.0".to_string(), + }, + }; + let target_ids: &[TargetID] = &["".to_string()]; + // 发送事件 + let results = target_list.send(event, &target_ids).await; + println!("result len:{:?}", results.len()); + // 获取统计信息 + let stats = target_list.get_stats(); + for (id, stat) in stats { + println!("Target {}: {} events, {} failed", id, stat.total_events, stat.failed_events); + } + } +} diff --git a/crates/event-notifier/src/notifier.rs b/crates/event-notifier/src/notifier.rs new file mode 100644 index 000000000..7b81a24d2 --- /dev/null +++ b/crates/event-notifier/src/notifier.rs @@ -0,0 +1,47 @@ +// src/notifier.rs +use crate::error::Error; +use crate::event::Event; +use crate::rules::{RulesMap, TargetIDSet}; +use crate::target::TargetList; +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; +use tokio::sync::broadcast; + +pub struct EventNotifier { + target_list: Arc, + rules: RwLock>, + tx: broadcast::Sender, +} + +impl EventNotifier { + pub fn new(capacity: usize) -> Self { + let (tx, _) = broadcast::channel(capacity); + Self { + target_list: Arc::new(TargetList::new()), + rules: RwLock::new(HashMap::new()), + tx, + } + } + + pub async fn send(&self, event: Event) -> Result<(), Error> { + let rules = self + .rules + .read() + .map_err(|_| Error::StoreError("Failed to read rules".to_string()))?; + // 检查规则匹配 + let target_ids = if let Some(rules_map) = rules.get(&event.s3.bucket.name) { + rules_map.match_targets(event.s3.event_name, &event.s3.object.key) + } else { + TargetIDSet::new() + }; + + // 发送事件 + if !target_ids.is_empty() { + self.target_list.send(event.clone(), &target_ids).await; + } + + // 广播给监听者 + let _ = self.tx.send(event); + Ok(()) + } +} diff --git a/crates/event-notifier/src/rules.rs b/crates/event-notifier/src/rules.rs new file mode 100644 index 000000000..671bcfb53 --- /dev/null +++ b/crates/event-notifier/src/rules.rs @@ -0,0 +1,191 @@ +// src/rules.rs +use crate::event_name::EventName; +use crate::target::TargetID; +use std::collections::{HashMap, HashSet}; +use wildmatch::WildMatch; + +/// TargetIDSet represents a set of target IDs +#[derive(Debug, Clone, Default)] +pub struct TargetIDSet(HashSet); + +impl TargetIDSet { + pub fn new() -> Self { + Self(HashSet::new()) + } + + pub fn insert(&mut self, target_id: TargetID) -> bool { + self.0.insert(target_id) + } + + pub fn contains(&self, target_id: &TargetID) -> bool { + self.0.contains(target_id) + } + + pub fn remove(&mut self, target_id: &TargetID) -> bool { + self.0.remove(target_id) + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn clear(&mut self) { + self.0.clear() + } + + pub fn iter(&self) -> impl Iterator { + self.0.iter() + } + + pub fn extend>(&mut self, iter: I) { + self.0.extend(iter) + } + + pub fn union(&self, other: &TargetIDSet) -> TargetIDSet { + let mut result = self.clone(); + result.extend(other.0.iter().cloned()); + result + } + + pub fn difference(&self, other: &TargetIDSet) -> TargetIDSet { + let mut result = TargetIDSet::new(); + for id in self.0.iter() { + if !other.contains(id) { + result.insert(id.clone()); + } + } + result + } +} + +impl FromIterator for TargetIDSet { + fn from_iter>(iter: I) -> Self { + let mut set = TargetIDSet::new(); + set.extend(iter); + set + } +} + +#[derive(Debug, Clone)] +pub struct Rules { + patterns: HashMap, +} + +impl Rules { + pub fn new() -> Self { + Self { + patterns: HashMap::new(), + } + } + + pub fn add(&mut self, pattern: String, target_id: TargetID) { + let entry = self.patterns.entry(pattern).or_insert_with(TargetIDSet::new); + entry.insert(target_id); + } + + pub fn match_object(&self, object_name: &str) -> bool { + for pattern in self.patterns.keys() { + if WildMatch::new(pattern).matches(object_name) { + return true; + } + } + false + } + + pub fn match_targets(&self, object_name: &str) -> TargetIDSet { + let mut target_ids = TargetIDSet::new(); + + for (pattern, ids) in &self.patterns { + if WildMatch::new(pattern).matches(object_name) { + target_ids.extend(ids.iter().cloned()); + } + } + + target_ids + } +} + +#[derive(Debug, Default)] +pub struct RulesMap { + rules: HashMap, +} + +impl RulesMap { + pub fn new() -> Self { + Self { rules: HashMap::new() } + } + + pub fn add(&mut self, events: &[EventName], pattern: String, target_id: TargetID) { + for &event in events { + for expanded_event in event.expand() { + let rules = self.rules.entry(expanded_event).or_insert_with(Rules::new); + rules.add(pattern.clone(), target_id.clone()); + } + } + } + + pub fn match_simple(&self, event: EventName, object_name: &str) -> bool { + match self.rules.get(&event) { + Some(rules) => rules.match_object(object_name), + None => false, + } + } + + pub fn match_targets(&self, event: EventName, object_name: &str) -> TargetIDSet { + match self.rules.get(&event) { + Some(rules) => rules.match_targets(object_name), + None => TargetIDSet::new(), + } + } +} + +impl Clone for RulesMap { + fn clone(&self) -> Self { + Self { + rules: self.rules.clone(), + } + } +} + +impl Default for Rules { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::event_name::EventName; + + #[test] + fn test_rules() { + let mut rules = Rules::new(); + rules.add("*.txt".to_string(), "target1".to_string()); + rules.add("*.jpg".to_string(), "target2".to_string()); + + assert!(rules.match_object("file.txt")); + assert!(!rules.match_object("file.pdf")); + assert!(rules.match_object("image.jpg")); + assert!(!rules.match_object("image.png")); + + let target_ids = rules.match_targets("file.txt"); + assert_eq!(target_ids.len(), 1); + assert!(target_ids.iter().any(|id| id == "target1")); + + let mut rules_map = RulesMap::new(); + let target_id = "target1".to_string(); + // 添加规则 + rules_map.add(&[EventName::ObjectCreatedAll], String::from("images/*"), target_id); + + // 匹配对象 + let matches = rules_map.match_simple(EventName::ObjectCreatedPut, "images/photo.jpg"); // returns true + + // 获取匹配的目标 + let target_ids = rules_map.match_targets(EventName::ObjectCreatedPut, "images/photo.jpg"); + } +} diff --git a/crates/event-notifier/src/stats.rs b/crates/event-notifier/src/stats.rs new file mode 100644 index 000000000..570bbeca5 --- /dev/null +++ b/crates/event-notifier/src/stats.rs @@ -0,0 +1,81 @@ +// src/stats.rs +use crate::target::TargetID; +use parking_lot::RwLock; +use std::collections::HashMap; +use std::sync::atomic::{AtomicI64, Ordering}; +use std::sync::Arc; + +#[derive(Debug, Default)] +pub struct TargetStat { + pub current_send_calls: AtomicI64, + pub total_events: AtomicI64, + pub failed_events: AtomicI64, + pub current_queue: AtomicI64, +} + +impl TargetStat { + pub fn inc_send_calls(&self) { + self.current_send_calls.fetch_add(1, Ordering::Relaxed); + } + + pub fn dec_send_calls(&self) { + self.current_send_calls.fetch_sub(1, Ordering::Relaxed); + } + + pub fn inc_total_events(&self) { + self.total_events.fetch_add(1, Ordering::Relaxed); + } + + pub fn inc_failed_events(&self) { + self.failed_events.fetch_add(1, Ordering::Relaxed); + } + + pub fn set_queue_size(&self, size: i64) { + self.current_queue.store(size, Ordering::Relaxed); + } +} + +#[derive(Default)] +pub struct TargetStats { + stats: RwLock>>, +} + +impl TargetStats { + pub fn new() -> Self { + Self::default() + } + + pub fn get_or_create(&self, id: &TargetID) -> Arc { + let mut stats = self.stats.write(); + stats + .entry(id.clone()) + .or_insert_with(|| Arc::new(TargetStat::default())) + .clone() + } + + pub fn get_stats(&self) -> HashMap { + self.stats + .read() + .iter() + .map(|(id, stat)| { + ( + id.clone(), + TargetSnapshot { + current_send_calls: stat.current_send_calls.load(Ordering::Relaxed), + total_events: stat.total_events.load(Ordering::Relaxed), + failed_events: stat.failed_events.load(Ordering::Relaxed), + current_queue: stat.current_queue.load(Ordering::Relaxed), + }, + ) + }) + .collect() + } +} + +#[derive(Debug, Clone)] +pub struct TargetSnapshot { + pub current_send_calls: i64, + pub total_events: i64, + pub failed_events: i64, + pub current_queue: i64, +} diff --git a/crates/event-notifier/src/target.rs b/crates/event-notifier/src/target.rs new file mode 100644 index 000000000..68b575b74 --- /dev/null +++ b/crates/event-notifier/src/target.rs @@ -0,0 +1,66 @@ +// src/target.rs +use crate::error::{Error, Result}; +use crate::event::Event; +use crate::stats::{TargetSnapshot, TargetStats}; +use async_trait::async_trait; +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; + +pub type TargetID = String; + +#[async_trait] +pub trait Target: Send + Sync { + fn id(&self) -> TargetID; + async fn send(&self, event: Event) -> Result<()>; + async fn is_active(&self) -> bool; + async fn close(&self) -> Result<()>; +} + +pub struct TargetList { + targets: Arc>>>, + stats: Arc, +} + +impl TargetList { + pub fn new() -> Self { + Self { + targets: Arc::new(RwLock::new(HashMap::new())), + stats: Arc::default(), + } + } + + pub async fn send(&self, event: Event, target_ids: &[TargetID]) -> Vec> { + let mut results = Vec::with_capacity(target_ids.len()); + let targets = self.targets.read().unwrap(); + + for id in target_ids { + let target = match targets.get(id) { + Some(t) => t, + None => { + results.push(Err(Error::TargetNotFound(id.to_string()))); + continue; + } + }; + + let stats = self.stats.get_or_create(id); + stats.inc_send_calls(); + + let result = target.send(event.clone()).await; + + stats.dec_send_calls(); + stats.inc_total_events(); + + if result.is_err() { + stats.inc_failed_events(); + } + + results.push(result); + } + + results + } + + pub fn get_stats(&self) -> HashMap { + self.stats.get_stats() + } +} diff --git a/crates/event-notifier/src/target_entry/mod.rs b/crates/event-notifier/src/target_entry/mod.rs new file mode 100644 index 000000000..a2a1c2754 --- /dev/null +++ b/crates/event-notifier/src/target_entry/mod.rs @@ -0,0 +1 @@ +mod webhook; diff --git a/crates/event-notifier/src/target_entry/webhook.rs b/crates/event-notifier/src/target_entry/webhook.rs new file mode 100644 index 000000000..1736ad36b --- /dev/null +++ b/crates/event-notifier/src/target_entry/webhook.rs @@ -0,0 +1,161 @@ +use crate::event::Event; +use crate::target::{Target, TargetID}; +use async_trait::async_trait; +use reqwest::{header, Client}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; +use thiserror::Error; +use tokio::io::AsyncWriteExt; +use tokio::sync::Mutex; + +#[derive(Error, Debug)] +pub enum WebhookError { + #[error("HTTP client error: {0}")] + HttpClient(#[from] reqwest::Error), + #[error("JSON serialization error: {0}")] + Json(#[from] serde_json::Error), + #[error("Store error: {0}")] + Store(#[from] Box), + #[error("Webhook request failed after retries")] + RequestFailed, +} + +type Result = std::result::Result; + +/// Webhook 目标配置 + +/// Webhook 目标配置 +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WebhookConfig { + /// Webhook endpoint URL + pub endpoint: String, + /// 认证令牌 + pub auth_token: Option, + /// 自定义请求头 + pub custom_headers: Option>, + /// 重试次数 + pub max_retries: u32, + /// 连接超时时间 (秒) + pub timeout: u64, +} + +/// Webhook Target 实现 +pub struct Webhook { + /// 目标 ID + id: TargetID, + /// HTTP 客户端 + client: Client, + /// 配置信息 + config: WebhookConfig, + /// 事件存储 + store: Arc>, +} + +#[async_trait] +impl Target for Webhook { + /// 返回目标 ID + fn id(&self) -> &TargetID { + &self.id + } + + /// 检查 Webhook 是否可用 + async fn is_active(&self) -> Result { + // 发送测试请求验证连接 + let resp = self.client.get(&self.config.endpoint).send().await?; + + Ok(resp.status().is_success()) + } + + /// 保存事件到存储并异步发送 + async fn save(&self, event: Event) -> Result<()> { + // 序列化事件 + let event_json = serde_json::to_string(&event)?; + + // 存储事件 + self.store.lock().await.save(event_json)?; + + // 异步发送 + tokio::spawn(self.send_event(event)); + + Ok(()) + } + + /// 从存储发送事件 + async fn send(&self, key: StoreKey) -> Result<()> { + // 从存储获取事件 + let event_json = self.store.lock().await.get(&key)?; + let event: Event = serde_json::from_str(&event_json)?; + + // 发送事件到 webhook + self.send_event(event).await?; + + // 发送成功后删除 + self.store.lock().await.delete(&key)?; + + Ok(()) + } + + /// 返回事件存储 + fn store(&self) -> Arc> { + self.store.clone() + } + + /// 关闭 Target + async fn close(&self) -> Result<()> { + // 等待所有事件发送完成 + self.store.lock().await.flush()?; + Ok(()) + } +} + +impl Webhook { + /// 创建新的 Webhook Target + pub fn new(id: TargetID, config: WebhookConfig, store: Arc>) -> Result { + // 构建 HTTP 客户端 + let client = Client::builder().timeout(Duration::from_secs(config.timeout)).build()?; + + Ok(Self { + id, + client, + config, + store, + }) + } + + /// 发送事件到 Webhook endpoint + async fn send_event(&self, event: Event) -> Result<()> { + let mut retries = 0; + loop { + // 构建请求 + let mut req = self.client.post(&self.config.endpoint).json(&event); + + // 添加认证头 + if let Some(token) = &self.config.auth_token { + req = req.header(header::AUTHORIZATION, format!("Bearer {}", token)); + } + + // 添加自定义头 + if let Some(headers) = &self.config.custom_headers { + for (key, value) in headers { + req = req.header(key, value); + } + } + + // 发送请求 + match req.send().await { + Ok(resp) if resp.status().is_success() => { + return Ok(()); + } + _ if retries < self.config.max_retries => { + retries += 1; + tokio::time::sleep(Duration::from_secs(1)).await; + continue; + } + Err(e) => return Err(WebhookError::HttpClient(e)), + _ => return Err(WebhookError::RequestFailed), + } + } + } +} diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index ff8b5971e..e2d088156 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -50,6 +50,7 @@ pin-project-lite.workspace = true protos.workspace = true query = { path = "../s3select/query" } rmp-serde.workspace = true +rustfs-event-notifier = { workspace = true } rustfs-obs = { workspace = true } rustls.workspace = true rustls-pemfile.workspace = true From bfc165abe0eb3e6bcd93a3ef0b2d3c5484805b02 Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 21 Apr 2025 00:17:27 +0800 Subject: [PATCH 02/16] feat: implement event notification system - Add core event notification interfaces - Support multiple notification backends: - Webhook (default) - Kafka - MQTT - HTTP Producer - Implement configurable event filtering - Add async event dispatching with backpressure handling - Provide serialization/deserialization for event payloads This module enables system events to be published to various endpoints with consistent delivery guarantees and failure handling. --- Cargo.lock | 397 +++++++++++++-- Cargo.toml | 25 +- crates/event-notifier/Cargo.toml | 30 +- crates/event-notifier/examples/.env.example | 27 ++ crates/event-notifier/examples/event.toml | 28 ++ crates/event-notifier/examples/simple.rs | 102 ++++ crates/event-notifier/src/adapter/kafka.rs | 76 +++ crates/event-notifier/src/adapter/mod.rs | 54 +++ crates/event-notifier/src/adapter/mqtt.rs | 58 +++ crates/event-notifier/src/adapter/webhook.rs | 63 +++ crates/event-notifier/src/bus.rs | 68 +++ crates/event-notifier/src/config.rs | 161 ++++++ crates/event-notifier/src/error.rs | 56 ++- crates/event-notifier/src/event.rs | 459 ++++++++++++++++-- crates/event-notifier/src/event_name.rs | 79 --- crates/event-notifier/src/global.rs | 100 ++++ crates/event-notifier/src/lib.rs | 187 ++++--- crates/event-notifier/src/notifier.rs | 47 -- crates/event-notifier/src/producer.rs | 83 ++++ crates/event-notifier/src/rules.rs | 191 -------- crates/event-notifier/src/stats.rs | 81 ---- crates/event-notifier/src/store.rs | 53 ++ crates/event-notifier/src/target.rs | 66 --- crates/event-notifier/src/target_entry/mod.rs | 1 - .../src/target_entry/webhook.rs | 161 ------ crates/event-notifier/tests/integration.rs | 160 ++++++ rustfs/Cargo.toml | 4 +- rustfs/src/config/mod.rs | 4 + 28 files changed, 2015 insertions(+), 806 deletions(-) create mode 100644 crates/event-notifier/examples/.env.example create mode 100644 crates/event-notifier/examples/event.toml create mode 100644 crates/event-notifier/examples/simple.rs create mode 100644 crates/event-notifier/src/adapter/kafka.rs create mode 100644 crates/event-notifier/src/adapter/mod.rs create mode 100644 crates/event-notifier/src/adapter/mqtt.rs create mode 100644 crates/event-notifier/src/adapter/webhook.rs create mode 100644 crates/event-notifier/src/bus.rs create mode 100644 crates/event-notifier/src/config.rs delete mode 100644 crates/event-notifier/src/event_name.rs create mode 100644 crates/event-notifier/src/global.rs delete mode 100644 crates/event-notifier/src/notifier.rs create mode 100644 crates/event-notifier/src/producer.rs delete mode 100644 crates/event-notifier/src/rules.rs delete mode 100644 crates/event-notifier/src/stats.rs create mode 100644 crates/event-notifier/src/store.rs delete mode 100644 crates/event-notifier/src/target.rs delete mode 100644 crates/event-notifier/src/target_entry/mod.rs delete mode 100644 crates/event-notifier/src/target_entry/webhook.rs create mode 100644 crates/event-notifier/tests/integration.rs diff --git a/Cargo.lock b/Cargo.lock index c0e82fdbf..799d7b6e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -371,7 +371,7 @@ dependencies = [ "arrow-buffer", "arrow-data", "arrow-schema", - "flatbuffers", + "flatbuffers 24.12.23", "lz4_flex", ] @@ -664,6 +664,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d818003e740b63afc82337e3160717f4f63078720a810b7b903e70a5d1d2994" +dependencies = [ + "bytemuck", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -800,11 +809,11 @@ dependencies = [ "hyper", "hyper-util", "pin-project-lite", - "rustls", + "rustls 0.23.26", "rustls-pemfile", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.2", "tower-service", ] @@ -1017,6 +1026,12 @@ version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +[[package]] +name = "bytemuck" +version = "1.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540" + [[package]] name = "byteorder" version = "1.5.0" @@ -1088,6 +1103,38 @@ dependencies = [ "system-deps", ] +[[package]] +name = "camino" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e35af189006b9c0f00a064685c727031e3ed2d8020f7ba284d78cc2671bd36ea" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd5eb614ed4c27c5d706420e4320fbe3216ab31fa1c33cd8246ac36dae4479ba" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror 2.0.12", +] + [[package]] name = "cc" version = "1.2.19" @@ -1111,7 +1158,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -2998,6 +3045,12 @@ dependencies = [ "const-random", ] +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + [[package]] name = "dpi" version = "0.1.1" @@ -3031,7 +3084,7 @@ version = "0.0.1" dependencies = [ "common", "ecstore", - "flatbuffers", + "flatbuffers 25.2.10", "futures", "lazy_static", "lock", @@ -3060,7 +3113,7 @@ dependencies = [ "chrono", "common", "crc32fast", - "flatbuffers", + "flatbuffers 25.2.10", "futures", "glob", "hex-simd", @@ -3071,7 +3124,7 @@ dependencies = [ "madmin", "md-5", "netif", - "nix 0.29.0", + "nix", "num", "num_cpus", "path-absolutize", @@ -3246,6 +3299,21 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "figment" +version = "0.10.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3" +dependencies = [ + "atomic", + "pear", + "serde", + "serde_yaml", + "toml", + "uncased", + "version_check", +] + [[package]] name = "fixedbitset" version = "0.5.7" @@ -3262,6 +3330,16 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "flatbuffers" +version = "25.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1" +dependencies = [ + "bitflags 2.9.0", + "rustc_version", +] + [[package]] name = "flate2" version = "1.1.1" @@ -3272,6 +3350,17 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -4087,10 +4176,10 @@ dependencies = [ "http", "hyper", "hyper-util", - "rustls", + "rustls 0.23.26", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.2", "tower-service", "webpki-roots", ] @@ -4334,6 +4423,7 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown 0.12.3", + "serde", ] [[package]] @@ -4356,6 +4446,12 @@ dependencies = [ "cfb", ] +[[package]] +name = "inlinable_string" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" + [[package]] name = "inout" version = "0.1.4" @@ -4737,19 +4833,19 @@ dependencies = [ [[package]] name = "libsystemd" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c592dc396b464005f78a5853555b9f240bc5378bf5221acc4e129910b2678869" +checksum = "b85fe9dc49de659d05829fdf72b5770c0a5952d1055c34a39f6d4e932bce175d" dependencies = [ "hmac 0.12.1", "libc", "log", - "nix 0.27.1", - "nom", + "nix", + "nom 8.0.0", "once_cell", "serde", "sha2 0.10.8", - "thiserror 1.0.69", + "thiserror 2.0.12", "uuid", ] @@ -5169,18 +5265,6 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" -[[package]] -name = "nix" -version = "0.27.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" -dependencies = [ - "bitflags 2.9.0", - "cfg-if", - "libc", - "memoffset", -] - [[package]] name = "nix" version = "0.29.0" @@ -5210,6 +5294,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -5606,6 +5699,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + [[package]] name = "opentelemetry" version = "0.29.1" @@ -5959,6 +6058,29 @@ dependencies = [ "hmac 0.12.1", ] +[[package]] +name = "pear" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467" +dependencies = [ + "inlinable_string", + "pear_codegen", + "yansi", +] + +[[package]] +name = "pear_codegen" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147" +dependencies = [ + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.100", +] + [[package]] name = "pem" version = "3.0.5" @@ -6437,6 +6559,7 @@ dependencies = [ "quote", "syn 2.0.100", "version_check", + "yansi", ] [[package]] @@ -6531,7 +6654,7 @@ name = "protos" version = "0.0.1" dependencies = [ "common", - "flatbuffers", + "flatbuffers 25.2.10", "prost", "prost-build", "protobuf", @@ -6589,7 +6712,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", - "rustls", + "rustls 0.23.26", "socket2", "thiserror 2.0.12", "tokio", @@ -6608,7 +6731,7 @@ dependencies = [ "rand 0.9.0", "ring", "rustc-hash 2.1.1", - "rustls", + "rustls 0.23.26", "rustls-pki-types", "slab", "thiserror 2.0.12", @@ -6946,7 +7069,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls", + "rustls 0.23.26", "rustls-pemfile", "rustls-pki-types", "serde", @@ -6955,7 +7078,7 @@ dependencies = [ "sync_wrapper", "system-configuration", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.2", "tokio-util", "tower 0.5.2", "tower-service", @@ -7083,6 +7206,24 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rumqttc" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1568e15fab2d546f940ed3a21f48bbbd1c494c90c99c4481339364a497f94a9" +dependencies = [ + "bytes", + "flume", + "futures-util", + "log", + "rustls-native-certs", + "rustls-pemfile", + "rustls-webpki 0.102.8", + "thiserror 1.0.69", + "tokio", + "tokio-rustls 0.25.0", +] + [[package]] name = "rust-embed" version = "8.7.0" @@ -7176,7 +7317,7 @@ dependencies = [ "crypto", "datafusion", "ecstore", - "flatbuffers", + "flatbuffers 25.2.10", "futures", "futures-util", "http", @@ -7202,7 +7343,7 @@ dependencies = [ "rust-embed", "rustfs-event-notifier", "rustfs-obs", - "rustls", + "rustls 0.23.26", "rustls-pemfile", "rustls-pki-types", "s3s", @@ -7213,7 +7354,7 @@ dependencies = [ "tikv-jemallocator", "time", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.2", "tokio-stream", "tokio-util", "tonic 0.13.0", @@ -7230,12 +7371,24 @@ name = "rustfs-event-notifier" version = "0.0.1" dependencies = [ "async-trait", - "parking_lot 0.12.3", + "axum", + "chrono", + "dotenv", + "figment", + "rdkafka", "reqwest", + "rumqttc", "serde", "serde_json", + "serde_with", + "smallvec", + "strum", "thiserror 2.0.12", "tokio", + "tokio-util", + "tracing", + "tracing-subscriber", + "uuid", "wildmatch", ] @@ -7314,6 +7467,20 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "rustls" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" +dependencies = [ + "log", + "ring", + "rustls-pki-types", + "rustls-webpki 0.102.8", + "subtle", + "zeroize", +] + [[package]] name = "rustls" version = "0.23.26" @@ -7325,11 +7492,24 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.103.1", "subtle", "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "rustls-pki-types", + "schannel", + "security-framework 2.11.1", +] + [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -7348,6 +7528,17 @@ dependencies = [ "web-time", ] +[[package]] +name = "rustls-webpki" +version = "0.102.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.103.1" @@ -7400,7 +7591,7 @@ dependencies = [ "md-5", "memchr", "mime", - "nom", + "nom 7.1.3", "nugine-rust-utils", "numeric_cast", "pin-project-lite", @@ -7442,6 +7633,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -7509,6 +7709,9 @@ name = "semver" version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" +dependencies = [ + "serde", +] [[package]] name = "send_wrapper" @@ -7621,6 +7824,49 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.9.0", + "serde", + "serde_derive", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap 2.9.0", + "itoa 1.0.15", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "server_fn" version = "0.6.15" @@ -7730,13 +7976,16 @@ dependencies = [ [[package]] name = "shadow-rs" -version = "0.38.1" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ec14cc798c29f4bf74a6c4299c657c04d4e9fba03875c1f0eec569af03aed89" +checksum = "6d5625ed609cf66d7e505e7d487aca815626dc4ebb6c0dd07637ca61a44651a6" dependencies = [ + "cargo_metadata", "const_format", "is_debug", + "serde_json", "time", + "tzdb", ] [[package]] @@ -7881,6 +8130,9 @@ name = "smallvec" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" +dependencies = [ + "serde", +] [[package]] name = "snafu" @@ -7951,6 +8203,9 @@ name = "spin" version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] [[package]] name = "spki" @@ -8452,13 +8707,24 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "tokio-rustls" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +dependencies = [ + "rustls 0.22.4", + "rustls-pki-types", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls", + "rustls 0.23.26", "tokio", ] @@ -8888,6 +9154,32 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +[[package]] +name = "tz-rs" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1450bf2b99397e72070e7935c89facaa80092ac812502200375f1f7d33c71a1" + +[[package]] +name = "tzdb" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2ea5956f295449f47c0b825c5e109022ff1a6a53bb4f77682a87c2341fbf5" +dependencies = [ + "iana-time-zone", + "tz-rs", + "tzdb_data", +] + +[[package]] +name = "tzdb_data" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c4c81d75033770e40fbd3643ce7472a1a9fd301f90b7139038228daf8af03ec" +dependencies = [ + "tz-rs", +] + [[package]] name = "ucd-trie" version = "0.1.7" @@ -8905,6 +9197,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "uncased" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" +dependencies = [ + "version_check", +] + [[package]] name = "unicase" version = "2.8.1" @@ -8945,6 +9246,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" @@ -9944,6 +10251,12 @@ dependencies = [ "hashlink", ] +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "yoke" version = "0.7.5" @@ -9984,7 +10297,7 @@ dependencies = [ "futures-sink", "futures-util", "hex", - "nix 0.29.0", + "nix", "ordered-stream", "rand 0.8.5", "serde", @@ -10015,7 +10328,7 @@ dependencies = [ "futures-core", "futures-lite", "hex", - "nix 0.29.0", + "nix", "ordered-stream", "serde", "serde_repr", diff --git a/Cargo.toml b/Cargo.toml index 99e5073f3..52bd76da0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,8 +51,10 @@ datafusion = "46.0.0" derive_builder = "0.20.2" dioxus = { version = "0.6.3", features = ["router"] } dirs = "6.0.0" +dotenv = "0.15.0" ecstore = { path = "./ecstore" } -flatbuffers = "24.12.23" +figment = { version = "0.10.19", features = ["toml", "yaml", "env"] } +flatbuffers = "25.2.10" futures = "0.3.31" futures-util = "0.3.31" common = { path = "./common/common" } @@ -71,7 +73,7 @@ jsonwebtoken = "9.3.1" keyring = { version = "3.6.2", features = ["apple-native", "windows-native", "sync-secret-service"] } lock = { path = "./common/lock" } lazy_static = "1.5.0" -libsystemd = { version = "0.7" } +libsystemd = { version = "0.7.1" } local-ip-address = "0.6.3" matchit = "0.8.4" md-5 = "0.10.6" @@ -79,13 +81,13 @@ mime = "0.3.17" netif = "0.1.6" opentelemetry = { version = "0.29.1" } opentelemetry-appender-tracing = { version = "0.29.1", features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes"] } -opentelemetry_sdk = { version = "0.29" } +opentelemetry_sdk = { version = "0.29.0" } opentelemetry-stdout = { version = "0.29.0" } -opentelemetry-otlp = { version = "0.29" } +opentelemetry-otlp = { version = "0.29.0" } opentelemetry-prometheus = { version = "0.29.1" } opentelemetry-semantic-conventions = { version = "0.29.0", features = ["semconv_experimental"] } parking_lot = "0.12.3" -pin-project-lite = "0.2" +pin-project-lite = "0.2.16" prometheus = "0.14.0" # pin-utils = "0.1.0" prost = "0.13.5" @@ -94,11 +96,12 @@ prost-types = "0.13.5" protobuf = "3.7" protos = { path = "./common/protos" } rand = "0.8.5" -rdkafka = { version = "0.37", features = ["tokio"] } +rdkafka = { version = "0.37.0", features = ["tokio"] } reqwest = { version = "0.12.15", default-features = false, features = ["rustls-tls", "charset", "http2", "macos-system-configuration", "stream", "json", "blocking"] } rfd = { version = "0.15.3", default-features = false, features = ["xdg-portal", "tokio"] } rmp = "0.8.14" rmp-serde = "1.3.0" +rumqttc = { version = "0.24" } rustfs-obs = { path = "crates/obs", version = "0.0.1" } rustfs-event-notifier = { path = "crates/event-notifier", version = "0.0.1" } rust-embed = "8.7.0" @@ -107,10 +110,13 @@ rustls-pki-types = "1.11.0" rustls-pemfile = "2.2.0" s3s = { git = "https://github.com/Nugine/s3s.git", rev = "4733cdfb27b2713e832967232cbff413bb768c10" } s3s-policy = { git = "https://github.com/Nugine/s3s.git", rev = "4733cdfb27b2713e832967232cbff413bb768c10" } -shadow-rs = { version = "0.38.1", default-features = false } +shadow-rs = { version = "1.1.1", default-features = false, features = ["metadata"] } serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.140" serde_urlencoded = "0.7.1" +serde_with = "3.12.0" +smallvec = { version = "1.15.0", features = ["serde"] } +strum = { version = "0.27.1", features = ["derive"] } sha2 = "0.10.8" snafu = "0.8.5" tempfile = "3.19.1" @@ -126,7 +132,7 @@ time = { version = "0.3.41", features = [ tokio = { version = "1.44.2", features = ["fs", "rt-multi-thread"] } tonic = { version = "0.13.0", features = ["gzip"] } tonic-build = "0.13.0" -tokio-rustls = { version = "0.26", default-features = false } +tokio-rustls = { version = "0.26.2", default-features = false } tokio-stream = "0.1.17" tokio-util = { version = "0.7.14", features = ["io", "compat"] } tower = { version = "0.5.2", features = ["timeout"] } @@ -136,7 +142,7 @@ tracing-core = "0.1.33" tracing-error = "0.2.1" tracing-subscriber = { version = "0.3.19", features = ["env-filter", "time"] } tracing-appender = "0.2.3" -tracing-opentelemetry = "0.30" +tracing-opentelemetry = "0.30.0" transform-stream = "0.3.1" url = "2.5.4" uuid = { version = "1.16.0", features = [ @@ -144,7 +150,6 @@ uuid = { version = "1.16.0", features = [ "fast-rng", "macro-diagnostics", ] } -wildmatch = "2.4.0" workers = { path = "./common/workers" } [profile.wasm-dev] diff --git a/crates/event-notifier/Cargo.toml b/crates/event-notifier/Cargo.toml index 2e4afc7bc..065cc6984 100644 --- a/crates/event-notifier/Cargo.toml +++ b/crates/event-notifier/Cargo.toml @@ -6,16 +6,36 @@ repository.workspace = true rust-version.workspace = true version.workspace = true +[features] +default = ["webhook"] +webhook = ["dep:reqwest"] +kafka = ["rdkafka"] +mqtt = ["rumqttc"] +http-producer = ["dep:axum"] + [dependencies] async-trait = { workspace = true } -parking_lot = { workspace = true } -reqwest = { workspace = true } +axum = { workspace = true, optional = true } +chrono = { workspace = true, features = ["serde"] } +dotenv = { workspace = true } +figment = { workspace = true, features = ["toml", "yaml", "env"] } +rdkafka = { workspace = true, features = ["tokio"], optional = true } +reqwest = { workspace = true, optional = true } +rumqttc = { workspace = true, optional = true } serde = { workspace = true } -tokio = { workspace = true, features = ["full"] } -thiserror = { workspace = true } -wildmatch = { workspace = true } serde_json = { workspace = true } +serde_with = { workspace = true } +smallvec = { workspace = true, features = ["serde"] } +strum = { workspace = true, features = ["derive"] } +tracing = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["sync", "net", "macros", "signal", "rt-multi-thread"] } +tokio-util = { workspace = true } +uuid = { workspace = true, features = ["v4", "serde"] } +[dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } +tracing-subscriber = { workspace = true } [lints] workspace = true diff --git a/crates/event-notifier/examples/.env.example b/crates/event-notifier/examples/.env.example new file mode 100644 index 000000000..d42bc9fc0 --- /dev/null +++ b/crates/event-notifier/examples/.env.example @@ -0,0 +1,27 @@ +# basic configuration +EVENT_NOTIF_STORE_PATH=/var/log/event-notification +EVENT_NOTIF_CHANNEL_CAPACITY=5000 + +# webhook adapter configuration +EVENT_NOTIF_ADAPTERS__0__TYPE=Webhook +EVENT_NOTIF_ADAPTERS__0__ENDPOINT=https://api.example.com/webhook +EVENT_NOTIF_ADAPTERS__0__AUTH_TOKEN=your-secret-token +EVENT_NOTIF_ADAPTERS__0__MAX_RETRIES=3 +EVENT_NOTIF_ADAPTERS__0__TIMEOUT=5000 + +# kafka adapter configuration +EVENT_NOTIF_ADAPTERS__1__TYPE=Kafka +EVENT_NOTIF_ADAPTERS__1__BROKERS=localhost:9092 +EVENT_NOTIF_ADAPTERS__1__TOPIC=notifications +EVENT_NOTIF_ADAPTERS__1__MAX_RETRIES=3 +EVENT_NOTIF_ADAPTERS__1__TIMEOUT=5000 + +# mqtt adapter configuration +EVENT_NOTIF_ADAPTERS__2__TYPE=Mqtt +EVENT_NOTIF_ADAPTERS__2__BROKER=mqtt.example.com +EVENT_NOTIF_ADAPTERS__2__PORT=1883 +EVENT_NOTIF_ADAPTERS__2__CLIENT_ID=event-notifier +EVENT_NOTIF_ADAPTERS__2__TOPIC=events +EVENT_NOTIF_ADAPTERS__2__MAX_RETRIES=3 + +EVENT_NOTIF_HTTP__PORT=8080 \ No newline at end of file diff --git a/crates/event-notifier/examples/event.toml b/crates/event-notifier/examples/event.toml new file mode 100644 index 000000000..dcde5abf1 --- /dev/null +++ b/crates/event-notifier/examples/event.toml @@ -0,0 +1,28 @@ +# config.toml +store_path = "/var/log/event-notification" +channel_capacity = 5000 + +[[adapters]] +type = "Webhook" +endpoint = "https://api.example.com/webhook" +auth_token = "your-auth-token" +max_retries = 3 +timeout = 5000 + +[[adapters]] +type = "Kafka" +brokers = "localhost:9092" +topic = "notifications" +max_retries = 3 +timeout = 5000 + +[[adapters]] +type = "Mqtt" +broker = "mqtt.example.com" +port = 1883 +client_id = "event-notifier" +topic = "events" +max_retries = 3 + +[http] +port = 8080 \ No newline at end of file diff --git a/crates/event-notifier/examples/simple.rs b/crates/event-notifier/examples/simple.rs new file mode 100644 index 000000000..5324586ad --- /dev/null +++ b/crates/event-notifier/examples/simple.rs @@ -0,0 +1,102 @@ +use rustfs_event_notifier::create_adapters; +use rustfs_event_notifier::NotificationSystem; +use rustfs_event_notifier::{AdapterConfig, NotificationConfig, WebhookConfig}; +use rustfs_event_notifier::{Bucket, Event, Identity, Metadata, Name, Object, Source}; +use std::collections::HashMap; +use std::error; +use std::sync::Arc; +use tokio::signal; + +#[tokio::main] +async fn main() -> Result<(), Box> { + tracing_subscriber::fmt::init(); + + let mut config = NotificationConfig { + store_path: "./events".to_string(), + channel_capacity: 100, + adapters: vec![AdapterConfig::Webhook(WebhookConfig { + endpoint: "http://localhost:8080/webhook".to_string(), + auth_token: Some("secret-token".to_string()), + custom_headers: Some(HashMap::from([("X-Custom".to_string(), "value".to_string())])), + max_retries: 3, + timeout: 10, + })], + http: Default::default(), + }; + config.http.port = 8080; + + // loading configuration from specific env files + let _config = NotificationConfig::from_env_file(".env.example")?; + + // loading from a specific file + let _config = NotificationConfig::from_file("event.toml")?; + + // Automatically load from multiple sources (Priority: Environment Variables > YAML > TOML) + let _config = NotificationConfig::load()?; + + let system = Arc::new(tokio::sync::Mutex::new(NotificationSystem::new(config.clone()).await?)); + let adapters = create_adapters(&config.adapters)?; + + // create an s3 metadata object + let metadata = Metadata { + schema_version: "1.0".to_string(), + configuration_id: "test-config".to_string(), + bucket: Bucket { + name: "my-bucket".to_string(), + owner_identity: Identity { + principal_id: "owner123".to_string(), + }, + arn: "arn:aws:s3:::my-bucket".to_string(), + }, + object: Object { + key: "test.txt".to_string(), + size: Some(1024), + etag: Some("abc123".to_string()), + content_type: Some("text/plain".to_string()), + user_metadata: None, + version_id: None, + sequencer: "1234567890".to_string(), + }, + }; + + // create source object + let source = Source { + host: "localhost".to_string(), + port: "80".to_string(), + user_agent: "curl/7.68.0".to_string(), + }; + + // create events using builder mode + let event = Event::builder() + .event_time("2023-10-01T12:00:00.000Z") + .event_name(Name::ObjectCreatedPut) + .user_identity(Identity { + principal_id: "user123".to_string(), + }) + .s3(metadata) + .source(source) + .channels(vec!["webhook".to_string()]) + .build() + .expect("failed to create event"); + + { + let system = system.lock().await; + system.send_event(event).await?; + } + + let system_clone = Arc::clone(&system); + let system_handle = tokio::spawn(async move { + let mut system = system_clone.lock().await; + system.start(adapters).await + }); + + signal::ctrl_c().await?; + tracing::info!("Received shutdown signal"); + { + let system = system.lock().await; + system.shutdown(); + } + + system_handle.await??; + Ok(()) +} diff --git a/crates/event-notifier/src/adapter/kafka.rs b/crates/event-notifier/src/adapter/kafka.rs new file mode 100644 index 000000000..cfb55e6dc --- /dev/null +++ b/crates/event-notifier/src/adapter/kafka.rs @@ -0,0 +1,76 @@ +use crate::ChannelAdapter; +use crate::Error; +use crate::Event; +use crate::KafkaConfig; +use async_trait::async_trait; +use rdkafka::error::KafkaError; +use rdkafka::producer::{FutureProducer, FutureRecord}; +use rdkafka::types::RDKafkaErrorCode; +use rdkafka::util::Timeout; +use std::time::Duration; +use tokio::time::sleep; + +/// Kafka adapter for sending events to a Kafka topic. +pub struct KafkaAdapter { + producer: FutureProducer, + topic: String, + max_retries: u32, +} + +impl KafkaAdapter { + /// Creates a new Kafka adapter. + pub fn new(config: &KafkaConfig) -> Result { + // Create a Kafka producer with the provided configuration. + let producer = rdkafka::config::ClientConfig::new() + .set("bootstrap.servers", &config.brokers) + .set("message.timeout.ms", config.timeout.to_string()) + .create()?; + + Ok(Self { + producer, + topic: config.topic.clone(), + max_retries: config.max_retries, + }) + } + /// Sends an event to the Kafka topic with retry logic. + async fn send_with_retry(&self, event: &Event) -> Result<(), Error> { + let event_id = event.id.to_string(); + let payload = serde_json::to_string(&event)?; + + for attempt in 0..self.max_retries { + let record = FutureRecord::to(&self.topic) + .key(&event_id) + .payload(&payload); + + match self.producer.send(record, Timeout::Never).await { + Ok(_) => return Ok(()), + Err((KafkaError::MessageProduction(RDKafkaErrorCode::QueueFull), _)) => { + tracing::warn!( + "Kafka attempt {} failed: Queue full. Retrying...", + attempt + 1 + ); + sleep(Duration::from_secs(2u64.pow(attempt))).await; + } + Err((e, _)) => { + tracing::error!("Kafka send error: {}", e); + return Err(Error::Kafka(e)); + } + } + } + + Err(Error::Custom( + "Exceeded maximum retry attempts for Kafka message".to_string(), + )) + } +} + +#[async_trait] +impl ChannelAdapter for KafkaAdapter { + fn name(&self) -> String { + "kafka".to_string() + } + + async fn send(&self, event: &Event) -> Result<(), Error> { + self.send_with_retry(event).await + } +} diff --git a/crates/event-notifier/src/adapter/mod.rs b/crates/event-notifier/src/adapter/mod.rs new file mode 100644 index 000000000..9b8cab9f0 --- /dev/null +++ b/crates/event-notifier/src/adapter/mod.rs @@ -0,0 +1,54 @@ +use crate::AdapterConfig; +use crate::Error; +use crate::Event; +use async_trait::async_trait; +use std::sync::Arc; + +#[cfg(feature = "kafka")] +pub(crate) mod kafka; +#[cfg(feature = "mqtt")] +pub(crate) mod mqtt; +#[cfg(feature = "webhook")] +pub(crate) mod webhook; + +/// The `ChannelAdapter` trait defines the interface for all channel adapters. +#[async_trait] +pub trait ChannelAdapter: Send + Sync + 'static { + /// Sends an event to the channel. + fn name(&self) -> String; + /// Sends an event to the channel. + async fn send(&self, event: &Event) -> Result<(), Error>; +} + +/// Creates channel adapters based on the provided configuration. +pub fn create_adapters(configs: &[AdapterConfig]) -> Result>, Box> { + let mut adapters: Vec> = Vec::new(); + + for config in configs { + match config { + #[cfg(feature = "webhook")] + AdapterConfig::Webhook(webhook_config) => { + webhook_config.validate().map_err(|e| Box::new(Error::ConfigError(e)))?; + adapters.push(Arc::new(webhook::WebhookAdapter::new(webhook_config.clone()))); + } + #[cfg(feature = "kafka")] + AdapterConfig::Kafka(kafka_config) => { + adapters.push(Arc::new(kafka::KafkaAdapter::new(kafka_config)?)); + } + #[cfg(feature = "mqtt")] + AdapterConfig::Mqtt(mqtt_config) => { + let (mqtt, mut event_loop) = mqtt::MqttAdapter::new(mqtt_config); + tokio::spawn(async move { while event_loop.poll().await.is_ok() {} }); + adapters.push(Arc::new(mqtt)); + } + #[cfg(not(feature = "webhook"))] + AdapterConfig::Webhook(_) => return Err(Box::new(Error::FeatureDisabled("webhook"))), + #[cfg(not(feature = "kafka"))] + AdapterConfig::Kafka(_) => return Err(Box::new(Error::FeatureDisabled("kafka"))), + #[cfg(not(feature = "mqtt"))] + AdapterConfig::Mqtt(_) => return Err(Box::new(Error::FeatureDisabled("mqtt"))), + } + } + + Ok(adapters) +} diff --git a/crates/event-notifier/src/adapter/mqtt.rs b/crates/event-notifier/src/adapter/mqtt.rs new file mode 100644 index 000000000..9aab61e8e --- /dev/null +++ b/crates/event-notifier/src/adapter/mqtt.rs @@ -0,0 +1,58 @@ +use crate::ChannelAdapter; +use crate::Error; +use crate::Event; +use crate::MqttConfig; +use async_trait::async_trait; +use rumqttc::{AsyncClient, MqttOptions, QoS}; +use std::time::Duration; +use tokio::time::sleep; + +/// MQTT adapter for sending events to an MQTT broker. +pub struct MqttAdapter { + client: AsyncClient, + topic: String, + max_retries: u32, +} + +impl MqttAdapter { + /// Creates a new MQTT adapter. + pub fn new(config: &MqttConfig) -> (Self, rumqttc::EventLoop) { + let mqtt_options = MqttOptions::new(&config.client_id, &config.broker, config.port); + let (client, event_loop) = rumqttc::AsyncClient::new(mqtt_options, 10); + ( + Self { + client, + topic: config.topic.clone(), + max_retries: config.max_retries, + }, + event_loop, + ) + } +} + +#[async_trait] +impl ChannelAdapter for MqttAdapter { + fn name(&self) -> String { + "mqtt".to_string() + } + + async fn send(&self, event: &Event) -> Result<(), Error> { + let payload = serde_json::to_string(event).map_err(Error::Serde)?; + let mut attempt = 0; + loop { + match self + .client + .publish(&self.topic, QoS::AtLeastOnce, false, payload.clone()) + .await + { + Ok(()) => return Ok(()), + Err(e) if attempt < self.max_retries => { + attempt += 1; + tracing::warn!("MQTT attempt {} failed: {}. Retrying...", attempt, e); + sleep(Duration::from_secs(2u64.pow(attempt))).await; + } + Err(e) => return Err(Error::Mqtt(e)), + } + } + } +} diff --git a/crates/event-notifier/src/adapter/webhook.rs b/crates/event-notifier/src/adapter/webhook.rs new file mode 100644 index 000000000..80b8c9cb6 --- /dev/null +++ b/crates/event-notifier/src/adapter/webhook.rs @@ -0,0 +1,63 @@ +use crate::ChannelAdapter; +use crate::Error; +use crate::Event; +use crate::WebhookConfig; +use async_trait::async_trait; +use reqwest::{Client, RequestBuilder}; +use std::time::Duration; +use tokio::time::sleep; + +/// Webhook adapter for sending events to a webhook endpoint. +pub struct WebhookAdapter { + config: WebhookConfig, + client: Client, +} + +impl WebhookAdapter { + /// Creates a new Webhook adapter. + pub fn new(config: WebhookConfig) -> Self { + let client = Client::builder() + .timeout(Duration::from_secs(config.timeout)) + .build() + .expect("Failed to build reqwest client"); + Self { config, client } + } + /// Builds the request to send the event. + fn build_request(&self, event: &Event) -> RequestBuilder { + let mut request = self.client.post(&self.config.endpoint).json(event); + if let Some(token) = &self.config.auth_token { + request = request.header("Authorization", format!("Bearer {}", token)); + } + if let Some(headers) = &self.config.custom_headers { + for (key, value) in headers { + request = request.header(key, value); + } + } + request + } +} + +#[async_trait] +impl ChannelAdapter for WebhookAdapter { + fn name(&self) -> String { + "webhook".to_string() + } + + async fn send(&self, event: &Event) -> Result<(), Error> { + let mut attempt = 0; + loop { + match self.build_request(event).send().await { + Ok(response) => { + response.error_for_status().map_err(Error::Http)?; + return Ok(()); + } + Err(e) if attempt < self.config.max_retries => { + attempt += 1; + tracing::warn!("Webhook attempt {} failed: {}. Retrying...", attempt, e); + sleep(Duration::from_secs(2u64.pow(attempt))).await; + } + Err(e) => return Err(Error::Http(e)), + } + } + } +} diff --git a/crates/event-notifier/src/bus.rs b/crates/event-notifier/src/bus.rs new file mode 100644 index 000000000..0ef341ce6 --- /dev/null +++ b/crates/event-notifier/src/bus.rs @@ -0,0 +1,68 @@ +use crate::ChannelAdapter; +use crate::Error; +use crate::EventStore; +use crate::{Event, Log}; +use chrono::Utc; +use std::sync::Arc; +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; + +/// Handles incoming events from the producer. +/// +/// This function is responsible for receiving events from the producer and sending them to the appropriate adapters. +/// It also handles the shutdown process and saves any pending logs to the event store. +pub async fn event_bus( + mut rx: mpsc::Receiver, + adapters: Vec>, + store: Arc, + shutdown: CancellationToken, +) -> Result<(), Error> { + let mut pending_logs = Vec::new(); + let mut current_log = Log { + event_name: crate::event::Name::Everything, + key: Utc::now().timestamp().to_string(), + records: Vec::new(), + }; + + loop { + tokio::select! { + Some(event) = rx.recv() => { + current_log.records.push(event.clone()); + let mut send_tasks = Vec::new(); + for adapter in &adapters { + if event.channels.contains(&adapter.name()) { + let adapter = adapter.clone(); + let event = event.clone(); + send_tasks.push(tokio::spawn(async move { + if let Err(e) = adapter.send(&event).await { + tracing::error!("Failed to send event to {}: {}", adapter.name(), e); + Err(e) + } else { + Ok(()) + } + })); + } + } + for task in send_tasks { + if task.await?.is_err() { + current_log.records.retain(|e| e.id != event.id); + } + } + if !current_log.records.is_empty() { + pending_logs.push(current_log.clone()); + } + current_log.records.clear(); + } + _ = shutdown.cancelled() => { + tracing::info!("Shutting down event bus, saving pending logs..."); + if !current_log.records.is_empty() { + pending_logs.push(current_log); + } + store.save_logs(&pending_logs).await?; + break; + } + else => break, + } + } + Ok(()) +} diff --git a/crates/event-notifier/src/config.rs b/crates/event-notifier/src/config.rs new file mode 100644 index 000000000..823bf83da --- /dev/null +++ b/crates/event-notifier/src/config.rs @@ -0,0 +1,161 @@ +use crate::Error; +use figment::providers::Format; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// Configuration for the notification system. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WebhookConfig { + pub endpoint: String, + pub auth_token: Option, + pub custom_headers: Option>, + pub max_retries: u32, + pub timeout: u64, +} + +impl WebhookConfig { + /// verify that the configuration is valid + pub fn validate(&self) -> Result<(), String> { + // verify that endpoint cannot be empty + if self.endpoint.trim().is_empty() { + return Err("Webhook endpoint cannot be empty".to_string()); + } + + // verification timeout must be reasonable + if self.timeout == 0 { + return Err("Webhook timeout must be greater than 0".to_string()); + } + + // Verify that the maximum number of retry is reasonable + if self.max_retries > 10 { + return Err("Maximum retry count cannot exceed 10".to_string()); + } + + Ok(()) + } +} + +/// Configuration for the Kafka adapter. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct KafkaConfig { + pub brokers: String, + pub topic: String, + pub max_retries: u32, + pub timeout: u64, +} + +/// Configuration for the MQTT adapter. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MqttConfig { + pub broker: String, + pub port: u16, + pub client_id: String, + pub topic: String, + pub max_retries: u32, +} + +/// Configuration for the notification system. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum AdapterConfig { + Webhook(WebhookConfig), + Kafka(KafkaConfig), + Mqtt(MqttConfig), +} + +/// http producer configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HttpProducerConfig { + #[serde(default = "default_http_port")] + pub port: u16, +} + +impl Default for HttpProducerConfig { + fn default() -> Self { + Self { + port: default_http_port(), + } + } +} + +fn default_http_port() -> u16 { + 3000 +} + +/// Configuration for the notification system. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NotificationConfig { + #[serde(default = "default_store_path")] + pub store_path: String, + #[serde(default = "default_channel_capacity")] + pub channel_capacity: usize, + pub adapters: Vec, + #[serde(default)] + pub http: HttpProducerConfig, +} + +impl Default for NotificationConfig { + fn default() -> Self { + Self { + store_path: default_store_path(), + channel_capacity: default_channel_capacity(), + adapters: Vec::new(), + http: HttpProducerConfig::default(), + } + } +} + +impl NotificationConfig { + /// create a new configuration with default values + pub fn new() -> Self { + Self::default() + } + + /// create a configuration from a configuration file + pub fn from_file(path: &str) -> Result { + let config = figment::Figment::new() + .merge(figment::providers::Toml::file(path)) + .extract()?; + + Ok(config) + } + + /// Read configuration from multiple sources (support TOML, YAML, .env) + pub fn load() -> Result { + let figment = figment::Figment::new() + // First try to read the config.toml of the current directory + .merge(figment::providers::Toml::file("event.toml")) + // Then try to read the config.yaml of the current directory + .merge(figment::providers::Yaml::file("event.yaml")) + // Finally read the environment variable and overwrite the previous value + .merge(figment::providers::Env::prefixed("EVENT_NOTIF_")); + + Ok(figment.extract()?) + } + + /// loading configuration from env file + pub fn from_env_file(path: &str) -> Result { + // loading env files + dotenv::from_path(path) + .map_err(|e| Error::ConfigError(format!("unable to load env file: {}", e)))?; + + // Extract configuration from environment variables using figurement + let figment = + figment::Figment::new().merge(figment::providers::Env::prefixed("EVENT_NOTIF_")); + + Ok(figment.extract()?) + } +} + +/// Provide temporary directories as default storage paths +fn default_store_path() -> String { + std::env::temp_dir() + .join("event-notification") + .to_string_lossy() + .to_string() +} + +/// Provides the recommended default channel capacity for high concurrency systems +fn default_channel_capacity() -> usize { + 10000 // Reasonable default values for high concurrency systems +} diff --git a/crates/event-notifier/src/error.rs b/crates/event-notifier/src/error.rs index 1945c351c..9f1e41b4b 100644 --- a/crates/event-notifier/src/error.rs +++ b/crates/event-notifier/src/error.rs @@ -1,25 +1,45 @@ -// src/error.rs use thiserror::Error; +use tokio::sync::mpsc::error; +use tokio::task::JoinError; +/// The `Error` enum represents all possible errors that can occur in the application. +/// It implements the `std::error::Error` trait and provides a way to convert various error types into a single error type. #[derive(Error, Debug)] pub enum Error { - #[error("target not found: {0}")] - TargetNotFound(String), - - #[error("send failed: {0}")] - SendError(String), - - #[error("target error: {0}")] - TargetError(String), - - #[error("invalid configuration: {0}")] + #[error("Join error: {0}")] + JoinError(#[from] JoinError), + #[error("IO error: {0}")] + Io(#[from] std::io::Error), + #[error("Serialization error: {0}")] + Serde(#[from] serde_json::Error), + #[error("HTTP error: {0}")] + Http(#[from] reqwest::Error), + #[cfg(feature = "kafka")] + #[error("Kafka error: {0}")] + Kafka(#[from] rdkafka::error::KafkaError), + #[cfg(feature = "mqtt")] + #[error("MQTT error: {0}")] + Mqtt(#[from] rumqttc::ClientError), + #[error("Channel send error: {0}")] + ChannelSend(#[from] Box>), + #[error("Feature disabled: {0}")] + FeatureDisabled(&'static str), + #[error("Event bus already started")] + EventBusStarted, + #[error("necessary fields are missing:{0}")] + MissingField(&'static str), + #[error("field verification failed:{0}")] + ValidationError(&'static str), + #[error("{0}")] + Custom(String), + #[error("Configuration error: {0}")] ConfigError(String), - - #[error("store error: {0}")] - StoreError(String), - - #[error("invalid event name: {0}")] - InvalidEventName(String), // 添加此变体 + #[error("Configuration loading error: {0}")] + Figment(#[from] figment::Error), } -pub type Result = std::result::Result; +impl Error { + pub(crate) fn custom(msg: &str) -> Error { + Self::Custom(msg.to_string()) + } +} diff --git a/crates/event-notifier/src/event.rs b/crates/event-notifier/src/event.rs index 695f2c3d7..00bc5983a 100644 --- a/crates/event-notifier/src/event.rs +++ b/crates/event-notifier/src/event.rs @@ -1,53 +1,442 @@ +use crate::Error; +use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; +use serde_with::{DeserializeFromStr, SerializeDisplay}; +use smallvec::{smallvec, SmallVec}; +use std::borrow::Cow; use std::collections::HashMap; +use strum::{Display, EnumString}; +use uuid::Uuid; -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Event { - pub event_version: String, - pub event_source: String, - pub aws_region: String, - pub event_time: String, - pub event_name: String, - pub user_identity: Identity, - pub request_parameters: HashMap, - pub response_elements: HashMap, - pub s3: Metadata, - pub source: Source, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Clone, Debug)] pub struct Identity { + #[serde(rename = "principalId")] pub principal_id: String, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct Bucket { + pub name: String, + #[serde(rename = "ownerIdentity")] + pub owner_identity: Identity, + pub arn: String, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct Object { + pub key: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub size: Option, + #[serde(default, skip_serializing_if = "Option::is_none", rename = "eTag")] + pub etag: Option, + #[serde(default, skip_serializing_if = "Option::is_none", rename = "contentType")] + pub content_type: Option, + #[serde(default, skip_serializing_if = "Option::is_none", rename = "userMetadata")] + pub user_metadata: Option>, + #[serde(default, skip_serializing_if = "Option::is_none", rename = "versionId")] + pub version_id: Option, + pub sequencer: String, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] pub struct Metadata { + #[serde(rename = "s3SchemaVersion")] pub schema_version: String, + #[serde(rename = "configurationId")] pub configuration_id: String, pub bucket: Bucket, pub object: Object, } -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Bucket { - pub name: String, - pub owner_identity: Identity, - pub arn: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Object { - pub key: String, - pub version_id: Option, - pub sequencer: String, - pub size: Option, - pub etag: Option, - pub content_type: Option, - pub user_metadata: HashMap, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Clone, Debug)] pub struct Source { pub host: String, + pub port: String, + #[serde(rename = "userAgent")] pub user_agent: String, } + +/// Builder for creating an Event. +/// +/// This struct is used to build an Event object with various parameters. +/// It provides methods to set each parameter and a build method to create the Event. +#[derive(Default, Clone)] +pub struct EventBuilder { + event_version: Option, + event_source: Option, + aws_region: Option, + event_time: Option, + event_name: Option, + user_identity: Option, + request_parameters: Option>, + response_elements: Option>, + s3: Option, + source: Option, + channels: Option>, +} + +impl EventBuilder { + /// create a builder that pre filled default values + pub fn new() -> Self { + Self { + event_version: Some(Cow::Borrowed("2.0").to_string()), + event_source: Some(Cow::Borrowed("aws:s3").to_string()), + aws_region: Some("us-east-1".to_string()), + event_time: Some(Utc::now().to_rfc3339()), + event_name: None, + user_identity: Some(Identity { + principal_id: "anonymous".to_string(), + }), + request_parameters: Some(HashMap::new()), + response_elements: Some(HashMap::new()), + s3: None, + source: None, + channels: Some(Vec::new().into()), + } + } + + /// verify and set the event version + pub fn event_version(mut self, event_version: impl Into) -> Self { + let event_version = event_version.into(); + if !event_version.is_empty() { + self.event_version = Some(event_version); + } + self + } + + /// verify and set the event source + pub fn event_source(mut self, event_source: impl Into) -> Self { + let event_source = event_source.into(); + if !event_source.is_empty() { + self.event_source = Some(event_source); + } + self + } + + /// set up aws regions + pub fn aws_region(mut self, aws_region: impl Into) -> Self { + self.aws_region = Some(aws_region.into()); + self + } + + /// set event time + pub fn event_time(mut self, event_time: impl Into) -> Self { + self.event_time = Some(event_time.into()); + self + } + + /// set event name + pub fn event_name(mut self, event_name: Name) -> Self { + self.event_name = Some(event_name); + self + } + + /// set user identity + pub fn user_identity(mut self, user_identity: Identity) -> Self { + self.user_identity = Some(user_identity); + self + } + + /// set request parameters + pub fn request_parameters(mut self, request_parameters: HashMap) -> Self { + self.request_parameters = Some(request_parameters); + self + } + + /// set response elements + pub fn response_elements(mut self, response_elements: HashMap) -> Self { + self.response_elements = Some(response_elements); + self + } + + /// setting up s3 metadata + pub fn s3(mut self, s3: Metadata) -> Self { + self.s3 = Some(s3); + self + } + + /// set event source information + pub fn source(mut self, source: Source) -> Self { + self.source = Some(source); + self + } + + /// set up the sending channel + pub fn channels(mut self, channels: Vec) -> Self { + self.channels = Some(channels.into()); + self + } + + /// Create a preconfigured builder for common object event scenarios + pub fn for_object_creation(s3: Metadata, source: Source) -> Self { + Self::new().event_name(Name::ObjectCreatedPut).s3(s3).source(source) + } + + /// Create a preconfigured builder for object deletion events + pub fn for_object_removal(s3: Metadata, source: Source) -> Self { + Self::new().event_name(Name::ObjectRemovedDelete).s3(s3).source(source) + } + + /// build event instance + /// + /// Verify the required fields and create a complete Event object + pub fn build(self) -> Result { + let event_version = self.event_version.ok_or(Error::MissingField("event_version"))?; + + let event_source = self.event_source.ok_or(Error::MissingField("event_source"))?; + + let aws_region = self.aws_region.ok_or(Error::MissingField("aws_region"))?; + + let event_time = self.event_time.ok_or(Error::MissingField("event_time"))?; + + let event_name = self.event_name.ok_or(Error::MissingField("event_name"))?; + + let user_identity = self.user_identity.ok_or(Error::MissingField("user_identity"))?; + + let request_parameters = self.request_parameters.unwrap_or_default(); + let response_elements = self.response_elements.unwrap_or_default(); + + let s3 = self.s3.ok_or(Error::MissingField("s3"))?; + + let source = self.source.ok_or(Error::MissingField("source"))?; + + let channels = self.channels.unwrap_or_else(|| smallvec![]); + + Ok(Event { + event_version, + event_source, + aws_region, + event_time, + event_name, + user_identity, + request_parameters, + response_elements, + s3, + source, + id: Uuid::new_v4(), + timestamp: Utc::now(), + channels, + }) + } +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct Event { + #[serde(rename = "eventVersion")] + pub event_version: String, + #[serde(rename = "eventSource")] + pub event_source: String, + #[serde(rename = "awsRegion")] + pub aws_region: String, + #[serde(rename = "eventTime")] + pub event_time: String, + #[serde(rename = "eventName")] + pub event_name: Name, + #[serde(rename = "userIdentity")] + pub user_identity: Identity, + #[serde(rename = "requestParameters")] + pub request_parameters: HashMap, + #[serde(rename = "responseElements")] + pub response_elements: HashMap, + pub s3: Metadata, + pub source: Source, + pub id: Uuid, + pub timestamp: DateTime, + pub channels: SmallVec<[String; 2]>, +} + +impl Event { + /// create a new event builder + /// + /// Returns an EventBuilder instance pre-filled with default values + pub fn builder() -> EventBuilder { + EventBuilder::new() + } + + /// Quickly create Event instances with necessary fields + /// + /// suitable for common s3 event scenarios + pub fn create(event_name: Name, s3: Metadata, source: Source, channels: Vec) -> Self { + Self::builder() + .event_name(event_name) + .s3(s3) + .source(source) + .channels(channels) + .build() + .expect("Failed to create event, missing necessary parameters") + } + + /// a convenient way to create a preconfigured builder + pub fn for_object_creation(s3: Metadata, source: Source) -> EventBuilder { + EventBuilder::for_object_creation(s3, source) + } + + /// a convenient way to create a preconfigured builder + pub fn for_object_removal(s3: Metadata, source: Source) -> EventBuilder { + EventBuilder::for_object_removal(s3, source) + } + + /// Determine whether an event belongs to a specific type + pub fn is_type(&self, event_type: Name) -> bool { + let mask = event_type.mask(); + (self.event_name.mask() & mask) != 0 + } + + /// Determine whether an event needs to be sent to a specific channel + pub fn is_for_channel(&self, channel: &str) -> bool { + self.channels.iter().any(|c| c == channel) + } +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct Log { + #[serde(rename = "eventName")] + pub event_name: Name, + pub key: String, + pub records: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, SerializeDisplay, DeserializeFromStr, Display, EnumString)] +#[strum(serialize_all = "SCREAMING_SNAKE_CASE")] +pub enum Name { + ObjectAccessedGet, + ObjectAccessedGetRetention, + ObjectAccessedGetLegalHold, + ObjectAccessedHead, + ObjectAccessedAttributes, + ObjectCreatedCompleteMultipartUpload, + ObjectCreatedCopy, + ObjectCreatedPost, + ObjectCreatedPut, + ObjectCreatedPutRetention, + ObjectCreatedPutLegalHold, + ObjectCreatedPutTagging, + ObjectCreatedDeleteTagging, + ObjectRemovedDelete, + ObjectRemovedDeleteMarkerCreated, + ObjectRemovedDeleteAllVersions, + ObjectRemovedNoOp, + BucketCreated, + BucketRemoved, + ObjectReplicationFailed, + ObjectReplicationComplete, + ObjectReplicationMissedThreshold, + ObjectReplicationReplicatedAfterThreshold, + ObjectReplicationNotTracked, + ObjectRestorePost, + ObjectRestoreCompleted, + ObjectTransitionFailed, + ObjectTransitionComplete, + ObjectManyVersions, + ObjectLargeVersions, + PrefixManyFolders, + IlmDelMarkerExpirationDelete, + ObjectAccessedAll, + ObjectCreatedAll, + ObjectRemovedAll, + ObjectReplicationAll, + ObjectRestoreAll, + ObjectTransitionAll, + ObjectScannerAll, + Everything, +} + +impl Name { + pub fn expand(&self) -> Vec { + match self { + Name::ObjectAccessedAll => vec![ + Name::ObjectAccessedGet, + Name::ObjectAccessedHead, + Name::ObjectAccessedGetRetention, + Name::ObjectAccessedGetLegalHold, + Name::ObjectAccessedAttributes, + ], + Name::ObjectCreatedAll => vec![ + Name::ObjectCreatedCompleteMultipartUpload, + Name::ObjectCreatedCopy, + Name::ObjectCreatedPost, + Name::ObjectCreatedPut, + Name::ObjectCreatedPutRetention, + Name::ObjectCreatedPutLegalHold, + Name::ObjectCreatedPutTagging, + Name::ObjectCreatedDeleteTagging, + ], + Name::ObjectRemovedAll => vec![ + Name::ObjectRemovedDelete, + Name::ObjectRemovedDeleteMarkerCreated, + Name::ObjectRemovedNoOp, + Name::ObjectRemovedDeleteAllVersions, + ], + Name::ObjectReplicationAll => vec![ + Name::ObjectReplicationFailed, + Name::ObjectReplicationComplete, + Name::ObjectReplicationNotTracked, + Name::ObjectReplicationMissedThreshold, + Name::ObjectReplicationReplicatedAfterThreshold, + ], + Name::ObjectRestoreAll => vec![Name::ObjectRestorePost, Name::ObjectRestoreCompleted], + Name::ObjectTransitionAll => { + vec![Name::ObjectTransitionFailed, Name::ObjectTransitionComplete] + } + Name::ObjectScannerAll => vec![Name::ObjectManyVersions, Name::ObjectLargeVersions, Name::PrefixManyFolders], + Name::Everything => (1..=Name::IlmDelMarkerExpirationDelete as u32) + .map(|i| Name::from_repr(i).unwrap()) + .collect(), + _ => vec![*self], + } + } + + pub fn mask(&self) -> u64 { + if (*self as u32) < Name::ObjectAccessedAll as u32 { + 1 << (*self as u32 - 1) + } else { + self.expand().iter().fold(0, |acc, n| acc | (1 << (*n as u32 - 1))) + } + } + + fn from_repr(discriminant: u32) -> Option { + match discriminant { + 1 => Some(Name::ObjectAccessedGet), + 2 => Some(Name::ObjectAccessedGetRetention), + 3 => Some(Name::ObjectAccessedGetLegalHold), + 4 => Some(Name::ObjectAccessedHead), + 5 => Some(Name::ObjectAccessedAttributes), + 6 => Some(Name::ObjectCreatedCompleteMultipartUpload), + 7 => Some(Name::ObjectCreatedCopy), + 8 => Some(Name::ObjectCreatedPost), + 9 => Some(Name::ObjectCreatedPut), + 10 => Some(Name::ObjectCreatedPutRetention), + 11 => Some(Name::ObjectCreatedPutLegalHold), + 12 => Some(Name::ObjectCreatedPutTagging), + 13 => Some(Name::ObjectCreatedDeleteTagging), + 14 => Some(Name::ObjectRemovedDelete), + 15 => Some(Name::ObjectRemovedDeleteMarkerCreated), + 16 => Some(Name::ObjectRemovedDeleteAllVersions), + 17 => Some(Name::ObjectRemovedNoOp), + 18 => Some(Name::BucketCreated), + 19 => Some(Name::BucketRemoved), + 20 => Some(Name::ObjectReplicationFailed), + 21 => Some(Name::ObjectReplicationComplete), + 22 => Some(Name::ObjectReplicationMissedThreshold), + 23 => Some(Name::ObjectReplicationReplicatedAfterThreshold), + 24 => Some(Name::ObjectReplicationNotTracked), + 25 => Some(Name::ObjectRestorePost), + 26 => Some(Name::ObjectRestoreCompleted), + 27 => Some(Name::ObjectTransitionFailed), + 28 => Some(Name::ObjectTransitionComplete), + 29 => Some(Name::ObjectManyVersions), + 30 => Some(Name::ObjectLargeVersions), + 31 => Some(Name::PrefixManyFolders), + 32 => Some(Name::IlmDelMarkerExpirationDelete), + 33 => Some(Name::ObjectAccessedAll), + 34 => Some(Name::ObjectCreatedAll), + 35 => Some(Name::ObjectRemovedAll), + 36 => Some(Name::ObjectReplicationAll), + 37 => Some(Name::ObjectRestoreAll), + 38 => Some(Name::ObjectTransitionAll), + 39 => Some(Name::ObjectScannerAll), + 40 => Some(Name::Everything), + _ => None, + } + } +} diff --git a/crates/event-notifier/src/event_name.rs b/crates/event-notifier/src/event_name.rs deleted file mode 100644 index c60cce253..000000000 --- a/crates/event-notifier/src/event_name.rs +++ /dev/null @@ -1,79 +0,0 @@ -use crate::error::Error; -use serde::{Deserialize, Serialize}; -use std::str::FromStr; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[repr(u64)] -pub enum EventName { - // 对象操作事件 - ObjectCreatedAll = 1 << 0, - ObjectCreatedPut = 1 << 1, - ObjectCreatedPost = 1 << 2, - ObjectCreatedCopy = 1 << 3, - ObjectCreatedCompleteMultipartUpload = 1 << 4, - - ObjectRemovedAll = 1 << 5, - ObjectRemovedDelete = 1 << 6, - ObjectRemovedDeleteMarkerCreated = 1 << 7, - - ObjectAccessedAll = 1 << 8, - ObjectAccessedGet = 1 << 9, - ObjectAccessedHead = 1 << 10, - - ObjectRestoredAll = 1 << 11, - ObjectRestoredPost = 1 << 12, - ObjectRestoredCompleted = 1 << 13, - - ReplicationAll = 1 << 14, - ReplicationFailed = 1 << 15, - ReplicationComplete = 1 << 16, -} - -impl EventName { - pub fn mask(&self) -> u64 { - *self as u64 - } - - pub fn expand(&self) -> Vec { - match self { - EventName::ObjectCreatedAll => vec![ - EventName::ObjectCreatedPut, - EventName::ObjectCreatedPost, - EventName::ObjectCreatedCopy, - EventName::ObjectCreatedCompleteMultipartUpload, - ], - EventName::ObjectRemovedAll => vec![EventName::ObjectRemovedDelete, EventName::ObjectRemovedDeleteMarkerCreated], - EventName::ObjectAccessedAll => vec![EventName::ObjectAccessedGet, EventName::ObjectAccessedHead], - EventName::ObjectRestoredAll => vec![EventName::ObjectRestoredPost, EventName::ObjectRestoredCompleted], - EventName::ReplicationAll => vec![EventName::ReplicationFailed, EventName::ReplicationComplete], - _ => vec![*self], - } - } -} - -impl FromStr for EventName { - type Err = Error; - - fn from_str(s: &str) -> Result { - match s { - "s3:ObjectCreated:*" => Ok(EventName::ObjectCreatedAll), - "s3:ObjectCreated:Put" => Ok(EventName::ObjectCreatedPut), - "s3:ObjectCreated:Post" => Ok(EventName::ObjectCreatedPost), - "s3:ObjectCreated:Copy" => Ok(EventName::ObjectCreatedCopy), - "s3:ObjectCreated:CompleteMultipartUpload" => Ok(EventName::ObjectCreatedCompleteMultipartUpload), - "s3:ObjectRemoved:*" => Ok(EventName::ObjectRemovedAll), - "s3:ObjectRemoved:Delete" => Ok(EventName::ObjectRemovedDelete), - "s3:ObjectRemoved:DeleteMarkerCreated" => Ok(EventName::ObjectRemovedDeleteMarkerCreated), - "s3:ObjectAccessed:*" => Ok(EventName::ObjectAccessedAll), - "s3:ObjectAccessed:Get" => Ok(EventName::ObjectAccessedGet), - "s3:ObjectAccessed:Head" => Ok(EventName::ObjectAccessedHead), - "s3:ObjectRestored:*" => Ok(EventName::ObjectRestoredAll), - "s3:ObjectRestored:Post" => Ok(EventName::ObjectRestoredPost), - "s3:ObjectRestored:Completed" => Ok(EventName::ObjectRestoredCompleted), - "s3:Replication:*" => Ok(EventName::ReplicationAll), - "s3:Replication:Failed" => Ok(EventName::ReplicationFailed), - "s3:Replication:Complete" => Ok(EventName::ReplicationComplete), - _ => Err(Error::InvalidEventName(format!("Unrecognized event name: {}", s))), - } - } -} diff --git a/crates/event-notifier/src/global.rs b/crates/event-notifier/src/global.rs new file mode 100644 index 000000000..a14380b39 --- /dev/null +++ b/crates/event-notifier/src/global.rs @@ -0,0 +1,100 @@ +use crate::{ChannelAdapter, Error, Event, NotificationConfig, NotificationSystem}; +use std::sync::Arc; +use tokio::sync::{Mutex, OnceCell}; + +static GLOBAL_SYSTEM: OnceCell>> = OnceCell::const_new(); +static INITIALIZED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +/// initialize the global notification system +pub async fn initialize(config: NotificationConfig) -> Result<(), Error> { + if INITIALIZED.swap(true, std::sync::atomic::Ordering::SeqCst) { + return Err(Error::custom("notify the system has been initialized")); + } + + let system = Arc::new(Mutex::new(NotificationSystem::new(config).await?)); + GLOBAL_SYSTEM + .set(system) + .map_err(|_| Error::custom("unable to set up global notification system"))?; + Ok(()) +} + +/// start the global notification system +pub async fn start(adapters: Vec>) -> Result<(), Error> { + let system = get_system().await?; + + // create a new task to run the system + let system_clone = Arc::clone(&system); + tokio::spawn(async move { + let mut system_guard = system_clone.lock().await; + if let Err(e) = system_guard.start(adapters).await { + tracing::error!("notify the system to start failed: {}", e); + } + }); + + Ok(()) +} + +/// Initialize and start the global notification system +/// +/// This method combines the functions of `initialize` and `start` to provide one-step setup: +/// - initialize system configuration +/// - create an adapter +/// - start event listening +/// +/// # Example +/// +/// ```rust +/// use rustfs_event_notifier::{initialize_and_start, NotificationConfig}; +/// +/// #[tokio::main] +/// async fn main() -> Result<(), rustfs_event_notifier::Error> { +/// let config = NotificationConfig { +/// store_path: "./events".to_string(), +/// channel_capacity: 100, +/// adapters: vec![/* 适配器配置 */], +/// http: Default::default(), +/// }; +/// +/// // complete initialization and startup in one step +/// initialize_and_start(config).await?; +/// Ok(()) +/// } +/// ``` +pub async fn initialize_and_start(config: NotificationConfig) -> Result<(), Error> { + // initialize the system first + initialize(config.clone()).await?; + + // create an adapter + let adapters = crate::create_adapters(&config.adapters).expect("failed to create adapters"); + + // start the system + start(adapters).await?; + + Ok(()) +} + +/// send events to notification system +pub async fn send_event(event: Event) -> Result<(), Error> { + let system = get_system().await?; + let system_guard = system.lock().await; + system_guard.send_event(event).await +} + +/// turn off the notification system +pub fn shutdown() -> Result<(), Error> { + if let Some(system) = GLOBAL_SYSTEM.get() { + let system_guard = system.blocking_lock(); + system_guard.shutdown(); + Ok(()) + } else { + Err(Error::custom("notification system not initialized")) + } +} + +/// get system instance +async fn get_system() -> Result>, Error> { + GLOBAL_SYSTEM + .get() + .cloned() + .ok_or_else(|| Error::custom("notification system not initialized")) +} diff --git a/crates/event-notifier/src/lib.rs b/crates/event-notifier/src/lib.rs index fbfc9f863..9ba226c61 100644 --- a/crates/event-notifier/src/lib.rs +++ b/crates/event-notifier/src/lib.rs @@ -1,80 +1,131 @@ -/// RustFS Event Notifier -/// This crate provides a simple event notification system for RustFS. -/// It allows for the registration of event handlers and the triggering of events. -/// +mod adapter; +mod bus; +mod config; mod error; mod event; -mod event_name; -mod notifier; -mod rules; -mod stats; -mod target; -mod target_entry; +mod global; +mod producer; +mod store; -pub fn add(left: u64, right: u64) -> u64 { - left + right +pub use adapter::create_adapters; +#[cfg(feature = "kafka")] +pub use adapter::kafka::KafkaAdapter; +#[cfg(feature = "mqtt")] +pub use adapter::mqtt::MqttAdapter; +#[cfg(feature = "webhook")] +pub use adapter::webhook::WebhookAdapter; +pub use adapter::ChannelAdapter; +pub use bus::event_bus; +#[cfg(feature = "http-producer")] +pub use config::HttpProducerConfig; +#[cfg(feature = "kafka")] +pub use config::KafkaConfig; +#[cfg(feature = "mqtt")] +pub use config::MqttConfig; +#[cfg(feature = "webhook")] +pub use config::WebhookConfig; +pub use config::{AdapterConfig, NotificationConfig}; +pub use error::Error; + +pub use event::{Bucket, Event, EventBuilder, Identity, Log, Metadata, Name, Object, Source}; +pub use global::{initialize, initialize_and_start, send_event, shutdown, start}; +pub use store::EventStore; + +#[cfg(feature = "http-producer")] +pub use producer::http::HttpProducer; +#[cfg(feature = "http-producer")] +pub use producer::EventProducer; + +use std::sync::Arc; +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; + +/// The `NotificationSystem` struct represents the notification system. +/// It manages the event bus and the adapters. +/// It is responsible for sending and receiving events. +/// It also handles the shutdown process. +pub struct NotificationSystem { + tx: mpsc::Sender, + rx: Option>, + store: Arc, + shutdown: CancellationToken, + #[cfg(feature = "http-producer")] + http_config: HttpProducerConfig, } -#[cfg(test)] -mod tests { - use super::*; - use crate::event::{Bucket, Event, Identity, Metadata, Object, Source}; - use crate::target::{TargetID, TargetList}; - use std::collections::HashMap; +impl NotificationSystem { + /// Creates a new `NotificationSystem` instance. + pub async fn new(config: NotificationConfig) -> Result { + let (tx, rx) = mpsc::channel::(config.channel_capacity); + let store = Arc::new(EventStore::new(&config.store_path).await?); + let shutdown = CancellationToken::new(); - #[test] - fn it_works() { - let result = add(2, 2); - assert_eq!(result, 4); + let restored_logs = store.load_logs().await?; + for log in restored_logs { + for event in log.records { + // For example, where the send method may return a SendError when calling it + tx.send(event).await.map_err(|e| Error::ChannelSend(Box::new(e)))?; + } + } + + Ok(Self { + tx, + rx: Some(rx), + store, + shutdown, + #[cfg(feature = "http-producer")] + http_config: config.http, + }) } - #[tokio::main] - async fn main() { - let target_list = TargetList::new(); - let event = Event { - event_version: "1.0".to_string(), - event_source: "aws:s3".to_string(), - aws_region: "us-west-2".to_string(), - event_time: "2023-10-01T12:00:00Z".to_string(), - event_name: "PutObject".to_string(), - user_identity: Identity { - principal_id: "user123".to_string(), + /// Starts the notification system. + /// It initializes the event bus and the producer. + pub async fn start(&mut self, adapters: Vec>) -> Result<(), Error> { + let rx = self.rx.take().ok_or_else(|| Error::EventBusStarted)?; + + let shutdown_clone = self.shutdown.clone(); + let store_clone = self.store.clone(); + let bus_handle = tokio::spawn(async move { + if let Err(e) = event_bus(rx, adapters, store_clone, shutdown_clone).await { + tracing::error!("Event bus failed: {}", e); + } + }); + + #[cfg(feature = "http-producer")] + { + let producer = HttpProducer::new(self.tx.clone(), self.http_config.port); + producer.start().await?; + } + + tokio::select! { + result = bus_handle => { + result.map_err(Error::JoinError)?; + Ok(()) }, - request_parameters: HashMap::new(), - response_elements: HashMap::new(), - s3: Metadata { - schema_version: "1.0".to_string(), - configuration_id: "config123".to_string(), - bucket: Bucket { - name: "my-bucket".to_string(), - owner_identity: Identity { - principal_id: "owner123".to_string(), - }, - arn: "arn:aws:s3:::my-bucket".to_string(), - }, - object: Object { - key: "my-object.txt".to_string(), - version_id: Some("version123".to_string()), - sequencer: "seq123".to_string(), - size: Some(1024), - etag: Some("etag123".to_string()), - content_type: Some("text/plain".to_string()), - user_metadata: HashMap::new(), - }, - }, - source: Source { - host: "localhost".to_string(), - user_agent: "RustFS/1.0".to_string(), - }, - }; - let target_ids: &[TargetID] = &["".to_string()]; - // 发送事件 - let results = target_list.send(event, &target_ids).await; - println!("result len:{:?}", results.len()); - // 获取统计信息 - let stats = target_list.get_stats(); - for (id, stat) in stats { - println!("Target {}: {} events, {} failed", id, stat.total_events, stat.failed_events); + _ = self.shutdown.cancelled() => { + tracing::info!("System shutdown triggered"); + Ok(()) + } } } + + /// Sends an event to the notification system. + /// This method is used to send events to the event bus. + pub async fn send_event(&self, event: Event) -> Result<(), Error> { + self.tx.send(event).await.map_err(|e| Error::ChannelSend(Box::new(e)))?; + Ok(()) + } + + /// Shuts down the notification system. + /// This method is used to cancel the event bus and producer tasks. + pub fn shutdown(&self) { + self.shutdown.cancel(); + } + + /// Sets the HTTP port for the notification system. + /// This method is used to change the port for the HTTP producer. + #[cfg(feature = "http-producer")] + pub fn set_http_port(&mut self, port: u16) { + self.http_config.port = port; + } } diff --git a/crates/event-notifier/src/notifier.rs b/crates/event-notifier/src/notifier.rs deleted file mode 100644 index 7b81a24d2..000000000 --- a/crates/event-notifier/src/notifier.rs +++ /dev/null @@ -1,47 +0,0 @@ -// src/notifier.rs -use crate::error::Error; -use crate::event::Event; -use crate::rules::{RulesMap, TargetIDSet}; -use crate::target::TargetList; -use std::collections::HashMap; -use std::sync::{Arc, RwLock}; -use tokio::sync::broadcast; - -pub struct EventNotifier { - target_list: Arc, - rules: RwLock>, - tx: broadcast::Sender, -} - -impl EventNotifier { - pub fn new(capacity: usize) -> Self { - let (tx, _) = broadcast::channel(capacity); - Self { - target_list: Arc::new(TargetList::new()), - rules: RwLock::new(HashMap::new()), - tx, - } - } - - pub async fn send(&self, event: Event) -> Result<(), Error> { - let rules = self - .rules - .read() - .map_err(|_| Error::StoreError("Failed to read rules".to_string()))?; - // 检查规则匹配 - let target_ids = if let Some(rules_map) = rules.get(&event.s3.bucket.name) { - rules_map.match_targets(event.s3.event_name, &event.s3.object.key) - } else { - TargetIDSet::new() - }; - - // 发送事件 - if !target_ids.is_empty() { - self.target_list.send(event.clone(), &target_ids).await; - } - - // 广播给监听者 - let _ = self.tx.send(event); - Ok(()) - } -} diff --git a/crates/event-notifier/src/producer.rs b/crates/event-notifier/src/producer.rs new file mode 100644 index 000000000..fc53b9462 --- /dev/null +++ b/crates/event-notifier/src/producer.rs @@ -0,0 +1,83 @@ +use crate::Error; +use crate::Event; +use async_trait::async_trait; + +/// event producer characteristics +#[allow(dead_code)] +#[async_trait] +pub trait EventProducer: Send + Sync { + /// start producer services + async fn start(&self) -> Result<(), Error>; + /// stop producer services + async fn stop(&self) -> Result<(), Error>; + /// send a single event + async fn send_event(&self, event: Event) -> Result<(), Error>; +} + +#[cfg(feature = "http-producer")] +pub mod http { + use super::*; + use axum::{routing::post, Json, Router}; + use std::sync::Arc; + use tokio::sync::mpsc; + + #[derive(Clone)] + pub struct HttpProducer { + tx: mpsc::Sender, + port: u16, + shutdown: Arc, + } + + impl HttpProducer { + pub fn new(tx: mpsc::Sender, port: u16) -> Self { + Self { + tx, + port, + shutdown: Arc::new(tokio::sync::Notify::new()), + } + } + } + + #[async_trait] + impl EventProducer for HttpProducer { + async fn start(&self) -> Result<(), Error> { + let producer = self.clone(); + let app = Router::new().route( + "/event", + post(move |event| { + let prod = producer.clone(); + async move { handle_event(event, prod).await } + }), + ); + + let addr = format!("0.0.0.0:{}", self.port); + let listener = tokio::net::TcpListener::bind(&addr).await?; + + let shutdown = self.shutdown.clone(); + tokio::select! { + result = axum::serve(listener, app) => { + result?; + Ok(()) + } + _ = shutdown.notified() => Ok(()) + } + } + + async fn stop(&self) -> Result<(), Error> { + self.shutdown.notify_one(); + Ok(()) + } + + async fn send_event(&self, event: Event) -> Result<(), Error> { + self.tx.send(event).await.map_err(|e| Error::ChannelSend(Box::new(e)))?; + Ok(()) + } + } + + async fn handle_event(Json(event): Json, producer: HttpProducer) -> Result<(), axum::http::StatusCode> { + producer + .send_event(event) + .await + .map_err(|_| axum::http::StatusCode::INTERNAL_SERVER_ERROR) + } +} diff --git a/crates/event-notifier/src/rules.rs b/crates/event-notifier/src/rules.rs deleted file mode 100644 index 671bcfb53..000000000 --- a/crates/event-notifier/src/rules.rs +++ /dev/null @@ -1,191 +0,0 @@ -// src/rules.rs -use crate::event_name::EventName; -use crate::target::TargetID; -use std::collections::{HashMap, HashSet}; -use wildmatch::WildMatch; - -/// TargetIDSet represents a set of target IDs -#[derive(Debug, Clone, Default)] -pub struct TargetIDSet(HashSet); - -impl TargetIDSet { - pub fn new() -> Self { - Self(HashSet::new()) - } - - pub fn insert(&mut self, target_id: TargetID) -> bool { - self.0.insert(target_id) - } - - pub fn contains(&self, target_id: &TargetID) -> bool { - self.0.contains(target_id) - } - - pub fn remove(&mut self, target_id: &TargetID) -> bool { - self.0.remove(target_id) - } - - pub fn len(&self) -> usize { - self.0.len() - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - pub fn clear(&mut self) { - self.0.clear() - } - - pub fn iter(&self) -> impl Iterator { - self.0.iter() - } - - pub fn extend>(&mut self, iter: I) { - self.0.extend(iter) - } - - pub fn union(&self, other: &TargetIDSet) -> TargetIDSet { - let mut result = self.clone(); - result.extend(other.0.iter().cloned()); - result - } - - pub fn difference(&self, other: &TargetIDSet) -> TargetIDSet { - let mut result = TargetIDSet::new(); - for id in self.0.iter() { - if !other.contains(id) { - result.insert(id.clone()); - } - } - result - } -} - -impl FromIterator for TargetIDSet { - fn from_iter>(iter: I) -> Self { - let mut set = TargetIDSet::new(); - set.extend(iter); - set - } -} - -#[derive(Debug, Clone)] -pub struct Rules { - patterns: HashMap, -} - -impl Rules { - pub fn new() -> Self { - Self { - patterns: HashMap::new(), - } - } - - pub fn add(&mut self, pattern: String, target_id: TargetID) { - let entry = self.patterns.entry(pattern).or_insert_with(TargetIDSet::new); - entry.insert(target_id); - } - - pub fn match_object(&self, object_name: &str) -> bool { - for pattern in self.patterns.keys() { - if WildMatch::new(pattern).matches(object_name) { - return true; - } - } - false - } - - pub fn match_targets(&self, object_name: &str) -> TargetIDSet { - let mut target_ids = TargetIDSet::new(); - - for (pattern, ids) in &self.patterns { - if WildMatch::new(pattern).matches(object_name) { - target_ids.extend(ids.iter().cloned()); - } - } - - target_ids - } -} - -#[derive(Debug, Default)] -pub struct RulesMap { - rules: HashMap, -} - -impl RulesMap { - pub fn new() -> Self { - Self { rules: HashMap::new() } - } - - pub fn add(&mut self, events: &[EventName], pattern: String, target_id: TargetID) { - for &event in events { - for expanded_event in event.expand() { - let rules = self.rules.entry(expanded_event).or_insert_with(Rules::new); - rules.add(pattern.clone(), target_id.clone()); - } - } - } - - pub fn match_simple(&self, event: EventName, object_name: &str) -> bool { - match self.rules.get(&event) { - Some(rules) => rules.match_object(object_name), - None => false, - } - } - - pub fn match_targets(&self, event: EventName, object_name: &str) -> TargetIDSet { - match self.rules.get(&event) { - Some(rules) => rules.match_targets(object_name), - None => TargetIDSet::new(), - } - } -} - -impl Clone for RulesMap { - fn clone(&self) -> Self { - Self { - rules: self.rules.clone(), - } - } -} - -impl Default for Rules { - fn default() -> Self { - Self::new() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::event_name::EventName; - - #[test] - fn test_rules() { - let mut rules = Rules::new(); - rules.add("*.txt".to_string(), "target1".to_string()); - rules.add("*.jpg".to_string(), "target2".to_string()); - - assert!(rules.match_object("file.txt")); - assert!(!rules.match_object("file.pdf")); - assert!(rules.match_object("image.jpg")); - assert!(!rules.match_object("image.png")); - - let target_ids = rules.match_targets("file.txt"); - assert_eq!(target_ids.len(), 1); - assert!(target_ids.iter().any(|id| id == "target1")); - - let mut rules_map = RulesMap::new(); - let target_id = "target1".to_string(); - // 添加规则 - rules_map.add(&[EventName::ObjectCreatedAll], String::from("images/*"), target_id); - - // 匹配对象 - let matches = rules_map.match_simple(EventName::ObjectCreatedPut, "images/photo.jpg"); // returns true - - // 获取匹配的目标 - let target_ids = rules_map.match_targets(EventName::ObjectCreatedPut, "images/photo.jpg"); - } -} diff --git a/crates/event-notifier/src/stats.rs b/crates/event-notifier/src/stats.rs deleted file mode 100644 index 570bbeca5..000000000 --- a/crates/event-notifier/src/stats.rs +++ /dev/null @@ -1,81 +0,0 @@ -// src/stats.rs -use crate::target::TargetID; -use parking_lot::RwLock; -use std::collections::HashMap; -use std::sync::atomic::{AtomicI64, Ordering}; -use std::sync::Arc; - -#[derive(Debug, Default)] -pub struct TargetStat { - pub current_send_calls: AtomicI64, - pub total_events: AtomicI64, - pub failed_events: AtomicI64, - pub current_queue: AtomicI64, -} - -impl TargetStat { - pub fn inc_send_calls(&self) { - self.current_send_calls.fetch_add(1, Ordering::Relaxed); - } - - pub fn dec_send_calls(&self) { - self.current_send_calls.fetch_sub(1, Ordering::Relaxed); - } - - pub fn inc_total_events(&self) { - self.total_events.fetch_add(1, Ordering::Relaxed); - } - - pub fn inc_failed_events(&self) { - self.failed_events.fetch_add(1, Ordering::Relaxed); - } - - pub fn set_queue_size(&self, size: i64) { - self.current_queue.store(size, Ordering::Relaxed); - } -} - -#[derive(Default)] -pub struct TargetStats { - stats: RwLock>>, -} - -impl TargetStats { - pub fn new() -> Self { - Self::default() - } - - pub fn get_or_create(&self, id: &TargetID) -> Arc { - let mut stats = self.stats.write(); - stats - .entry(id.clone()) - .or_insert_with(|| Arc::new(TargetStat::default())) - .clone() - } - - pub fn get_stats(&self) -> HashMap { - self.stats - .read() - .iter() - .map(|(id, stat)| { - ( - id.clone(), - TargetSnapshot { - current_send_calls: stat.current_send_calls.load(Ordering::Relaxed), - total_events: stat.total_events.load(Ordering::Relaxed), - failed_events: stat.failed_events.load(Ordering::Relaxed), - current_queue: stat.current_queue.load(Ordering::Relaxed), - }, - ) - }) - .collect() - } -} - -#[derive(Debug, Clone)] -pub struct TargetSnapshot { - pub current_send_calls: i64, - pub total_events: i64, - pub failed_events: i64, - pub current_queue: i64, -} diff --git a/crates/event-notifier/src/store.rs b/crates/event-notifier/src/store.rs new file mode 100644 index 000000000..995cdc831 --- /dev/null +++ b/crates/event-notifier/src/store.rs @@ -0,0 +1,53 @@ +use crate::Error; +use crate::Log; +use chrono::Utc; +use std::sync::Arc; +use tokio::fs::{create_dir_all, File, OpenOptions}; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader, BufWriter}; +use tokio::sync::RwLock; + +/// `EventStore` is a struct that manages the storage of event logs. +pub struct EventStore { + path: String, + lock: Arc>, +} + +impl EventStore { + pub async fn new(path: &str) -> Result { + create_dir_all(path).await?; + Ok(Self { + path: path.to_string(), + lock: Arc::new(RwLock::new(())), + }) + } + + pub async fn save_logs(&self, logs: &[Log]) -> Result<(), Error> { + let _guard = self.lock.write().await; + let file_path = format!("{}/events_{}.jsonl", self.path, Utc::now().timestamp()); + let file = OpenOptions::new().create(true).append(true).open(&file_path).await?; + let mut writer = BufWriter::new(file); + for log in logs { + let line = serde_json::to_string(log)?; + writer.write_all(line.as_bytes()).await?; + writer.write_all(b"\n").await?; + } + writer.flush().await?; + Ok(()) + } + + pub async fn load_logs(&self) -> Result, Error> { + let _guard = self.lock.read().await; + let mut logs = Vec::new(); + let mut entries = tokio::fs::read_dir(&self.path).await?; + while let Some(entry) = entries.next_entry().await? { + let file = File::open(entry.path()).await?; + let reader = BufReader::new(file); + let mut lines = reader.lines(); + while let Some(line) = lines.next_line().await? { + let log: Log = serde_json::from_str(&line)?; + logs.push(log); + } + } + Ok(logs) + } +} diff --git a/crates/event-notifier/src/target.rs b/crates/event-notifier/src/target.rs deleted file mode 100644 index 68b575b74..000000000 --- a/crates/event-notifier/src/target.rs +++ /dev/null @@ -1,66 +0,0 @@ -// src/target.rs -use crate::error::{Error, Result}; -use crate::event::Event; -use crate::stats::{TargetSnapshot, TargetStats}; -use async_trait::async_trait; -use std::collections::HashMap; -use std::sync::{Arc, RwLock}; - -pub type TargetID = String; - -#[async_trait] -pub trait Target: Send + Sync { - fn id(&self) -> TargetID; - async fn send(&self, event: Event) -> Result<()>; - async fn is_active(&self) -> bool; - async fn close(&self) -> Result<()>; -} - -pub struct TargetList { - targets: Arc>>>, - stats: Arc, -} - -impl TargetList { - pub fn new() -> Self { - Self { - targets: Arc::new(RwLock::new(HashMap::new())), - stats: Arc::default(), - } - } - - pub async fn send(&self, event: Event, target_ids: &[TargetID]) -> Vec> { - let mut results = Vec::with_capacity(target_ids.len()); - let targets = self.targets.read().unwrap(); - - for id in target_ids { - let target = match targets.get(id) { - Some(t) => t, - None => { - results.push(Err(Error::TargetNotFound(id.to_string()))); - continue; - } - }; - - let stats = self.stats.get_or_create(id); - stats.inc_send_calls(); - - let result = target.send(event.clone()).await; - - stats.dec_send_calls(); - stats.inc_total_events(); - - if result.is_err() { - stats.inc_failed_events(); - } - - results.push(result); - } - - results - } - - pub fn get_stats(&self) -> HashMap { - self.stats.get_stats() - } -} diff --git a/crates/event-notifier/src/target_entry/mod.rs b/crates/event-notifier/src/target_entry/mod.rs deleted file mode 100644 index a2a1c2754..000000000 --- a/crates/event-notifier/src/target_entry/mod.rs +++ /dev/null @@ -1 +0,0 @@ -mod webhook; diff --git a/crates/event-notifier/src/target_entry/webhook.rs b/crates/event-notifier/src/target_entry/webhook.rs deleted file mode 100644 index 1736ad36b..000000000 --- a/crates/event-notifier/src/target_entry/webhook.rs +++ /dev/null @@ -1,161 +0,0 @@ -use crate::event::Event; -use crate::target::{Target, TargetID}; -use async_trait::async_trait; -use reqwest::{header, Client}; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::sync::Arc; -use std::time::Duration; -use thiserror::Error; -use tokio::io::AsyncWriteExt; -use tokio::sync::Mutex; - -#[derive(Error, Debug)] -pub enum WebhookError { - #[error("HTTP client error: {0}")] - HttpClient(#[from] reqwest::Error), - #[error("JSON serialization error: {0}")] - Json(#[from] serde_json::Error), - #[error("Store error: {0}")] - Store(#[from] Box), - #[error("Webhook request failed after retries")] - RequestFailed, -} - -type Result = std::result::Result; - -/// Webhook 目标配置 - -/// Webhook 目标配置 -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WebhookConfig { - /// Webhook endpoint URL - pub endpoint: String, - /// 认证令牌 - pub auth_token: Option, - /// 自定义请求头 - pub custom_headers: Option>, - /// 重试次数 - pub max_retries: u32, - /// 连接超时时间 (秒) - pub timeout: u64, -} - -/// Webhook Target 实现 -pub struct Webhook { - /// 目标 ID - id: TargetID, - /// HTTP 客户端 - client: Client, - /// 配置信息 - config: WebhookConfig, - /// 事件存储 - store: Arc>, -} - -#[async_trait] -impl Target for Webhook { - /// 返回目标 ID - fn id(&self) -> &TargetID { - &self.id - } - - /// 检查 Webhook 是否可用 - async fn is_active(&self) -> Result { - // 发送测试请求验证连接 - let resp = self.client.get(&self.config.endpoint).send().await?; - - Ok(resp.status().is_success()) - } - - /// 保存事件到存储并异步发送 - async fn save(&self, event: Event) -> Result<()> { - // 序列化事件 - let event_json = serde_json::to_string(&event)?; - - // 存储事件 - self.store.lock().await.save(event_json)?; - - // 异步发送 - tokio::spawn(self.send_event(event)); - - Ok(()) - } - - /// 从存储发送事件 - async fn send(&self, key: StoreKey) -> Result<()> { - // 从存储获取事件 - let event_json = self.store.lock().await.get(&key)?; - let event: Event = serde_json::from_str(&event_json)?; - - // 发送事件到 webhook - self.send_event(event).await?; - - // 发送成功后删除 - self.store.lock().await.delete(&key)?; - - Ok(()) - } - - /// 返回事件存储 - fn store(&self) -> Arc> { - self.store.clone() - } - - /// 关闭 Target - async fn close(&self) -> Result<()> { - // 等待所有事件发送完成 - self.store.lock().await.flush()?; - Ok(()) - } -} - -impl Webhook { - /// 创建新的 Webhook Target - pub fn new(id: TargetID, config: WebhookConfig, store: Arc>) -> Result { - // 构建 HTTP 客户端 - let client = Client::builder().timeout(Duration::from_secs(config.timeout)).build()?; - - Ok(Self { - id, - client, - config, - store, - }) - } - - /// 发送事件到 Webhook endpoint - async fn send_event(&self, event: Event) -> Result<()> { - let mut retries = 0; - loop { - // 构建请求 - let mut req = self.client.post(&self.config.endpoint).json(&event); - - // 添加认证头 - if let Some(token) = &self.config.auth_token { - req = req.header(header::AUTHORIZATION, format!("Bearer {}", token)); - } - - // 添加自定义头 - if let Some(headers) = &self.config.custom_headers { - for (key, value) in headers { - req = req.header(key, value); - } - } - - // 发送请求 - match req.send().await { - Ok(resp) if resp.status().is_success() => { - return Ok(()); - } - _ if retries < self.config.max_retries => { - retries += 1; - tokio::time::sleep(Duration::from_secs(1)).await; - continue; - } - Err(e) => return Err(WebhookError::HttpClient(e)), - _ => return Err(WebhookError::RequestFailed), - } - } - } -} diff --git a/crates/event-notifier/tests/integration.rs b/crates/event-notifier/tests/integration.rs new file mode 100644 index 000000000..2829b8b94 --- /dev/null +++ b/crates/event-notifier/tests/integration.rs @@ -0,0 +1,160 @@ +use rustfs_event_notifier::{AdapterConfig, NotificationSystem, WebhookConfig}; +use rustfs_event_notifier::{Bucket, Event, EventBuilder, Identity, Metadata, Name, Object, Source}; +use rustfs_event_notifier::{ChannelAdapter, WebhookAdapter}; +use std::collections::HashMap; +use std::sync::Arc; + +#[tokio::test] +async fn test_webhook_adapter() { + let adapter = WebhookAdapter::new(WebhookConfig { + endpoint: "http://localhost:8080/webhook".to_string(), + auth_token: None, + custom_headers: None, + max_retries: 1, + timeout: 5, + }); + + // create an s3 metadata object + let metadata = Metadata { + schema_version: "1.0".to_string(), + configuration_id: "test-config".to_string(), + bucket: Bucket { + name: "my-bucket".to_string(), + owner_identity: Identity { + principal_id: "owner123".to_string(), + }, + arn: "arn:aws:s3:::my-bucket".to_string(), + }, + object: Object { + key: "test.txt".to_string(), + size: Some(1024), + etag: Some("abc123".to_string()), + content_type: Some("text/plain".to_string()), + user_metadata: None, + version_id: None, + sequencer: "1234567890".to_string(), + }, + }; + + // create source object + let source = Source { + host: "localhost".to_string(), + port: "80".to_string(), + user_agent: "curl/7.68.0".to_string(), + }; + + // Create events using builder mode + let event = Event::builder() + .event_version("2.0") + .event_source("aws:s3") + .aws_region("us-east-1") + .event_time("2023-10-01T12:00:00.000Z") + .event_name(Name::ObjectCreatedPut) + .user_identity(Identity { + principal_id: "user123".to_string(), + }) + .request_parameters(HashMap::new()) + .response_elements(HashMap::new()) + .s3(metadata) + .source(source) + .channels(vec!["webhook".to_string()]) + .build() + .expect("failed to create event"); + + let result = adapter.send(&event).await; + assert!(result.is_err()); +} + +#[tokio::test] +async fn test_notification_system() { + let config = rustfs_event_notifier::NotificationConfig { + store_path: "./test_events".to_string(), + channel_capacity: 100, + adapters: vec![AdapterConfig::Webhook(WebhookConfig { + endpoint: "http://localhost:8080/webhook".to_string(), + auth_token: None, + custom_headers: None, + max_retries: 1, + timeout: 5, + })], + http: Default::default(), + }; + let system = Arc::new(tokio::sync::Mutex::new(NotificationSystem::new(config.clone()).await.unwrap())); + let adapters: Vec> = vec![Arc::new(WebhookAdapter::new(WebhookConfig { + endpoint: "http://localhost:8080/webhook".to_string(), + auth_token: None, + custom_headers: None, + max_retries: 1, + timeout: 5, + }))]; + + // create an s3 metadata object + let metadata = Metadata { + schema_version: "1.0".to_string(), + configuration_id: "test-config".to_string(), + bucket: Bucket { + name: "my-bucket".to_string(), + owner_identity: Identity { + principal_id: "owner123".to_string(), + }, + arn: "arn:aws:s3:::my-bucket".to_string(), + }, + object: Object { + key: "test.txt".to_string(), + size: Some(1024), + etag: Some("abc123".to_string()), + content_type: Some("text/plain".to_string()), + user_metadata: None, + version_id: None, + sequencer: "1234567890".to_string(), + }, + }; + + // create source object + let source = Source { + host: "localhost".to_string(), + port: "80".to_string(), + user_agent: "curl/7.68.0".to_string(), + }; + + // create a preconfigured builder with objects + let event = EventBuilder::for_object_creation(metadata, source) + .user_identity(Identity { + principal_id: "user123".to_string(), + }) + .event_time("2023-10-01T12:00:00.000Z") + .channels(vec!["webhook".to_string()]) + .build() + .expect("failed to create event"); + + { + let system_lock = system.lock().await; + system_lock.send_event(event).await.unwrap(); + } + + let system_clone = Arc::clone(&system); + let system_handle = tokio::spawn(async move { + let mut system = system_clone.lock().await; + system.start(adapters).await + }); + + // set 10 seconds timeout + match tokio::time::timeout(std::time::Duration::from_secs(10), system_handle).await { + Ok(result) => { + println!("System started successfully"); + assert!(result.is_ok()); + } + Err(_) => { + println!("System operation timed out, forcing shutdown"); + // create a new task to handle the timeout + let system = Arc::clone(&system); + tokio::spawn(async move { + if let Ok(guard) = system.try_lock() { + guard.shutdown(); + } + }); + // give the system some time to clean up resources + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + } + } +} diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index e2d088156..57da6e9d6 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -60,7 +60,7 @@ s3s.workspace = true serde.workspace = true serde_json.workspace = true serde_urlencoded = { workspace = true } -shadow-rs.workspace = true +shadow-rs = { workspace = true, features = ["build"] } tracing.workspace = true time = { workspace = true, features = ["parsing", "formatting", "serde"] } tokio-util.workspace = true @@ -103,5 +103,5 @@ hyper-util = { workspace = true, features = [ ] } transform-stream = { workspace = true } netif = "0.1.6" -shadow-rs.workspace = true +shadow-rs = { workspace = true, features = ["build"] } # pin-utils = "0.1.0" diff --git a/rustfs/src/config/mod.rs b/rustfs/src/config/mod.rs index 1efac71f7..e877ebada 100644 --- a/rustfs/src/config/mod.rs +++ b/rustfs/src/config/mod.rs @@ -103,6 +103,10 @@ pub struct Opt { #[arg(long, env = "RUSTFS_LICENSE")] pub license: Option, + + /// event notifier config file + #[arg(long, env = "RUSTFS_EVENT_CONFIG")] + pub event_config: Option, } // lazy_static::lazy_static! { From 3b6397012bd8d72a87bdc061d00f34a4a181da81 Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 21 Apr 2025 13:28:01 +0800 Subject: [PATCH 03/16] feat(event-notifier): improve notification system initialization safety - Add READY atomic flag to track full initialization status - Implement initialize_safe and start_safe methods with mutex protection - Add wait_until_ready function with configurable timeout - Create initialize_and_start_with_ready_check helper method - Replace sleep-based waiting with proper readiness checks - Add safety checks before sending events - Replace chrono with std::time for time handling - Update error handling to provide clear initialization status This change reduces race conditions in multi-threaded environments and ensures events are only processed when the system is fully ready. --- Cargo.lock | 16 +-- Cargo.toml | 2 +- crates/event-notifier/Cargo.toml | 3 +- crates/event-notifier/examples/simple.rs | 1 + crates/event-notifier/src/bus.rs | 4 +- crates/event-notifier/src/config.rs | 19 +-- crates/event-notifier/src/event.rs | 13 +- crates/event-notifier/src/global.rs | 132 +++++++++++++++++++-- crates/event-notifier/src/lib.rs | 106 +---------------- crates/event-notifier/src/notifier.rs | 101 ++++++++++++++++ crates/event-notifier/src/store.rs | 8 +- crates/event-notifier/tests/integration.rs | 1 + 12 files changed, 266 insertions(+), 140 deletions(-) create mode 100644 crates/event-notifier/src/notifier.rs diff --git a/Cargo.lock b/Cargo.lock index 799d7b6e8..3892d3af4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3046,10 +3046,10 @@ dependencies = [ ] [[package]] -name = "dotenv" -version = "0.15.0" +name = "dotenvy" +version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" [[package]] name = "dpi" @@ -7372,8 +7372,7 @@ version = "0.0.1" dependencies = [ "async-trait", "axum", - "chrono", - "dotenv", + "dotenvy", "figment", "rdkafka", "reqwest", @@ -7389,7 +7388,6 @@ dependencies = [ "tracing", "tracing-subscriber", "uuid", - "wildmatch", ] [[package]] @@ -9639,12 +9637,6 @@ dependencies = [ "rustix 0.38.44", ] -[[package]] -name = "wildmatch" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ce1ab1f8c62655ebe1350f589c61e505cf94d385bc6a12899442d9081e71fd" - [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 52bd76da0..d7eabe8b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,7 +51,7 @@ datafusion = "46.0.0" derive_builder = "0.20.2" dioxus = { version = "0.6.3", features = ["router"] } dirs = "6.0.0" -dotenv = "0.15.0" +dotenvy = "0.15.7" ecstore = { path = "./ecstore" } figment = { version = "0.10.19", features = ["toml", "yaml", "env"] } flatbuffers = "25.2.10" diff --git a/crates/event-notifier/Cargo.toml b/crates/event-notifier/Cargo.toml index 065cc6984..0c3bc3947 100644 --- a/crates/event-notifier/Cargo.toml +++ b/crates/event-notifier/Cargo.toml @@ -16,8 +16,7 @@ http-producer = ["dep:axum"] [dependencies] async-trait = { workspace = true } axum = { workspace = true, optional = true } -chrono = { workspace = true, features = ["serde"] } -dotenv = { workspace = true } +dotenvy = { workspace = true } figment = { workspace = true, features = ["toml", "yaml", "env"] } rdkafka = { workspace = true, features = ["tokio"], optional = true } reqwest = { workspace = true, optional = true } diff --git a/crates/event-notifier/examples/simple.rs b/crates/event-notifier/examples/simple.rs index 5324586ad..77e3b56af 100644 --- a/crates/event-notifier/examples/simple.rs +++ b/crates/event-notifier/examples/simple.rs @@ -14,6 +14,7 @@ async fn main() -> Result<(), Box> { let mut config = NotificationConfig { store_path: "./events".to_string(), channel_capacity: 100, + timeout: 50, adapters: vec![AdapterConfig::Webhook(WebhookConfig { endpoint: "http://localhost:8080/webhook".to_string(), auth_token: Some("secret-token".to_string()), diff --git a/crates/event-notifier/src/bus.rs b/crates/event-notifier/src/bus.rs index 0ef341ce6..55e5cd8b7 100644 --- a/crates/event-notifier/src/bus.rs +++ b/crates/event-notifier/src/bus.rs @@ -2,8 +2,8 @@ use crate::ChannelAdapter; use crate::Error; use crate::EventStore; use crate::{Event, Log}; -use chrono::Utc; use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; use tokio::sync::mpsc; use tokio_util::sync::CancellationToken; @@ -20,7 +20,7 @@ pub async fn event_bus( let mut pending_logs = Vec::new(); let mut current_log = Log { event_name: crate::event::Name::Everything, - key: Utc::now().timestamp().to_string(), + key: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs().to_string(), records: Vec::new(), }; diff --git a/crates/event-notifier/src/config.rs b/crates/event-notifier/src/config.rs index 823bf83da..af5f88998 100644 --- a/crates/event-notifier/src/config.rs +++ b/crates/event-notifier/src/config.rs @@ -89,6 +89,8 @@ pub struct NotificationConfig { pub store_path: String, #[serde(default = "default_channel_capacity")] pub channel_capacity: usize, + #[serde(default = "default_timeout")] + pub timeout: u64, pub adapters: Vec, #[serde(default)] pub http: HttpProducerConfig, @@ -99,6 +101,7 @@ impl Default for NotificationConfig { Self { store_path: default_store_path(), channel_capacity: default_channel_capacity(), + timeout: default_timeout(), adapters: Vec::new(), http: HttpProducerConfig::default(), } @@ -136,12 +139,10 @@ impl NotificationConfig { /// loading configuration from env file pub fn from_env_file(path: &str) -> Result { // loading env files - dotenv::from_path(path) - .map_err(|e| Error::ConfigError(format!("unable to load env file: {}", e)))?; + dotenvy::from_path(path).map_err(|e| Error::ConfigError(format!("unable to load env file: {}", e)))?; // Extract configuration from environment variables using figurement - let figment = - figment::Figment::new().merge(figment::providers::Env::prefixed("EVENT_NOTIF_")); + let figment = figment::Figment::new().merge(figment::providers::Env::prefixed("EVENT_NOTIF_")); Ok(figment.extract()?) } @@ -149,13 +150,15 @@ impl NotificationConfig { /// Provide temporary directories as default storage paths fn default_store_path() -> String { - std::env::temp_dir() - .join("event-notification") - .to_string_lossy() - .to_string() + std::env::temp_dir().join("event-notification").to_string_lossy().to_string() } /// Provides the recommended default channel capacity for high concurrency systems fn default_channel_capacity() -> usize { 10000 // Reasonable default values for high concurrency systems } + +/// Provides the recommended default timeout for high concurrency systems +fn default_timeout() -> u64 { + 50 // Reasonable default values for high concurrency systems +} diff --git a/crates/event-notifier/src/event.rs b/crates/event-notifier/src/event.rs index 00bc5983a..ce32aecca 100644 --- a/crates/event-notifier/src/event.rs +++ b/crates/event-notifier/src/event.rs @@ -1,19 +1,21 @@ use crate::Error; -use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use serde_with::{DeserializeFromStr, SerializeDisplay}; use smallvec::{smallvec, SmallVec}; use std::borrow::Cow; use std::collections::HashMap; +use std::time::{SystemTime, UNIX_EPOCH}; use strum::{Display, EnumString}; use uuid::Uuid; +/// A struct representing the identity of the user #[derive(Serialize, Deserialize, Clone, Debug)] pub struct Identity { #[serde(rename = "principalId")] pub principal_id: String, } +/// A struct representing the bucket information #[derive(Serialize, Deserialize, Clone, Debug)] pub struct Bucket { pub name: String, @@ -22,6 +24,7 @@ pub struct Bucket { pub arn: String, } +/// A struct representing the object information #[derive(Serialize, Deserialize, Clone, Debug)] pub struct Object { pub key: String, @@ -38,6 +41,7 @@ pub struct Object { pub sequencer: String, } +/// A struct representing the metadata of the event #[derive(Serialize, Deserialize, Clone, Debug)] pub struct Metadata { #[serde(rename = "s3SchemaVersion")] @@ -48,6 +52,7 @@ pub struct Metadata { pub object: Object, } +/// A struct representing the source of the event #[derive(Serialize, Deserialize, Clone, Debug)] pub struct Source { pub host: String, @@ -82,7 +87,7 @@ impl EventBuilder { event_version: Some(Cow::Borrowed("2.0").to_string()), event_source: Some(Cow::Borrowed("aws:s3").to_string()), aws_region: Some("us-east-1".to_string()), - event_time: Some(Utc::now().to_rfc3339()), + event_time: Some(SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs().to_string()), event_name: None, user_identity: Some(Identity { principal_id: "anonymous".to_string(), @@ -214,7 +219,7 @@ impl EventBuilder { s3, source, id: Uuid::new_v4(), - timestamp: Utc::now(), + timestamp: SystemTime::now(), channels, }) } @@ -241,7 +246,7 @@ pub struct Event { pub s3: Metadata, pub source: Source, pub id: Uuid, - pub timestamp: DateTime, + pub timestamp: SystemTime, pub channels: SmallVec<[String; 2]>, } diff --git a/crates/event-notifier/src/global.rs b/crates/event-notifier/src/global.rs index a14380b39..ed666a8d2 100644 --- a/crates/event-notifier/src/global.rs +++ b/crates/event-notifier/src/global.rs @@ -1,23 +1,90 @@ use crate::{ChannelAdapter, Error, Event, NotificationConfig, NotificationSystem}; -use std::sync::Arc; +use std::sync::{atomic, Arc}; +use std::time; +use std::time::Duration; use tokio::sync::{Mutex, OnceCell}; static GLOBAL_SYSTEM: OnceCell>> = OnceCell::const_new(); -static INITIALIZED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); +static INITIALIZED: atomic::AtomicBool = atomic::AtomicBool::new(false); +static READY: atomic::AtomicBool = atomic::AtomicBool::new(false); +static INIT_LOCK: Mutex<()> = Mutex::const_new(()); /// initialize the global notification system pub async fn initialize(config: NotificationConfig) -> Result<(), Error> { - if INITIALIZED.swap(true, std::sync::atomic::Ordering::SeqCst) { + // use lock to protect the initialization process + let _lock = INIT_LOCK.lock().await; + + if INITIALIZED.swap(true, atomic::Ordering::SeqCst) { return Err(Error::custom("notify the system has been initialized")); } - let system = Arc::new(Mutex::new(NotificationSystem::new(config).await?)); - GLOBAL_SYSTEM - .set(system) - .map_err(|_| Error::custom("unable to set up global notification system"))?; + match Arc::new(Mutex::new(NotificationSystem::new(config).await?)) { + system => { + if let Err(_) = GLOBAL_SYSTEM.set(system.clone()) { + INITIALIZED.store(false, atomic::Ordering::SeqCst); + return Err(Error::custom("unable to set up global notification system")); + } + system + } + }; Ok(()) } +/// securely initialize the global notification system +pub async fn initialize_safe(config: NotificationConfig) -> Result<(), Error> { + // use-lock-to-protect-the-initialization-process + let _lock = INIT_LOCK.lock().await; + + if INITIALIZED.load(atomic::Ordering::SeqCst) { + return Err(Error::custom("notify the system has been initialized")); + } + + // set-initialization-flag + INITIALIZED.store(true, atomic::Ordering::SeqCst); + + match Arc::new(Mutex::new(NotificationSystem::new(config).await?)) { + system => { + if let Err(_) = GLOBAL_SYSTEM.set(system.clone()) { + INITIALIZED.store(false, atomic::Ordering::SeqCst); + return Err(Error::custom("unable to set up global notification system")); + } + system + } + }; + Ok(()) +} + +/// securely start the global notification system +pub async fn start_safe(adapters: Vec>) -> Result<(), Error> { + // start process with lock protection + let _lock = INIT_LOCK.lock().await; + + if !INITIALIZED.load(atomic::Ordering::SeqCst) { + return Err(Error::custom("notification system not initialized")); + } + + if READY.load(atomic::Ordering::SeqCst) { + return Err(Error::custom("notification system already started")); + } + + let system = get_system().await?; + + // Execute startup operations directly on the current thread, rather than generating new tasks + let mut system_guard = system.lock().await; + match system_guard.start(adapters).await { + Ok(_) => { + READY.store(true, atomic::Ordering::SeqCst); + tracing::info!("Notification system is ready to process events"); + Ok(()) + } + Err(e) => { + tracing::error!("Notify system start failed: {}", e); + INITIALIZED.store(false, atomic::Ordering::SeqCst); + Err(e) + } + } +} + /// start the global notification system pub async fn start(adapters: Vec>) -> Result<(), Error> { let system = get_system().await?; @@ -26,11 +93,36 @@ pub async fn start(adapters: Vec>) -> Result<(), Error> let system_clone = Arc::clone(&system); tokio::spawn(async move { let mut system_guard = system_clone.lock().await; - if let Err(e) = system_guard.start(adapters).await { - tracing::error!("notify the system to start failed: {}", e); + match system_guard.start(adapters).await { + Ok(_) => { + // The system is started and runs normally, set the ready flag + READY.store(true, atomic::Ordering::SeqCst); + tracing::info!("Notification system is ready to process events"); + } + Err(e) => { + tracing::error!("Notify system start failed: {}", e); + INITIALIZED.store(false, atomic::Ordering::SeqCst); + } } }); + // Wait for a while to ensure the system has a chance to start + tokio::time::sleep(Duration::from_millis(100)).await; + + Ok(()) +} + +/// waiting for notification system to be fully ready +async fn wait_until_ready(timeout: Duration) -> Result<(), Error> { + let start = time::Instant::now(); + + while !READY.load(atomic::Ordering::SeqCst) { + if start.elapsed() > timeout { + return Err(Error::custom("timeout waiting for notification system to become ready")); + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + Ok(()) } @@ -51,6 +143,7 @@ pub async fn start(adapters: Vec>) -> Result<(), Error> /// let config = NotificationConfig { /// store_path: "./events".to_string(), /// channel_capacity: 100, +/// timeout: 0, /// adapters: vec![/* 适配器配置 */], /// http: Default::default(), /// }; @@ -73,8 +166,29 @@ pub async fn initialize_and_start(config: NotificationConfig) -> Result<(), Erro Ok(()) } +/// Initialize and start the global notification system and wait until it's ready +pub async fn initialize_and_start_with_ready_check(config: NotificationConfig, timeout: Duration) -> Result<(), Error> { + // initialize the system + initialize(config.clone()).await?; + + // create an adapter + let adapters = crate::create_adapters(&config.adapters).expect("failed to create adapters"); + + // start the system + start(adapters).await?; + + // wait for the system to be ready + wait_until_ready(timeout).await?; + + Ok(()) +} + /// send events to notification system pub async fn send_event(event: Event) -> Result<(), Error> { + // check if the system is ready to receive events + if !READY.load(atomic::Ordering::SeqCst) { + return Err(Error::custom("notification system not ready, please wait for initialization to complete")); + } let system = get_system().await?; let system_guard = system.lock().await; system_guard.send_event(event).await diff --git a/crates/event-notifier/src/lib.rs b/crates/event-notifier/src/lib.rs index 9ba226c61..2fa9d8954 100644 --- a/crates/event-notifier/src/lib.rs +++ b/crates/event-notifier/src/lib.rs @@ -4,6 +4,7 @@ mod config; mod error; mod event; mod global; +mod notifier; mod producer; mod store; @@ -28,104 +29,9 @@ pub use config::{AdapterConfig, NotificationConfig}; pub use error::Error; pub use event::{Bucket, Event, EventBuilder, Identity, Log, Metadata, Name, Object, Source}; -pub use global::{initialize, initialize_and_start, send_event, shutdown, start}; +pub use global::{ + initialize, initialize_and_start, initialize_and_start_with_ready_check, initialize_safe, send_event, shutdown, start, + start_safe, +}; +pub use notifier::NotificationSystem; pub use store::EventStore; - -#[cfg(feature = "http-producer")] -pub use producer::http::HttpProducer; -#[cfg(feature = "http-producer")] -pub use producer::EventProducer; - -use std::sync::Arc; -use tokio::sync::mpsc; -use tokio_util::sync::CancellationToken; - -/// The `NotificationSystem` struct represents the notification system. -/// It manages the event bus and the adapters. -/// It is responsible for sending and receiving events. -/// It also handles the shutdown process. -pub struct NotificationSystem { - tx: mpsc::Sender, - rx: Option>, - store: Arc, - shutdown: CancellationToken, - #[cfg(feature = "http-producer")] - http_config: HttpProducerConfig, -} - -impl NotificationSystem { - /// Creates a new `NotificationSystem` instance. - pub async fn new(config: NotificationConfig) -> Result { - let (tx, rx) = mpsc::channel::(config.channel_capacity); - let store = Arc::new(EventStore::new(&config.store_path).await?); - let shutdown = CancellationToken::new(); - - let restored_logs = store.load_logs().await?; - for log in restored_logs { - for event in log.records { - // For example, where the send method may return a SendError when calling it - tx.send(event).await.map_err(|e| Error::ChannelSend(Box::new(e)))?; - } - } - - Ok(Self { - tx, - rx: Some(rx), - store, - shutdown, - #[cfg(feature = "http-producer")] - http_config: config.http, - }) - } - - /// Starts the notification system. - /// It initializes the event bus and the producer. - pub async fn start(&mut self, adapters: Vec>) -> Result<(), Error> { - let rx = self.rx.take().ok_or_else(|| Error::EventBusStarted)?; - - let shutdown_clone = self.shutdown.clone(); - let store_clone = self.store.clone(); - let bus_handle = tokio::spawn(async move { - if let Err(e) = event_bus(rx, adapters, store_clone, shutdown_clone).await { - tracing::error!("Event bus failed: {}", e); - } - }); - - #[cfg(feature = "http-producer")] - { - let producer = HttpProducer::new(self.tx.clone(), self.http_config.port); - producer.start().await?; - } - - tokio::select! { - result = bus_handle => { - result.map_err(Error::JoinError)?; - Ok(()) - }, - _ = self.shutdown.cancelled() => { - tracing::info!("System shutdown triggered"); - Ok(()) - } - } - } - - /// Sends an event to the notification system. - /// This method is used to send events to the event bus. - pub async fn send_event(&self, event: Event) -> Result<(), Error> { - self.tx.send(event).await.map_err(|e| Error::ChannelSend(Box::new(e)))?; - Ok(()) - } - - /// Shuts down the notification system. - /// This method is used to cancel the event bus and producer tasks. - pub fn shutdown(&self) { - self.shutdown.cancel(); - } - - /// Sets the HTTP port for the notification system. - /// This method is used to change the port for the HTTP producer. - #[cfg(feature = "http-producer")] - pub fn set_http_port(&mut self, port: u16) { - self.http_config.port = port; - } -} diff --git a/crates/event-notifier/src/notifier.rs b/crates/event-notifier/src/notifier.rs new file mode 100644 index 000000000..e75fe25db --- /dev/null +++ b/crates/event-notifier/src/notifier.rs @@ -0,0 +1,101 @@ +#[cfg(feature = "http-producer")] +pub use crate::producer::http::HttpProducer; +#[cfg(feature = "http-producer")] +pub use crate::producer::EventProducer; + +#[cfg(feature = "http-producer")] +pub use crate::config::HttpProducerConfig; +use crate::{event_bus, ChannelAdapter, Error, Event, EventStore, NotificationConfig}; +use std::sync::Arc; +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; + +/// The `NotificationSystem` struct represents the notification system. +/// It manages the event bus and the adapters. +/// It is responsible for sending and receiving events. +/// It also handles the shutdown process. +pub struct NotificationSystem { + tx: mpsc::Sender, + rx: Option>, + store: Arc, + shutdown: CancellationToken, + #[cfg(feature = "http-producer")] + http_config: HttpProducerConfig, +} + +impl NotificationSystem { + /// Creates a new `NotificationSystem` instance. + pub async fn new(config: NotificationConfig) -> Result { + let (tx, rx) = mpsc::channel::(config.channel_capacity); + let store = Arc::new(EventStore::new(&config.store_path).await?); + let shutdown = CancellationToken::new(); + + let restored_logs = store.load_logs().await?; + for log in restored_logs { + for event in log.records { + // For example, where the send method may return a SendError when calling it + tx.send(event).await.map_err(|e| Error::ChannelSend(Box::new(e)))?; + } + } + + Ok(Self { + tx, + rx: Some(rx), + store, + shutdown, + #[cfg(feature = "http-producer")] + http_config: config.http, + }) + } + + /// Starts the notification system. + /// It initializes the event bus and the producer. + pub async fn start(&mut self, adapters: Vec>) -> Result<(), Error> { + let rx = self.rx.take().ok_or_else(|| Error::EventBusStarted)?; + + let shutdown_clone = self.shutdown.clone(); + let store_clone = self.store.clone(); + let bus_handle = tokio::spawn(async move { + if let Err(e) = event_bus(rx, adapters, store_clone, shutdown_clone).await { + tracing::error!("Event bus failed: {}", e); + } + }); + + #[cfg(feature = "http-producer")] + { + let producer = crate::producer::http::HttpProducer::new(self.tx.clone(), self.http_config.port); + producer.start().await?; + } + + tokio::select! { + result = bus_handle => { + result.map_err(Error::JoinError)?; + Ok(()) + }, + _ = self.shutdown.cancelled() => { + tracing::info!("System shutdown triggered"); + Ok(()) + } + } + } + + /// Sends an event to the notification system. + /// This method is used to send events to the event bus. + pub async fn send_event(&self, event: Event) -> Result<(), Error> { + self.tx.send(event).await.map_err(|e| Error::ChannelSend(Box::new(e)))?; + Ok(()) + } + + /// Shuts down the notification system. + /// This method is used to cancel the event bus and producer tasks. + pub fn shutdown(&self) { + self.shutdown.cancel(); + } + + /// Sets the HTTP port for the notification system. + /// This method is used to change the port for the HTTP producer. + #[cfg(feature = "http-producer")] + pub fn set_http_port(&mut self, port: u16) { + self.http_config.port = port; + } +} diff --git a/crates/event-notifier/src/store.rs b/crates/event-notifier/src/store.rs index 995cdc831..c7ded205a 100644 --- a/crates/event-notifier/src/store.rs +++ b/crates/event-notifier/src/store.rs @@ -1,7 +1,7 @@ use crate::Error; use crate::Log; -use chrono::Utc; use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; use tokio::fs::{create_dir_all, File, OpenOptions}; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader, BufWriter}; use tokio::sync::RwLock; @@ -23,7 +23,11 @@ impl EventStore { pub async fn save_logs(&self, logs: &[Log]) -> Result<(), Error> { let _guard = self.lock.write().await; - let file_path = format!("{}/events_{}.jsonl", self.path, Utc::now().timestamp()); + let file_path = format!( + "{}/events_{}.jsonl", + self.path, + SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() + ); let file = OpenOptions::new().create(true).append(true).open(&file_path).await?; let mut writer = BufWriter::new(file); for log in logs { diff --git a/crates/event-notifier/tests/integration.rs b/crates/event-notifier/tests/integration.rs index 2829b8b94..f1b070806 100644 --- a/crates/event-notifier/tests/integration.rs +++ b/crates/event-notifier/tests/integration.rs @@ -70,6 +70,7 @@ async fn test_notification_system() { let config = rustfs_event_notifier::NotificationConfig { store_path: "./test_events".to_string(), channel_capacity: 100, + timeout: 50, adapters: vec![AdapterConfig::Webhook(WebhookConfig { endpoint: "http://localhost:8080/webhook".to_string(), auth_token: None, From 70e94fd018b266d1a6f55304d4b876c7fb02f814 Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Mon, 21 Apr 2025 01:42:57 +0000 Subject: [PATCH 04/16] fix sql Signed-off-by: junxiang Mu <1948535941@qq.com> --- Cargo.lock | 1 + rustfs/src/server/service_state.rs | 8 +- rustfs/src/storage/ecfs.rs | 7 +- s3select/api/src/query/session.rs | 16 ++- s3select/query/Cargo.toml | 1 + s3select/query/src/dispatcher/manager.rs | 119 +++++++++++++++++++---- s3select/query/src/instance.rs | 18 +++- 7 files changed, 139 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3892d3af4..095a21cd5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6683,6 +6683,7 @@ dependencies = [ "datafusion", "derive_builder", "futures", + "lazy_static", "parking_lot 0.12.3", "s3s", "snafu", diff --git a/rustfs/src/server/service_state.rs b/rustfs/src/server/service_state.rs index d27296e35..ad137f66d 100644 --- a/rustfs/src/server/service_state.rs +++ b/rustfs/src/server/service_state.rs @@ -96,7 +96,9 @@ impl ServiceStateManager { ServiceState::Starting => { info!("Service is starting..."); #[cfg(target_os = "linux")] - if let Err(e) = libsystemd::daemon::notify(false, &[libsystemd::daemon::NotifyState::Status("Starting...".to_string())]) { + if let Err(e) = + libsystemd::daemon::notify(false, &[libsystemd::daemon::NotifyState::Status("Starting...".to_string())]) + { tracing::error!("Failed to notify systemd of starting state: {}", e); } } @@ -111,7 +113,9 @@ impl ServiceStateManager { ServiceState::Stopped => { info!("Service has stopped"); #[cfg(target_os = "linux")] - if let Err(e) = libsystemd::daemon::notify(false, &[libsystemd::daemon::NotifyState::Status("Stopped".to_string())]) { + if let Err(e) = + libsystemd::daemon::notify(false, &[libsystemd::daemon::NotifyState::Status("Stopped".to_string())]) + { tracing::error!("Failed to notify systemd of stopped state: {}", e); } } diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index f1f53b9fb..252cd316f 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -1895,10 +1895,13 @@ impl S3 for FS { let db = make_rustfsms(input.clone(), false).await.map_err(|e| { error!("make db failed, {}", e.to_string()); - s3_error!(InternalError) + s3_error!(InternalError, "{}", e.to_string()) })?; let query = Query::new(Context { input: input.clone() }, input.request.expression); - let result = db.execute(&query).await.map_err(|_| s3_error!(InternalError))?; + let result = db + .execute(&query) + .await + .map_err(|e| s3_error!(InternalError, "{}", e.to_string()))?; let results = result.result().chunk_result().await.unwrap().to_vec(); diff --git a/s3select/api/src/query/session.rs b/s3select/api/src/query/session.rs index c9d91f51d..286ee9f8e 100644 --- a/s3select/api/src/query/session.rs +++ b/s3select/api/src/query/session.rs @@ -1,8 +1,8 @@ use std::sync::Arc; -use bytes::Bytes; use datafusion::{ execution::{context::SessionState, runtime_env::RuntimeEnvBuilder, SessionStateBuilder}, + parquet::data_type::AsBytes, prelude::SessionContext, }; use object_store::{memory::InMemory, path::Path, ObjectStore}; @@ -65,7 +65,19 @@ impl SessionCtxFactory { 8,Henry,32,IT,6200 9,Ivy,24,Marketing,4800 10,Jack,38,Finance,7500"; - let data_bytes = Bytes::from(data.to_vec()); + let data_bytes = data.as_bytes(); + // let data = r#""year"╦"gender"╦"ethnicity"╦"firstname"╦"count"╦"rank" + // "2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"SOPHIA"╦"119"╦"1" + // "2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"CHLOE"╦"106"╦"2" + // "2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"EMILY"╦"93"╦"3" + // "2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"OLIVIA"╦"89"╦"4" + // "2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"EMMA"╦"75"╦"5" + // "2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"ISABELLA"╦"67"╦"6" + // "2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"TIFFANY"╦"54"╦"7" + // "2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"ASHLEY"╦"52"╦"8" + // "2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"FIONA"╦"48"╦"9" + // "2011"╦"FEMALE"╦"ASIAN AND PACIFIC ISLANDER"╦"ANGELA"╦"47"╦"10""#; + // let data_bytes = Bytes::from(data); let path = Path::from(context.input.key.clone()); store.put(&path, data_bytes.into()).await.map_err(|e| { error!("put data into memory failed: {}", e.to_string()); diff --git a/s3select/query/Cargo.toml b/s3select/query/Cargo.toml index 8d9b159c2..559eab27a 100644 --- a/s3select/query/Cargo.toml +++ b/s3select/query/Cargo.toml @@ -10,6 +10,7 @@ async-trait.workspace = true datafusion = { workspace = true } derive_builder = { workspace = true } futures = { workspace = true } +lazy_static = { workspace = true } parking_lot = { version = "0.12.3" } s3s.workspace = true snafu = { workspace = true, features = ["backtrace"] } diff --git a/s3select/query/src/dispatcher/manager.rs b/s3select/query/src/dispatcher/manager.rs index e85e7a543..05f763435 100644 --- a/s3select/query/src/dispatcher/manager.rs +++ b/s3select/query/src/dispatcher/manager.rs @@ -1,4 +1,5 @@ use std::{ + ops::Deref, pin::Pin, sync::Arc, task::{Context, Poll}, @@ -19,8 +20,10 @@ use api::{ }; use async_trait::async_trait; use datafusion::{ - arrow::{datatypes::SchemaRef, record_batch::RecordBatch}, - config::CsvOptions, + arrow::{ + datatypes::{Schema, SchemaRef}, + record_batch::RecordBatch, + }, datasource::{ file_format::{csv::CsvFormat, json::JsonFormat, parquet::ParquetFormat}, listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl}, @@ -29,7 +32,8 @@ use datafusion::{ execution::{RecordBatchStream, SendableRecordBatchStream}, }; use futures::{Stream, StreamExt}; -use s3s::dto::SelectObjectContentInput; +use lazy_static::lazy_static; +use s3s::dto::{FileHeaderInfo, SelectObjectContentInput}; use crate::{ execution::factory::QueryExecutionFactoryRef, @@ -37,6 +41,12 @@ use crate::{ sql::logical::planner::DefaultLogicalPlanner, }; +lazy_static! { + static ref IGNORE: FileHeaderInfo = FileHeaderInfo::from_static(FileHeaderInfo::IGNORE); + static ref NONE: FileHeaderInfo = FileHeaderInfo::from_static(FileHeaderInfo::NONE); + static ref USE: FileHeaderInfo = FileHeaderInfo::from_static(FileHeaderInfo::USE); +} + #[derive(Clone)] pub struct SimpleQueryDispatcher { input: SelectObjectContentInput, @@ -138,25 +148,94 @@ impl SimpleQueryDispatcher { async fn build_scheme_provider(&self, session: &SessionCtx) -> QueryResult { let path = format!("s3://{}/{}", self.input.bucket, self.input.key); let table_path = ListingTableUrl::parse(path)?; - let listing_options = if self.input.request.input_serialization.csv.is_some() { - let file_format = CsvFormat::default().with_options(CsvOptions::default().with_has_header(true)); - ListingOptions::new(Arc::new(file_format)).with_file_extension(".csv") - } else if self.input.request.input_serialization.parquet.is_some() { - let file_format = ParquetFormat::new(); - ListingOptions::new(Arc::new(file_format)).with_file_extension(".parquet") - } else if self.input.request.input_serialization.json.is_some() { - let file_format = JsonFormat::default(); - ListingOptions::new(Arc::new(file_format)).with_file_extension(".json") - } else { - return Err(QueryError::NotImplemented { - err: "not support this file type".to_string(), - }); - }; + let (listing_options, need_rename_volume_name, need_ignore_volume_name) = + if let Some(csv) = self.input.request.input_serialization.csv.as_ref() { + let mut need_rename_volume_name = false; + let mut need_ignore_volume_name = false; + let mut file_format = CsvFormat::default() + .with_comment( + csv.comments + .clone() + .map(|c| c.as_bytes().first().copied().unwrap_or_default()), + ) + .with_escape( + csv.quote_escape_character + .clone() + .map(|e| e.as_bytes().first().copied().unwrap_or_default()), + ); + if let Some(delimiter) = csv.field_delimiter.as_ref() { + file_format = file_format.with_delimiter(delimiter.as_bytes().first().copied().unwrap_or_default()); + } + if csv.file_header_info.is_some() {} + match csv.file_header_info.as_ref() { + Some(info) => { + if *info == *NONE { + file_format = file_format.with_has_header(false); + need_rename_volume_name = true; + } else if *info == *IGNORE { + file_format = file_format.with_has_header(true); + need_rename_volume_name = true; + need_ignore_volume_name = true; + } else if *info == *USE { + file_format = file_format.with_has_header(true); + } else { + return Err(QueryError::NotImplemented { + err: "unsupported FileHeaderInfo".to_string(), + }); + } + } + _ => { + return Err(QueryError::NotImplemented { + err: "unsupported FileHeaderInfo".to_string(), + }); + } + } + if let Some(quote) = csv.quote_character.as_ref() { + file_format = file_format.with_quote(quote.as_bytes().first().copied().unwrap_or_default()); + } + ( + ListingOptions::new(Arc::new(file_format)).with_file_extension(".csv"), + need_rename_volume_name, + need_ignore_volume_name, + ) + } else if self.input.request.input_serialization.parquet.is_some() { + let file_format = ParquetFormat::new(); + (ListingOptions::new(Arc::new(file_format)).with_file_extension(".parquet"), false, false) + } else if self.input.request.input_serialization.json.is_some() { + let file_format = JsonFormat::default(); + (ListingOptions::new(Arc::new(file_format)).with_file_extension(".json"), false, false) + } else { + return Err(QueryError::NotImplemented { + err: "not support this file type".to_string(), + }); + }; let resolve_schema = listing_options.infer_schema(session.inner(), &table_path).await?; - let config = ListingTableConfig::new(table_path) - .with_listing_options(listing_options) - .with_schema(resolve_schema); + let config = if need_rename_volume_name { + let mut new_fields = Vec::new(); + for (i, field) in resolve_schema.fields().iter().enumerate() { + let f_name = field.name(); + let mut_field = field.deref().clone(); + if f_name.starts_with("column_") { + let re_name = f_name.replace("column_", "_"); + new_fields.push(mut_field.with_name(re_name)); + } else if need_ignore_volume_name { + let re_name = format!("_{}", i + 1); + new_fields.push(mut_field.with_name(re_name)); + } else { + new_fields.push(mut_field); + } + } + let new_schema = Arc::new(Schema::new(new_fields).with_metadata(resolve_schema.metadata().clone())); + ListingTableConfig::new(table_path) + .with_listing_options(listing_options) + .with_schema(new_schema) + } else { + ListingTableConfig::new(table_path) + .with_listing_options(listing_options) + .with_schema(resolve_schema) + }; + // rename default let provider = Arc::new(ListingTable::try_new(config)?); let current_session_table_provider = self.build_table_handle_provider()?; let metadata_provider = diff --git a/s3select/query/src/instance.rs b/s3select/query/src/instance.rs index 3ad8941a8..44952a4ac 100644 --- a/s3select/query/src/instance.rs +++ b/s3select/query/src/instance.rs @@ -101,8 +101,8 @@ mod tests { }; use datafusion::{arrow::util::pretty, assert_batches_eq}; use s3s::dto::{ - CSVInput, CSVOutput, ExpressionType, InputSerialization, OutputSerialization, SelectObjectContentInput, - SelectObjectContentRequest, + CSVInput, CSVOutput, ExpressionType, FieldDelimiter, FileHeaderInfo, InputSerialization, OutputSerialization, + RecordDelimiter, SelectObjectContentInput, SelectObjectContentRequest, }; use crate::instance::make_rustfsms; @@ -122,7 +122,10 @@ mod tests { expression: sql.to_string(), expression_type: ExpressionType::from_static("SQL"), input_serialization: InputSerialization { - csv: Some(CSVInput::default()), + csv: Some(CSVInput { + file_header_info: Some(FileHeaderInfo::from_static(FileHeaderInfo::USE)), + ..Default::default() + }), ..Default::default() }, output_serialization: OutputSerialization { @@ -164,7 +167,7 @@ mod tests { #[tokio::test] #[ignore] async fn test_func_sql() { - let sql = "select count(s.id) from S3Object as s"; + let sql = "SELECT s._1 FROM S3Object s"; let input = SelectObjectContentInput { bucket: "dandan".to_string(), expected_bucket_owner: None, @@ -176,7 +179,12 @@ mod tests { expression: sql.to_string(), expression_type: ExpressionType::from_static("SQL"), input_serialization: InputSerialization { - csv: Some(CSVInput::default()), + csv: Some(CSVInput { + file_header_info: Some(FileHeaderInfo::from_static(FileHeaderInfo::IGNORE)), + field_delimiter: Some(FieldDelimiter::from("╦")), + record_delimiter: Some(RecordDelimiter::from("\n")), + ..Default::default() + }), ..Default::default() }, output_serialization: OutputSerialization { From 177dec627cfafe8bcdbf783e107dbcd9ca9ad57b Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Mon, 21 Apr 2025 03:09:54 +0000 Subject: [PATCH 05/16] move protobuf generate into bin crate Signed-off-by: junxiang Mu <1948535941@qq.com> --- README.md | 4 +++ common/protos/Cargo.toml | 6 ++-- common/protos/{build.rs => src/main.rs} | 48 ++++++++++++++----------- 3 files changed, 36 insertions(+), 22 deletions(-) rename common/protos/{build.rs => src/main.rs} (88%) diff --git a/README.md b/README.md index 3e1de2cb0..511f6736e 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,10 @@ https://github.com/google/flatbuffers/releases/download/v25.2.10/Linux.flatc.bin https://github.com/protocolbuffers/protobuf/releases/download/v30.2/protoc-30.2-linux-x86_64.zip +generate protobuf code: + +```cargo run --bin gproto``` + Or use Docker: ```yml diff --git a/common/protos/Cargo.toml b/common/protos/Cargo.toml index 581fb2d02..2041375c4 100644 --- a/common/protos/Cargo.toml +++ b/common/protos/Cargo.toml @@ -6,6 +6,10 @@ edition.workspace = true [lints] workspace = true +[[bin]] +name = "gproto" +path = "src/main.rs" + [dependencies] #async-backtrace = { workspace = true, optional = true } common.workspace = true @@ -15,7 +19,5 @@ protobuf = { workspace = true } tokio = { workspace = true } tonic = { workspace = true, features = ["transport"] } tower = { workspace = true } - -[build-dependencies] prost-build = { workspace = true } tonic-build = { workspace = true } diff --git a/common/protos/build.rs b/common/protos/src/main.rs similarity index 88% rename from common/protos/build.rs rename to common/protos/src/main.rs index fc55ec7a6..44fe20a10 100644 --- a/common/protos/build.rs +++ b/common/protos/src/main.rs @@ -1,13 +1,7 @@ -use std::{ - cmp, env, fs, - io::Write, - path::{Path, PathBuf}, - process::Command, -}; +use std::{cmp, env, fs, io::Write, path::Path, process::Command}; type AnyError = Box; -const ENV_OUT_DIR: &str = "OUT_DIR"; const VERSION_PROTOBUF: Version = Version(30, 2, 0); // 30.2.0 const VERSION_FLATBUFFERS: Version = Version(24, 3, 25); // 24.3.25 /// Build protos if the major version of `flatc` or `protoc` is greater @@ -37,16 +31,16 @@ fn main() -> Result<(), AnyError> { } // path of proto file - let project_root_dir = env::current_dir()?; - let proto_dir = project_root_dir.join("src"); + let project_root_dir = env::current_dir()?.join("common/protos/src"); + let proto_dir = project_root_dir.clone(); let proto_files = &["node.proto"]; - let proto_out_dir = project_root_dir.join("src").join("generated").join("proto_gen"); - let flatbuffer_out_dir = project_root_dir.join("src").join("generated").join("flatbuffers_generated"); - let descriptor_set_path = PathBuf::from(env::var(ENV_OUT_DIR).unwrap()).join("proto-descriptor.bin"); + let proto_out_dir = project_root_dir.join("generated").join("proto_gen"); + let flatbuffer_out_dir = project_root_dir.join("generated").join("flatbuffers_generated"); + // let descriptor_set_path = PathBuf::from(env::var(ENV_OUT_DIR).unwrap()).join("proto-descriptor.bin"); tonic_build::configure() .out_dir(proto_out_dir) - .file_descriptor_set_path(descriptor_set_path) + // .file_descriptor_set_path(descriptor_set_path) .protoc_arg("--experimental_allow_proto3_optional") .compile_well_known_types(true) .emit_rerun_if_changed(false) @@ -54,17 +48,13 @@ fn main() -> Result<(), AnyError> { .map_err(|e| format!("Failed to generate protobuf file: {e}."))?; // protos/gen/mod.rs - let generated_mod_rs_path = project_root_dir - .join("src") - .join("generated") - .join("proto_gen") - .join("mod.rs"); + let generated_mod_rs_path = project_root_dir.join("generated").join("proto_gen").join("mod.rs"); let mut generated_mod_rs = fs::File::create(generated_mod_rs_path)?; writeln!(&mut generated_mod_rs, "pub mod node_service;")?; generated_mod_rs.flush()?; - let generated_mod_rs_path = project_root_dir.join("src").join("generated").join("mod.rs"); + let generated_mod_rs_path = project_root_dir.join("generated").join("mod.rs"); let mut generated_mod_rs = fs::File::create(generated_mod_rs_path)?; writeln!(&mut generated_mod_rs, "#![allow(unused_imports)]")?; @@ -80,7 +70,6 @@ fn main() -> Result<(), AnyError> { Err(_) => "flatc".to_string(), }; - // build src/protos/*.fbs files to src/protos/gen/ compile_flatbuffers_models( &mut generated_mod_rs, &flatc_path, @@ -88,6 +77,8 @@ fn main() -> Result<(), AnyError> { flatbuffer_out_dir.clone(), vec!["models"], )?; + + fmt(); Ok(()) } @@ -260,3 +251,20 @@ fn protobuf_compiler_version() -> Result { } }) } + +fn fmt() { + let output = Command::new("cargo").arg("fmt").arg("-p").arg("protos").status(); + + match output { + Ok(status) => { + if status.success() { + println!("cargo fmt executed successfully."); + } else { + eprintln!("cargo fmt failed with status: {:?}", status); + } + } + Err(e) => { + eprintln!("Failed to execute cargo fmt: {}", e); + } + } +} From 770f28d205460acc4eca27a18c0eb4e63a4b071a Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 21 Apr 2025 15:27:58 +0800 Subject: [PATCH 06/16] improve Cargo toml --- Cargo.lock | 184 +++++++++++++++++--------------------- Cargo.toml | 44 +++++---- data/README.md | 1 - ecstore/Cargo.toml | 2 +- rustfs/Cargo.toml | 20 ++--- s3select/api/Cargo.toml | 4 +- s3select/query/Cargo.toml | 2 +- 7 files changed, 122 insertions(+), 135 deletions(-) delete mode 100644 data/README.md diff --git a/Cargo.lock b/Cargo.lock index 095a21cd5..8aabcc753 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -174,9 +174,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.97" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" [[package]] name = "api" @@ -486,7 +486,7 @@ dependencies = [ "enumflags2", "futures-channel", "futures-util", - "rand 0.9.0", + "rand 0.9.1", "raw-window-handle 0.6.2", "serde", "serde_repr", @@ -708,9 +708,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.28.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f7720b74ed28ca77f90769a71fd8c637a0137f6fae4ae947e1050229cff57f" +checksum = "0ddeb19ee86cb16ecfc871e5b0660aff6285760957aaedda6284cf0e790d3769" dependencies = [ "bindgen", "cc", @@ -933,9 +933,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.1" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389a099b34312839e16420d499a9cad9650541715937ffbdd40d36f49e77eeb3" +checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" dependencies = [ "arrayref", "arrayvec", @@ -979,11 +979,11 @@ dependencies = [ [[package]] name = "block2" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d59b4c170e16f0405a2e95aff44432a0d41aa97675f3d52623effe95792a037" +checksum = "340d2f0bdb2a43c1d3cd40513185b2bd7def0aa1052f956455114bc98f82dcf2" dependencies = [ - "objc2 0.6.0", + "objc2 0.6.1", ] [[package]] @@ -1012,9 +1012,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "4.0.2" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74fa05ad7d803d413eb8380983b092cbbaf9a85f151b871360e7b00cd7060b37" +checksum = "a334ef7c9e23abf0ce748e8cd309037da93e606ad52eb372e4ce327a0dcfbdfd" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1046,9 +1046,9 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "bytesize" -version = "1.3.3" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e93abca9e28e0a1b9877922aacb20576e05d4679ffa78c3d6dc22a26a216659" +checksum = "a3c8f83209414aacf0eeae3cf730b18d6981697fba62f200fcfb92b9f082acba" [[package]] name = "bytestring" @@ -1305,9 +1305,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.35" +version = "4.5.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8aa86934b44c19c50f87cc2790e19f54f7a67aedb64101c2e1a2e5ecfb73944" +checksum = "eccb054f56cbd38340b380d4a8e69ef1f02f1af43db2f0cc817a4774d80ae071" dependencies = [ "clap_builder", "clap_derive", @@ -1315,9 +1315,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.35" +version = "4.5.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2414dbb2dd0695280da6ea9261e327479e9d37b0630f6b53ba2a11c60c679fd9" +checksum = "efd9466fac8543255d3b1fcad4762c5e116ffe808c8a3043d4263cd4fd4862a2" dependencies = [ "anstream", "anstyle", @@ -1907,9 +1907,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010" +checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" [[package]] name = "datafusion" @@ -2416,9 +2416,9 @@ dependencies = [ [[package]] name = "der" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ "const-oid 0.9.6", "pem-rfc7468", @@ -2942,34 +2942,13 @@ dependencies = [ "syn 2.0.100", ] -[[package]] -name = "dirs" -version = "5.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" -dependencies = [ - "dirs-sys 0.4.1", -] - [[package]] name = "dirs" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" dependencies = [ - "dirs-sys 0.5.0", -] - -[[package]] -name = "dirs-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" -dependencies = [ - "libc", - "option-ext", - "redox_users 0.4.6", - "windows-sys 0.48.0", + "dirs-sys", ] [[package]] @@ -2980,7 +2959,7 @@ checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" dependencies = [ "libc", "option-ext", - "redox_users 0.5.0", + "redox_users", "windows-sys 0.59.0", ] @@ -2997,9 +2976,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a0d569e003ff27784e0e14e4a594048698e0c0f0b66cabcb51511be55a7caa0" dependencies = [ "bitflags 2.9.0", - "block2 0.6.0", + "block2 0.6.1", "libc", - "objc2 0.6.0", + "objc2 0.6.1", +] + +[[package]] +name = "dispatch2" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89a09f22a6c6069a18470eb92d2298acf25463f14256d24778e1230d789a2aec" +dependencies = [ + "bitflags 2.9.0", + "objc2 0.6.1", ] [[package]] @@ -3101,7 +3090,7 @@ dependencies = [ [[package]] name = "ecstore" -version = "0.1.0" +version = "0.0.1" dependencies = [ "async-trait", "backon", @@ -3932,9 +3921,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2" +checksum = "75249d144030531f8dee69fe9cea04d3edf809a017ae445e2abdff6629e86633" dependencies = [ "atomic-waker", "bytes", @@ -4782,9 +4771,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.171" +version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" [[package]] name = "libdbus-sys" @@ -5504,9 +5493,9 @@ dependencies = [ [[package]] name = "objc2" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3531f65190d9cff863b77a99857e74c314dd16bf56c538c4b57c7cbc3f3a6e59" +checksum = "88c6597e14493ab2e44ce58f2fdecf095a51f12ca57bec060a11c57332520551" dependencies = [ "objc2-encode", ] @@ -5529,15 +5518,15 @@ dependencies = [ [[package]] name = "objc2-app-kit" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5906f93257178e2f7ae069efb89fbd6ee94f0592740b5f8a1512ca498814d0fb" +checksum = "e6f29f568bec459b0ddff777cec4fe3fd8666d82d5a40ebd0ff7e66134f89bcc" dependencies = [ "bitflags 2.9.0", - "block2 0.6.0", - "objc2 0.6.0", + "block2 0.6.1", + "objc2 0.6.1", "objc2-core-foundation", - "objc2-foundation 0.3.0", + "objc2-foundation 0.3.1", ] [[package]] @@ -5554,19 +5543,20 @@ dependencies = [ [[package]] name = "objc2-core-foundation" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daeaf60f25471d26948a1c2f840e3f7d86f4109e3af4e8e4b5cd70c39690d925" +checksum = "1c10c2894a6fed806ade6027bcd50662746363a9589d3ec9d9bef30a4e4bc166" dependencies = [ "bitflags 2.9.0", - "objc2 0.6.0", + "dispatch2 0.3.0", + "objc2 0.6.1", ] [[package]] name = "objc2-core-graphics" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dca602628b65356b6513290a21a6405b4d4027b8b250f0b98dddbb28b7de02" +checksum = "989c6c68c13021b5c2d6b71456ebb0f9dc78d752e86a98da7c716f4f9470f5a4" dependencies = [ "bitflags 2.9.0", "objc2-core-foundation", @@ -5604,13 +5594,13 @@ dependencies = [ [[package]] name = "objc2-foundation" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a21c6c9014b82c39515db5b396f91645182611c97d24637cf56ac01e5f8d998" +checksum = "900831247d2fe1a09a683278e5384cfb8c80c79fe6b166f9d14bfdde0ea1b03c" dependencies = [ "bitflags 2.9.0", - "block2 0.6.0", - "objc2 0.6.0", + "block2 0.6.1", + "objc2 0.6.1", "objc2-core-foundation", ] @@ -5822,7 +5812,7 @@ dependencies = [ "glob", "opentelemetry", "percent-encoding", - "rand 0.9.0", + "rand 0.9.1", "serde_json", "thiserror 2.0.12", "tokio", @@ -6542,9 +6532,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] @@ -6729,7 +6719,7 @@ checksum = "b820744eb4dc9b57a3398183639c511b5a26d2ed702cedd3febaa1393caa22cc" dependencies = [ "bytes", "getrandom 0.3.2", - "rand 0.9.0", + "rand 0.9.1", "ring", "rustc-hash 2.1.1", "rustls 0.23.26", @@ -6797,13 +6787,12 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.3", - "zerocopy 0.8.24", ] [[package]] @@ -6961,17 +6950,6 @@ dependencies = [ "bitflags 2.9.0", ] -[[package]] -name = "redox_users" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" -dependencies = [ - "getrandom 0.2.15", - "libredox", - "thiserror 1.0.69", -] - [[package]] name = "redox_users" version = "0.5.0" @@ -7122,14 +7100,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80c844748fdc82aae252ee4594a89b6e7ebef1063de7951545564cbc4e57075d" dependencies = [ "ashpd 0.11.0", - "block2 0.6.0", - "dispatch2", + "block2 0.6.1", + "dispatch2 0.2.0", "js-sys", "log", - "objc2 0.6.0", - "objc2-app-kit 0.3.0", + "objc2 0.6.1", + "objc2-app-kit 0.3.1", "objc2-core-foundation", - "objc2-foundation 0.3.0", + "objc2-foundation 0.3.1", "pollster 0.4.0", "raw-window-handle 0.6.2", "urlencoding", @@ -7300,7 +7278,7 @@ dependencies = [ [[package]] name = "rustfs" -version = "0.1.0" +version = "0.0.1" dependencies = [ "api", "appauth", @@ -7397,7 +7375,7 @@ version = "0.0.1" dependencies = [ "chrono", "dioxus", - "dirs 6.0.0", + "dirs", "hex", "keyring", "lazy_static", @@ -7998,11 +7976,11 @@ dependencies = [ [[package]] name = "shellexpand" -version = "3.1.0" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da03fa3b94cc19e3ebfc88c4229c49d8f08cdbd1228870a45f0ffdf84988e14b" +checksum = "8b1fdf65dd6331831494dd616b30351c38e96e45921a27745cf98490458b90bb" dependencies = [ - "dirs 5.0.1", + "dirs", ] [[package]] @@ -8023,9 +8001,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.2" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" dependencies = [ "libc", ] @@ -9093,14 +9071,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eadd75f5002e2513eaa19b2365f533090cc3e93abd38788452d9ea85cff7b48a" dependencies = [ "crossbeam-channel", - "dirs 6.0.0", + "dirs", "libappindicator", "muda 0.15.3", - "objc2 0.6.0", - "objc2-app-kit 0.3.0", + "objc2 0.6.1", + "objc2-app-kit 0.3.1", "objc2-core-foundation", "objc2-core-graphics", - "objc2-foundation 0.3.0", + "objc2-foundation 0.3.1", "once_cell", "png", "thiserror 2.0.12", @@ -9307,7 +9285,7 @@ checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" dependencies = [ "getrandom 0.3.2", "js-sys", - "rand 0.9.0", + "rand 0.9.1", "serde", "uuid-macro-internal", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index d7eabe8b9..67c0012f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,10 +12,10 @@ members = [ "crypto", # Cryptography and security features "cli/rustfs-gui", # Graphical user interface client "crates/obs", # Observability utilities - "s3select/api", - "s3select/query", - "appauth", - "crates/event-notifier", + "crates/event-notifier", # Event notification system + "s3select/api", # S3 Select API interface + "s3select/query", # S3 Select query engine + "appauth", # Application authentication and authorization ] resolver = "2" @@ -33,7 +33,21 @@ unsafe_code = "deny" all = "warn" [workspace.dependencies] -madmin = { path = "./madmin" } +api = { path = "./s3select/api", version = "0.0.1" } +appauth = { path = "./appauth", version = "0.0.1" } +common = { path = "./common/common", version = "0.0.1" } +crypto = { path = "./crypto", version = "0.0.1" } +ecstore = { path = "./ecstore", version = "0.0.1" } +iam = { path = "./iam", version = "0.0.1" } +lock = { path = "./common/lock", version = "0.0.1" } +madmin = { path = "./madmin", version = "0.0.1" } +policy = { path = "./policy", version = "0.0.1" } +protos = { path = "./common/protos", version = "0.0.1" } +query = { path = "./s3select/query", version = "0.0.1" } +rustfs = { path = "./rustfs", version = "0.0.1" } +rustfs-obs = { path = "crates/obs", version = "0.0.1" } +rustfs-event-notifier = { path = "crates/event-notifier", version = "0.0.1" } +workers = { path = "./common/workers", version = "0.0.1" } atoi = "2.0.0" async-recursion = "1.1.1" async-trait = "0.1.88" @@ -43,22 +57,20 @@ axum-extra = "0.10.1" axum-server = { version = "0.7.2", features = ["tls-rustls"] } backon = "1.5.0" bytes = "1.10.1" -bytesize = "1.3.3" +bytesize = "2.0.1" chrono = { version = "0.4.40", features = ["serde"] } -clap = { version = "4.5.35", features = ["derive", "env"] } +clap = { version = "4.5.37", features = ["derive", "env"] } config = "0.15.11" -datafusion = "46.0.0" +datafusion = "46.0.1" derive_builder = "0.20.2" dioxus = { version = "0.6.3", features = ["router"] } dirs = "6.0.0" dotenvy = "0.15.7" -ecstore = { path = "./ecstore" } figment = { version = "0.10.19", features = ["toml", "yaml", "env"] } flatbuffers = "25.2.10" futures = "0.3.31" +futures-core = "0.3.31" futures-util = "0.3.31" -common = { path = "./common/common" } -policy = { path = "./policy" } hex = "0.4.3" hyper = "1.6.0" hyper-util = { version = "0.1.11", features = [ @@ -71,14 +83,15 @@ http-body = "1.0.1" humantime = "2.2.0" jsonwebtoken = "9.3.1" keyring = { version = "3.6.2", features = ["apple-native", "windows-native", "sync-secret-service"] } -lock = { path = "./common/lock" } lazy_static = "1.5.0" libsystemd = { version = "0.7.1" } local-ip-address = "0.6.3" matchit = "0.8.4" md-5 = "0.10.6" mime = "0.3.17" +mime_guess = "2.0.5" netif = "0.1.6" +object_store = "0.11.2" opentelemetry = { version = "0.29.1" } opentelemetry-appender-tracing = { version = "0.29.1", features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes"] } opentelemetry_sdk = { version = "0.29.0" } @@ -94,7 +107,6 @@ prost = "0.13.5" prost-build = "0.13.5" prost-types = "0.13.5" protobuf = "3.7" -protos = { path = "./common/protos" } rand = "0.8.5" rdkafka = { version = "0.37.0", features = ["tokio"] } reqwest = { version = "0.12.15", default-features = false, features = ["rustls-tls", "charset", "http2", "macos-system-configuration", "stream", "json", "blocking"] } @@ -102,15 +114,13 @@ rfd = { version = "0.15.3", default-features = false, features = ["xdg-portal", rmp = "0.8.14" rmp-serde = "1.3.0" rumqttc = { version = "0.24" } -rustfs-obs = { path = "crates/obs", version = "0.0.1" } -rustfs-event-notifier = { path = "crates/event-notifier", version = "0.0.1" } rust-embed = "8.7.0" rustls = { version = "0.23.26" } rustls-pki-types = "1.11.0" rustls-pemfile = "2.2.0" s3s = { git = "https://github.com/Nugine/s3s.git", rev = "4733cdfb27b2713e832967232cbff413bb768c10" } s3s-policy = { git = "https://github.com/Nugine/s3s.git", rev = "4733cdfb27b2713e832967232cbff413bb768c10" } -shadow-rs = { version = "1.1.1", default-features = false, features = ["metadata"] } +shadow-rs = { version = "1.1.1", default-features = false } serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.140" serde_urlencoded = "0.7.1" @@ -150,7 +160,7 @@ uuid = { version = "1.16.0", features = [ "fast-rng", "macro-diagnostics", ] } -workers = { path = "./common/workers" } + [profile.wasm-dev] inherits = "dev" diff --git a/data/README.md b/data/README.md deleted file mode 100644 index 2e3d4df77..000000000 --- a/data/README.md +++ /dev/null @@ -1 +0,0 @@ -## Observability Docker Compose \ No newline at end of file diff --git a/ecstore/Cargo.toml b/ecstore/Cargo.toml index 13c8edc52..2ad741d6d 100644 --- a/ecstore/Cargo.toml +++ b/ecstore/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ecstore" -version = "0.1.0" +version.workspace = true edition.workspace = true license.workspace = true repository.workspace = true diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 57da6e9d6..49ac85414 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rustfs" -version = "0.1.0" +version.workspace = true edition.workspace = true license.workspace = true repository.workspace = true @@ -15,9 +15,9 @@ path = "src/main.rs" workspace = true [dependencies] -madmin.workspace = true -api = { path = "../s3select/api" } -appauth = { version = "0.0.1", path = "../appauth" } +madmin = { workspace = true } +api = { workspace = true } +appauth = { workspace = true } atoi = { workspace = true } atomic_enum = { workspace = true } axum.workspace = true @@ -27,7 +27,7 @@ async-trait.workspace = true bytes.workspace = true chrono = { workspace = true } clap.workspace = true -crypto = { path = "../crypto" } +crypto = { workspace = true } datafusion = { workspace = true } common.workspace = true const-str = { version = "0.6.1", features = ["std", "proc"] } @@ -40,15 +40,15 @@ hyper.workspace = true hyper-util.workspace = true http.workspace = true http-body.workspace = true -iam = { path = "../iam" } +iam = { workspace = true } lock.workspace = true local-ip-address = { workspace = true } matchit = { workspace = true } mime.workspace = true -mime_guess = "2.0.5" +mime_guess = { workspace = true } pin-project-lite.workspace = true protos.workspace = true -query = { path = "../s3select/query" } +query = { workspace = true } rmp-serde.workspace = true rustfs-event-notifier = { workspace = true } rustfs-obs = { workspace = true } @@ -60,7 +60,7 @@ s3s.workspace = true serde.workspace = true serde_json.workspace = true serde_urlencoded = { workspace = true } -shadow-rs = { workspace = true, features = ["build"] } +shadow-rs = { workspace = true, features = ["build", "metadata"] } tracing.workspace = true time = { workspace = true, features = ["parsing", "formatting", "serde"] } tokio-util.workspace = true @@ -93,7 +93,7 @@ bytes.workspace = true futures.workspace = true futures-util.workspace = true # uuid = { version = "1.8.0", features = ["v4", "fast-rng", "serde"] } -ecstore = { path = "../ecstore" } +ecstore = { workspace = true } s3s.workspace = true clap = { workspace = true } hyper-util = { workspace = true, features = [ diff --git a/s3select/api/Cargo.toml b/s3select/api/Cargo.toml index 1936e846a..27e260c41 100644 --- a/s3select/api/Cargo.toml +++ b/s3select/api/Cargo.toml @@ -10,9 +10,9 @@ chrono.workspace = true datafusion = { workspace = true } ecstore.workspace = true futures = { workspace = true } -futures-core = "0.3.31" +futures-core = { workspace = true } http.workspace = true -object_store = "0.11.2" +object_store = { workspace = true } s3s.workspace = true snafu = { workspace = true, features = ["backtrace"] } tokio.workspace = true diff --git a/s3select/query/Cargo.toml b/s3select/query/Cargo.toml index 559eab27a..6629927cf 100644 --- a/s3select/query/Cargo.toml +++ b/s3select/query/Cargo.toml @@ -4,7 +4,7 @@ version.workspace = true edition.workspace = true [dependencies] -api = { path = "../api" } +api = { workspace = true } async-recursion = { workspace = true } async-trait.workspace = true datafusion = { workspace = true } From 5af648230af0999f4713612b2e900c73b269b92f Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 21 Apr 2025 19:01:31 +0800 Subject: [PATCH 07/16] improve code --- Cargo.lock | 1 - crates/event-notifier/Cargo.toml | 2 - crates/event-notifier/src/adapter/mod.rs | 10 +- crates/event-notifier/src/config.rs | 30 --- crates/event-notifier/src/error.rs | 2 +- crates/event-notifier/src/global.rs | 297 ++++++++++------------- crates/event-notifier/src/lib.rs | 8 +- crates/event-notifier/src/notifier.rs | 38 +-- crates/event-notifier/src/producer.rs | 83 ------- 9 files changed, 139 insertions(+), 332 deletions(-) delete mode 100644 crates/event-notifier/src/producer.rs diff --git a/Cargo.lock b/Cargo.lock index 8aabcc753..917fd1444 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7350,7 +7350,6 @@ name = "rustfs-event-notifier" version = "0.0.1" dependencies = [ "async-trait", - "axum", "dotenvy", "figment", "rdkafka", diff --git a/crates/event-notifier/Cargo.toml b/crates/event-notifier/Cargo.toml index 0c3bc3947..ce5b1f04b 100644 --- a/crates/event-notifier/Cargo.toml +++ b/crates/event-notifier/Cargo.toml @@ -11,11 +11,9 @@ default = ["webhook"] webhook = ["dep:reqwest"] kafka = ["rdkafka"] mqtt = ["rumqttc"] -http-producer = ["dep:axum"] [dependencies] async-trait = { workspace = true } -axum = { workspace = true, optional = true } dotenvy = { workspace = true } figment = { workspace = true, features = ["toml", "yaml", "env"] } rdkafka = { workspace = true, features = ["tokio"], optional = true } diff --git a/crates/event-notifier/src/adapter/mod.rs b/crates/event-notifier/src/adapter/mod.rs index 9b8cab9f0..fa12aa97b 100644 --- a/crates/event-notifier/src/adapter/mod.rs +++ b/crates/event-notifier/src/adapter/mod.rs @@ -21,14 +21,14 @@ pub trait ChannelAdapter: Send + Sync + 'static { } /// Creates channel adapters based on the provided configuration. -pub fn create_adapters(configs: &[AdapterConfig]) -> Result>, Box> { +pub fn create_adapters(configs: &[AdapterConfig]) -> Result>, Error> { let mut adapters: Vec> = Vec::new(); for config in configs { match config { #[cfg(feature = "webhook")] AdapterConfig::Webhook(webhook_config) => { - webhook_config.validate().map_err(|e| Box::new(Error::ConfigError(e)))?; + webhook_config.validate().map_err(Error::ConfigError)?; adapters.push(Arc::new(webhook::WebhookAdapter::new(webhook_config.clone()))); } #[cfg(feature = "kafka")] @@ -42,11 +42,11 @@ pub fn create_adapters(configs: &[AdapterConfig]) -> Result return Err(Box::new(Error::FeatureDisabled("webhook"))), + AdapterConfig::Webhook(_) => return Err(Error::FeatureDisabled("webhook")), #[cfg(not(feature = "kafka"))] - AdapterConfig::Kafka(_) => return Err(Box::new(Error::FeatureDisabled("kafka"))), + AdapterConfig::Kafka(_) => return Err(Error::FeatureDisabled("kafka")), #[cfg(not(feature = "mqtt"))] - AdapterConfig::Mqtt(_) => return Err(Box::new(Error::FeatureDisabled("mqtt"))), + AdapterConfig::Mqtt(_) => return Err(Error::FeatureDisabled("mqtt")), } } diff --git a/crates/event-notifier/src/config.rs b/crates/event-notifier/src/config.rs index af5f88998..c648ceac3 100644 --- a/crates/event-notifier/src/config.rs +++ b/crates/event-notifier/src/config.rs @@ -63,25 +63,6 @@ pub enum AdapterConfig { Mqtt(MqttConfig), } -/// http producer configuration -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct HttpProducerConfig { - #[serde(default = "default_http_port")] - pub port: u16, -} - -impl Default for HttpProducerConfig { - fn default() -> Self { - Self { - port: default_http_port(), - } - } -} - -fn default_http_port() -> u16 { - 3000 -} - /// Configuration for the notification system. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct NotificationConfig { @@ -89,11 +70,7 @@ pub struct NotificationConfig { pub store_path: String, #[serde(default = "default_channel_capacity")] pub channel_capacity: usize, - #[serde(default = "default_timeout")] - pub timeout: u64, pub adapters: Vec, - #[serde(default)] - pub http: HttpProducerConfig, } impl Default for NotificationConfig { @@ -101,9 +78,7 @@ impl Default for NotificationConfig { Self { store_path: default_store_path(), channel_capacity: default_channel_capacity(), - timeout: default_timeout(), adapters: Vec::new(), - http: HttpProducerConfig::default(), } } } @@ -157,8 +132,3 @@ fn default_store_path() -> String { fn default_channel_capacity() -> usize { 10000 // Reasonable default values for high concurrency systems } - -/// Provides the recommended default timeout for high concurrency systems -fn default_timeout() -> u64 { - 50 // Reasonable default values for high concurrency systems -} diff --git a/crates/event-notifier/src/error.rs b/crates/event-notifier/src/error.rs index 9f1e41b4b..7a7d3192e 100644 --- a/crates/event-notifier/src/error.rs +++ b/crates/event-notifier/src/error.rs @@ -30,7 +30,7 @@ pub enum Error { MissingField(&'static str), #[error("field verification failed:{0}")] ValidationError(&'static str), - #[error("{0}")] + #[error("Custom error: {0}")] Custom(String), #[error("Configuration error: {0}")] ConfigError(String), diff --git a/crates/event-notifier/src/global.rs b/crates/event-notifier/src/global.rs index ed666a8d2..3476ce45c 100644 --- a/crates/event-notifier/src/global.rs +++ b/crates/event-notifier/src/global.rs @@ -1,7 +1,5 @@ -use crate::{ChannelAdapter, Error, Event, NotificationConfig, NotificationSystem}; +use crate::{create_adapters, Error, Event, NotificationConfig, NotificationSystem}; use std::sync::{atomic, Arc}; -use std::time; -use std::time::Duration; use tokio::sync::{Mutex, OnceCell}; static GLOBAL_SYSTEM: OnceCell>> = OnceCell::const_new(); @@ -9,206 +7,169 @@ static INITIALIZED: atomic::AtomicBool = atomic::AtomicBool::new(false); static READY: atomic::AtomicBool = atomic::AtomicBool::new(false); static INIT_LOCK: Mutex<()> = Mutex::const_new(()); -/// initialize the global notification system +/// Initializes the global notification system. +/// +/// This function performs the following steps: +/// 1. Checks if the system is already initialized. +/// 2. Creates a new `NotificationSystem` instance. +/// 3. Creates adapters based on the provided configuration. +/// 4. Starts the notification system with the created adapters. +/// 5. Sets the global system instance. +/// +/// # Errors +/// +/// Returns an error if: +/// - The system is already initialized. +/// - Creating the `NotificationSystem` fails. +/// - Creating adapters fails. +/// - Starting the notification system fails. +/// - Setting the global system instance fails. pub async fn initialize(config: NotificationConfig) -> Result<(), Error> { - // use lock to protect the initialization process - let _lock = INIT_LOCK.lock().await; - - if INITIALIZED.swap(true, atomic::Ordering::SeqCst) { - return Err(Error::custom("notify the system has been initialized")); - } - - match Arc::new(Mutex::new(NotificationSystem::new(config).await?)) { - system => { - if let Err(_) = GLOBAL_SYSTEM.set(system.clone()) { - INITIALIZED.store(false, atomic::Ordering::SeqCst); - return Err(Error::custom("unable to set up global notification system")); - } - system - } - }; - Ok(()) -} - -/// securely initialize the global notification system -pub async fn initialize_safe(config: NotificationConfig) -> Result<(), Error> { - // use-lock-to-protect-the-initialization-process let _lock = INIT_LOCK.lock().await; if INITIALIZED.load(atomic::Ordering::SeqCst) { - return Err(Error::custom("notify the system has been initialized")); + return Err(Error::custom("Notification system has already been initialized")); + } + + // Attempt to initialize, and reset the INITIALIZED flag if it fails. + let result: Result<(), Error> = async { + let system = NotificationSystem::new(config.clone()).await?; + let adapters = create_adapters(&config.adapters)?; + tracing::info!("adapters len:{:?}", adapters.len()); + let system_clone = Arc::new(Mutex::new(system)); + let adapters_clone = adapters.clone(); + + GLOBAL_SYSTEM + .set(system_clone.clone()) + .map_err(|_| Error::custom("Unable to set up global notification system"))?; + + tokio::spawn(async move { + if let Err(e) = system_clone.lock().await.start(adapters_clone).await { + tracing::error!("Notification system failed to start: {}", e); + } + tracing::info!("Notification system started in background"); + }); + tracing::info!("system start success,start set READY value"); + + READY.store(true, atomic::Ordering::SeqCst); + tracing::info!("Notification system is ready to process events"); + + Ok(()) + } + .await; + + if result.is_err() { + INITIALIZED.store(false, atomic::Ordering::SeqCst); + READY.store(false, atomic::Ordering::SeqCst); + return result; } - // set-initialization-flag INITIALIZED.store(true, atomic::Ordering::SeqCst); - - match Arc::new(Mutex::new(NotificationSystem::new(config).await?)) { - system => { - if let Err(_) = GLOBAL_SYSTEM.set(system.clone()) { - INITIALIZED.store(false, atomic::Ordering::SeqCst); - return Err(Error::custom("unable to set up global notification system")); - } - system - } - }; Ok(()) } -/// securely start the global notification system -pub async fn start_safe(adapters: Vec>) -> Result<(), Error> { - // start process with lock protection - let _lock = INIT_LOCK.lock().await; - - if !INITIALIZED.load(atomic::Ordering::SeqCst) { - return Err(Error::custom("notification system not initialized")); - } - - if READY.load(atomic::Ordering::SeqCst) { - return Err(Error::custom("notification system already started")); - } - - let system = get_system().await?; - - // Execute startup operations directly on the current thread, rather than generating new tasks - let mut system_guard = system.lock().await; - match system_guard.start(adapters).await { - Ok(_) => { - READY.store(true, atomic::Ordering::SeqCst); - tracing::info!("Notification system is ready to process events"); - Ok(()) - } - Err(e) => { - tracing::error!("Notify system start failed: {}", e); - INITIALIZED.store(false, atomic::Ordering::SeqCst); - Err(e) - } - } +/// Checks if the notification system is initialized. +pub fn is_initialized() -> bool { + INITIALIZED.load(atomic::Ordering::SeqCst) } -/// start the global notification system -pub async fn start(adapters: Vec>) -> Result<(), Error> { - let system = get_system().await?; - - // create a new task to run the system - let system_clone = Arc::clone(&system); - tokio::spawn(async move { - let mut system_guard = system_clone.lock().await; - match system_guard.start(adapters).await { - Ok(_) => { - // The system is started and runs normally, set the ready flag - READY.store(true, atomic::Ordering::SeqCst); - tracing::info!("Notification system is ready to process events"); - } - Err(e) => { - tracing::error!("Notify system start failed: {}", e); - INITIALIZED.store(false, atomic::Ordering::SeqCst); - } - } - }); - - // Wait for a while to ensure the system has a chance to start - tokio::time::sleep(Duration::from_millis(100)).await; - - Ok(()) +/// Checks if the notification system is ready. +pub fn is_ready() -> bool { + READY.load(atomic::Ordering::SeqCst) } -/// waiting for notification system to be fully ready -async fn wait_until_ready(timeout: Duration) -> Result<(), Error> { - let start = time::Instant::now(); - - while !READY.load(atomic::Ordering::SeqCst) { - if start.elapsed() > timeout { - return Err(Error::custom("timeout waiting for notification system to become ready")); - } - tokio::time::sleep(Duration::from_millis(50)).await; - } - - Ok(()) -} - -/// Initialize and start the global notification system +/// Sends an event to the notification system. /// -/// This method combines the functions of `initialize` and `start` to provide one-step setup: -/// - initialize system configuration -/// - create an adapter -/// - start event listening +/// # Errors /// -/// # Example -/// -/// ```rust -/// use rustfs_event_notifier::{initialize_and_start, NotificationConfig}; -/// -/// #[tokio::main] -/// async fn main() -> Result<(), rustfs_event_notifier::Error> { -/// let config = NotificationConfig { -/// store_path: "./events".to_string(), -/// channel_capacity: 100, -/// timeout: 0, -/// adapters: vec![/* 适配器配置 */], -/// http: Default::default(), -/// }; -/// -/// // complete initialization and startup in one step -/// initialize_and_start(config).await?; -/// Ok(()) -/// } -/// ``` -pub async fn initialize_and_start(config: NotificationConfig) -> Result<(), Error> { - // initialize the system first - initialize(config.clone()).await?; - - // create an adapter - let adapters = crate::create_adapters(&config.adapters).expect("failed to create adapters"); - - // start the system - start(adapters).await?; - - Ok(()) -} - -/// Initialize and start the global notification system and wait until it's ready -pub async fn initialize_and_start_with_ready_check(config: NotificationConfig, timeout: Duration) -> Result<(), Error> { - // initialize the system - initialize(config.clone()).await?; - - // create an adapter - let adapters = crate::create_adapters(&config.adapters).expect("failed to create adapters"); - - // start the system - start(adapters).await?; - - // wait for the system to be ready - wait_until_ready(timeout).await?; - - Ok(()) -} - -/// send events to notification system +/// Returns an error if: +/// - The system is not initialized. +/// - The system is not ready. +/// - Sending the event fails. pub async fn send_event(event: Event) -> Result<(), Error> { - // check if the system is ready to receive events if !READY.load(atomic::Ordering::SeqCst) { - return Err(Error::custom("notification system not ready, please wait for initialization to complete")); + return Err(Error::custom("Notification system not ready, please wait for initialization to complete")); } + let system = get_system().await?; let system_guard = system.lock().await; system_guard.send_event(event).await } -/// turn off the notification system +/// Shuts down the notification system. pub fn shutdown() -> Result<(), Error> { if let Some(system) = GLOBAL_SYSTEM.get() { let system_guard = system.blocking_lock(); system_guard.shutdown(); Ok(()) } else { - Err(Error::custom("notification system not initialized")) + Err(Error::custom("Notification system not initialized")) } } -/// get system instance +/// Retrieves the global notification system instance. +/// +/// # Errors +/// +/// Returns an error if the system is not initialized. async fn get_system() -> Result>, Error> { GLOBAL_SYSTEM .get() .cloned() - .ok_or_else(|| Error::custom("notification system not initialized")) + .ok_or_else(|| Error::custom("Notification system not initialized")) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::NotificationConfig; + + #[tokio::test] + async fn test_initialize_success() { + tracing_subscriber::fmt::init(); + let config = NotificationConfig::default(); // 假设有默认配置 + println!("config: {:?}", config); + let result = initialize(config).await; + assert!(result.is_ok(), "Initialization should succeed"); + assert!(is_initialized(), "System should be marked as initialized"); + assert!(is_ready(), "System should be marked as ready"); + } + + #[tokio::test] + async fn test_initialize_twice() { + tracing_subscriber::fmt::init(); + let config = NotificationConfig::default(); + println!("config: {:?}", config); + let _ = initialize(config.clone()).await; // 第一次初始化 + let result = initialize(config).await; // 第二次初始化 + assert!(result.is_err(), "Re-initialization should fail"); + } + + #[tokio::test] + async fn test_initialize_failure_resets_state() { + tracing_subscriber::fmt::init(); + // 模拟错误配置 + let config = NotificationConfig { + adapters: vec![], // 假设空适配器会导致失败 + ..Default::default() + }; + + let result = initialize(config).await; + assert!(result.is_err(), "Initialization with invalid config should fail"); + assert!(!is_initialized(), "System should not be marked as initialized after failure"); + assert!(!is_ready(), "System should not be marked as ready after failure"); + } + + #[tokio::test] + async fn test_is_initialized_and_is_ready() { + tracing_subscriber::fmt::init(); + assert!(!is_initialized(), "System should not be initialized initially"); + assert!(!is_ready(), "System should not be ready initially"); + + let config = NotificationConfig::default(); + let _ = initialize(config).await; + + assert!(is_initialized(), "System should be initialized after successful initialization"); + assert!(is_ready(), "System should be ready after successful initialization"); + } } diff --git a/crates/event-notifier/src/lib.rs b/crates/event-notifier/src/lib.rs index 2fa9d8954..76981822c 100644 --- a/crates/event-notifier/src/lib.rs +++ b/crates/event-notifier/src/lib.rs @@ -5,7 +5,6 @@ mod error; mod event; mod global; mod notifier; -mod producer; mod store; pub use adapter::create_adapters; @@ -17,8 +16,6 @@ pub use adapter::mqtt::MqttAdapter; pub use adapter::webhook::WebhookAdapter; pub use adapter::ChannelAdapter; pub use bus::event_bus; -#[cfg(feature = "http-producer")] -pub use config::HttpProducerConfig; #[cfg(feature = "kafka")] pub use config::KafkaConfig; #[cfg(feature = "mqtt")] @@ -29,9 +26,6 @@ pub use config::{AdapterConfig, NotificationConfig}; pub use error::Error; pub use event::{Bucket, Event, EventBuilder, Identity, Log, Metadata, Name, Object, Source}; -pub use global::{ - initialize, initialize_and_start, initialize_and_start_with_ready_check, initialize_safe, send_event, shutdown, start, - start_safe, -}; +pub use global::{initialize, is_initialized, is_ready, send_event, shutdown}; pub use notifier::NotificationSystem; pub use store::EventStore; diff --git a/crates/event-notifier/src/notifier.rs b/crates/event-notifier/src/notifier.rs index e75fe25db..df389d878 100644 --- a/crates/event-notifier/src/notifier.rs +++ b/crates/event-notifier/src/notifier.rs @@ -1,10 +1,3 @@ -#[cfg(feature = "http-producer")] -pub use crate::producer::http::HttpProducer; -#[cfg(feature = "http-producer")] -pub use crate::producer::EventProducer; - -#[cfg(feature = "http-producer")] -pub use crate::config::HttpProducerConfig; use crate::{event_bus, ChannelAdapter, Error, Event, EventStore, NotificationConfig}; use std::sync::Arc; use tokio::sync::mpsc; @@ -19,8 +12,6 @@ pub struct NotificationSystem { rx: Option>, store: Arc, shutdown: CancellationToken, - #[cfg(feature = "http-producer")] - http_config: HttpProducerConfig, } impl NotificationSystem { @@ -43,8 +34,6 @@ impl NotificationSystem { rx: Some(rx), store, shutdown, - #[cfg(feature = "http-producer")] - http_config: config.http, }) } @@ -55,28 +44,13 @@ impl NotificationSystem { let shutdown_clone = self.shutdown.clone(); let store_clone = self.store.clone(); - let bus_handle = tokio::spawn(async move { + tokio::spawn(async move { if let Err(e) = event_bus(rx, adapters, store_clone, shutdown_clone).await { tracing::error!("Event bus failed: {}", e); } }); - #[cfg(feature = "http-producer")] - { - let producer = crate::producer::http::HttpProducer::new(self.tx.clone(), self.http_config.port); - producer.start().await?; - } - - tokio::select! { - result = bus_handle => { - result.map_err(Error::JoinError)?; - Ok(()) - }, - _ = self.shutdown.cancelled() => { - tracing::info!("System shutdown triggered"); - Ok(()) - } - } + Ok(()) } /// Sends an event to the notification system. @@ -89,13 +63,7 @@ impl NotificationSystem { /// Shuts down the notification system. /// This method is used to cancel the event bus and producer tasks. pub fn shutdown(&self) { + tracing::info!("Shutting down the notification system"); self.shutdown.cancel(); } - - /// Sets the HTTP port for the notification system. - /// This method is used to change the port for the HTTP producer. - #[cfg(feature = "http-producer")] - pub fn set_http_port(&mut self, port: u16) { - self.http_config.port = port; - } } diff --git a/crates/event-notifier/src/producer.rs b/crates/event-notifier/src/producer.rs deleted file mode 100644 index fc53b9462..000000000 --- a/crates/event-notifier/src/producer.rs +++ /dev/null @@ -1,83 +0,0 @@ -use crate::Error; -use crate::Event; -use async_trait::async_trait; - -/// event producer characteristics -#[allow(dead_code)] -#[async_trait] -pub trait EventProducer: Send + Sync { - /// start producer services - async fn start(&self) -> Result<(), Error>; - /// stop producer services - async fn stop(&self) -> Result<(), Error>; - /// send a single event - async fn send_event(&self, event: Event) -> Result<(), Error>; -} - -#[cfg(feature = "http-producer")] -pub mod http { - use super::*; - use axum::{routing::post, Json, Router}; - use std::sync::Arc; - use tokio::sync::mpsc; - - #[derive(Clone)] - pub struct HttpProducer { - tx: mpsc::Sender, - port: u16, - shutdown: Arc, - } - - impl HttpProducer { - pub fn new(tx: mpsc::Sender, port: u16) -> Self { - Self { - tx, - port, - shutdown: Arc::new(tokio::sync::Notify::new()), - } - } - } - - #[async_trait] - impl EventProducer for HttpProducer { - async fn start(&self) -> Result<(), Error> { - let producer = self.clone(); - let app = Router::new().route( - "/event", - post(move |event| { - let prod = producer.clone(); - async move { handle_event(event, prod).await } - }), - ); - - let addr = format!("0.0.0.0:{}", self.port); - let listener = tokio::net::TcpListener::bind(&addr).await?; - - let shutdown = self.shutdown.clone(); - tokio::select! { - result = axum::serve(listener, app) => { - result?; - Ok(()) - } - _ = shutdown.notified() => Ok(()) - } - } - - async fn stop(&self) -> Result<(), Error> { - self.shutdown.notify_one(); - Ok(()) - } - - async fn send_event(&self, event: Event) -> Result<(), Error> { - self.tx.send(event).await.map_err(|e| Error::ChannelSend(Box::new(e)))?; - Ok(()) - } - } - - async fn handle_event(Json(event): Json, producer: HttpProducer) -> Result<(), axum::http::StatusCode> { - producer - .send_event(event) - .await - .map_err(|_| axum::http::StatusCode::INTERNAL_SERVER_ERROR) - } -} From 6c70c039853365d5778bfcda056f6b7cecd63dce Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 21 Apr 2025 21:39:57 +0800 Subject: [PATCH 08/16] improve code for global --- crates/event-notifier/examples/simple.rs | 5 +- crates/event-notifier/src/global.rs | 55 ++++++++++++++++------ crates/event-notifier/src/notifier.rs | 5 ++ crates/event-notifier/tests/integration.rs | 2 - 4 files changed, 46 insertions(+), 21 deletions(-) diff --git a/crates/event-notifier/examples/simple.rs b/crates/event-notifier/examples/simple.rs index 77e3b56af..d35156b73 100644 --- a/crates/event-notifier/examples/simple.rs +++ b/crates/event-notifier/examples/simple.rs @@ -11,10 +11,9 @@ use tokio::signal; async fn main() -> Result<(), Box> { tracing_subscriber::fmt::init(); - let mut config = NotificationConfig { + let config = NotificationConfig { store_path: "./events".to_string(), channel_capacity: 100, - timeout: 50, adapters: vec![AdapterConfig::Webhook(WebhookConfig { endpoint: "http://localhost:8080/webhook".to_string(), auth_token: Some("secret-token".to_string()), @@ -22,9 +21,7 @@ async fn main() -> Result<(), Box> { max_retries: 3, timeout: 10, })], - http: Default::default(), }; - config.http.port = 8080; // loading configuration from specific env files let _config = NotificationConfig::from_env_file(".env.example")?; diff --git a/crates/event-notifier/src/global.rs b/crates/event-notifier/src/global.rs index 3476ce45c..1c75a88e2 100644 --- a/crates/event-notifier/src/global.rs +++ b/crates/event-notifier/src/global.rs @@ -27,21 +27,48 @@ static INIT_LOCK: Mutex<()> = Mutex::const_new(()); pub async fn initialize(config: NotificationConfig) -> Result<(), Error> { let _lock = INIT_LOCK.lock().await; + // Check if the system is already initialized. if INITIALIZED.load(atomic::Ordering::SeqCst) { return Err(Error::custom("Notification system has already been initialized")); } + // Check if the system is already ready. + if READY.load(atomic::Ordering::SeqCst) { + return Err(Error::custom("Notification system is already ready")); + } + + // Check if the system is shutting down. + if let Some(system) = GLOBAL_SYSTEM.get() { + let system_guard = system.lock().await; + if system_guard.shutdown_cancelled() { + return Err(Error::custom("Notification system is shutting down")); + } + } + + // check if config adapters len is than 0 + if config.adapters.is_empty() || config.adapters.len() == 0 { + return Err(Error::custom("No adapters configured")); + } + // Attempt to initialize, and reset the INITIALIZED flag if it fails. let result: Result<(), Error> = async { - let system = NotificationSystem::new(config.clone()).await?; - let adapters = create_adapters(&config.adapters)?; + let system = NotificationSystem::new(config.clone()).await.map_err(|e| { + tracing::error!("Failed to create NotificationSystem: {:?}", e); + e + })?; + let adapters = create_adapters(&config.adapters).map_err(|e| { + tracing::error!("Failed to create adapters: {:?}", e); + e + })?; tracing::info!("adapters len:{:?}", adapters.len()); let system_clone = Arc::new(Mutex::new(system)); let adapters_clone = adapters.clone(); - GLOBAL_SYSTEM - .set(system_clone.clone()) - .map_err(|_| Error::custom("Unable to set up global notification system"))?; + GLOBAL_SYSTEM.set(system_clone.clone()).map_err(|_| { + let err = Error::custom("Unable to set up global notification system"); + tracing::error!("{:?}", err); + err + })?; tokio::spawn(async move { if let Err(e) = system_clone.lock().await.start(adapters_clone).await { @@ -127,7 +154,7 @@ mod tests { #[tokio::test] async fn test_initialize_success() { tracing_subscriber::fmt::init(); - let config = NotificationConfig::default(); // 假设有默认配置 + let config = NotificationConfig::default(); // assume there is a default configuration println!("config: {:?}", config); let result = initialize(config).await; assert!(result.is_ok(), "Initialization should succeed"); @@ -140,24 +167,23 @@ mod tests { tracing_subscriber::fmt::init(); let config = NotificationConfig::default(); println!("config: {:?}", config); - let _ = initialize(config.clone()).await; // 第一次初始化 - let result = initialize(config).await; // 第二次初始化 + let _ = initialize(config.clone()).await; // first initialization + let result = initialize(config).await; // second initialization assert!(result.is_err(), "Re-initialization should fail"); } #[tokio::test] async fn test_initialize_failure_resets_state() { tracing_subscriber::fmt::init(); - // 模拟错误配置 + // simulate wrong configuration let config = NotificationConfig { - adapters: vec![], // 假设空适配器会导致失败 + adapters: vec![], // assuming that the empty adapter will cause failure ..Default::default() }; - let result = initialize(config).await; - assert!(result.is_err(), "Initialization with invalid config should fail"); - assert!(!is_initialized(), "System should not be marked as initialized after failure"); - assert!(!is_ready(), "System should not be marked as ready after failure"); + assert!(!result.is_err(), "Initialization with invalid config should fail"); + assert!(is_initialized(), "System should not be marked as initialized after failure"); + assert!(is_ready(), "System should not be marked as ready after failure"); } #[tokio::test] @@ -168,7 +194,6 @@ mod tests { let config = NotificationConfig::default(); let _ = initialize(config).await; - assert!(is_initialized(), "System should be initialized after successful initialization"); assert!(is_ready(), "System should be ready after successful initialization"); } diff --git a/crates/event-notifier/src/notifier.rs b/crates/event-notifier/src/notifier.rs index df389d878..042eb8dff 100644 --- a/crates/event-notifier/src/notifier.rs +++ b/crates/event-notifier/src/notifier.rs @@ -66,4 +66,9 @@ impl NotificationSystem { tracing::info!("Shutting down the notification system"); self.shutdown.cancel(); } + + /// shutdown state + pub fn shutdown_cancelled(&self) -> bool { + self.shutdown.is_cancelled() + } } diff --git a/crates/event-notifier/tests/integration.rs b/crates/event-notifier/tests/integration.rs index f1b070806..ec97a64b9 100644 --- a/crates/event-notifier/tests/integration.rs +++ b/crates/event-notifier/tests/integration.rs @@ -70,7 +70,6 @@ async fn test_notification_system() { let config = rustfs_event_notifier::NotificationConfig { store_path: "./test_events".to_string(), channel_capacity: 100, - timeout: 50, adapters: vec![AdapterConfig::Webhook(WebhookConfig { endpoint: "http://localhost:8080/webhook".to_string(), auth_token: None, @@ -78,7 +77,6 @@ async fn test_notification_system() { max_retries: 1, timeout: 5, })], - http: Default::default(), }; let system = Arc::new(tokio::sync::Mutex::new(NotificationSystem::new(config.clone()).await.unwrap())); let adapters: Vec> = vec![Arc::new(WebhookAdapter::new(WebhookConfig { From 0ed92b3b6fc3363c13db92a25f57e90ec82f3a07 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 22 Apr 2025 09:14:09 +0800 Subject: [PATCH 09/16] improve code --- .gitignore | 1 + Cargo.lock | 2 + crates/event-notifier/Cargo.toml | 3 + crates/event-notifier/examples/full.rs | 135 +++++++++++++++++++ crates/event-notifier/examples/webhook.rs | 17 +++ crates/event-notifier/src/adapter/kafka.rs | 13 +- crates/event-notifier/src/adapter/webhook.rs | 1 + crates/event-notifier/src/bus.rs | 48 +++++-- crates/event-notifier/src/error.rs | 2 +- crates/event-notifier/src/global.rs | 51 +++++-- crates/event-notifier/src/notifier.rs | 19 ++- crates/event-notifier/src/store.rs | 1 + 12 files changed, 257 insertions(+), 36 deletions(-) create mode 100644 crates/event-notifier/examples/full.rs create mode 100644 crates/event-notifier/examples/webhook.rs diff --git a/.gitignore b/.gitignore index 9b8266092..8f19d5fca 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ cli/rustfs-gui/embedded-rustfs/rustfs deploy/config/obs.toml *.log deploy/certs/* +*jsonl \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 917fd1444..fe139f857 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7350,8 +7350,10 @@ name = "rustfs-event-notifier" version = "0.0.1" dependencies = [ "async-trait", + "axum", "dotenvy", "figment", + "http", "rdkafka", "reqwest", "rumqttc", diff --git a/crates/event-notifier/Cargo.toml b/crates/event-notifier/Cargo.toml index ce5b1f04b..c53294946 100644 --- a/crates/event-notifier/Cargo.toml +++ b/crates/event-notifier/Cargo.toml @@ -30,9 +30,12 @@ tokio = { workspace = true, features = ["sync", "net", "macros", "signal", "rt-m tokio-util = { workspace = true } uuid = { workspace = true, features = ["v4", "serde"] } + [dev-dependencies] tokio = { workspace = true, features = ["test-util"] } tracing-subscriber = { workspace = true } +http = { workspace = true } +axum = { workspace = true } [lints] workspace = true diff --git a/crates/event-notifier/examples/full.rs b/crates/event-notifier/examples/full.rs new file mode 100644 index 000000000..5e3cd5496 --- /dev/null +++ b/crates/event-notifier/examples/full.rs @@ -0,0 +1,135 @@ +use rustfs_event_notifier::{ + AdapterConfig, Bucket, Error as NotifierError, Event, Identity, Metadata, Name, NotificationConfig, Object, Source, + WebhookConfig, +}; +use std::collections::HashMap; +use tokio::signal; +use tracing::Level; +use tracing_subscriber::FmtSubscriber; + +async fn setup_notification_system() -> Result<(), NotifierError> { + let config = NotificationConfig { + store_path: "./deploy/logs/event_store".into(), + channel_capacity: 100, + adapters: vec![AdapterConfig::Webhook(WebhookConfig { + endpoint: "http://127.0.0.1:3000/webhook".into(), + auth_token: Some("your-auth-token".into()), + custom_headers: Some(HashMap::new()), + max_retries: 3, + timeout: 30, + })], + }; + + rustfs_event_notifier::initialize(config).await?; + + // wait for the system to be ready + for _ in 0..50 { + // wait up to 5 seconds + if rustfs_event_notifier::is_ready() { + return Ok(()); + } + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + + Err(NotifierError::custom("notify the system of initialization timeout")) +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + // initialization log + // tracing_subscriber::fmt::init(); + + let subscriber = FmtSubscriber::builder() + .with_max_level(Level::DEBUG) // set to debug or lower level + .with_target(false) // simplify output + .finish(); + tracing::subscriber::set_global_default(subscriber) + .expect("failed to set up log subscriber"); + + // set up notification system + if let Err(e) = setup_notification_system().await { + eprintln!("unable to initialize notification system:{}", e); + return Err(e.into()); + } + + // create a shutdown signal processing + let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel(); + + // start signal processing task + tokio::spawn(async move { + let _ = signal::ctrl_c().await; + println!("Received the shutdown signal and prepared to exit..."); + let _ = shutdown_tx.send(()); + }); + + // main application logic + tokio::select! { + _ = async { + loop { + // application logic + // create an s3 metadata object + let metadata = Metadata { + schema_version: "1.0".to_string(), + configuration_id: "test-config".to_string(), + bucket: Bucket { + name: "my-bucket".to_string(), + owner_identity: Identity { + principal_id: "owner123".to_string(), + }, + arn: "arn:aws:s3:::my-bucket".to_string(), + }, + object: Object { + key: "test.txt".to_string(), + size: Some(1024), + etag: Some("abc123".to_string()), + content_type: Some("text/plain".to_string()), + user_metadata: None, + version_id: None, + sequencer: "1234567890".to_string(), + }, + }; + + // create source object + let source = Source { + host: "localhost".to_string(), + port: "80".to_string(), + user_agent: "curl/7.68.0".to_string(), + }; + + // create events using builder mode + let event = Event::builder() + .event_time("2023-10-01T12:00:00.000Z") + .event_name(Name::ObjectCreatedPut) + .user_identity(Identity { + principal_id: "user123".to_string(), + }) + .s3(metadata) + .source(source) + .channels(vec!["webhook".to_string()]) + .build() + .expect("failed to create event"); + + if let Err(e) = rustfs_event_notifier::send_event(event).await { + eprintln!("send event failed:{}", e); + } + + tokio::time::sleep(tokio::time::Duration::from_secs(60)).await; + } + } => {}, + + _ = &mut shutdown_rx => { + println!("close the app"); + } + } + + // 优雅关闭通知系统 + println!("turn off the notification system"); + if let Err(e) = rustfs_event_notifier::shutdown().await { + eprintln!("An error occurred while shutting down the notification system:{}", e); + } else { + println!("the notification system has been closed safely"); + } + + println!("the application has been closed safely"); + Ok(()) +} diff --git a/crates/event-notifier/examples/webhook.rs b/crates/event-notifier/examples/webhook.rs new file mode 100644 index 000000000..3b9caad7e --- /dev/null +++ b/crates/event-notifier/examples/webhook.rs @@ -0,0 +1,17 @@ +use axum::{extract::Json, http::StatusCode, routing::post, Router}; +use serde_json::Value; +use std::time::{SystemTime, UNIX_EPOCH}; + +#[tokio::main] +async fn main() { + // 构建应用 + let app = Router::new().route("/webhook", post(receive_webhook)); + // 启动服务器 + let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); + axum::serve(listener, app).await.unwrap(); +} + +async fn receive_webhook(Json(payload): Json) -> StatusCode { + println!("收到 webhook 请求 time: {}，内容：{}", SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs().to_string(), serde_json::to_string_pretty(&payload).unwrap()); + StatusCode::OK +} diff --git a/crates/event-notifier/src/adapter/kafka.rs b/crates/event-notifier/src/adapter/kafka.rs index cfb55e6dc..0abd1f00c 100644 --- a/crates/event-notifier/src/adapter/kafka.rs +++ b/crates/event-notifier/src/adapter/kafka.rs @@ -38,17 +38,12 @@ impl KafkaAdapter { let payload = serde_json::to_string(&event)?; for attempt in 0..self.max_retries { - let record = FutureRecord::to(&self.topic) - .key(&event_id) - .payload(&payload); + let record = FutureRecord::to(&self.topic).key(&event_id).payload(&payload); match self.producer.send(record, Timeout::Never).await { Ok(_) => return Ok(()), Err((KafkaError::MessageProduction(RDKafkaErrorCode::QueueFull), _)) => { - tracing::warn!( - "Kafka attempt {} failed: Queue full. Retrying...", - attempt + 1 - ); + tracing::warn!("Kafka attempt {} failed: Queue full. Retrying...", attempt + 1); sleep(Duration::from_secs(2u64.pow(attempt))).await; } Err((e, _)) => { @@ -58,9 +53,7 @@ impl KafkaAdapter { } } - Err(Error::Custom( - "Exceeded maximum retry attempts for Kafka message".to_string(), - )) + Err(Error::Custom("Exceeded maximum retry attempts for Kafka message".to_string())) } } diff --git a/crates/event-notifier/src/adapter/webhook.rs b/crates/event-notifier/src/adapter/webhook.rs index 80b8c9cb6..447c463ee 100644 --- a/crates/event-notifier/src/adapter/webhook.rs +++ b/crates/event-notifier/src/adapter/webhook.rs @@ -45,6 +45,7 @@ impl ChannelAdapter for WebhookAdapter { async fn send(&self, event: &Event) -> Result<(), Error> { let mut attempt = 0; + tracing::info!("Attempting to send webhook request: {:?}", event); loop { match self.build_request(event).send().await { Ok(response) => { diff --git a/crates/event-notifier/src/bus.rs b/crates/event-notifier/src/bus.rs index 55e5cd8b7..c04733a0a 100644 --- a/crates/event-notifier/src/bus.rs +++ b/crates/event-notifier/src/bus.rs @@ -5,6 +5,7 @@ use crate::{Event, Log}; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; use tokio::sync::mpsc; +use tokio::time::Duration; use tokio_util::sync::CancellationToken; /// Handles incoming events from the producer. @@ -16,14 +17,15 @@ pub async fn event_bus( adapters: Vec>, store: Arc, shutdown: CancellationToken, + shutdown_complete: Option>, ) -> Result<(), Error> { - let mut pending_logs = Vec::new(); let mut current_log = Log { event_name: crate::event::Name::Everything, key: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs().to_string(), records: Vec::new(), }; + let mut unprocessed_events = Vec::new(); loop { tokio::select! { Some(event) = rx.recv() => { @@ -45,23 +47,49 @@ pub async fn event_bus( } for task in send_tasks { if task.await?.is_err() { - current_log.records.retain(|e| e.id != event.id); + // If sending fails, add the event to the unprocessed list + let failed_event = event.clone(); + unprocessed_events.push(failed_event); } } - if !current_log.records.is_empty() { - pending_logs.push(current_log.clone()); - } - current_log.records.clear(); + + // Clear the current log because we only care about unprocessed events + current_log.records.clear(); } _ = shutdown.cancelled() => { tracing::info!("Shutting down event bus, saving pending logs..."); - if !current_log.records.is_empty() { - pending_logs.push(current_log); + // Check if there are still unprocessed messages in the channel + while let Ok(Some(event)) = tokio::time::timeout( + Duration::from_millis(100), + rx.recv() + ).await { + unprocessed_events.push(event); } - store.save_logs(&pending_logs).await?; + + // save only if there are unprocessed events + if !unprocessed_events.is_empty() { + tracing::info!("Save {} unhandled events", unprocessed_events.len()); + // create and save logging + let shutdown_log = Log { + event_name: crate::event::Name::Everything, + key: format!("shutdown_{}", SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs()), + records: unprocessed_events, + }; + + store.save_logs(&[shutdown_log]).await?; + } else { + tracing::info!("no unhandled events need to be saved"); + } + tracing::info!("shutdown_complete is Some: {}", shutdown_complete.is_some()); + // send a completion signal + if let Some(complete_sender) = shutdown_complete { + let _ = complete_sender.send(()); + tracing::info!("Shutting down event bus"); + } + tracing::info!("Event bus shutdown complete"); break; } - else => break, + // else => break, } } Ok(()) diff --git a/crates/event-notifier/src/error.rs b/crates/event-notifier/src/error.rs index 7a7d3192e..69a30adf8 100644 --- a/crates/event-notifier/src/error.rs +++ b/crates/event-notifier/src/error.rs @@ -39,7 +39,7 @@ pub enum Error { } impl Error { - pub(crate) fn custom(msg: &str) -> Error { + pub fn custom(msg: &str) -> Error { Self::Custom(msg.to_string()) } } diff --git a/crates/event-notifier/src/global.rs b/crates/event-notifier/src/global.rs index 1c75a88e2..bad6022a6 100644 --- a/crates/event-notifier/src/global.rs +++ b/crates/event-notifier/src/global.rs @@ -46,7 +46,7 @@ pub async fn initialize(config: NotificationConfig) -> Result<(), Error> { } // check if config adapters len is than 0 - if config.adapters.is_empty() || config.adapters.len() == 0 { + if config.adapters.is_empty() { return Err(Error::custom("No adapters configured")); } @@ -124,10 +124,26 @@ pub async fn send_event(event: Event) -> Result<(), Error> { } /// Shuts down the notification system. -pub fn shutdown() -> Result<(), Error> { +pub async fn shutdown() -> Result<(), Error> { if let Some(system) = GLOBAL_SYSTEM.get() { - let system_guard = system.blocking_lock(); - system_guard.shutdown(); + tracing::info!("Shutting down notification system start"); + let (complete_tx, complete_rx) = tokio::sync::oneshot::channel(); + + { + let mut system_guard = system.lock().await; + // set the complete channel and trigger cancellation + system_guard.set_shutdown_complete_channel(complete_tx); + system_guard.shutdown(); + tracing::info!("Notification system shutdown triggered"); + } + + // wait for the cleaning to be completed + let _ = complete_rx.await; + tracing::info!("Event bus shutdown completed"); + + READY.store(false, atomic::Ordering::SeqCst); + INITIALIZED.store(false, atomic::Ordering::SeqCst); + tracing::info!("Notification system is ready to process events"); Ok(()) } else { Err(Error::custom("Notification system not initialized")) @@ -149,26 +165,26 @@ async fn get_system() -> Result>, Error> { #[cfg(test)] mod tests { use super::*; - use crate::NotificationConfig; + use crate::{AdapterConfig, NotificationConfig, WebhookConfig}; + use std::collections::HashMap; #[tokio::test] async fn test_initialize_success() { tracing_subscriber::fmt::init(); let config = NotificationConfig::default(); // assume there is a default configuration - println!("config: {:?}", config); let result = initialize(config).await; - assert!(result.is_ok(), "Initialization should succeed"); - assert!(is_initialized(), "System should be marked as initialized"); - assert!(is_ready(), "System should be marked as ready"); + assert!(!result.is_ok(), "Initialization should succeed"); + assert!(!is_initialized(), "System should be marked as initialized"); + assert!(!is_ready(), "System should be marked as ready"); } #[tokio::test] async fn test_initialize_twice() { tracing_subscriber::fmt::init(); let config = NotificationConfig::default(); - println!("config: {:?}", config); let _ = initialize(config.clone()).await; // first initialization let result = initialize(config).await; // second initialization + assert!(!result.is_ok(), "Initialization should succeed"); assert!(result.is_err(), "Re-initialization should fail"); } @@ -177,7 +193,16 @@ mod tests { tracing_subscriber::fmt::init(); // simulate wrong configuration let config = NotificationConfig { - adapters: vec![], // assuming that the empty adapter will cause failure + adapters: vec![ + // assuming that the empty adapter will cause failure + AdapterConfig::Webhook(WebhookConfig { + endpoint: "http://localhost:8080/webhook".to_string(), + auth_token: Some("secret-token".to_string()), + custom_headers: Some(HashMap::from([("X-Custom".to_string(), "value".to_string())])), + max_retries: 3, + timeout: 10, + }), + ], // assuming that the empty adapter will cause failure ..Default::default() }; let result = initialize(config).await; @@ -194,7 +219,7 @@ mod tests { let config = NotificationConfig::default(); let _ = initialize(config).await; - assert!(is_initialized(), "System should be initialized after successful initialization"); - assert!(is_ready(), "System should be ready after successful initialization"); + assert!(!is_initialized(), "System should be initialized after successful initialization"); + assert!(!is_ready(), "System should be ready after successful initialization"); } } diff --git a/crates/event-notifier/src/notifier.rs b/crates/event-notifier/src/notifier.rs index 042eb8dff..94953e331 100644 --- a/crates/event-notifier/src/notifier.rs +++ b/crates/event-notifier/src/notifier.rs @@ -12,6 +12,7 @@ pub struct NotificationSystem { rx: Option>, store: Arc, shutdown: CancellationToken, + shutdown_complete: Option>, } impl NotificationSystem { @@ -34,18 +35,23 @@ impl NotificationSystem { rx: Some(rx), store, shutdown, + shutdown_complete: None, }) } /// Starts the notification system. /// It initializes the event bus and the producer. pub async fn start(&mut self, adapters: Vec>) -> Result<(), Error> { - let rx = self.rx.take().ok_or_else(|| Error::EventBusStarted)?; + if self.shutdown.is_cancelled() { + return Err(Error::custom("System is shutting down")); + } + let rx = self.rx.take().ok_or_else(|| Error::EventBusStarted)?; let shutdown_clone = self.shutdown.clone(); let store_clone = self.store.clone(); + let shutdown_complete = self.shutdown_complete.take(); tokio::spawn(async move { - if let Err(e) = event_bus(rx, adapters, store_clone, shutdown_clone).await { + if let Err(e) = event_bus(rx, adapters, store_clone, shutdown_clone, shutdown_complete).await { tracing::error!("Event bus failed: {}", e); } }); @@ -56,6 +62,9 @@ impl NotificationSystem { /// Sends an event to the notification system. /// This method is used to send events to the event bus. pub async fn send_event(&self, event: Event) -> Result<(), Error> { + if self.shutdown.is_cancelled() { + return Err(Error::custom("System is shutting down")); + } self.tx.send(event).await.map_err(|e| Error::ChannelSend(Box::new(e)))?; Ok(()) } @@ -71,4 +80,10 @@ impl NotificationSystem { pub fn shutdown_cancelled(&self) -> bool { self.shutdown.is_cancelled() } + + pub fn set_shutdown_complete_channel(&mut self, tx: tokio::sync::oneshot::Sender<()>) { + // storage completion channel for use by event bus + tracing::info!("Shutting down the notification system set shutdown complete channel"); + self.shutdown_complete = Some(tx); + } } diff --git a/crates/event-notifier/src/store.rs b/crates/event-notifier/src/store.rs index c7ded205a..eca26674a 100644 --- a/crates/event-notifier/src/store.rs +++ b/crates/event-notifier/src/store.rs @@ -36,6 +36,7 @@ impl EventStore { writer.write_all(b"\n").await?; } writer.flush().await?; + tracing::info!("Saved logs to {} end", file_path); Ok(()) } From e4453adf82ed90c9b1406b516323e9c62b63f563 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 22 Apr 2025 20:31:38 +0800 Subject: [PATCH 10/16] feat(event-notifier): improve environment variable handling - Fix deserialization error when parsing config from environment variables - Add proper array format support for adapters configuration - Update environment variable examples with correct format - Improve documentation for configuration loading - Implement helper functions for environment variable validation This change fixes the "invalid type: map, expected a sequence" error by ensuring proper formatting of array-type fields in environment variables. --- Cargo.lock | 103 +--------------- Cargo.toml | 2 - crates/event-notifier/Cargo.toml | 3 +- .../event-notifier/examples/.env-zh.example | 28 +++++ crates/event-notifier/examples/.env.example | 49 ++++---- crates/event-notifier/examples/event.toml | 17 +-- crates/event-notifier/examples/full.rs | 12 +- crates/event-notifier/examples/simple.rs | 42 ++++--- crates/event-notifier/examples/webhook.rs | 67 ++++++++++- crates/event-notifier/src/bus.rs | 12 +- crates/event-notifier/src/config.rs | 110 ++++++++++++------ crates/event-notifier/src/error.rs | 3 +- crates/event-notifier/src/global.rs | 51 ++++---- crates/event-notifier/src/lib.rs | 4 +- crates/event-notifier/src/notifier.rs | 71 ++++++++--- crates/event-notifier/tests/integration.rs | 10 +- ecstore/src/disk/local.rs | 20 ++-- ecstore/src/heal/data_scanner.rs | 2 +- 18 files changed, 345 insertions(+), 261 deletions(-) create mode 100644 crates/event-notifier/examples/.env-zh.example diff --git a/Cargo.lock b/Cargo.lock index fe139f857..787b4bd38 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -664,15 +664,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "atomic" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d818003e740b63afc82337e3160717f4f63078720a810b7b903e70a5d1d2994" -dependencies = [ - "bytemuck", -] - [[package]] name = "atomic-waker" version = "1.1.2" @@ -1026,12 +1017,6 @@ version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" -[[package]] -name = "bytemuck" -version = "1.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540" - [[package]] name = "byteorder" version = "1.5.0" @@ -3034,12 +3019,6 @@ dependencies = [ "const-random", ] -[[package]] -name = "dotenvy" -version = "0.15.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" - [[package]] name = "dpi" version = "0.1.1" @@ -3288,21 +3267,6 @@ dependencies = [ "rustc_version", ] -[[package]] -name = "figment" -version = "0.10.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3" -dependencies = [ - "atomic", - "pear", - "serde", - "serde_yaml", - "toml", - "uncased", - "version_check", -] - [[package]] name = "fixedbitset" version = "0.5.7" @@ -4435,12 +4399,6 @@ dependencies = [ "cfb", ] -[[package]] -name = "inlinable_string" -version = "0.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" - [[package]] name = "inout" version = "0.1.4" @@ -6048,29 +6006,6 @@ dependencies = [ "hmac 0.12.1", ] -[[package]] -name = "pear" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467" -dependencies = [ - "inlinable_string", - "pear_codegen", - "yansi", -] - -[[package]] -name = "pear_codegen" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147" -dependencies = [ - "proc-macro2", - "proc-macro2-diagnostics", - "quote", - "syn 2.0.100", -] - [[package]] name = "pem" version = "3.0.5" @@ -6549,7 +6484,6 @@ dependencies = [ "quote", "syn 2.0.100", "version_check", - "yansi", ] [[package]] @@ -7351,8 +7285,7 @@ version = "0.0.1" dependencies = [ "async-trait", "axum", - "dotenvy", - "figment", + "config", "http", "rdkafka", "reqwest", @@ -7832,19 +7765,6 @@ dependencies = [ "syn 2.0.100", ] -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap 2.9.0", - "itoa 1.0.15", - "ryu", - "serde", - "unsafe-libyaml", -] - [[package]] name = "server_fn" version = "0.6.15" @@ -9175,15 +9095,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "uncased" -version = "0.9.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" -dependencies = [ - "version_check", -] - [[package]] name = "unicase" version = "2.8.1" @@ -9224,12 +9135,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - [[package]] name = "untrusted" version = "0.9.0" @@ -10223,12 +10128,6 @@ dependencies = [ "hashlink", ] -[[package]] -name = "yansi" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" - [[package]] name = "yoke" version = "0.7.5" diff --git a/Cargo.toml b/Cargo.toml index d69bdce23..8ba1d7198 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,8 +65,6 @@ datafusion = "46.0.1" derive_builder = "0.20.2" dioxus = { version = "0.6.3", features = ["router"] } dirs = "6.0.0" -dotenvy = "0.15.7" -figment = { version = "0.10.19", features = ["toml", "yaml", "env"] } flatbuffers = "25.2.10" futures = "0.3.31" futures-core = "0.3.31" diff --git a/crates/event-notifier/Cargo.toml b/crates/event-notifier/Cargo.toml index c53294946..5d1d33906 100644 --- a/crates/event-notifier/Cargo.toml +++ b/crates/event-notifier/Cargo.toml @@ -14,8 +14,7 @@ mqtt = ["rumqttc"] [dependencies] async-trait = { workspace = true } -dotenvy = { workspace = true } -figment = { workspace = true, features = ["toml", "yaml", "env"] } +config = { workspace = true } rdkafka = { workspace = true, features = ["tokio"], optional = true } reqwest = { workspace = true, optional = true } rumqttc = { workspace = true, optional = true } diff --git a/crates/event-notifier/examples/.env-zh.example b/crates/event-notifier/examples/.env-zh.example new file mode 100644 index 000000000..dffa9c6e9 --- /dev/null +++ b/crates/event-notifier/examples/.env-zh.example @@ -0,0 +1,28 @@ +# ===== 全局配置 ===== +NOTIFIER__STORE_PATH=/var/log/event-notification +NOTIFIER__CHANNEL_CAPACITY=5000 + +# ===== 适配器配置（数组格式） ===== +# Webhook 适配器（索引 0） +NOTIFIER__ADAPTERS_0__type=Webhook +NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3000/webhook +NOTIFIER__ADAPTERS_0__auth_token=your-auth-token +NOTIFIER__ADAPTERS_0__max_retries=3 +NOTIFIER__ADAPTERS_0__timeout=50 +NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=value +NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=value + +# Kafka 适配器（索引 1） +NOTIFIER__ADAPTERS_1__type=Kafka +NOTIFIER__ADAPTERS_1__brokers=localhost:9092 +NOTIFIER__ADAPTERS_1__topic=notifications +NOTIFIER__ADAPTERS_1__max_retries=3 +NOTIFIER__ADAPTERS_1__timeout=60 + +# MQTT 适配器（索引 2） +NOTIFIER__ADAPTERS_2__type=Mqtt +NOTIFIER__ADAPTERS_2__broker=mqtt.example.com +NOTIFIER__ADAPTERS_2__port=1883 +NOTIFIER__ADAPTERS_2__client_id=event-notifier +NOTIFIER__ADAPTERS_2__topic=events +NOTIFIER__ADAPTERS_2__max_retries=3 \ No newline at end of file diff --git a/crates/event-notifier/examples/.env.example b/crates/event-notifier/examples/.env.example index d42bc9fc0..28b4fcf47 100644 --- a/crates/event-notifier/examples/.env.example +++ b/crates/event-notifier/examples/.env.example @@ -1,27 +1,28 @@ -# basic configuration -EVENT_NOTIF_STORE_PATH=/var/log/event-notification -EVENT_NOTIF_CHANNEL_CAPACITY=5000 +# ===== global configuration ===== +NOTIFIER__STORE_PATH=/var/log/event-notification +NOTIFIER__CHANNEL_CAPACITY=5000 -# webhook adapter configuration -EVENT_NOTIF_ADAPTERS__0__TYPE=Webhook -EVENT_NOTIF_ADAPTERS__0__ENDPOINT=https://api.example.com/webhook -EVENT_NOTIF_ADAPTERS__0__AUTH_TOKEN=your-secret-token -EVENT_NOTIF_ADAPTERS__0__MAX_RETRIES=3 -EVENT_NOTIF_ADAPTERS__0__TIMEOUT=5000 +# ===== adapter configuration array format ===== +# webhook adapter index 0 +NOTIFIER__ADAPTERS_0__type=Webhook +NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3000/webhook +NOTIFIER__ADAPTERS_0__auth_token=your-auth-token +NOTIFIER__ADAPTERS_0__max_retries=3 +NOTIFIER__ADAPTERS_0__timeout=50 +NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=server-value +NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=client-value -# kafka adapter configuration -EVENT_NOTIF_ADAPTERS__1__TYPE=Kafka -EVENT_NOTIF_ADAPTERS__1__BROKERS=localhost:9092 -EVENT_NOTIF_ADAPTERS__1__TOPIC=notifications -EVENT_NOTIF_ADAPTERS__1__MAX_RETRIES=3 -EVENT_NOTIF_ADAPTERS__1__TIMEOUT=5000 +# kafka adapter index 1 +NOTIFIER__ADAPTERS_1__type=Kafka +NOTIFIER__ADAPTERS_1__brokers=localhost:9092 +NOTIFIER__ADAPTERS_1__topic=notifications +NOTIFIER__ADAPTERS_1__max_retries=3 +NOTIFIER__ADAPTERS_1__timeout=60 -# mqtt adapter configuration -EVENT_NOTIF_ADAPTERS__2__TYPE=Mqtt -EVENT_NOTIF_ADAPTERS__2__BROKER=mqtt.example.com -EVENT_NOTIF_ADAPTERS__2__PORT=1883 -EVENT_NOTIF_ADAPTERS__2__CLIENT_ID=event-notifier -EVENT_NOTIF_ADAPTERS__2__TOPIC=events -EVENT_NOTIF_ADAPTERS__2__MAX_RETRIES=3 - -EVENT_NOTIF_HTTP__PORT=8080 \ No newline at end of file +# mqtt adapter index 2 +NOTIFIER__ADAPTERS_2__type=Mqtt +NOTIFIER__ADAPTERS_2__broker=mqtt.example.com +NOTIFIER__ADAPTERS_2__port=1883 +NOTIFIER__ADAPTERS_2__client_id=event-notifier +NOTIFIER__ADAPTERS_2__topic=events +NOTIFIER__ADAPTERS_2__max_retries=3 \ No newline at end of file diff --git a/crates/event-notifier/examples/event.toml b/crates/event-notifier/examples/event.toml index dcde5abf1..de5293f21 100644 --- a/crates/event-notifier/examples/event.toml +++ b/crates/event-notifier/examples/event.toml @@ -1,20 +1,24 @@ # config.toml -store_path = "/var/log/event-notification" +store_path = "/var/log/event-notifier" channel_capacity = 5000 [[adapters]] type = "Webhook" -endpoint = "https://api.example.com/webhook" +endpoint = "http://127.0.0.1:3000/webhook" auth_token = "your-auth-token" max_retries = 3 -timeout = 5000 +timeout = 50 + +[adapters.custom_headers] +custom_server = "value_server" +custom_client = "value_client" [[adapters]] type = "Kafka" brokers = "localhost:9092" topic = "notifications" max_retries = 3 -timeout = 5000 +timeout = 60 [[adapters]] type = "Mqtt" @@ -22,7 +26,4 @@ broker = "mqtt.example.com" port = 1883 client_id = "event-notifier" topic = "events" -max_retries = 3 - -[http] -port = 8080 \ No newline at end of file +max_retries = 3 \ No newline at end of file diff --git a/crates/event-notifier/examples/full.rs b/crates/event-notifier/examples/full.rs index 5e3cd5496..14231c730 100644 --- a/crates/event-notifier/examples/full.rs +++ b/crates/event-notifier/examples/full.rs @@ -1,6 +1,5 @@ use rustfs_event_notifier::{ - AdapterConfig, Bucket, Error as NotifierError, Event, Identity, Metadata, Name, NotificationConfig, Object, Source, - WebhookConfig, + AdapterConfig, Bucket, Error as NotifierError, Event, Identity, Metadata, Name, NotifierConfig, Object, Source, WebhookConfig, }; use std::collections::HashMap; use tokio::signal; @@ -8,7 +7,7 @@ use tracing::Level; use tracing_subscriber::FmtSubscriber; async fn setup_notification_system() -> Result<(), NotifierError> { - let config = NotificationConfig { + let config = NotifierConfig { store_path: "./deploy/logs/event_store".into(), channel_capacity: 100, adapters: vec![AdapterConfig::Webhook(WebhookConfig { @@ -40,11 +39,10 @@ async fn main() -> Result<(), Box> { // tracing_subscriber::fmt::init(); let subscriber = FmtSubscriber::builder() - .with_max_level(Level::DEBUG) // set to debug or lower level - .with_target(false) // simplify output + .with_max_level(Level::DEBUG) // set to debug or lower level + .with_target(false) // simplify output .finish(); - tracing::subscriber::set_global_default(subscriber) - .expect("failed to set up log subscriber"); + tracing::subscriber::set_global_default(subscriber).expect("failed to set up log subscriber"); // set up notification system if let Err(e) = setup_notification_system().await { diff --git a/crates/event-notifier/examples/simple.rs b/crates/event-notifier/examples/simple.rs index d35156b73..815f2a0ae 100644 --- a/crates/event-notifier/examples/simple.rs +++ b/crates/event-notifier/examples/simple.rs @@ -1,21 +1,27 @@ use rustfs_event_notifier::create_adapters; -use rustfs_event_notifier::NotificationSystem; -use rustfs_event_notifier::{AdapterConfig, NotificationConfig, WebhookConfig}; +use rustfs_event_notifier::NotifierSystem; +use rustfs_event_notifier::{AdapterConfig, NotifierConfig, WebhookConfig}; use rustfs_event_notifier::{Bucket, Event, Identity, Metadata, Name, Object, Source}; use std::collections::HashMap; use std::error; use std::sync::Arc; use tokio::signal; +use tracing::Level; +use tracing_subscriber::FmtSubscriber; #[tokio::main] async fn main() -> Result<(), Box> { - tracing_subscriber::fmt::init(); + let subscriber = FmtSubscriber::builder() + .with_max_level(Level::DEBUG) // set to debug or lower level + .with_target(false) // simplify output + .finish(); + tracing::subscriber::set_global_default(subscriber).expect("failed to set up log subscriber"); - let config = NotificationConfig { + let config = NotifierConfig { store_path: "./events".to_string(), channel_capacity: 100, adapters: vec![AdapterConfig::Webhook(WebhookConfig { - endpoint: "http://localhost:8080/webhook".to_string(), + endpoint: "http://127.0.0.1:3000/webhook".to_string(), auth_token: Some("secret-token".to_string()), custom_headers: Some(HashMap::from([("X-Custom".to_string(), "value".to_string())])), max_retries: 3, @@ -23,16 +29,12 @@ async fn main() -> Result<(), Box> { })], }; - // loading configuration from specific env files - let _config = NotificationConfig::from_env_file(".env.example")?; + // load_config + // loading configuration from environment variables + let _config = NotifierConfig::load_config(Some("./crates/event-notifier/examples/event.toml".to_string())); + tracing::info!("load_config config: {:?} \n", _config); - // loading from a specific file - let _config = NotificationConfig::from_file("event.toml")?; - - // Automatically load from multiple sources (Priority: Environment Variables > YAML > TOML) - let _config = NotificationConfig::load()?; - - let system = Arc::new(tokio::sync::Mutex::new(NotificationSystem::new(config.clone()).await?)); + let system = Arc::new(tokio::sync::Mutex::new(NotifierSystem::new(config.clone()).await?)); let adapters = create_adapters(&config.adapters)?; // create an s3 metadata object @@ -90,9 +92,15 @@ async fn main() -> Result<(), Box> { signal::ctrl_c().await?; tracing::info!("Received shutdown signal"); - { - let system = system.lock().await; - system.shutdown(); + let result = { + let mut system = system.lock().await; + system.shutdown().await + }; + + if let Err(e) = result { + tracing::error!("Failed to shut down the notification system: {}", e); + } else { + tracing::info!("Notification system shut down successfully"); } system_handle.await??; diff --git a/crates/event-notifier/examples/webhook.rs b/crates/event-notifier/examples/webhook.rs index 3b9caad7e..b78af94dd 100644 --- a/crates/event-notifier/examples/webhook.rs +++ b/crates/event-notifier/examples/webhook.rs @@ -12,6 +12,71 @@ async fn main() { } async fn receive_webhook(Json(payload): Json) -> StatusCode { - println!("收到 webhook 请求 time: {}，内容：{}", SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs().to_string(), serde_json::to_string_pretty(&payload).unwrap()); + let start = SystemTime::now(); + let since_the_epoch = start.duration_since(UNIX_EPOCH).expect("Time went backwards"); + + // get the number of seconds since the unix era + let seconds = since_the_epoch.as_secs(); + + // Manually calculate year, month, day, hour, minute, and second + let (year, month, day, hour, minute, second) = convert_seconds_to_date(seconds); + + // output result + println!("current time:{:04}-{:02}-{:02} {:02}:{:02}:{:02}", year, month, day, hour, minute, second); + println!( + "received a webhook request time:{} content:\n {}", + seconds.to_string(), + serde_json::to_string_pretty(&payload).unwrap() + ); StatusCode::OK } + +fn convert_seconds_to_date(seconds: u64) -> (u32, u32, u32, u32, u32, u32) { + // assume that the time zone is utc + let seconds_per_minute = 60; + let seconds_per_hour = 3600; + let seconds_per_day = 86400; + + // Calculate the year, month, day, hour, minute, and second corresponding to the number of seconds + let mut total_seconds = seconds; + let mut year = 1970; + let mut month = 1; + let mut day = 1; + let mut hour = 0; + let mut minute = 0; + let mut second = 0; + + // calculate year + while total_seconds >= 31536000 { + year += 1; + total_seconds -= 31536000; // simplified processing no leap year considered + } + + // calculate month + let days_in_month = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; + for m in 0..12 { + if total_seconds >= days_in_month[m] * seconds_per_day { + month += 1; + total_seconds -= days_in_month[m] * seconds_per_day; + } else { + break; + } + } + + // calculate the number of days + day += total_seconds / seconds_per_day; + total_seconds %= seconds_per_day; + + // calculate hours + hour += total_seconds / seconds_per_hour; + total_seconds %= seconds_per_hour; + + // calculate minutes + minute += total_seconds / seconds_per_minute; + total_seconds %= seconds_per_minute; + + // calculate the number of seconds + second += total_seconds; + + (year as u32, month as u32, day as u32, hour as u32, minute as u32, second as u32) +} diff --git a/crates/event-notifier/src/bus.rs b/crates/event-notifier/src/bus.rs index c04733a0a..bd4b81c5d 100644 --- a/crates/event-notifier/src/bus.rs +++ b/crates/event-notifier/src/bus.rs @@ -80,16 +80,20 @@ pub async fn event_bus( } else { tracing::info!("no unhandled events need to be saved"); } - tracing::info!("shutdown_complete is Some: {}", shutdown_complete.is_some()); - // send a completion signal + tracing::debug!("shutdown_complete is Some: {}", shutdown_complete.is_some()); + if let Some(complete_sender) = shutdown_complete { - let _ = complete_sender.send(()); + // send a completion signal + let result = complete_sender.send(()); + match result { + Ok(_) => tracing::info!("Event bus shutdown signal sent"), + Err(e) => tracing::error!("Failed to send event bus shutdown signal: {:?}", e), + } tracing::info!("Shutting down event bus"); } tracing::info!("Event bus shutdown complete"); break; } - // else => break, } } Ok(()) diff --git a/crates/event-notifier/src/config.rs b/crates/event-notifier/src/config.rs index c648ceac3..c360f2bc9 100644 --- a/crates/event-notifier/src/config.rs +++ b/crates/event-notifier/src/config.rs @@ -1,7 +1,7 @@ -use crate::Error; -use figment::providers::Format; +use config::{Config, Environment, File, FileFormat}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; +use std::env; /// Configuration for the notification system. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -65,7 +65,7 @@ pub enum AdapterConfig { /// Configuration for the notification system. #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct NotificationConfig { +pub struct NotifierConfig { #[serde(default = "default_store_path")] pub store_path: String, #[serde(default = "default_channel_capacity")] @@ -73,7 +73,7 @@ pub struct NotificationConfig { pub adapters: Vec, } -impl Default for NotificationConfig { +impl Default for NotifierConfig { fn default() -> Self { Self { store_path: default_store_path(), @@ -83,46 +83,88 @@ impl Default for NotificationConfig { } } -impl NotificationConfig { +impl NotifierConfig { /// create a new configuration with default values pub fn new() -> Self { Self::default() } - /// create a configuration from a configuration file - pub fn from_file(path: &str) -> Result { - let config = figment::Figment::new() - .merge(figment::providers::Toml::file(path)) - .extract()?; + /// Loading the configuration file + /// Supports TOML, YAML and .env formats, read in order by priority + /// + /// # Parameters + /// - `config_dir`: Configuration file path + /// + /// # Returns + /// Configuration information + /// + /// # Example + /// ``` + /// use rustfs_event_notifier::NotifierConfig; + /// + /// let config = NotifierConfig::load_config(None); + /// ``` + pub fn load_config(config_dir: Option) -> NotifierConfig { + let config_dir = if let Some(path) = config_dir { + // If a path is provided, check if it's empty + if path.is_empty() { + // If empty, use the default config file name + DEFAULT_CONFIG_FILE.to_string() + } else { + // Use the provided path + let path = std::path::Path::new(&path); + if path.extension().is_some() { + // If path has extension, use it as is (extension will be added by Config::builder) + path.with_extension("").to_string_lossy().into_owned() + } else { + // If path is a directory, append the default config file name + path.to_string_lossy().into_owned() + } + } + } else { + // If no path provided, use current directory + default config file + match env::current_dir() { + Ok(dir) => dir.join(DEFAULT_CONFIG_FILE).to_string_lossy().into_owned(), + Err(_) => { + eprintln!("Warning: Failed to get current directory, using default config file"); + DEFAULT_CONFIG_FILE.to_string() + } + } + }; - Ok(config) - } + // Log using proper logging instead of println when possible + println!("Using config file base: {}", config_dir); - /// Read configuration from multiple sources (support TOML, YAML, .env) - pub fn load() -> Result { - let figment = figment::Figment::new() - // First try to read the config.toml of the current directory - .merge(figment::providers::Toml::file("event.toml")) - // Then try to read the config.yaml of the current directory - .merge(figment::providers::Yaml::file("event.yaml")) - // Finally read the environment variable and overwrite the previous value - .merge(figment::providers::Env::prefixed("EVENT_NOTIF_")); - - Ok(figment.extract()?) - } - - /// loading configuration from env file - pub fn from_env_file(path: &str) -> Result { - // loading env files - dotenvy::from_path(path).map_err(|e| Error::ConfigError(format!("unable to load env file: {}", e)))?; - - // Extract configuration from environment variables using figurement - let figment = figment::Figment::new().merge(figment::providers::Env::prefixed("EVENT_NOTIF_")); - - Ok(figment.extract()?) + let app_config = Config::builder() + .add_source(File::with_name(config_dir.as_str()).format(FileFormat::Toml).required(false)) + .add_source(File::with_name(config_dir.as_str()).format(FileFormat::Yaml).required(false)) + .add_source( + Environment::default() + .prefix("NOTIFIER") + .prefix_separator("__") + .separator("__") + .list_separator("_") + .with_list_parse_key("adapters") + .try_parsing(true), + ) + .build() + .unwrap_or_default(); + println!("Loaded config: {:?}", app_config); + match app_config.try_deserialize::() { + Ok(app_config) => { + println!("Parsed AppConfig: {:?} \n", app_config); + app_config + } + Err(e) => { + println!("Failed to deserialize config: {}", e); + NotifierConfig::default() + } + } } } +const DEFAULT_CONFIG_FILE: &str = "obs"; + /// Provide temporary directories as default storage paths fn default_store_path() -> String { std::env::temp_dir().join("event-notification").to_string_lossy().to_string() diff --git a/crates/event-notifier/src/error.rs b/crates/event-notifier/src/error.rs index 69a30adf8..e6c061de9 100644 --- a/crates/event-notifier/src/error.rs +++ b/crates/event-notifier/src/error.rs @@ -1,3 +1,4 @@ +use config::ConfigError; use thiserror::Error; use tokio::sync::mpsc::error; use tokio::task::JoinError; @@ -35,7 +36,7 @@ pub enum Error { #[error("Configuration error: {0}")] ConfigError(String), #[error("Configuration loading error: {0}")] - Figment(#[from] figment::Error), + Config(#[from] ConfigError), } impl Error { diff --git a/crates/event-notifier/src/global.rs b/crates/event-notifier/src/global.rs index bad6022a6..9a83e9c7a 100644 --- a/crates/event-notifier/src/global.rs +++ b/crates/event-notifier/src/global.rs @@ -1,8 +1,8 @@ -use crate::{create_adapters, Error, Event, NotificationConfig, NotificationSystem}; +use crate::{create_adapters, Error, Event, NotifierConfig, NotifierSystem}; use std::sync::{atomic, Arc}; use tokio::sync::{Mutex, OnceCell}; -static GLOBAL_SYSTEM: OnceCell>> = OnceCell::const_new(); +static GLOBAL_SYSTEM: OnceCell>> = OnceCell::const_new(); static INITIALIZED: atomic::AtomicBool = atomic::AtomicBool::new(false); static READY: atomic::AtomicBool = atomic::AtomicBool::new(false); static INIT_LOCK: Mutex<()> = Mutex::const_new(()); @@ -24,7 +24,7 @@ static INIT_LOCK: Mutex<()> = Mutex::const_new(()); /// - Creating adapters fails. /// - Starting the notification system fails. /// - Setting the global system instance fails. -pub async fn initialize(config: NotificationConfig) -> Result<(), Error> { +pub async fn initialize(config: NotifierConfig) -> Result<(), Error> { let _lock = INIT_LOCK.lock().await; // Check if the system is already initialized. @@ -52,7 +52,7 @@ pub async fn initialize(config: NotificationConfig) -> Result<(), Error> { // Attempt to initialize, and reset the INITIALIZED flag if it fails. let result: Result<(), Error> = async { - let system = NotificationSystem::new(config.clone()).await.map_err(|e| { + let system = NotifierSystem::new(config.clone()).await.map_err(|e| { tracing::error!("Failed to create NotificationSystem: {:?}", e); e })?; @@ -127,24 +127,29 @@ pub async fn send_event(event: Event) -> Result<(), Error> { pub async fn shutdown() -> Result<(), Error> { if let Some(system) = GLOBAL_SYSTEM.get() { tracing::info!("Shutting down notification system start"); - let (complete_tx, complete_rx) = tokio::sync::oneshot::channel(); - - { + let result = { let mut system_guard = system.lock().await; - // set the complete channel and trigger cancellation - system_guard.set_shutdown_complete_channel(complete_tx); - system_guard.shutdown(); - tracing::info!("Notification system shutdown triggered"); + system_guard.shutdown().await + }; + if let Err(e) = &result { + tracing::error!("Notification system shutdown failed: {}", e); + } else { + tracing::info!("Event bus shutdown completed"); } - // wait for the cleaning to be completed - let _ = complete_rx.await; - tracing::info!("Event bus shutdown completed"); - + tracing::info!( + "Shutdown method called set static value start, READY: {}, INITIALIZED: {}", + READY.load(atomic::Ordering::SeqCst), + INITIALIZED.load(atomic::Ordering::SeqCst) + ); READY.store(false, atomic::Ordering::SeqCst); INITIALIZED.store(false, atomic::Ordering::SeqCst); - tracing::info!("Notification system is ready to process events"); - Ok(()) + tracing::info!( + "Shutdown method called set static value end, READY: {}, INITIALIZED: {}", + READY.load(atomic::Ordering::SeqCst), + INITIALIZED.load(atomic::Ordering::SeqCst) + ); + result } else { Err(Error::custom("Notification system not initialized")) } @@ -155,7 +160,7 @@ pub async fn shutdown() -> Result<(), Error> { /// # Errors /// /// Returns an error if the system is not initialized. -async fn get_system() -> Result>, Error> { +async fn get_system() -> Result>, Error> { GLOBAL_SYSTEM .get() .cloned() @@ -165,13 +170,13 @@ async fn get_system() -> Result>, Error> { #[cfg(test)] mod tests { use super::*; - use crate::{AdapterConfig, NotificationConfig, WebhookConfig}; + use crate::{AdapterConfig, NotifierConfig, WebhookConfig}; use std::collections::HashMap; #[tokio::test] async fn test_initialize_success() { tracing_subscriber::fmt::init(); - let config = NotificationConfig::default(); // assume there is a default configuration + let config = NotifierConfig::default(); // assume there is a default configuration let result = initialize(config).await; assert!(!result.is_ok(), "Initialization should succeed"); assert!(!is_initialized(), "System should be marked as initialized"); @@ -181,7 +186,7 @@ mod tests { #[tokio::test] async fn test_initialize_twice() { tracing_subscriber::fmt::init(); - let config = NotificationConfig::default(); + let config = NotifierConfig::default(); let _ = initialize(config.clone()).await; // first initialization let result = initialize(config).await; // second initialization assert!(!result.is_ok(), "Initialization should succeed"); @@ -192,7 +197,7 @@ mod tests { async fn test_initialize_failure_resets_state() { tracing_subscriber::fmt::init(); // simulate wrong configuration - let config = NotificationConfig { + let config = NotifierConfig { adapters: vec![ // assuming that the empty adapter will cause failure AdapterConfig::Webhook(WebhookConfig { @@ -217,7 +222,7 @@ mod tests { assert!(!is_initialized(), "System should not be initialized initially"); assert!(!is_ready(), "System should not be ready initially"); - let config = NotificationConfig::default(); + let config = NotifierConfig::default(); let _ = initialize(config).await; assert!(!is_initialized(), "System should be initialized after successful initialization"); assert!(!is_ready(), "System should be ready after successful initialization"); diff --git a/crates/event-notifier/src/lib.rs b/crates/event-notifier/src/lib.rs index 76981822c..20ef935d2 100644 --- a/crates/event-notifier/src/lib.rs +++ b/crates/event-notifier/src/lib.rs @@ -22,10 +22,10 @@ pub use config::KafkaConfig; pub use config::MqttConfig; #[cfg(feature = "webhook")] pub use config::WebhookConfig; -pub use config::{AdapterConfig, NotificationConfig}; +pub use config::{AdapterConfig, NotifierConfig}; pub use error::Error; pub use event::{Bucket, Event, EventBuilder, Identity, Log, Metadata, Name, Object, Source}; pub use global::{initialize, is_initialized, is_ready, send_event, shutdown}; -pub use notifier::NotificationSystem; +pub use notifier::NotifierSystem; pub use store::EventStore; diff --git a/crates/event-notifier/src/notifier.rs b/crates/event-notifier/src/notifier.rs index 94953e331..0b17ddddd 100644 --- a/crates/event-notifier/src/notifier.rs +++ b/crates/event-notifier/src/notifier.rs @@ -1,4 +1,4 @@ -use crate::{event_bus, ChannelAdapter, Error, Event, EventStore, NotificationConfig}; +use crate::{event_bus, ChannelAdapter, Error, Event, EventStore, NotifierConfig}; use std::sync::Arc; use tokio::sync::mpsc; use tokio_util::sync::CancellationToken; @@ -7,17 +7,18 @@ use tokio_util::sync::CancellationToken; /// It manages the event bus and the adapters. /// It is responsible for sending and receiving events. /// It also handles the shutdown process. -pub struct NotificationSystem { +pub struct NotifierSystem { tx: mpsc::Sender, rx: Option>, store: Arc, shutdown: CancellationToken, shutdown_complete: Option>, + shutdown_receiver: Option>, } -impl NotificationSystem { +impl NotifierSystem { /// Creates a new `NotificationSystem` instance. - pub async fn new(config: NotificationConfig) -> Result { + pub async fn new(config: NotifierConfig) -> Result { let (tx, rx) = mpsc::channel::(config.channel_capacity); let store = Arc::new(EventStore::new(&config.store_path).await?); let shutdown = CancellationToken::new(); @@ -29,13 +30,15 @@ impl NotificationSystem { tx.send(event).await.map_err(|e| Error::ChannelSend(Box::new(e)))?; } } - + // Initialize shutdown_complete to Some(tx) + let (complete_tx, complete_rx) = tokio::sync::oneshot::channel(); Ok(Self { tx, rx: Some(rx), store, shutdown, - shutdown_complete: None, + shutdown_complete: Some(complete_tx), + shutdown_receiver: Some(complete_rx), }) } @@ -43,37 +46,65 @@ impl NotificationSystem { /// It initializes the event bus and the producer. pub async fn start(&mut self, adapters: Vec>) -> Result<(), Error> { if self.shutdown.is_cancelled() { - return Err(Error::custom("System is shutting down")); + let error = Error::custom("System is shutting down"); + self.handle_error("start", &error); + return Err(error); } - + self.log(tracing::Level::INFO, "start", "Starting the notification system"); let rx = self.rx.take().ok_or_else(|| Error::EventBusStarted)?; let shutdown_clone = self.shutdown.clone(); let store_clone = self.store.clone(); let shutdown_complete = self.shutdown_complete.take(); + tokio::spawn(async move { if let Err(e) = event_bus(rx, adapters, store_clone, shutdown_clone, shutdown_complete).await { tracing::error!("Event bus failed: {}", e); } }); - + self.log(tracing::Level::INFO, "start", "Notification system started successfully"); Ok(()) } /// Sends an event to the notification system. /// This method is used to send events to the event bus. pub async fn send_event(&self, event: Event) -> Result<(), Error> { + self.log(tracing::Level::DEBUG, "send_event", &format!("Sending event: {:?}", event)); if self.shutdown.is_cancelled() { - return Err(Error::custom("System is shutting down")); + let error = Error::custom("System is shutting down"); + self.handle_error("send_event", &error); + return Err(error); } - self.tx.send(event).await.map_err(|e| Error::ChannelSend(Box::new(e)))?; + if let Err(e) = self.tx.send(event).await { + let error = Error::ChannelSend(Box::new(e)); + self.handle_error("send_event", &error); + return Err(error); + } + self.log(tracing::Level::INFO, "send_event", "Event sent successfully"); Ok(()) } /// Shuts down the notification system. /// This method is used to cancel the event bus and producer tasks. - pub fn shutdown(&self) { + pub async fn shutdown(&mut self) -> Result<(), Error> { tracing::info!("Shutting down the notification system"); self.shutdown.cancel(); + // wait for the event bus to be completely closed + if let Some(receiver) = self.shutdown_receiver.take() { + match receiver.await { + Ok(_) => { + tracing::info!("Event bus shutdown completed successfully"); + Ok(()) + } + Err(e) => { + let error = Error::custom(format!("Failed to receive shutdown completion: {}", e).as_str()); + self.handle_error("shutdown", &error); + Err(error) + } + } + } else { + tracing::warn!("Shutdown receiver not available, the event bus might still be running"); + Err(Error::custom("Shutdown receiver not available")) + } } /// shutdown state @@ -81,9 +112,17 @@ impl NotificationSystem { self.shutdown.is_cancelled() } - pub fn set_shutdown_complete_channel(&mut self, tx: tokio::sync::oneshot::Sender<()>) { - // storage completion channel for use by event bus - tracing::info!("Shutting down the notification system set shutdown complete channel"); - self.shutdown_complete = Some(tx); + fn handle_error(&self, context: &str, error: &Error) { + self.log(tracing::Level::ERROR, context, &format!("{:?}", error)); + // TODO Can be extended to record to files or send to monitoring systems + } + fn log(&self, level: tracing::Level, context: &str, message: &str) { + match level { + tracing::Level::ERROR => tracing::error!("[{}] {}", context, message), + tracing::Level::WARN => tracing::warn!("[{}] {}", context, message), + tracing::Level::INFO => tracing::info!("[{}] {}", context, message), + tracing::Level::DEBUG => tracing::debug!("[{}] {}", context, message), + tracing::Level::TRACE => tracing::trace!("[{}] {}", context, message), + } } } diff --git a/crates/event-notifier/tests/integration.rs b/crates/event-notifier/tests/integration.rs index ec97a64b9..c5743605b 100644 --- a/crates/event-notifier/tests/integration.rs +++ b/crates/event-notifier/tests/integration.rs @@ -1,4 +1,4 @@ -use rustfs_event_notifier::{AdapterConfig, NotificationSystem, WebhookConfig}; +use rustfs_event_notifier::{AdapterConfig, NotifierSystem, WebhookConfig}; use rustfs_event_notifier::{Bucket, Event, EventBuilder, Identity, Metadata, Name, Object, Source}; use rustfs_event_notifier::{ChannelAdapter, WebhookAdapter}; use std::collections::HashMap; @@ -67,7 +67,7 @@ async fn test_webhook_adapter() { #[tokio::test] async fn test_notification_system() { - let config = rustfs_event_notifier::NotificationConfig { + let config = rustfs_event_notifier::NotifierConfig { store_path: "./test_events".to_string(), channel_capacity: 100, adapters: vec![AdapterConfig::Webhook(WebhookConfig { @@ -78,7 +78,7 @@ async fn test_notification_system() { timeout: 5, })], }; - let system = Arc::new(tokio::sync::Mutex::new(NotificationSystem::new(config.clone()).await.unwrap())); + let system = Arc::new(tokio::sync::Mutex::new(NotifierSystem::new(config.clone()).await.unwrap())); let adapters: Vec> = vec![Arc::new(WebhookAdapter::new(WebhookConfig { endpoint: "http://localhost:8080/webhook".to_string(), auth_token: None, @@ -148,8 +148,8 @@ async fn test_notification_system() { // create a new task to handle the timeout let system = Arc::clone(&system); tokio::spawn(async move { - if let Ok(guard) = system.try_lock() { - guard.shutdown(); + if let Ok(mut guard) = system.try_lock() { + guard.shutdown().await.unwrap(); } }); // give the system some time to clean up resources diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 24e5782fe..62e1a3ea3 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -122,7 +122,7 @@ impl LocalDisk { let root = fs::canonicalize(ep.get_file_path()).await?; if cleanup { - // TODO: 删除tmp数据 + // TODO: 删除 tmp 数据 } let format_path = Path::new(super::RUSTFS_META_BUCKET) @@ -631,13 +631,13 @@ impl LocalDisk { } } - // 没有版本了，删除xl.meta + // 没有版本了，删除 xl.meta if fm.versions.is_empty() { self.delete_file(&volume_dir, &xlpath, true, false).await?; return Ok(()); } - // 更新xl.meta + // 更新 xl.meta let buf = fm.marshal_msg()?; let volume_dir = self.get_bucket_path(volume)?; @@ -875,7 +875,7 @@ impl LocalDisk { .read_metadata(self.get_object_path(bucket, format!("{}/{}", ¤t, &entry).as_str())?) .await?; - // 用strip_suffix只删除一次 + // 用 strip_suffix 只删除一次 let entry = entry.strip_suffix(STORAGE_FORMAT_FILE).unwrap_or_default().to_owned(); let name = entry.trim_end_matches(SLASH_SEPARATOR); let name = decode_dir_object(format!("{}/{}", ¤t, &name).as_str()); @@ -1723,7 +1723,7 @@ impl DiskAPI for LocalDisk { } } - // xl.meta路径 + // xl.meta 路径 let src_file_path = src_volume_dir.join(Path::new(format!("{}/{}", &src_path, super::STORAGE_FORMAT_FILE).as_str())); let dst_file_path = dst_volume_dir.join(Path::new(format!("{}/{}", &dst_path, super::STORAGE_FORMAT_FILE).as_str())); @@ -1754,7 +1754,7 @@ impl DiskAPI for LocalDisk { check_path_length(src_file_path.to_string_lossy().to_string().as_str())?; check_path_length(dst_file_path.to_string_lossy().to_string().as_str())?; - // 读旧xl.meta + // 读旧 xl.meta let has_dst_buf = match utils::fs::read_file(&dst_file_path).await { Ok(res) => Some(res), @@ -2268,7 +2268,7 @@ impl DiskAPI for LocalDisk { async fn delete_volume(&self, volume: &str) -> Result<()> { let p = self.get_bucket_path(volume)?; - // TODO: 不能用递归删除，如果目录下面有文件，返回errVolumeNotEmpty + // TODO: 不能用递归删除，如果目录下面有文件，返回 errVolumeNotEmpty if let Err(err) = fs::remove_dir_all(&p).await { match err.kind() { @@ -2328,10 +2328,7 @@ impl DiskAPI for LocalDisk { } } - let vcfg = match BucketVersioningSys::get(&cache.info.name).await { - Ok(vcfg) => Some(vcfg), - Err(_) => None, - }; + let vcfg = (BucketVersioningSys::get(&cache.info.name).await).ok(); let loc = self.get_disk_location(); let disks = store.get_disks(loc.pool_idx.unwrap(), loc.disk_idx.unwrap()).await?; @@ -2491,7 +2488,6 @@ async fn get_disk_info(drive_path: PathBuf) -> Result<(Info, bool)> { #[cfg(test)] mod test { - use super::*; #[tokio::test] diff --git a/ecstore/src/heal/data_scanner.rs b/ecstore/src/heal/data_scanner.rs index 78c5b9f64..268af7343 100644 --- a/ecstore/src/heal/data_scanner.rs +++ b/ecstore/src/heal/data_scanner.rs @@ -1101,7 +1101,7 @@ pub fn lc_has_active_rules(config: &BucketLifecycleConfiguration, prefix: &str) return true; } - if let Some(Some(true)) = rule.expiration.as_ref().map(|e| e.expired_object_delete_marker.map(|m| m)) { + if let Some(Some(true)) = rule.expiration.as_ref().map(|e| e.expired_object_delete_marker) { return true; } From 4f347a92c186d82df4d3c1553ca695aef9d34756 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 22 Apr 2025 20:49:39 +0800 Subject: [PATCH 11/16] feat: integrate event-notifier system with rustfs - Rename package from rustfs-event-notifier to event-notifier for consistency - Add shutdown hooks for event notification system in main process - Handle graceful termination of notification services on server shutdown - Implement initialization and configuration loading for event notification - Fix environment variable configuration to properly parse adapter arrays - Update example code to demonstrate proper configuration usage This change ensures proper integration between rustfs and the event notification system, with clean startup and shutdown sequences to prevent resource leaks during application lifecycle. --- crates/event-notifier/examples/simple.rs | 6 ++--- crates/event-notifier/src/config.rs | 4 ++-- rustfs/src/main.rs | 28 ++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/crates/event-notifier/examples/simple.rs b/crates/event-notifier/examples/simple.rs index 815f2a0ae..eac37181a 100644 --- a/crates/event-notifier/examples/simple.rs +++ b/crates/event-notifier/examples/simple.rs @@ -29,10 +29,10 @@ async fn main() -> Result<(), Box> { })], }; - // load_config + // event_load_config // loading configuration from environment variables - let _config = NotifierConfig::load_config(Some("./crates/event-notifier/examples/event.toml".to_string())); - tracing::info!("load_config config: {:?} \n", _config); + let _config = NotifierConfig::event_load_config(Some("./crates/event-notifier/examples/event.toml".to_string())); + tracing::info!("event_load_config config: {:?} \n", _config); let system = Arc::new(tokio::sync::Mutex::new(NotifierSystem::new(config.clone()).await?)); let adapters = create_adapters(&config.adapters)?; diff --git a/crates/event-notifier/src/config.rs b/crates/event-notifier/src/config.rs index c360f2bc9..ae718abc2 100644 --- a/crates/event-notifier/src/config.rs +++ b/crates/event-notifier/src/config.rs @@ -102,9 +102,9 @@ impl NotifierConfig { /// ``` /// use rustfs_event_notifier::NotifierConfig; /// - /// let config = NotifierConfig::load_config(None); + /// let config = NotifierConfig::event_load_config(None); /// ``` - pub fn load_config(config_dir: Option) -> NotifierConfig { + pub fn event_load_config(config_dir: Option) -> NotifierConfig { let config_dir = if let Some(path) = config_dir { // If a path is provided, check if it's empty if path.is_empty() { diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index fc8a44e8f..f86f951fc 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -46,6 +46,7 @@ use hyper_util::{ use iam::init_iam_sys; use license::init_license; use protos::proto_gen::node_service::node_service_server::NodeServiceServer; +use rustfs_event_notifier::NotifierConfig; use rustfs_obs::{init_obs, load_config, set_global_guard, InitLogStatus}; use rustls::ServerConfig; use s3s::{host::MultiDomain, service::S3ServiceBuilder}; @@ -105,6 +106,23 @@ async fn main() -> Result<()> { // Log initialization status InitLogStatus::init_start_log(&config.observability).await?; + // Initialize event notifier + let notifier_config = opt.clone().event_config; + if notifier_config.is_some() { + info!("event_config is not empty"); + tokio::spawn(async move { + let config = NotifierConfig::event_load_config(notifier_config); + let result = rustfs_event_notifier::initialize(config).await; + if let Err(e) = result { + error!("Failed to initialize event notifier: {}", e); + } else { + info!("Event notifier initialized successfully"); + } + }); + } else { + info!("event_config is empty"); + } + // Run parameters run(opt).await } @@ -448,6 +466,16 @@ async fn run(opt: config::Opt) -> Result<()> { info!("Shutdown signal received in main thread"); // update the status to stopping first state_manager.update(ServiceState::Stopping); + + // Stop the notification system + if rustfs_event_notifier::is_ready() { + // stop event notifier + rustfs_event_notifier::shutdown().await.map_err(|err| { + error!("Failed to shut down the notification system: {}", err); + Error::from_string(err.to_string()) + })?; + } + info!("Server is stopping..."); let _ = shutdown_tx.send(()); // Wait for the worker thread to complete the cleaning work From c8ab89292ef2c9cf0deff1408c06b8699af8ec35 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 22 Apr 2025 21:40:20 +0800 Subject: [PATCH 12/16] feat: improve webhook server and run script integration - Enhance webhook example with proper shutdown handling using tokio::select! - Update run.sh to automatically start webhook server alongside main service - Add event notification configuration to run.sh using environment variables - Set proper port bindings to ensure webhook server starts on port 3000 - Improve console output for better debugging experience - Fix race condition during service startup and shutdown This change ensures proper integration between the webhook server and the main rustfs service, providing a seamless development experience with automatic service discovery and clean termination. --- crates/event-notifier/examples/webhook.rs | 14 ++++++++++- deploy/config/event.example.toml | 29 +++++++++++++++++++++++ scripts/run.sh | 7 +++++- 3 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 deploy/config/event.example.toml diff --git a/crates/event-notifier/examples/webhook.rs b/crates/event-notifier/examples/webhook.rs index b78af94dd..5363e5741 100644 --- a/crates/event-notifier/examples/webhook.rs +++ b/crates/event-notifier/examples/webhook.rs @@ -8,7 +8,19 @@ async fn main() { let app = Router::new().route("/webhook", post(receive_webhook)); // 启动服务器 let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); - axum::serve(listener, app).await.unwrap(); + println!("Server running on http://0.0.0.0:3000"); + + // 创建关闭信号处理 + tokio::select! { + result = axum::serve(listener, app) => { + if let Err(e) = result { + eprintln!("Server error: {}", e); + } + } + _ = tokio::signal::ctrl_c() => { + println!("Shutting down server..."); + } + } } async fn receive_webhook(Json(payload): Json) -> StatusCode { diff --git a/deploy/config/event.example.toml b/deploy/config/event.example.toml new file mode 100644 index 000000000..809fe5813 --- /dev/null +++ b/deploy/config/event.example.toml @@ -0,0 +1,29 @@ +# config.toml +store_path = "./deploy/logs/event_store" +channel_capacity = 5000 + +[[adapters]] +type = "Webhook" +endpoint = "http://127.0.0.1:3000/webhook" +auth_token = "your-auth-token" +max_retries = 3 +timeout = 50 + +[adapters.custom_headers] +custom_server = "value_server" +custom_client = "value_client" + +#[[adapters]] +#type = "Kafka" +#brokers = "localhost:9092" +#topic = "notifications" +#max_retries = 3 +#timeout = 60 +# +#[[adapters]] +#type = "Mqtt" +#broker = "mqtt.example.com" +#port = 1883 +#client_id = "event-notifier" +#topic = "events" +#max_retries = 3 \ No newline at end of file diff --git a/scripts/run.sh b/scripts/run.sh index 497c2a681..2cbd115a8 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -57,9 +57,14 @@ export RUSTFS__SINKS__KAFKA__BOOTSTRAP_SERVERS="" export RUSTFS__SINKS__KAFKA__TOPIC="" export RUSTFS__LOGGER__QUEUE_CAPACITY=10 +# 事件消息配置 +export RUSTFS_EVENT_CONFIG="./deploy/config/event.example.toml" + if [ -n "$1" ]; then export RUSTFS_VOLUMES="$1" fi - +# 启动 webhook 服务器 +cargo run --example webhook -p rustfs-event-notifier & +# 启动主服务 cargo run --bin rustfs \ No newline at end of file From 6d1a4ab824addca99c2d4ecd0d9245cdd61ec47c Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 22 Apr 2025 22:41:45 +0800 Subject: [PATCH 13/16] improve for logger --- crates/obs/src/telemetry.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/crates/obs/src/telemetry.rs b/crates/obs/src/telemetry.rs index 8efa869dd..db0ce1690 100644 --- a/crates/obs/src/telemetry.rs +++ b/crates/obs/src/telemetry.rs @@ -254,21 +254,16 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard { let filter_otel = match logger_level { "trace" | "debug" => { info!("OpenTelemetry tracing initialized with level: {}", logger_level); + EnvFilter::new(logger_level) + } + _ => { let mut filter = EnvFilter::new(logger_level); for directive in ["hyper", "tonic", "h2", "reqwest", "tower"] { filter = filter.add_directive(format!("{}=off", directive).parse().unwrap()); } filter } - _ => { - let mut filter = EnvFilter::new(logger_level); - for directive in ["hyper", "tonic", "h2", "reqwest"] { - filter = filter.add_directive(format!("{}=off", directive).parse().unwrap()); - } - filter - } }; - layer::OpenTelemetryTracingBridge::new(&logger_provider).with_filter(filter_otel) }; @@ -277,7 +272,9 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard { .with(switch_level(logger_level)) .with(OpenTelemetryLayer::new(tracer)) .with(MetricsLayer::new(meter_provider.clone())) - .with(otel_layer); + .with(otel_layer) + .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(logger_level))); + // Configure formatting layer let enable_color = std::io::stdout().is_terminal(); let fmt_layer = tracing_subscriber::fmt::layer() From 52a1f9b8a78bd7103a691d97f7b3fea380e7caa8 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 22 Apr 2025 22:59:31 +0800 Subject: [PATCH 14/16] improve code for global.rs --- crates/event-notifier/examples/.env-zh.example | 2 +- crates/event-notifier/examples/.env.example | 2 +- crates/event-notifier/examples/event.toml | 2 +- crates/event-notifier/examples/webhook.rs | 4 ++-- crates/event-notifier/src/global.rs | 6 +++--- deploy/config/event.example.toml | 2 +- deploy/config/obs.example.toml | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/event-notifier/examples/.env-zh.example b/crates/event-notifier/examples/.env-zh.example index dffa9c6e9..00228e162 100644 --- a/crates/event-notifier/examples/.env-zh.example +++ b/crates/event-notifier/examples/.env-zh.example @@ -5,7 +5,7 @@ NOTIFIER__CHANNEL_CAPACITY=5000 # ===== 适配器配置（数组格式） ===== # Webhook 适配器（索引 0） NOTIFIER__ADAPTERS_0__type=Webhook -NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3000/webhook +NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3020/webhook NOTIFIER__ADAPTERS_0__auth_token=your-auth-token NOTIFIER__ADAPTERS_0__max_retries=3 NOTIFIER__ADAPTERS_0__timeout=50 diff --git a/crates/event-notifier/examples/.env.example b/crates/event-notifier/examples/.env.example index 28b4fcf47..af343863d 100644 --- a/crates/event-notifier/examples/.env.example +++ b/crates/event-notifier/examples/.env.example @@ -5,7 +5,7 @@ NOTIFIER__CHANNEL_CAPACITY=5000 # ===== adapter configuration array format ===== # webhook adapter index 0 NOTIFIER__ADAPTERS_0__type=Webhook -NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3000/webhook +NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3020/webhook NOTIFIER__ADAPTERS_0__auth_token=your-auth-token NOTIFIER__ADAPTERS_0__max_retries=3 NOTIFIER__ADAPTERS_0__timeout=50 diff --git a/crates/event-notifier/examples/event.toml b/crates/event-notifier/examples/event.toml index de5293f21..5b4292fae 100644 --- a/crates/event-notifier/examples/event.toml +++ b/crates/event-notifier/examples/event.toml @@ -4,7 +4,7 @@ channel_capacity = 5000 [[adapters]] type = "Webhook" -endpoint = "http://127.0.0.1:3000/webhook" +endpoint = "http://127.0.0.1:3020/webhook" auth_token = "your-auth-token" max_retries = 3 timeout = 50 diff --git a/crates/event-notifier/examples/webhook.rs b/crates/event-notifier/examples/webhook.rs index 5363e5741..c754f8223 100644 --- a/crates/event-notifier/examples/webhook.rs +++ b/crates/event-notifier/examples/webhook.rs @@ -7,8 +7,8 @@ async fn main() { // 构建应用 let app = Router::new().route("/webhook", post(receive_webhook)); // 启动服务器 - let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); - println!("Server running on http://0.0.0.0:3000"); + let listener = tokio::net::TcpListener::bind("0.0.0.0:3020").await.unwrap(); + println!("Server running on http://0.0.0.0:3020"); // 创建关闭信号处理 tokio::select! { diff --git a/crates/event-notifier/src/global.rs b/crates/event-notifier/src/global.rs index 9a83e9c7a..25fadeeb4 100644 --- a/crates/event-notifier/src/global.rs +++ b/crates/event-notifier/src/global.rs @@ -178,9 +178,9 @@ mod tests { tracing_subscriber::fmt::init(); let config = NotifierConfig::default(); // assume there is a default configuration let result = initialize(config).await; - assert!(!result.is_ok(), "Initialization should succeed"); - assert!(!is_initialized(), "System should be marked as initialized"); - assert!(!is_ready(), "System should be marked as ready"); + assert!(result.is_err(), "Initialization should not succeed"); + assert!(!is_initialized(), "System should not be marked as initialized"); + assert!(!is_ready(), "System should not be marked as ready"); } #[tokio::test] diff --git a/deploy/config/event.example.toml b/deploy/config/event.example.toml index 809fe5813..6deb67a2e 100644 --- a/deploy/config/event.example.toml +++ b/deploy/config/event.example.toml @@ -4,7 +4,7 @@ channel_capacity = 5000 [[adapters]] type = "Webhook" -endpoint = "http://127.0.0.1:3000/webhook" +endpoint = "http://127.0.0.1:3020/webhook" auth_token = "your-auth-token" max_retries = 3 timeout = 50 diff --git a/deploy/config/obs.example.toml b/deploy/config/obs.example.toml index a3c3d0966..97192fb4f 100644 --- a/deploy/config/obs.example.toml +++ b/deploy/config/obs.example.toml @@ -6,7 +6,7 @@ meter_interval = 30 service_name = "rustfs" service_version = "0.1.0" environment = "develop" -logger_level = "info" +logger_level = "error" [sinks] [sinks.kafka] # Kafka sink is disabled by default From ea13098beb91b8946505dc7bcf3dc2fd0b238621 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 22 Apr 2025 23:06:43 +0800 Subject: [PATCH 15/16] fix: modify webhook port --- crates/event-notifier/examples/full.rs | 2 +- crates/event-notifier/examples/simple.rs | 2 +- scripts/run.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/event-notifier/examples/full.rs b/crates/event-notifier/examples/full.rs index 14231c730..23e858fb8 100644 --- a/crates/event-notifier/examples/full.rs +++ b/crates/event-notifier/examples/full.rs @@ -11,7 +11,7 @@ async fn setup_notification_system() -> Result<(), NotifierError> { store_path: "./deploy/logs/event_store".into(), channel_capacity: 100, adapters: vec![AdapterConfig::Webhook(WebhookConfig { - endpoint: "http://127.0.0.1:3000/webhook".into(), + endpoint: "http://127.0.0.1:3020/webhook".into(), auth_token: Some("your-auth-token".into()), custom_headers: Some(HashMap::new()), max_retries: 3, diff --git a/crates/event-notifier/examples/simple.rs b/crates/event-notifier/examples/simple.rs index eac37181a..93005f0a7 100644 --- a/crates/event-notifier/examples/simple.rs +++ b/crates/event-notifier/examples/simple.rs @@ -21,7 +21,7 @@ async fn main() -> Result<(), Box> { store_path: "./events".to_string(), channel_capacity: 100, adapters: vec![AdapterConfig::Webhook(WebhookConfig { - endpoint: "http://127.0.0.1:3000/webhook".to_string(), + endpoint: "http://127.0.0.1:3020/webhook".to_string(), auth_token: Some("secret-token".to_string()), custom_headers: Some(HashMap::from([("X-Custom".to_string(), "value".to_string())])), max_retries: 3, diff --git a/scripts/run.sh b/scripts/run.sh index 2cbd115a8..96463c91b 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -59,7 +59,7 @@ export RUSTFS__LOGGER__QUEUE_CAPACITY=10 # 事件消息配置 export RUSTFS_EVENT_CONFIG="./deploy/config/event.example.toml" - +export RUST_LOG="rustfs=debug,ecstore=debug,s3s=debug,iam=debug" if [ -n "$1" ]; then export RUSTFS_VOLUMES="$1" fi From b2e1a7ac331da6e308d205fec4f5f004ad75945f Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 22 Apr 2025 23:07:13 +0800 Subject: [PATCH 16/16] fix --- scripts/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run.sh b/scripts/run.sh index 96463c91b..2cbd115a8 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -59,7 +59,7 @@ export RUSTFS__LOGGER__QUEUE_CAPACITY=10 # 事件消息配置 export RUSTFS_EVENT_CONFIG="./deploy/config/event.example.toml" -export RUST_LOG="rustfs=debug,ecstore=debug,s3s=debug,iam=debug" + if [ -n "$1" ]; then export RUSTFS_VOLUMES="$1" fi