mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-12 08:06:54 +00:00
modify crates name
This commit is contained in:
houseme
@@ -1,48 +0,0 @@
|
||||
[package]
|
||||
name = "rustfs-event"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
version.workspace = true
|
||||
|
||||
[features]
|
||||
default = ["webhook"]
|
||||
webhook = ["dep:reqwest"]
|
||||
mqtt = ["rumqttc"]
|
||||
kafka = ["dep:rdkafka"]
|
||||
|
||||
[dependencies]
|
||||
async-trait = { workspace = true }
|
||||
config = { workspace = true }
|
||||
common = { workspace = true }
|
||||
ecstore = { workspace = true }
|
||||
lazy_static = { workspace = true }
|
||||
once_cell = { workspace = true }
|
||||
reqwest = { workspace = true, optional = true }
|
||||
rumqttc = { workspace = true, optional = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
serde_with = { workspace = true }
|
||||
smallvec = { workspace = true, features = ["serde"] }
|
||||
strum = { workspace = true, features = ["derive"] }
|
||||
tracing = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["sync", "net", "macros", "signal", "rt-multi-thread"] }
|
||||
tokio-util = { workspace = true }
|
||||
uuid = { workspace = true, features = ["v4", "serde"] }
|
||||
snap = { workspace = true }
|
||||
|
||||
|
||||
# Only enable kafka features and related dependencies on Linux
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
rdkafka = { workspace = true, features = ["tokio"], optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["test-util"] }
|
||||
tracing-subscriber = { workspace = true }
|
||||
axum = { workspace = true }
|
||||
dotenvy = { workspace = true }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -1,28 +0,0 @@
|
||||
## ===== global configuration =====
|
||||
#NOTIFIER__STORE_PATH=/var/log/event-notification
|
||||
#NOTIFIER__CHANNEL_CAPACITY=5000
|
||||
#
|
||||
## ===== adapter configuration array format =====
|
||||
## webhook adapter index 0
|
||||
#NOTIFIER__ADAPTERS_0__type=Webhook
|
||||
#NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3020/webhook
|
||||
#NOTIFIER__ADAPTERS_0__auth_token=your-auth-token
|
||||
#NOTIFIER__ADAPTERS_0__max_retries=3
|
||||
#NOTIFIER__ADAPTERS_0__timeout=50
|
||||
#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=server-value
|
||||
#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=client-value
|
||||
#
|
||||
## kafka adapter index 1
|
||||
#NOTIFIER__ADAPTERS_1__type=Kafka
|
||||
#NOTIFIER__ADAPTERS_1__brokers=localhost:9092
|
||||
#NOTIFIER__ADAPTERS_1__topic=notifications
|
||||
#NOTIFIER__ADAPTERS_1__max_retries=3
|
||||
#NOTIFIER__ADAPTERS_1__timeout=60
|
||||
#
|
||||
## mqtt adapter index 2
|
||||
#NOTIFIER__ADAPTERS_2__type=Mqtt
|
||||
#NOTIFIER__ADAPTERS_2__broker=mqtt.example.com
|
||||
#NOTIFIER__ADAPTERS_2__port=1883
|
||||
#NOTIFIER__ADAPTERS_2__client_id=event-notifier
|
||||
#NOTIFIER__ADAPTERS_2__topic=events
|
||||
#NOTIFIER__ADAPTERS_2__max_retries=3
|
||||
@@ -1,28 +0,0 @@
|
||||
## ===== 全局配置 =====
|
||||
#NOTIFIER__STORE_PATH=/var/log/event-notification
|
||||
#NOTIFIER__CHANNEL_CAPACITY=5000
|
||||
#
|
||||
## ===== 适配器配置（数组格式） =====
|
||||
## Webhook 适配器（索引 0）
|
||||
#NOTIFIER__ADAPTERS_0__type=Webhook
|
||||
#NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3020/webhook
|
||||
#NOTIFIER__ADAPTERS_0__auth_token=your-auth-token
|
||||
#NOTIFIER__ADAPTERS_0__max_retries=3
|
||||
#NOTIFIER__ADAPTERS_0__timeout=50
|
||||
#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=value
|
||||
#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=value
|
||||
#
|
||||
## Kafka 适配器（索引 1）
|
||||
#NOTIFIER__ADAPTERS_1__type=Kafka
|
||||
#NOTIFIER__ADAPTERS_1__brokers=localhost:9092
|
||||
#NOTIFIER__ADAPTERS_1__topic=notifications
|
||||
#NOTIFIER__ADAPTERS_1__max_retries=3
|
||||
#NOTIFIER__ADAPTERS_1__timeout=60
|
||||
#
|
||||
## MQTT 适配器（索引 2）
|
||||
#NOTIFIER__ADAPTERS_2__type=Mqtt
|
||||
#NOTIFIER__ADAPTERS_2__broker=mqtt.example.com
|
||||
#NOTIFIER__ADAPTERS_2__port=1883
|
||||
#NOTIFIER__ADAPTERS_2__client_id=event-notifier
|
||||
#NOTIFIER__ADAPTERS_2__topic=events
|
||||
#NOTIFIER__ADAPTERS_2__max_retries=3
|
||||
@@ -1,29 +0,0 @@
|
||||
# config.toml
|
||||
store_path = "/var/log/event-notifier"
|
||||
channel_capacity = 5000
|
||||
|
||||
[[adapters]]
|
||||
type = "Webhook"
|
||||
endpoint = "http://127.0.0.1:3020/webhook"
|
||||
auth_token = "your-auth-token"
|
||||
max_retries = 3
|
||||
timeout = 50
|
||||
|
||||
[adapters.custom_headers]
|
||||
custom_server = "value_server"
|
||||
custom_client = "value_client"
|
||||
|
||||
[[adapters]]
|
||||
type = "Kafka"
|
||||
brokers = "localhost:9092"
|
||||
topic = "notifications"
|
||||
max_retries = 3
|
||||
timeout = 60
|
||||
|
||||
[[adapters]]
|
||||
type = "Mqtt"
|
||||
broker = "mqtt.example.com"
|
||||
port = 1883
|
||||
client_id = "event-notifier"
|
||||
topic = "events"
|
||||
max_retries = 3
|
||||
@@ -1,144 +0,0 @@
|
||||
use rustfs_event::{
|
||||
AdapterConfig, Bucket, ChannelAdapterType, Error as NotifierError, Event, Identity, Metadata, Name, NotifierConfig, Object,
|
||||
Source, WebhookConfig,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use tokio::signal;
|
||||
use tracing::Level;
|
||||
use tracing_subscriber::FmtSubscriber;
|
||||
|
||||
async fn setup_notification_system() -> Result<(), NotifierError> {
|
||||
let config = NotifierConfig {
|
||||
store_path: "./deploy/logs/event_store".into(),
|
||||
channel_capacity: 100,
|
||||
adapters: vec![AdapterConfig::Webhook(WebhookConfig {
|
||||
endpoint: "http://127.0.0.1:3020/webhook".into(),
|
||||
auth_token: Some("your-auth-token".into()),
|
||||
custom_headers: Some(HashMap::new()),
|
||||
max_retries: 3,
|
||||
timeout: Some(30),
|
||||
retry_interval: Some(5),
|
||||
client_cert: None,
|
||||
client_key: None,
|
||||
common: rustfs_event::AdapterCommon {
|
||||
identifier: "webhook".into(),
|
||||
comment: "webhook".into(),
|
||||
enable: true,
|
||||
queue_dir: "./deploy/logs/event_queue".into(),
|
||||
queue_limit: 100,
|
||||
},
|
||||
})],
|
||||
};
|
||||
|
||||
rustfs_event::initialize(&config).await?;
|
||||
|
||||
// wait for the system to be ready
|
||||
for _ in 0..50 {
|
||||
// wait up to 5 seconds
|
||||
if rustfs_event::is_ready() {
|
||||
return Ok(());
|
||||
}
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
|
||||
}
|
||||
|
||||
Err(NotifierError::custom("notify the system of initialization timeout"))
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
// initialization log
|
||||
// tracing_subscriber::fmt::init();
|
||||
|
||||
let subscriber = FmtSubscriber::builder()
|
||||
.with_max_level(Level::DEBUG) // set to debug or lower level
|
||||
.with_target(false) // simplify output
|
||||
.finish();
|
||||
tracing::subscriber::set_global_default(subscriber).expect("failed to set up log subscriber");
|
||||
|
||||
// set up notification system
|
||||
if let Err(e) = setup_notification_system().await {
|
||||
eprintln!("unable to initialize notification system:{}", e);
|
||||
return Err(e.into());
|
||||
}
|
||||
|
||||
// create a shutdown signal processing
|
||||
let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel();
|
||||
|
||||
// start signal processing task
|
||||
tokio::spawn(async move {
|
||||
let _ = signal::ctrl_c().await;
|
||||
println!("Received the shutdown signal and prepared to exit...");
|
||||
let _ = shutdown_tx.send(());
|
||||
});
|
||||
|
||||
// main application logic
|
||||
tokio::select! {
|
||||
_ = async {
|
||||
loop {
|
||||
// application logic
|
||||
// create an s3 metadata object
|
||||
let metadata = Metadata {
|
||||
schema_version: "1.0".to_string(),
|
||||
configuration_id: "test-config".to_string(),
|
||||
bucket: Bucket {
|
||||
name: "my-bucket".to_string(),
|
||||
owner_identity: Identity {
|
||||
principal_id: "owner123".to_string(),
|
||||
},
|
||||
arn: "arn:aws:s3:::my-bucket".to_string(),
|
||||
},
|
||||
object: Object {
|
||||
key: "test.txt".to_string(),
|
||||
size: Some(1024),
|
||||
etag: Some("abc123".to_string()),
|
||||
content_type: Some("text/plain".to_string()),
|
||||
user_metadata: None,
|
||||
version_id: None,
|
||||
sequencer: "1234567890".to_string(),
|
||||
},
|
||||
};
|
||||
|
||||
// create source object
|
||||
let source = Source {
|
||||
host: "localhost".to_string(),
|
||||
port: "80".to_string(),
|
||||
user_agent: "curl/7.68.0".to_string(),
|
||||
};
|
||||
|
||||
// create events using builder mode
|
||||
let event = Event::builder()
|
||||
.event_time("2023-10-01T12:00:00.000Z")
|
||||
.event_name(Name::ObjectCreatedPut)
|
||||
.user_identity(Identity {
|
||||
principal_id: "user123".to_string(),
|
||||
})
|
||||
.s3(metadata)
|
||||
.source(source)
|
||||
.channels(vec![ChannelAdapterType::Webhook.to_string()])
|
||||
.build()
|
||||
.expect("failed to create event");
|
||||
|
||||
if let Err(e) = rustfs_event::send_event(event).await {
|
||||
eprintln!("send event failed:{}", e);
|
||||
}
|
||||
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(60)).await;
|
||||
}
|
||||
} => {},
|
||||
|
||||
_ = &mut shutdown_rx => {
|
||||
println!("close the app");
|
||||
}
|
||||
}
|
||||
|
||||
// 优雅关闭通知系统
|
||||
println!("turn off the notification system");
|
||||
if let Err(e) = rustfs_event::shutdown().await {
|
||||
eprintln!("An error occurred while shutting down the notification system:{}", e);
|
||||
} else {
|
||||
println!("the notification system has been closed safely");
|
||||
}
|
||||
|
||||
println!("the application has been closed safely");
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,120 +0,0 @@
|
||||
use rustfs_event::NotifierSystem;
|
||||
use rustfs_event::{create_adapters, ChannelAdapterType};
|
||||
use rustfs_event::{AdapterConfig, NotifierConfig, WebhookConfig};
|
||||
use rustfs_event::{Bucket, Event, Identity, Metadata, Name, Object, Source};
|
||||
use std::collections::HashMap;
|
||||
use std::error;
|
||||
use std::sync::Arc;
|
||||
use tokio::signal;
|
||||
use tracing::Level;
|
||||
use tracing_subscriber::FmtSubscriber;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn error::Error>> {
|
||||
let subscriber = FmtSubscriber::builder()
|
||||
.with_max_level(Level::DEBUG) // set to debug or lower level
|
||||
.with_target(false) // simplify output
|
||||
.finish();
|
||||
tracing::subscriber::set_global_default(subscriber).expect("failed to set up log subscriber");
|
||||
|
||||
let config = NotifierConfig {
|
||||
store_path: "./events".to_string(),
|
||||
channel_capacity: 100,
|
||||
adapters: vec![AdapterConfig::Webhook(WebhookConfig {
|
||||
endpoint: "http://127.0.0.1:3020/webhook".to_string(),
|
||||
auth_token: Some("secret-token".to_string()),
|
||||
custom_headers: Some(HashMap::from([("X-Custom".to_string(), "value".to_string())])),
|
||||
max_retries: 3,
|
||||
timeout: Some(30),
|
||||
retry_interval: Some(5),
|
||||
client_cert: None,
|
||||
client_key: None,
|
||||
common: rustfs_event::AdapterCommon {
|
||||
identifier: "webhook".to_string(),
|
||||
comment: "webhook".to_string(),
|
||||
enable: true,
|
||||
queue_dir: "./deploy/logs/event_queue".to_string(),
|
||||
queue_limit: 100,
|
||||
},
|
||||
})],
|
||||
};
|
||||
|
||||
// event_load_config
|
||||
// loading configuration from environment variables
|
||||
let _config = NotifierConfig::event_load_config(Some("./crates/event/examples/event.toml".to_string()));
|
||||
tracing::info!("event_load_config config: {:?} \n", _config);
|
||||
dotenvy::dotenv()?;
|
||||
let _config = NotifierConfig::event_load_config(None);
|
||||
tracing::info!("event_load_config config: {:?} \n", _config);
|
||||
let system = Arc::new(tokio::sync::Mutex::new(NotifierSystem::new(config.clone()).await?));
|
||||
let adapters = create_adapters(&config.adapters)?;
|
||||
|
||||
// create an s3 metadata object
|
||||
let metadata = Metadata {
|
||||
schema_version: "1.0".to_string(),
|
||||
configuration_id: "test-config".to_string(),
|
||||
bucket: Bucket {
|
||||
name: "my-bucket".to_string(),
|
||||
owner_identity: Identity {
|
||||
principal_id: "owner123".to_string(),
|
||||
},
|
||||
arn: "arn:aws:s3:::my-bucket".to_string(),
|
||||
},
|
||||
object: Object {
|
||||
key: "test.txt".to_string(),
|
||||
size: Some(1024),
|
||||
etag: Some("abc123".to_string()),
|
||||
content_type: Some("text/plain".to_string()),
|
||||
user_metadata: None,
|
||||
version_id: None,
|
||||
sequencer: "1234567890".to_string(),
|
||||
},
|
||||
};
|
||||
|
||||
// create source object
|
||||
let source = Source {
|
||||
host: "localhost".to_string(),
|
||||
port: "80".to_string(),
|
||||
user_agent: "curl/7.68.0".to_string(),
|
||||
};
|
||||
|
||||
// create events using builder mode
|
||||
let event = Event::builder()
|
||||
.event_time("2023-10-01T12:00:00.000Z")
|
||||
.event_name(Name::ObjectCreatedPut)
|
||||
.user_identity(Identity {
|
||||
principal_id: "user123".to_string(),
|
||||
})
|
||||
.s3(metadata)
|
||||
.source(source)
|
||||
.channels(vec![ChannelAdapterType::Webhook.to_string()])
|
||||
.build()
|
||||
.expect("failed to create event");
|
||||
|
||||
{
|
||||
let system = system.lock().await;
|
||||
system.send_event(event).await?;
|
||||
}
|
||||
|
||||
let system_clone = Arc::clone(&system);
|
||||
let system_handle = tokio::spawn(async move {
|
||||
let mut system = system_clone.lock().await;
|
||||
system.start(adapters).await
|
||||
});
|
||||
|
||||
signal::ctrl_c().await?;
|
||||
tracing::info!("Received shutdown signal");
|
||||
let result = {
|
||||
let mut system = system.lock().await;
|
||||
system.shutdown().await
|
||||
};
|
||||
|
||||
if let Err(e) = result {
|
||||
tracing::error!("Failed to shut down the notification system: {}", e);
|
||||
} else {
|
||||
tracing::info!("Notification system shut down successfully");
|
||||
}
|
||||
|
||||
system_handle.await??;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
use axum::routing::get;
|
||||
use axum::{extract::Json, http::StatusCode, routing::post, Router};
|
||||
use serde_json::Value;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
// 构建应用
|
||||
let app = Router::new()
|
||||
.route("/webhook", post(receive_webhook))
|
||||
.route("/webhook", get(receive_webhook));
|
||||
// 启动服务器
|
||||
let listener = tokio::net::TcpListener::bind("0.0.0.0:3020").await.unwrap();
|
||||
println!("Server running on http://0.0.0.0:3020");
|
||||
|
||||
// 创建关闭信号处理
|
||||
tokio::select! {
|
||||
result = axum::serve(listener, app) => {
|
||||
if let Err(e) = result {
|
||||
eprintln!("Server error: {}", e);
|
||||
}
|
||||
}
|
||||
_ = tokio::signal::ctrl_c() => {
|
||||
println!("Shutting down server...");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn receive_webhook(Json(payload): Json<Value>) -> StatusCode {
|
||||
let start = SystemTime::now();
|
||||
let since_the_epoch = start.duration_since(UNIX_EPOCH).expect("Time went backwards");
|
||||
|
||||
// get the number of seconds since the unix era
|
||||
let seconds = since_the_epoch.as_secs();
|
||||
|
||||
// Manually calculate year, month, day, hour, minute, and second
|
||||
let (year, month, day, hour, minute, second) = convert_seconds_to_date(seconds);
|
||||
|
||||
// output result
|
||||
println!("current time:{:04}-{:02}-{:02} {:02}:{:02}:{:02}", year, month, day, hour, minute, second);
|
||||
println!(
|
||||
"received a webhook request time:{} content:\n {}",
|
||||
seconds,
|
||||
serde_json::to_string_pretty(&payload).unwrap()
|
||||
);
|
||||
StatusCode::OK
|
||||
}
|
||||
|
||||
fn convert_seconds_to_date(seconds: u64) -> (u32, u32, u32, u32, u32, u32) {
|
||||
// assume that the time zone is utc
|
||||
let seconds_per_minute = 60;
|
||||
let seconds_per_hour = 3600;
|
||||
let seconds_per_day = 86400;
|
||||
|
||||
// Calculate the year, month, day, hour, minute, and second corresponding to the number of seconds
|
||||
let mut total_seconds = seconds;
|
||||
let mut year = 1970;
|
||||
let mut month = 1;
|
||||
let mut day = 1;
|
||||
let mut hour = 0;
|
||||
let mut minute = 0;
|
||||
let mut second = 0;
|
||||
|
||||
// calculate year
|
||||
while total_seconds >= 31536000 {
|
||||
year += 1;
|
||||
total_seconds -= 31536000; // simplified processing no leap year considered
|
||||
}
|
||||
|
||||
// calculate month
|
||||
let days_in_month = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
|
||||
for m in &days_in_month {
|
||||
if total_seconds >= m * seconds_per_day {
|
||||
month += 1;
|
||||
total_seconds -= m * seconds_per_day;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// calculate the number of days
|
||||
day += total_seconds / seconds_per_day;
|
||||
total_seconds %= seconds_per_day;
|
||||
|
||||
// calculate hours
|
||||
hour += total_seconds / seconds_per_hour;
|
||||
total_seconds %= seconds_per_hour;
|
||||
|
||||
// calculate minutes
|
||||
minute += total_seconds / seconds_per_minute;
|
||||
total_seconds %= seconds_per_minute;
|
||||
|
||||
// calculate the number of seconds
|
||||
second += total_seconds;
|
||||
|
||||
(year as u32, month as u32, day as u32, hour as u32, minute as u32, second as u32)
|
||||
}
|
||||
@@ -1,180 +0,0 @@
|
||||
use crate::config::kafka::KafkaConfig;
|
||||
use crate::config::{default_queue_limit, DEFAULT_RETRY_INTERVAL, STORE_PREFIX};
|
||||
use crate::{ChannelAdapter, ChannelAdapterType};
|
||||
use crate::{Error, Event, QueueStore};
|
||||
use async_trait::async_trait;
|
||||
use rdkafka::error::KafkaError;
|
||||
use rdkafka::producer::{FutureProducer, FutureRecord};
|
||||
use rdkafka::types::RDKafkaErrorCode;
|
||||
use rdkafka::util::Timeout;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::time::sleep;
|
||||
use ChannelAdapterType::Kafka;
|
||||
|
||||
/// Kafka adapter for sending events to a Kafka topic.
|
||||
pub struct KafkaAdapter {
|
||||
producer: FutureProducer,
|
||||
store: Option<Arc<QueueStore<Event>>>,
|
||||
config: KafkaConfig,
|
||||
}
|
||||
|
||||
impl KafkaAdapter {
|
||||
/// Creates a new Kafka adapter.
|
||||
pub fn new(config: &KafkaConfig) -> Result<Self, Error> {
|
||||
// Create a Kafka producer with the provided configuration.
|
||||
let producer = rdkafka::config::ClientConfig::new()
|
||||
.set("bootstrap.servers", &config.brokers)
|
||||
.set("message.timeout.ms", config.timeout.to_string())
|
||||
.create()
|
||||
.map_err(|e| Error::msg(format!("Failed to create a Kafka producer: {}", e)))?;
|
||||
|
||||
// create a queue store if enabled
|
||||
let store = if !config.common.queue_dir.is_empty() {
|
||||
let store_path = PathBuf::from(&config.common.queue_dir).join(format!(
|
||||
"{}-{}-{}",
|
||||
STORE_PREFIX,
|
||||
Kafka.as_str(),
|
||||
config.common.identifier
|
||||
));
|
||||
|
||||
let queue_limit = if config.queue_limit > 0 {
|
||||
config.queue_limit
|
||||
} else {
|
||||
default_queue_limit()
|
||||
};
|
||||
let store = QueueStore::new(store_path, config.queue_limit, Some(".event".to_string()));
|
||||
if let Err(e) = store.open() {
|
||||
tracing::error!("Unable to open queue storage: {}", e);
|
||||
None
|
||||
} else {
|
||||
Some(Arc::new(store))
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Ok(Self { config, producer, store })
|
||||
}
|
||||
|
||||
/// Handle backlog events in storage
|
||||
pub async fn process_backlog(&self) -> Result<(), Error> {
|
||||
if let Some(store) = &self.store {
|
||||
let keys = store.list();
|
||||
|
||||
for key in keys {
|
||||
match store.get_multiple(&key) {
|
||||
Ok(events) => {
|
||||
for event in events {
|
||||
// Use the retry interval to send events
|
||||
if let Err(e) = self.send_with_retry(&event).await {
|
||||
tracing::error!("Processing of backlog events failed: {}", e);
|
||||
// If it still fails, we remain in the queue
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// The event is deleted after it has been successfully processed
|
||||
if let Err(e) = store.del(&key) {
|
||||
tracing::error!("Failed to delete a handled event: {}", e);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("Fetch events from the queue failed: {}", e);
|
||||
|
||||
// If the event cannot be read, it may be corrupted, delete it
|
||||
if let Err(del_err) = store.del(&key) {
|
||||
tracing::error!("Failed to delete a corrupted event: {}", del_err);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Sends an event to the Kafka topic with retry logic.
|
||||
async fn send_with_retry(&self, event: &Event) -> Result<(), Error> {
|
||||
let retry_interval = match self.config.retry_interval {
|
||||
Some(t) => Duration::from_secs(t),
|
||||
None => Duration::from_secs(DEFAULT_RETRY_INTERVAL), // Default to 3 seconds if not set
|
||||
};
|
||||
|
||||
for attempt in 0..self.max_retries {
|
||||
match self.send_request(event).await {
|
||||
Ok(_) => return Ok(()),
|
||||
Err((KafkaError::MessageProduction(RDKafkaErrorCode::QueueFull), _)) => {
|
||||
tracing::warn!("Kafka attempt {} failed: Queue full. Retrying...", attempt + 1);
|
||||
// sleep(Duration::from_secs(2u64.pow(attempt))).await;
|
||||
sleep(retry_interval).await;
|
||||
}
|
||||
Err((e, _)) => {
|
||||
tracing::error!("Kafka send error: {}", e);
|
||||
return Err(Error::Kafka(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::Custom("Exceeded maximum retry attempts for Kafka message".to_string()))
|
||||
}
|
||||
|
||||
/// Send a single Kafka message
|
||||
async fn send_request(&self, event: &Event) -> Result<(), Error> {
|
||||
// Serialize events
|
||||
let payload = serde_json::to_string(event).map_err(|e| Error::Custom(format!("Serialization event failed: {}", e)))?;
|
||||
|
||||
// Create a Kafka record
|
||||
let record = FutureRecord::to(&self.config.topic).payload(&payload).key(&event.id); // Use the event ID as the key
|
||||
|
||||
// Send to Kafka
|
||||
let delivery_status = self
|
||||
.producer
|
||||
.send(record, Duration::from_millis(self.config.timeout))
|
||||
.await
|
||||
.map_err(|(e, _)| Error::Custom(format!("Failed to send to Kafka: {}", e)))?;
|
||||
// Check delivery status
|
||||
if let Some((err, _)) = delivery_status {
|
||||
return Err(Error::Kafka(err));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Save the event to the queue
|
||||
async fn save_to_queue(&self, event: &Event) -> Result<(), Error> {
|
||||
if let Some(store) = &self.store {
|
||||
store
|
||||
.put(event.clone())
|
||||
.map_err(|e| Error::Custom(format!("Saving events to queue failed: {}", e)))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ChannelAdapter for KafkaAdapter {
|
||||
fn name(&self) -> String {
|
||||
ChannelAdapterType::Kafka.to_string()
|
||||
}
|
||||
|
||||
async fn send(&self, event: &Event) -> Result<(), Error> {
|
||||
// Try to deal with the backlog of events first
|
||||
let _ = self.process_backlog().await;
|
||||
|
||||
// An attempt was made to send the current event
|
||||
match self.send_with_retry(event).await {
|
||||
Ok(_) => Ok(()),
|
||||
Err(e) => {
|
||||
// If the send fails and the queue is enabled, save to the queue
|
||||
if let Some(_) = &self.store {
|
||||
tracing::warn!("Failed to send events to Kafka and saved to a queue: {}", e);
|
||||
self.save_to_queue(event).await?;
|
||||
return Ok(());
|
||||
}
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,120 +0,0 @@
|
||||
use crate::config::adapter::AdapterConfig;
|
||||
use crate::{Error, Event};
|
||||
use async_trait::async_trait;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[cfg(all(feature = "kafka", target_os = "linux"))]
|
||||
pub(crate) mod kafka;
|
||||
#[cfg(feature = "mqtt")]
|
||||
pub(crate) mod mqtt;
|
||||
#[cfg(feature = "webhook")]
|
||||
pub(crate) mod webhook;
|
||||
|
||||
#[allow(dead_code)]
|
||||
const NOTIFY_KAFKA_SUB_SYS: &str = "notify_kafka";
|
||||
#[allow(dead_code)]
|
||||
const NOTIFY_MQTT_SUB_SYS: &str = "notify_mqtt";
|
||||
#[allow(dead_code)]
|
||||
const NOTIFY_MY_SQL_SUB_SYS: &str = "notify_mysql";
|
||||
#[allow(dead_code)]
|
||||
const NOTIFY_NATS_SUB_SYS: &str = "notify_nats";
|
||||
#[allow(dead_code)]
|
||||
const NOTIFY_NSQ_SUB_SYS: &str = "notify_nsq";
|
||||
#[allow(dead_code)]
|
||||
const NOTIFY_ES_SUB_SYS: &str = "notify_elasticsearch";
|
||||
#[allow(dead_code)]
|
||||
const NOTIFY_AMQP_SUB_SYS: &str = "notify_amqp";
|
||||
#[allow(dead_code)]
|
||||
const NOTIFY_POSTGRES_SUB_SYS: &str = "notify_postgres";
|
||||
#[allow(dead_code)]
|
||||
const NOTIFY_REDIS_SUB_SYS: &str = "notify_redis";
|
||||
const NOTIFY_WEBHOOK_SUB_SYS: &str = "notify_webhook";
|
||||
|
||||
/// The `ChannelAdapterType` enum represents the different types of channel adapters.
|
||||
///
|
||||
/// It is used to identify the type of adapter being used in the system.
|
||||
///
|
||||
/// # Variants
|
||||
///
|
||||
/// - `Webhook`: Represents a webhook adapter.
|
||||
/// - `Kafka`: Represents a Kafka adapter.
|
||||
/// - `Mqtt`: Represents an MQTT adapter.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```
|
||||
/// use rustfs_event::ChannelAdapterType;
|
||||
///
|
||||
/// let adapter_type = ChannelAdapterType::Webhook;
|
||||
/// match adapter_type {
|
||||
/// ChannelAdapterType::Webhook => println!("Using webhook adapter"),
|
||||
/// ChannelAdapterType::Kafka => println!("Using Kafka adapter"),
|
||||
/// ChannelAdapterType::Mqtt => println!("Using MQTT adapter"),
|
||||
/// }
|
||||
pub enum ChannelAdapterType {
|
||||
Webhook,
|
||||
Kafka,
|
||||
Mqtt,
|
||||
}
|
||||
|
||||
impl ChannelAdapterType {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
ChannelAdapterType::Webhook => "webhook",
|
||||
ChannelAdapterType::Kafka => "kafka",
|
||||
ChannelAdapterType::Mqtt => "mqtt",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ChannelAdapterType {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
ChannelAdapterType::Webhook => write!(f, "webhook"),
|
||||
ChannelAdapterType::Kafka => write!(f, "kafka"),
|
||||
ChannelAdapterType::Mqtt => write!(f, "mqtt"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The `ChannelAdapter` trait defines the interface for all channel adapters.
|
||||
#[async_trait]
|
||||
pub trait ChannelAdapter: Send + Sync + 'static {
|
||||
/// Sends an event to the channel.
|
||||
fn name(&self) -> String;
|
||||
/// Sends an event to the channel.
|
||||
async fn send(&self, event: &Event) -> Result<(), Error>;
|
||||
}
|
||||
|
||||
/// Creates channel adapters based on the provided configuration.
|
||||
pub fn create_adapters(configs: Vec<AdapterConfig>) -> Result<Vec<Arc<dyn ChannelAdapter>>, Error> {
|
||||
let mut adapters: Vec<Arc<dyn ChannelAdapter>> = Vec::new();
|
||||
|
||||
for config in configs {
|
||||
match config {
|
||||
#[cfg(feature = "webhook")]
|
||||
AdapterConfig::Webhook(webhook_config) => {
|
||||
webhook_config.validate().map_err(Error::ConfigError)?;
|
||||
adapters.push(Arc::new(webhook::WebhookAdapter::new(webhook_config.clone())));
|
||||
}
|
||||
#[cfg(all(feature = "kafka", target_os = "linux"))]
|
||||
AdapterConfig::Kafka(kafka_config) => {
|
||||
adapters.push(Arc::new(kafka::KafkaAdapter::new(kafka_config)?));
|
||||
}
|
||||
#[cfg(feature = "mqtt")]
|
||||
AdapterConfig::Mqtt(mqtt_config) => {
|
||||
let (mqtt, mut event_loop) = mqtt::MqttAdapter::new(mqtt_config);
|
||||
tokio::spawn(async move { while event_loop.poll().await.is_ok() {} });
|
||||
adapters.push(Arc::new(mqtt));
|
||||
}
|
||||
#[cfg(not(feature = "webhook"))]
|
||||
AdapterConfig::Webhook(_) => return Err(Error::FeatureDisabled("webhook")),
|
||||
#[cfg(any(not(feature = "kafka"), not(target_os = "linux")))]
|
||||
AdapterConfig::Kafka(_) => return Err(Error::FeatureDisabled("kafka")),
|
||||
#[cfg(not(feature = "mqtt"))]
|
||||
AdapterConfig::Mqtt(_) => return Err(Error::FeatureDisabled("mqtt")),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(adapters)
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
use crate::config::mqtt::MqttConfig;
|
||||
use crate::{ChannelAdapter, ChannelAdapterType};
|
||||
use crate::{Error, Event};
|
||||
use async_trait::async_trait;
|
||||
use rumqttc::{AsyncClient, MqttOptions, QoS};
|
||||
use std::time::Duration;
|
||||
use tokio::time::sleep;
|
||||
|
||||
/// MQTT adapter for sending events to an MQTT broker.
|
||||
pub struct MqttAdapter {
|
||||
client: AsyncClient,
|
||||
topic: String,
|
||||
max_retries: u32,
|
||||
}
|
||||
|
||||
impl MqttAdapter {
|
||||
/// Creates a new MQTT adapter.
|
||||
pub fn new(config: &MqttConfig) -> (Self, rumqttc::EventLoop) {
|
||||
let mqtt_options = MqttOptions::new(&config.client_id, &config.broker, config.port);
|
||||
let (client, event_loop) = rumqttc::AsyncClient::new(mqtt_options, 10);
|
||||
(
|
||||
Self {
|
||||
client,
|
||||
topic: config.topic.clone(),
|
||||
max_retries: config.max_retries,
|
||||
},
|
||||
event_loop,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ChannelAdapter for MqttAdapter {
|
||||
fn name(&self) -> String {
|
||||
ChannelAdapterType::Mqtt.to_string()
|
||||
}
|
||||
|
||||
async fn send(&self, event: &Event) -> Result<(), Error> {
|
||||
let payload = serde_json::to_string(event).map_err(Error::Serde)?;
|
||||
let mut attempt = 0;
|
||||
loop {
|
||||
match self
|
||||
.client
|
||||
.publish(&self.topic, QoS::AtLeastOnce, false, payload.clone())
|
||||
.await
|
||||
{
|
||||
Ok(()) => return Ok(()),
|
||||
Err(e) if attempt < self.max_retries => {
|
||||
attempt += 1;
|
||||
tracing::warn!("MQTT attempt {} failed: {}. Retrying...", attempt, e);
|
||||
sleep(Duration::from_secs(2u64.pow(attempt))).await;
|
||||
}
|
||||
Err(e) => return Err(Error::Mqtt(e)),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,271 +0,0 @@
|
||||
use crate::config::webhook::WebhookConfig;
|
||||
use crate::config::STORE_PREFIX;
|
||||
use crate::store::queue::Store;
|
||||
use crate::{ChannelAdapter, ChannelAdapterType};
|
||||
use crate::{Error, QueueStore};
|
||||
use crate::{Event, DEFAULT_RETRY_INTERVAL};
|
||||
use async_trait::async_trait;
|
||||
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
|
||||
use reqwest::{self, Client, Identity, RequestBuilder};
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::time::sleep;
|
||||
use ChannelAdapterType::Webhook;
|
||||
|
||||
// Webhook constants
|
||||
pub const WEBHOOK_ENDPOINT: &str = "endpoint";
|
||||
pub const WEBHOOK_AUTH_TOKEN: &str = "auth_token";
|
||||
pub const WEBHOOK_QUEUE_DIR: &str = "queue_dir";
|
||||
pub const WEBHOOK_QUEUE_LIMIT: &str = "queue_limit";
|
||||
pub const WEBHOOK_CLIENT_CERT: &str = "client_cert";
|
||||
pub const WEBHOOK_CLIENT_KEY: &str = "client_key";
|
||||
|
||||
pub const ENV_WEBHOOK_ENABLE: &str = "RUSTFS_NOTIFY_WEBHOOK_ENABLE";
|
||||
pub const ENV_WEBHOOK_ENDPOINT: &str = "RUSTFS_NOTIFY_WEBHOOK_ENDPOINT";
|
||||
pub const ENV_WEBHOOK_AUTH_TOKEN: &str = "RUSTFS_NOTIFY_WEBHOOK_AUTH_TOKEN";
|
||||
pub const ENV_WEBHOOK_QUEUE_DIR: &str = "RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR";
|
||||
pub const ENV_WEBHOOK_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_WEBHOOK_QUEUE_LIMIT";
|
||||
pub const ENV_WEBHOOK_CLIENT_CERT: &str = "RUSTFS_NOTIFY_WEBHOOK_CLIENT_CERT";
|
||||
pub const ENV_WEBHOOK_CLIENT_KEY: &str = "RUSTFS_NOTIFY_WEBHOOK_CLIENT_KEY";
|
||||
|
||||
/// Webhook adapter for sending events to a webhook endpoint.
|
||||
pub struct WebhookAdapter {
|
||||
/// Configuration information
|
||||
config: WebhookConfig,
|
||||
/// Event storage queues
|
||||
store: Option<Arc<QueueStore<Event>>>,
|
||||
/// HTTP client
|
||||
client: Client,
|
||||
}
|
||||
|
||||
impl WebhookAdapter {
|
||||
/// Creates a new Webhook adapter.
|
||||
pub fn new(config: WebhookConfig) -> Self {
|
||||
let mut builder = Client::builder();
|
||||
if config.timeout.is_some() {
|
||||
// Set the timeout for the client
|
||||
match config.timeout {
|
||||
Some(t) => builder = builder.timeout(Duration::from_secs(t)),
|
||||
None => tracing::warn!("Timeout is not set, using default timeout"),
|
||||
}
|
||||
}
|
||||
let client = if let (Some(cert_path), Some(key_path)) = (&config.client_cert, &config.client_key) {
|
||||
let cert_path = PathBuf::from(cert_path);
|
||||
let key_path = PathBuf::from(key_path);
|
||||
|
||||
// Check if the certificate file exists
|
||||
if !cert_path.exists() || !key_path.exists() {
|
||||
tracing::warn!("Certificate files not found, falling back to default client");
|
||||
builder.build()
|
||||
} else {
|
||||
// Try to read and load the certificate
|
||||
match (fs::read(&cert_path), fs::read(&key_path)) {
|
||||
(Ok(cert_data), Ok(key_data)) => {
|
||||
// Create an identity
|
||||
let mut pem_data = cert_data;
|
||||
pem_data.extend_from_slice(&key_data);
|
||||
|
||||
match Identity::from_pem(&pem_data) {
|
||||
Ok(identity) => {
|
||||
tracing::info!("Successfully loaded client certificate");
|
||||
builder.identity(identity).build()
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to create identity from PEM: {}, falling back to default client", e);
|
||||
builder.build()
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
tracing::warn!("Failed to read certificate files, falling back to default client");
|
||||
builder.build()
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
builder.build()
|
||||
}
|
||||
.unwrap_or_else(|e| {
|
||||
tracing::error!("Failed to create HTTP client: {}", e);
|
||||
reqwest::Client::new()
|
||||
});
|
||||
|
||||
// create a queue store if enabled
|
||||
let store = if !config.common.queue_dir.len() > 0 {
|
||||
let store_path = PathBuf::from(&config.common.queue_dir).join(format!(
|
||||
"{}-{}-{}",
|
||||
STORE_PREFIX,
|
||||
Webhook.as_str(),
|
||||
config.common.identifier
|
||||
));
|
||||
let queue_limit = if config.common.queue_limit > 0 {
|
||||
config.common.queue_limit
|
||||
} else {
|
||||
crate::config::default_queue_limit()
|
||||
};
|
||||
let store = QueueStore::new(store_path, queue_limit, Some(".event".to_string()));
|
||||
if let Err(e) = store.open() {
|
||||
tracing::error!("Unable to open queue storage: {}", e);
|
||||
None
|
||||
} else {
|
||||
Some(Arc::new(store))
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Self { config, store, client }
|
||||
}
|
||||
|
||||
/// Handle backlog events in storage
|
||||
pub async fn process_backlog(&self) -> Result<(), Error> {
|
||||
if let Some(store) = &self.store {
|
||||
let keys = store.list();
|
||||
for key in keys {
|
||||
match store.get_multiple(&key) {
|
||||
Ok(events) => {
|
||||
for event in events {
|
||||
if let Err(e) = self.send_with_retry(&event).await {
|
||||
tracing::error!("Processing of backlog events failed: {}", e);
|
||||
// If it still fails, we remain in the queue
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Deleted after successful processing
|
||||
if let Err(e) = store.del(&key) {
|
||||
tracing::error!("Failed to delete a handled event: {}", e);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to read events from storage: {}", e);
|
||||
// delete the broken entries
|
||||
// If the event cannot be read, it may be corrupted, delete it
|
||||
if let Err(del_err) = store.del(&key) {
|
||||
tracing::error!("Failed to delete a corrupted event: {}", del_err);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
///Send events to the webhook endpoint with retry logic
|
||||
async fn send_with_retry(&self, event: &Event) -> Result<(), Error> {
|
||||
let retry_interval = match self.config.retry_interval {
|
||||
Some(t) => Duration::from_secs(t),
|
||||
None => Duration::from_secs(DEFAULT_RETRY_INTERVAL), // Default to 3 seconds if not set
|
||||
};
|
||||
let mut attempts = 0;
|
||||
|
||||
loop {
|
||||
attempts += 1;
|
||||
match self.send_request(event).await {
|
||||
Ok(_) => return Ok(()),
|
||||
Err(e) => {
|
||||
if attempts <= self.config.max_retries {
|
||||
tracing::warn!("Send to webhook fails and will be retried after 3 seconds:{}", e);
|
||||
sleep(retry_interval).await;
|
||||
} else if let Some(store) = &self.store {
|
||||
// store in a queue for later processing
|
||||
tracing::warn!("The maximum number of retries is reached, and the event is stored in a queue:{}", e);
|
||||
if let Err(store_err) = store.put(event.clone()) {
|
||||
tracing::error!("Events cannot be stored to a queue:{}", store_err);
|
||||
}
|
||||
return Err(e);
|
||||
} else {
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Send a single HTTP request
|
||||
async fn send_request(&self, event: &Event) -> Result<(), Error> {
|
||||
// Send a request
|
||||
let response = self
|
||||
.build_request(event)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| Error::Custom(format!("Sending a webhook request failed:{}", e)))?;
|
||||
|
||||
// Check the response status
|
||||
if !response.status().is_success() {
|
||||
let status = response.status();
|
||||
let body = response
|
||||
.text()
|
||||
.await
|
||||
.unwrap_or_else(|_| "Unable to read response body".to_string());
|
||||
return Err(Error::Custom(format!("Webhook request failed, status code:{},response:{}", status, body)));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Builds the request to send the event.
|
||||
fn build_request(&self, event: &Event) -> RequestBuilder {
|
||||
let mut request = self
|
||||
.client
|
||||
.post(&self.config.endpoint)
|
||||
.json(event)
|
||||
.header("Content-Type", "application/json");
|
||||
if let Some(token) = &self.config.auth_token {
|
||||
let tokens: Vec<&str> = token.split_whitespace().collect();
|
||||
match tokens.len() {
|
||||
2 => request = request.header("Authorization", token),
|
||||
1 => request = request.header("Authorization", format!("Bearer {}", token)),
|
||||
_ => tracing::warn!("Invalid auth token format, skipping Authorization header"),
|
||||
}
|
||||
}
|
||||
if let Some(headers) = &self.config.custom_headers {
|
||||
let mut header_map = HeaderMap::new();
|
||||
for (key, value) in headers {
|
||||
if let (Ok(name), Ok(val)) = (HeaderName::from_bytes(key.as_bytes()), HeaderValue::from_str(value)) {
|
||||
header_map.insert(name, val);
|
||||
}
|
||||
}
|
||||
request = request.headers(header_map);
|
||||
}
|
||||
request
|
||||
}
|
||||
|
||||
/// Save the event to the queue
|
||||
async fn save_to_queue(&self, event: &Event) -> Result<(), Error> {
|
||||
if let Some(store) = &self.store {
|
||||
store
|
||||
.put(event.clone())
|
||||
.map_err(|e| Error::Custom(format!("Saving events to queue failed: {}", e)))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ChannelAdapter for WebhookAdapter {
|
||||
fn name(&self) -> String {
|
||||
Webhook.to_string()
|
||||
}
|
||||
|
||||
async fn send(&self, event: &Event) -> Result<(), Error> {
|
||||
// Deal with the backlog of events first
|
||||
let _ = self.process_backlog().await;
|
||||
|
||||
// Send the current event
|
||||
match self.send_with_retry(event).await {
|
||||
Ok(_) => Ok(()),
|
||||
Err(e) => {
|
||||
// If the send fails and the queue is enabled, save to the queue
|
||||
if let Some(_) = &self.store {
|
||||
tracing::warn!("Failed to send the event and saved to the queue: {}", e);
|
||||
self.save_to_queue(event).await?;
|
||||
return Ok(());
|
||||
}
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
use crate::config::kafka::KafkaConfig;
|
||||
use crate::config::mqtt::MqttConfig;
|
||||
use crate::config::webhook::WebhookConfig;
|
||||
use crate::config::{default_queue_dir, default_queue_limit};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Add a common field for the adapter configuration
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AdapterCommon {
|
||||
/// Adapter identifier for unique identification
|
||||
pub identifier: String,
|
||||
/// Adapter description information
|
||||
pub comment: String,
|
||||
/// Whether to enable this adapter
|
||||
#[serde(default)]
|
||||
pub enable: bool,
|
||||
#[serde(default = "default_queue_dir")]
|
||||
pub queue_dir: String,
|
||||
#[serde(default = "default_queue_limit")]
|
||||
pub queue_limit: u64,
|
||||
}
|
||||
|
||||
impl Default for AdapterCommon {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
identifier: String::new(),
|
||||
comment: String::new(),
|
||||
enable: false,
|
||||
queue_dir: default_queue_dir(),
|
||||
queue_limit: default_queue_limit(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Configuration for the adapter.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum AdapterConfig {
|
||||
Webhook(WebhookConfig),
|
||||
Kafka(KafkaConfig),
|
||||
Mqtt(MqttConfig),
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
use crate::config::adapter::AdapterCommon;
|
||||
use crate::config::{default_queue_dir, default_queue_limit};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Configuration for the Kafka adapter.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct KafkaConfig {
|
||||
#[serde(flatten)]
|
||||
pub common: AdapterCommon,
|
||||
pub brokers: String,
|
||||
pub topic: String,
|
||||
pub max_retries: u32,
|
||||
pub timeout: u64,
|
||||
}
|
||||
|
||||
impl Default for KafkaConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
common: AdapterCommon::default(),
|
||||
brokers: String::new(),
|
||||
topic: String::new(),
|
||||
max_retries: 3,
|
||||
timeout: 5000,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl KafkaConfig {
|
||||
/// Create a new Kafka configuration
|
||||
pub fn new(identifier: impl Into<String>, brokers: impl Into<String>, topic: impl Into<String>) -> Self {
|
||||
Self {
|
||||
common: AdapterCommon {
|
||||
identifier: identifier.into(),
|
||||
comment: String::new(),
|
||||
enable: true,
|
||||
queue_dir: default_queue_dir(),
|
||||
queue_limit: default_queue_limit(),
|
||||
},
|
||||
brokers: brokers.into(),
|
||||
topic: topic.into(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
use std::env;
|
||||
|
||||
pub mod adapter;
|
||||
pub mod kafka;
|
||||
pub mod mqtt;
|
||||
pub mod notifier;
|
||||
pub mod webhook;
|
||||
|
||||
/// The default configuration file name
|
||||
const DEFAULT_CONFIG_FILE: &str = "event";
|
||||
|
||||
/// The prefix for the configuration file
|
||||
pub const STORE_PREFIX: &str = "rustfs";
|
||||
|
||||
/// The default retry interval for the webhook adapter
|
||||
pub const DEFAULT_RETRY_INTERVAL: u64 = 3;
|
||||
|
||||
/// The default maximum retry count for the webhook adapter
|
||||
pub const DEFAULT_MAX_RETRIES: u32 = 3;
|
||||
|
||||
/// The default notification queue limit
|
||||
pub const DEFAULT_NOTIFY_QUEUE_LIMIT: u64 = 10000;
|
||||
|
||||
/// Provide temporary directories as default storage paths
|
||||
pub(crate) fn default_queue_dir() -> String {
|
||||
env::var("EVENT_QUEUE_DIR").unwrap_or_else(|e| {
|
||||
tracing::info!("Failed to get `EVENT_QUEUE_DIR` failed err: {}", e.to_string());
|
||||
env::temp_dir().join(DEFAULT_CONFIG_FILE).to_string_lossy().to_string()
|
||||
})
|
||||
}
|
||||
|
||||
/// Provides the recommended default channel capacity for high concurrency systems
|
||||
pub(crate) fn default_queue_limit() -> u64 {
|
||||
env::var("EVENT_CHANNEL_CAPACITY")
|
||||
.unwrap_or_else(|_| DEFAULT_NOTIFY_QUEUE_LIMIT.to_string())
|
||||
.parse()
|
||||
.unwrap_or(DEFAULT_NOTIFY_QUEUE_LIMIT) // Default to 10000 if parsing fails
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
use crate::config::adapter::AdapterCommon;
|
||||
use crate::config::{default_queue_dir, default_queue_limit};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Configuration for the MQTT adapter.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MqttConfig {
|
||||
#[serde(flatten)]
|
||||
pub common: AdapterCommon,
|
||||
pub broker: String,
|
||||
pub port: u16,
|
||||
pub client_id: String,
|
||||
pub topic: String,
|
||||
pub max_retries: u32,
|
||||
}
|
||||
|
||||
impl Default for MqttConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
common: AdapterCommon::default(),
|
||||
broker: String::new(),
|
||||
port: 1883,
|
||||
client_id: String::new(),
|
||||
topic: String::new(),
|
||||
max_retries: 3,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl MqttConfig {
|
||||
/// Create a new MQTT configuration
|
||||
pub fn new(identifier: impl Into<String>, broker: impl Into<String>, topic: impl Into<String>) -> Self {
|
||||
Self {
|
||||
common: AdapterCommon {
|
||||
identifier: identifier.into(),
|
||||
comment: String::new(),
|
||||
enable: true,
|
||||
queue_dir: default_queue_dir(),
|
||||
queue_limit: default_queue_limit(),
|
||||
},
|
||||
broker: broker.into(),
|
||||
topic: topic.into(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
use crate::config::{adapter::AdapterConfig, kafka::KafkaConfig, mqtt::MqttConfig, webhook::WebhookConfig};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Event Notifier Configuration
|
||||
/// This struct contains the configuration for the event notifier system,
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct EventNotifierConfig {
|
||||
/// A collection of webhook configurations, with the key being a unique identifier
|
||||
#[serde(default)]
|
||||
pub webhook: HashMap<String, WebhookConfig>,
|
||||
/// A collection of Kafka configurations, with the key being a unique identifier
|
||||
#[serde(default)]
|
||||
pub kafka: HashMap<String, KafkaConfig>,
|
||||
///MQTT configuration collection, with the key being a unique identifier
|
||||
#[serde(default)]
|
||||
pub mqtt: HashMap<String, MqttConfig>,
|
||||
}
|
||||
|
||||
impl EventNotifierConfig {
|
||||
/// Create a new default configuration
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Load the configuration from the file
|
||||
pub fn event_load_config(_config_dir: Option<String>) -> EventNotifierConfig {
|
||||
// The existing implementation remains the same, but returns EventNotifierConfig
|
||||
// ...
|
||||
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Deserialization configuration
|
||||
pub fn unmarshal(data: &[u8]) -> common::error::Result<EventNotifierConfig> {
|
||||
let m: EventNotifierConfig = serde_json::from_slice(data)?;
|
||||
Ok(m)
|
||||
}
|
||||
|
||||
/// Serialization configuration
|
||||
pub fn marshal(&self) -> common::error::Result<Vec<u8>> {
|
||||
let data = serde_json::to_vec(&self)?;
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
/// Convert this configuration to a list of adapter configurations
|
||||
pub fn to_adapter_configs(&self) -> Vec<AdapterConfig> {
|
||||
let mut adapters = Vec::new();
|
||||
|
||||
// Add all enabled webhook configurations
|
||||
for webhook in self.webhook.values() {
|
||||
if webhook.common.enable {
|
||||
adapters.push(AdapterConfig::Webhook(webhook.clone()));
|
||||
}
|
||||
}
|
||||
|
||||
// Add all enabled Kafka configurations
|
||||
for kafka in self.kafka.values() {
|
||||
if kafka.common.enable {
|
||||
adapters.push(AdapterConfig::Kafka(kafka.clone()));
|
||||
}
|
||||
}
|
||||
|
||||
// Add all enabled MQTT configurations
|
||||
for mqtt in self.mqtt.values() {
|
||||
if mqtt.common.enable {
|
||||
adapters.push(AdapterConfig::Mqtt(mqtt.clone()));
|
||||
}
|
||||
}
|
||||
|
||||
adapters
|
||||
}
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
use crate::config::adapter::AdapterCommon;
|
||||
use crate::config::{default_queue_dir, default_queue_limit};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use tracing::info;
|
||||
|
||||
/// Configuration for the webhook adapter.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct WebhookConfig {
|
||||
#[serde(flatten)]
|
||||
pub common: AdapterCommon,
|
||||
pub endpoint: String,
|
||||
pub auth_token: Option<String>,
|
||||
pub custom_headers: Option<HashMap<String, String>>,
|
||||
pub max_retries: u32,
|
||||
pub retry_interval: Option<u64>,
|
||||
pub timeout: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub client_cert: Option<String>,
|
||||
#[serde(default)]
|
||||
pub client_key: Option<String>,
|
||||
}
|
||||
|
||||
impl WebhookConfig {
|
||||
/// validate the configuration for the webhook adapter
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// - `Result<(), String>`: Ok if the configuration is valid, Err with a message if invalid.
|
||||
///
|
||||
/// # Errors
|
||||
/// - Returns an error if the configuration is invalid, such as empty endpoint, unreasonable timeout, or mismatched certificate and key.
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
// If not enabled, the other fields are not validated
|
||||
if !self.common.enable {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// verify that endpoint cannot be empty
|
||||
if self.endpoint.trim().is_empty() {
|
||||
return Err("Webhook endpoint cannot be empty".to_string());
|
||||
}
|
||||
|
||||
// verification timeout must be reasonable
|
||||
if self.timeout.is_some() {
|
||||
match self.timeout {
|
||||
Some(timeout) if timeout > 0 => {
|
||||
info!("Webhook timeout is set to {}", timeout);
|
||||
}
|
||||
_ => return Err("Webhook timeout must be greater than 0".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
// Verify that the maximum number of retry is reasonable
|
||||
if self.max_retries > 10 {
|
||||
return Err("Maximum retry count cannot exceed 10".to_string());
|
||||
}
|
||||
|
||||
// Verify the queue directory path
|
||||
if !self.common.queue_dir.is_empty() && !Path::new(&self.common.queue_dir).is_absolute() {
|
||||
return Err("Queue directory path should be absolute".to_string());
|
||||
}
|
||||
|
||||
// The authentication certificate and key must appear in pairs
|
||||
if (self.client_cert.is_some() && self.client_key.is_none()) || (self.client_cert.is_none() && self.client_key.is_some())
|
||||
{
|
||||
return Err("Certificate and key must be specified as a pair".to_string());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Create a new webhook configuration
|
||||
pub fn new(identifier: impl Into<String>, endpoint: impl Into<String>) -> Self {
|
||||
Self {
|
||||
common: AdapterCommon {
|
||||
identifier: identifier.into(),
|
||||
comment: String::new(),
|
||||
enable: true,
|
||||
queue_dir: default_queue_dir(),
|
||||
queue_limit: default_queue_limit(),
|
||||
},
|
||||
endpoint: endpoint.into(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,420 +0,0 @@
|
||||
use config::ConfigError;
|
||||
use thiserror::Error;
|
||||
use tokio::sync::mpsc::error;
|
||||
use tokio::task::JoinError;
|
||||
|
||||
/// The `Error` enum represents all possible errors that can occur in the application.
|
||||
/// It implements the `std::error::Error` trait and provides a way to convert various error types into a single error type.
|
||||
#[derive(Error, Debug)]
|
||||
pub enum Error {
|
||||
#[error("Join error: {0}")]
|
||||
JoinError(#[from] JoinError),
|
||||
#[error("IO error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
#[error("Serialization error: {0}")]
|
||||
Serde(#[from] serde_json::Error),
|
||||
#[error("HTTP error: {0}")]
|
||||
Http(#[from] reqwest::Error),
|
||||
#[cfg(all(feature = "kafka", target_os = "linux"))]
|
||||
#[error("Kafka error: {0}")]
|
||||
Kafka(#[from] rdkafka::error::KafkaError),
|
||||
#[cfg(feature = "mqtt")]
|
||||
#[error("MQTT error: {0}")]
|
||||
Mqtt(#[from] rumqttc::ClientError),
|
||||
#[error("Channel send error: {0}")]
|
||||
ChannelSend(#[from] Box<error::SendError<crate::event::Event>>),
|
||||
#[error("Feature disabled: {0}")]
|
||||
FeatureDisabled(&'static str),
|
||||
#[error("Event bus already started")]
|
||||
EventBusStarted,
|
||||
#[error("necessary fields are missing:{0}")]
|
||||
MissingField(&'static str),
|
||||
#[error("field verification failed:{0}")]
|
||||
ValidationError(&'static str),
|
||||
#[error("Custom error: {0}")]
|
||||
Custom(String),
|
||||
#[error("Configuration error: {0}")]
|
||||
ConfigError(String),
|
||||
#[error("Configuration loading error: {0}")]
|
||||
Config(#[from] ConfigError),
|
||||
#[error("create adapter failed error: {0}")]
|
||||
AdapterCreationFailed(String),
|
||||
}
|
||||
|
||||
impl Error {
|
||||
pub fn custom(msg: &str) -> Error {
|
||||
Self::Custom(msg.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::error::Error as StdError;
|
||||
use std::io;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
#[test]
|
||||
fn test_error_display() {
|
||||
// Test error message display
|
||||
let custom_error = Error::custom("test message");
|
||||
assert_eq!(custom_error.to_string(), "Custom error: test message");
|
||||
|
||||
let feature_error = Error::FeatureDisabled("test feature");
|
||||
assert_eq!(feature_error.to_string(), "Feature disabled: test feature");
|
||||
|
||||
let event_bus_error = Error::EventBusStarted;
|
||||
assert_eq!(event_bus_error.to_string(), "Event bus already started");
|
||||
|
||||
let missing_field_error = Error::MissingField("required_field");
|
||||
assert_eq!(missing_field_error.to_string(), "necessary fields are missing:required_field");
|
||||
|
||||
let validation_error = Error::ValidationError("invalid format");
|
||||
assert_eq!(validation_error.to_string(), "field verification failed:invalid format");
|
||||
|
||||
let config_error = Error::ConfigError("invalid config".to_string());
|
||||
assert_eq!(config_error.to_string(), "Configuration error: invalid config");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_debug() {
|
||||
// Test Debug trait implementation
|
||||
let custom_error = Error::custom("debug test");
|
||||
let debug_str = format!("{:?}", custom_error);
|
||||
assert!(debug_str.contains("Custom"));
|
||||
assert!(debug_str.contains("debug test"));
|
||||
|
||||
let feature_error = Error::FeatureDisabled("debug feature");
|
||||
let debug_str = format!("{:?}", feature_error);
|
||||
assert!(debug_str.contains("FeatureDisabled"));
|
||||
assert!(debug_str.contains("debug feature"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_custom_error_creation() {
|
||||
// Test custom error creation
|
||||
let error = Error::custom("test custom error");
|
||||
match error {
|
||||
Error::Custom(msg) => assert_eq!(msg, "test custom error"),
|
||||
_ => panic!("Expected Custom error variant"),
|
||||
}
|
||||
|
||||
// Test empty string
|
||||
let empty_error = Error::custom("");
|
||||
match empty_error {
|
||||
Error::Custom(msg) => assert_eq!(msg, ""),
|
||||
_ => panic!("Expected Custom error variant"),
|
||||
}
|
||||
|
||||
// Test special characters
|
||||
let special_error = Error::custom("Test Chinese 中文 & special chars: !@#$%");
|
||||
match special_error {
|
||||
Error::Custom(msg) => assert_eq!(msg, "Test Chinese 中文 & special chars: !@#$%"),
|
||||
_ => panic!("Expected Custom error variant"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_io_error_conversion() {
|
||||
// Test IO error conversion
|
||||
let io_error = io::Error::new(io::ErrorKind::NotFound, "file not found");
|
||||
let converted_error: Error = io_error.into();
|
||||
|
||||
match converted_error {
|
||||
Error::Io(err) => {
|
||||
assert_eq!(err.kind(), io::ErrorKind::NotFound);
|
||||
assert_eq!(err.to_string(), "file not found");
|
||||
}
|
||||
_ => panic!("Expected Io error variant"),
|
||||
}
|
||||
|
||||
// Test different types of IO errors
|
||||
let permission_error = io::Error::new(io::ErrorKind::PermissionDenied, "access denied");
|
||||
let converted: Error = permission_error.into();
|
||||
assert!(matches!(converted, Error::Io(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_serde_error_conversion() {
|
||||
// Test serialization error conversion
|
||||
let invalid_json = r#"{"invalid": json}"#;
|
||||
let serde_error = serde_json::from_str::<serde_json::Value>(invalid_json).unwrap_err();
|
||||
let converted_error: Error = serde_error.into();
|
||||
|
||||
match converted_error {
|
||||
Error::Serde(_) => {
|
||||
// Verify error type is correct
|
||||
assert!(converted_error.to_string().contains("Serialization error"));
|
||||
}
|
||||
_ => panic!("Expected Serde error variant"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_config_error_conversion() {
|
||||
// Test configuration error conversion
|
||||
let config_error = ConfigError::Message("invalid configuration".to_string());
|
||||
let converted_error: Error = config_error.into();
|
||||
|
||||
match converted_error {
|
||||
Error::Config(_) => {
|
||||
assert!(converted_error.to_string().contains("Configuration loading error"));
|
||||
}
|
||||
_ => panic!("Expected Config error variant"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_channel_send_error_conversion() {
|
||||
// Test channel send error conversion
|
||||
let (tx, rx) = mpsc::channel::<crate::event::Event>(1);
|
||||
drop(rx); // Close receiver
|
||||
|
||||
// Create a test event
|
||||
use crate::event::{Bucket, Identity, Metadata, Name, Object, Source};
|
||||
use std::collections::HashMap;
|
||||
|
||||
let identity = Identity::new("test-user".to_string());
|
||||
let bucket = Bucket::new("test-bucket".to_string(), identity.clone(), "arn:aws:s3:::test-bucket".to_string());
|
||||
let object = Object::new(
|
||||
"test-key".to_string(),
|
||||
Some(1024),
|
||||
Some("etag123".to_string()),
|
||||
Some("text/plain".to_string()),
|
||||
Some(HashMap::new()),
|
||||
None,
|
||||
"sequencer123".to_string(),
|
||||
);
|
||||
let metadata = Metadata::create("1.0".to_string(), "config1".to_string(), bucket, object);
|
||||
let source = Source::new("localhost".to_string(), "8080".to_string(), "test-agent".to_string());
|
||||
|
||||
let test_event = crate::event::Event::builder()
|
||||
.event_name(Name::ObjectCreatedPut)
|
||||
.s3(metadata)
|
||||
.source(source)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let send_result = tx.send(test_event).await;
|
||||
assert!(send_result.is_err());
|
||||
|
||||
let send_error = send_result.unwrap_err();
|
||||
let boxed_error = Box::new(send_error);
|
||||
let converted_error: Error = boxed_error.into();
|
||||
|
||||
match converted_error {
|
||||
Error::ChannelSend(_) => {
|
||||
assert!(converted_error.to_string().contains("Channel send error"));
|
||||
}
|
||||
_ => panic!("Expected ChannelSend error variant"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_source_chain() {
|
||||
// 测试错误源链
|
||||
let io_error = io::Error::new(io::ErrorKind::InvalidData, "invalid data");
|
||||
let converted_error: Error = io_error.into();
|
||||
|
||||
// 验证错误源
|
||||
assert!(converted_error.source().is_some());
|
||||
let source = converted_error.source().unwrap();
|
||||
assert_eq!(source.to_string(), "invalid data");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_variants_exhaustive() {
|
||||
// 测试所有错误变体的创建
|
||||
let errors = vec![
|
||||
Error::FeatureDisabled("test"),
|
||||
Error::EventBusStarted,
|
||||
Error::MissingField("field"),
|
||||
Error::ValidationError("validation"),
|
||||
Error::Custom("custom".to_string()),
|
||||
Error::ConfigError("config".to_string()),
|
||||
];
|
||||
|
||||
for error in errors {
|
||||
// 验证每个错误都能正确显示
|
||||
let error_str = error.to_string();
|
||||
assert!(!error_str.is_empty());
|
||||
|
||||
// 验证每个错误都能正确调试
|
||||
let debug_str = format!("{:?}", error);
|
||||
assert!(!debug_str.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_equality_and_matching() {
|
||||
// 测试错误的模式匹配
|
||||
let custom_error = Error::custom("test");
|
||||
match custom_error {
|
||||
Error::Custom(msg) => assert_eq!(msg, "test"),
|
||||
_ => panic!("Pattern matching failed"),
|
||||
}
|
||||
|
||||
let feature_error = Error::FeatureDisabled("feature");
|
||||
match feature_error {
|
||||
Error::FeatureDisabled(feature) => assert_eq!(feature, "feature"),
|
||||
_ => panic!("Pattern matching failed"),
|
||||
}
|
||||
|
||||
let event_bus_error = Error::EventBusStarted;
|
||||
match event_bus_error {
|
||||
Error::EventBusStarted => {} // 正确匹配
|
||||
_ => panic!("Pattern matching failed"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_message_formatting() {
|
||||
// 测试错误消息格式化
|
||||
let test_cases = vec![
|
||||
(Error::FeatureDisabled("kafka"), "Feature disabled: kafka"),
|
||||
(Error::MissingField("bucket_name"), "necessary fields are missing:bucket_name"),
|
||||
(Error::ValidationError("invalid email"), "field verification failed:invalid email"),
|
||||
(Error::ConfigError("missing file".to_string()), "Configuration error: missing file"),
|
||||
];
|
||||
|
||||
for (error, expected_message) in test_cases {
|
||||
assert_eq!(error.to_string(), expected_message);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_memory_efficiency() {
|
||||
// 测试错误类型的内存效率
|
||||
use std::mem;
|
||||
|
||||
let size = mem::size_of::<Error>();
|
||||
// 错误类型应该相对紧凑，考虑到包含多种错误类型，96 字节是合理的
|
||||
assert!(size <= 128, "Error size should be reasonable, got {} bytes", size);
|
||||
|
||||
// 测试 Option<Error>的大小
|
||||
let option_size = mem::size_of::<Option<Error>>();
|
||||
assert!(option_size <= 136, "Option<Error> should be efficient, got {} bytes", option_size);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_thread_safety() {
|
||||
// 测试错误类型的线程安全性
|
||||
fn assert_send<T: Send>() {}
|
||||
fn assert_sync<T: Sync>() {}
|
||||
|
||||
assert_send::<Error>();
|
||||
assert_sync::<Error>();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_custom_error_edge_cases() {
|
||||
// 测试自定义错误的边界情况
|
||||
let long_message = "a".repeat(1000);
|
||||
let long_error = Error::custom(&long_message);
|
||||
match long_error {
|
||||
Error::Custom(msg) => assert_eq!(msg.len(), 1000),
|
||||
_ => panic!("Expected Custom error variant"),
|
||||
}
|
||||
|
||||
// 测试包含换行符的消息
|
||||
let multiline_error = Error::custom("line1\nline2\nline3");
|
||||
match multiline_error {
|
||||
Error::Custom(msg) => assert!(msg.contains('\n')),
|
||||
_ => panic!("Expected Custom error variant"),
|
||||
}
|
||||
|
||||
// 测试包含 Unicode 字符的消息
|
||||
let unicode_error = Error::custom("🚀 Unicode test 测试 🎉");
|
||||
match unicode_error {
|
||||
Error::Custom(msg) => assert!(msg.contains('🚀')),
|
||||
_ => panic!("Expected Custom error variant"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_conversion_consistency() {
|
||||
// 测试错误转换的一致性
|
||||
let original_io_error = io::Error::new(io::ErrorKind::TimedOut, "timeout");
|
||||
let error_message = original_io_error.to_string();
|
||||
let converted: Error = original_io_error.into();
|
||||
|
||||
// 验证转换后的错误包含原始错误信息
|
||||
assert!(converted.to_string().contains(&error_message));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_downcast() {
|
||||
// 测试错误的向下转型
|
||||
let io_error = io::Error::other("test error");
|
||||
let converted: Error = io_error.into();
|
||||
|
||||
// 验证可以获取源错误
|
||||
if let Error::Io(ref inner) = converted {
|
||||
assert_eq!(inner.to_string(), "test error");
|
||||
assert_eq!(inner.kind(), io::ErrorKind::Other);
|
||||
} else {
|
||||
panic!("Expected Io error variant");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_chain_depth() {
|
||||
// 测试错误链的深度
|
||||
let root_cause = io::Error::other("root cause");
|
||||
let converted: Error = root_cause.into();
|
||||
|
||||
let mut depth = 0;
|
||||
let mut current_error: &dyn StdError = &converted;
|
||||
|
||||
while let Some(source) = current_error.source() {
|
||||
depth += 1;
|
||||
current_error = source;
|
||||
// 防止无限循环
|
||||
if depth > 10 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert!(depth > 0, "Error should have at least one source");
|
||||
assert!(depth <= 3, "Error chain should not be too deep");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_static_str_lifetime() {
|
||||
// 测试静态字符串生命周期
|
||||
fn create_feature_error() -> Error {
|
||||
Error::FeatureDisabled("static_feature")
|
||||
}
|
||||
|
||||
let error = create_feature_error();
|
||||
match error {
|
||||
Error::FeatureDisabled(feature) => assert_eq!(feature, "static_feature"),
|
||||
_ => panic!("Expected FeatureDisabled error variant"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_formatting_consistency() {
|
||||
// 测试错误格式化的一致性
|
||||
let errors = vec![
|
||||
Error::FeatureDisabled("test"),
|
||||
Error::MissingField("field"),
|
||||
Error::ValidationError("validation"),
|
||||
Error::Custom("custom".to_string()),
|
||||
];
|
||||
|
||||
for error in errors {
|
||||
let display_str = error.to_string();
|
||||
let debug_str = format!("{:?}", error);
|
||||
|
||||
// Display 和 Debug 都不应该为空
|
||||
assert!(!display_str.is_empty());
|
||||
assert!(!debug_str.is_empty());
|
||||
|
||||
// Debug 输出通常包含更多信息，但不是绝对的
|
||||
// 这里我们只验证两者都有内容即可
|
||||
assert!(!debug_str.is_empty());
|
||||
assert!(!display_str.is_empty());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,616 +0,0 @@
|
||||
use crate::Error;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_with::{DeserializeFromStr, SerializeDisplay};
|
||||
use smallvec::{smallvec, SmallVec};
|
||||
use std::borrow::Cow;
|
||||
use std::collections::HashMap;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use strum::{Display, EnumString};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// A struct representing the identity of the user
|
||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||
pub struct Identity {
|
||||
#[serde(rename = "principalId")]
|
||||
pub principal_id: String,
|
||||
}
|
||||
|
||||
impl Identity {
|
||||
/// Create a new Identity instance
|
||||
pub fn new(principal_id: String) -> Self {
|
||||
Self { principal_id }
|
||||
}
|
||||
|
||||
/// Set the principal ID
|
||||
pub fn set_principal_id(&mut self, principal_id: String) {
|
||||
self.principal_id = principal_id;
|
||||
}
|
||||
}
|
||||
|
||||
/// A struct representing the bucket information
|
||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||
pub struct Bucket {
|
||||
pub name: String,
|
||||
#[serde(rename = "ownerIdentity")]
|
||||
pub owner_identity: Identity,
|
||||
pub arn: String,
|
||||
}
|
||||
|
||||
impl Bucket {
|
||||
/// Create a new Bucket instance
|
||||
pub fn new(name: String, owner_identity: Identity, arn: String) -> Self {
|
||||
Self {
|
||||
name,
|
||||
owner_identity,
|
||||
arn,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the name of the bucket
|
||||
pub fn set_name(&mut self, name: String) {
|
||||
self.name = name;
|
||||
}
|
||||
|
||||
/// Set the ARN of the bucket
|
||||
pub fn set_arn(&mut self, arn: String) {
|
||||
self.arn = arn;
|
||||
}
|
||||
|
||||
/// Set the owner identity of the bucket
|
||||
pub fn set_owner_identity(&mut self, owner_identity: Identity) {
|
||||
self.owner_identity = owner_identity;
|
||||
}
|
||||
}
|
||||
|
||||
/// A struct representing the object information
|
||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||
pub struct Object {
|
||||
pub key: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub size: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none", rename = "eTag")]
|
||||
pub etag: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none", rename = "contentType")]
|
||||
pub content_type: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none", rename = "userMetadata")]
|
||||
pub user_metadata: Option<HashMap<String, String>>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none", rename = "versionId")]
|
||||
pub version_id: Option<String>,
|
||||
pub sequencer: String,
|
||||
}
|
||||
|
||||
impl Object {
|
||||
/// Create a new Object instance
|
||||
pub fn new(
|
||||
key: String,
|
||||
size: Option<i64>,
|
||||
etag: Option<String>,
|
||||
content_type: Option<String>,
|
||||
user_metadata: Option<HashMap<String, String>>,
|
||||
version_id: Option<String>,
|
||||
sequencer: String,
|
||||
) -> Self {
|
||||
Self {
|
||||
key,
|
||||
size,
|
||||
etag,
|
||||
content_type,
|
||||
user_metadata,
|
||||
version_id,
|
||||
sequencer,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the key
|
||||
pub fn set_key(&mut self, key: String) {
|
||||
self.key = key;
|
||||
}
|
||||
|
||||
/// Set the size
|
||||
pub fn set_size(&mut self, size: Option<i64>) {
|
||||
self.size = size;
|
||||
}
|
||||
|
||||
/// Set the etag
|
||||
pub fn set_etag(&mut self, etag: Option<String>) {
|
||||
self.etag = etag;
|
||||
}
|
||||
|
||||
/// Set the content type
|
||||
pub fn set_content_type(&mut self, content_type: Option<String>) {
|
||||
self.content_type = content_type;
|
||||
}
|
||||
|
||||
/// Set the user metadata
|
||||
pub fn set_user_metadata(&mut self, user_metadata: Option<HashMap<String, String>>) {
|
||||
self.user_metadata = user_metadata;
|
||||
}
|
||||
|
||||
/// Set the version ID
|
||||
pub fn set_version_id(&mut self, version_id: Option<String>) {
|
||||
self.version_id = version_id;
|
||||
}
|
||||
|
||||
/// Set the sequencer
|
||||
pub fn set_sequencer(&mut self, sequencer: String) {
|
||||
self.sequencer = sequencer;
|
||||
}
|
||||
}
|
||||
|
||||
/// A struct representing the metadata of the event
|
||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||
pub struct Metadata {
|
||||
#[serde(rename = "s3SchemaVersion")]
|
||||
pub schema_version: String,
|
||||
#[serde(rename = "configurationId")]
|
||||
pub configuration_id: String,
|
||||
pub bucket: Bucket,
|
||||
pub object: Object,
|
||||
}
|
||||
|
||||
impl Default for Metadata {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
impl Metadata {
|
||||
/// Create a new Metadata instance with default values
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
schema_version: "1.0".to_string(),
|
||||
configuration_id: "default".to_string(),
|
||||
bucket: Bucket::new(
|
||||
"default".to_string(),
|
||||
Identity::new("default".to_string()),
|
||||
"arn:aws:s3:::default".to_string(),
|
||||
),
|
||||
object: Object::new("default".to_string(), None, None, None, None, None, "default".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new Metadata instance
|
||||
pub fn create(schema_version: String, configuration_id: String, bucket: Bucket, object: Object) -> Self {
|
||||
Self {
|
||||
schema_version,
|
||||
configuration_id,
|
||||
bucket,
|
||||
object,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the schema version
|
||||
pub fn set_schema_version(&mut self, schema_version: String) {
|
||||
self.schema_version = schema_version;
|
||||
}
|
||||
|
||||
/// Set the configuration ID
|
||||
pub fn set_configuration_id(&mut self, configuration_id: String) {
|
||||
self.configuration_id = configuration_id;
|
||||
}
|
||||
|
||||
/// Set the bucket
|
||||
pub fn set_bucket(&mut self, bucket: Bucket) {
|
||||
self.bucket = bucket;
|
||||
}
|
||||
|
||||
/// Set the object
|
||||
pub fn set_object(&mut self, object: Object) {
|
||||
self.object = object;
|
||||
}
|
||||
}
|
||||
|
||||
/// A struct representing the source of the event
|
||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||
pub struct Source {
|
||||
pub host: String,
|
||||
pub port: String,
|
||||
#[serde(rename = "userAgent")]
|
||||
pub user_agent: String,
|
||||
}
|
||||
|
||||
impl Source {
|
||||
/// Create a new Source instance
|
||||
pub fn new(host: String, port: String, user_agent: String) -> Self {
|
||||
Self { host, port, user_agent }
|
||||
}
|
||||
|
||||
/// Set the host
|
||||
pub fn set_host(&mut self, host: String) {
|
||||
self.host = host;
|
||||
}
|
||||
|
||||
/// Set the port
|
||||
pub fn set_port(&mut self, port: String) {
|
||||
self.port = port;
|
||||
}
|
||||
|
||||
/// Set the user agent
|
||||
pub fn set_user_agent(&mut self, user_agent: String) {
|
||||
self.user_agent = user_agent;
|
||||
}
|
||||
}
|
||||
|
||||
/// Builder for creating an Event.
|
||||
///
|
||||
/// This struct is used to build an Event object with various parameters.
|
||||
/// It provides methods to set each parameter and a build method to create the Event.
|
||||
#[derive(Default, Clone)]
|
||||
pub struct EventBuilder {
|
||||
event_version: Option<String>,
|
||||
event_source: Option<String>,
|
||||
aws_region: Option<String>,
|
||||
event_time: Option<String>,
|
||||
event_name: Option<Name>,
|
||||
user_identity: Option<Identity>,
|
||||
request_parameters: Option<HashMap<String, String>>,
|
||||
response_elements: Option<HashMap<String, String>>,
|
||||
s3: Option<Metadata>,
|
||||
source: Option<Source>,
|
||||
channels: Option<SmallVec<[String; 2]>>,
|
||||
}
|
||||
|
||||
impl EventBuilder {
|
||||
/// create a builder that pre filled default values
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
event_version: Some(Cow::Borrowed("2.0").to_string()),
|
||||
event_source: Some(Cow::Borrowed("aws:s3").to_string()),
|
||||
aws_region: Some("us-east-1".to_string()),
|
||||
event_time: Some(SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs().to_string()),
|
||||
event_name: None,
|
||||
user_identity: Some(Identity {
|
||||
principal_id: "anonymous".to_string(),
|
||||
}),
|
||||
request_parameters: Some(HashMap::new()),
|
||||
response_elements: Some(HashMap::new()),
|
||||
s3: None,
|
||||
source: None,
|
||||
channels: Some(Vec::new().into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// verify and set the event version
|
||||
pub fn event_version(mut self, event_version: impl Into<String>) -> Self {
|
||||
let event_version = event_version.into();
|
||||
if !event_version.is_empty() {
|
||||
self.event_version = Some(event_version);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// verify and set the event source
|
||||
pub fn event_source(mut self, event_source: impl Into<String>) -> Self {
|
||||
let event_source = event_source.into();
|
||||
if !event_source.is_empty() {
|
||||
self.event_source = Some(event_source);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// set up aws regions
|
||||
pub fn aws_region(mut self, aws_region: impl Into<String>) -> Self {
|
||||
self.aws_region = Some(aws_region.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// set event time
|
||||
pub fn event_time(mut self, event_time: impl Into<String>) -> Self {
|
||||
self.event_time = Some(event_time.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// set event name
|
||||
pub fn event_name(mut self, event_name: Name) -> Self {
|
||||
self.event_name = Some(event_name);
|
||||
self
|
||||
}
|
||||
|
||||
/// set user identity
|
||||
pub fn user_identity(mut self, user_identity: Identity) -> Self {
|
||||
self.user_identity = Some(user_identity);
|
||||
self
|
||||
}
|
||||
|
||||
/// set request parameters
|
||||
pub fn request_parameters(mut self, request_parameters: HashMap<String, String>) -> Self {
|
||||
self.request_parameters = Some(request_parameters);
|
||||
self
|
||||
}
|
||||
|
||||
/// set response elements
|
||||
pub fn response_elements(mut self, response_elements: HashMap<String, String>) -> Self {
|
||||
self.response_elements = Some(response_elements);
|
||||
self
|
||||
}
|
||||
|
||||
/// setting up s3 metadata
|
||||
pub fn s3(mut self, s3: Metadata) -> Self {
|
||||
self.s3 = Some(s3);
|
||||
self
|
||||
}
|
||||
|
||||
/// set event source information
|
||||
pub fn source(mut self, source: Source) -> Self {
|
||||
self.source = Some(source);
|
||||
self
|
||||
}
|
||||
|
||||
/// set up the sending channel
|
||||
pub fn channels(mut self, channels: Vec<String>) -> Self {
|
||||
self.channels = Some(channels.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Create a preconfigured builder for common object event scenarios
|
||||
pub fn for_object_creation(s3: Metadata, source: Source) -> Self {
|
||||
Self::new().event_name(Name::ObjectCreatedPut).s3(s3).source(source)
|
||||
}
|
||||
|
||||
/// Create a preconfigured builder for object deletion events
|
||||
pub fn for_object_removal(s3: Metadata, source: Source) -> Self {
|
||||
Self::new().event_name(Name::ObjectRemovedDelete).s3(s3).source(source)
|
||||
}
|
||||
|
||||
/// build event instance
|
||||
///
|
||||
/// Verify the required fields and create a complete Event object
|
||||
pub fn build(self) -> Result<Event, Error> {
|
||||
let event_version = self.event_version.ok_or(Error::MissingField("event_version"))?;
|
||||
|
||||
let event_source = self.event_source.ok_or(Error::MissingField("event_source"))?;
|
||||
|
||||
let aws_region = self.aws_region.ok_or(Error::MissingField("aws_region"))?;
|
||||
|
||||
let event_time = self.event_time.ok_or(Error::MissingField("event_time"))?;
|
||||
|
||||
let event_name = self.event_name.ok_or(Error::MissingField("event_name"))?;
|
||||
|
||||
let user_identity = self.user_identity.ok_or(Error::MissingField("user_identity"))?;
|
||||
|
||||
let request_parameters = self.request_parameters.unwrap_or_default();
|
||||
let response_elements = self.response_elements.unwrap_or_default();
|
||||
|
||||
let s3 = self.s3.ok_or(Error::MissingField("s3"))?;
|
||||
|
||||
let source = self.source.ok_or(Error::MissingField("source"))?;
|
||||
|
||||
let channels = self.channels.unwrap_or_else(|| smallvec![]);
|
||||
|
||||
Ok(Event {
|
||||
event_version,
|
||||
event_source,
|
||||
aws_region,
|
||||
event_time,
|
||||
event_name,
|
||||
user_identity,
|
||||
request_parameters,
|
||||
response_elements,
|
||||
s3,
|
||||
source,
|
||||
id: Uuid::new_v4(),
|
||||
timestamp: SystemTime::now(),
|
||||
channels,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||
pub struct Event {
|
||||
#[serde(rename = "eventVersion")]
|
||||
pub event_version: String,
|
||||
#[serde(rename = "eventSource")]
|
||||
pub event_source: String,
|
||||
#[serde(rename = "awsRegion")]
|
||||
pub aws_region: String,
|
||||
#[serde(rename = "eventTime")]
|
||||
pub event_time: String,
|
||||
#[serde(rename = "eventName")]
|
||||
pub event_name: Name,
|
||||
#[serde(rename = "userIdentity")]
|
||||
pub user_identity: Identity,
|
||||
#[serde(rename = "requestParameters")]
|
||||
pub request_parameters: HashMap<String, String>,
|
||||
#[serde(rename = "responseElements")]
|
||||
pub response_elements: HashMap<String, String>,
|
||||
pub s3: Metadata,
|
||||
pub source: Source,
|
||||
pub id: Uuid,
|
||||
pub timestamp: SystemTime,
|
||||
pub channels: SmallVec<[String; 2]>,
|
||||
}
|
||||
|
||||
impl Event {
|
||||
/// create a new event builder
|
||||
///
|
||||
/// Returns an EventBuilder instance pre-filled with default values
|
||||
pub fn builder() -> EventBuilder {
|
||||
EventBuilder::new()
|
||||
}
|
||||
|
||||
/// Quickly create Event instances with necessary fields
|
||||
///
|
||||
/// suitable for common s3 event scenarios
|
||||
pub fn create(event_name: Name, s3: Metadata, source: Source, channels: Vec<String>) -> Self {
|
||||
Self::builder()
|
||||
.event_name(event_name)
|
||||
.s3(s3)
|
||||
.source(source)
|
||||
.channels(channels)
|
||||
.build()
|
||||
.expect("Failed to create event, missing necessary parameters")
|
||||
}
|
||||
|
||||
/// a convenient way to create a preconfigured builder
|
||||
pub fn for_object_creation(s3: Metadata, source: Source) -> EventBuilder {
|
||||
EventBuilder::for_object_creation(s3, source)
|
||||
}
|
||||
|
||||
/// a convenient way to create a preconfigured builder
|
||||
pub fn for_object_removal(s3: Metadata, source: Source) -> EventBuilder {
|
||||
EventBuilder::for_object_removal(s3, source)
|
||||
}
|
||||
|
||||
/// Determine whether an event belongs to a specific type
|
||||
pub fn is_type(&self, event_type: Name) -> bool {
|
||||
let mask = event_type.mask();
|
||||
(self.event_name.mask() & mask) != 0
|
||||
}
|
||||
|
||||
/// Determine whether an event needs to be sent to a specific channel
|
||||
pub fn is_for_channel(&self, channel: &str) -> bool {
|
||||
self.channels.iter().any(|c| c == channel)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone, Debug)]
|
||||
pub struct Log {
|
||||
#[serde(rename = "eventName")]
|
||||
pub event_name: Name,
|
||||
pub key: String,
|
||||
pub records: Vec<Event>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerializeDisplay, DeserializeFromStr, Display, EnumString)]
|
||||
#[strum(serialize_all = "SCREAMING_SNAKE_CASE")]
|
||||
pub enum Name {
|
||||
ObjectAccessedGet,
|
||||
ObjectAccessedGetRetention,
|
||||
ObjectAccessedGetLegalHold,
|
||||
ObjectAccessedHead,
|
||||
ObjectAccessedAttributes,
|
||||
ObjectCreatedCompleteMultipartUpload,
|
||||
ObjectCreatedCopy,
|
||||
ObjectCreatedPost,
|
||||
ObjectCreatedPut,
|
||||
ObjectCreatedPutRetention,
|
||||
ObjectCreatedPutLegalHold,
|
||||
ObjectCreatedPutTagging,
|
||||
ObjectCreatedDeleteTagging,
|
||||
ObjectRemovedDelete,
|
||||
ObjectRemovedDeleteMarkerCreated,
|
||||
ObjectRemovedDeleteAllVersions,
|
||||
ObjectRemovedNoOp,
|
||||
BucketCreated,
|
||||
BucketRemoved,
|
||||
ObjectReplicationFailed,
|
||||
ObjectReplicationComplete,
|
||||
ObjectReplicationMissedThreshold,
|
||||
ObjectReplicationReplicatedAfterThreshold,
|
||||
ObjectReplicationNotTracked,
|
||||
ObjectRestorePost,
|
||||
ObjectRestoreCompleted,
|
||||
ObjectTransitionFailed,
|
||||
ObjectTransitionComplete,
|
||||
ObjectManyVersions,
|
||||
ObjectLargeVersions,
|
||||
PrefixManyFolders,
|
||||
IlmDelMarkerExpirationDelete,
|
||||
ObjectAccessedAll,
|
||||
ObjectCreatedAll,
|
||||
ObjectRemovedAll,
|
||||
ObjectReplicationAll,
|
||||
ObjectRestoreAll,
|
||||
ObjectTransitionAll,
|
||||
ObjectScannerAll,
|
||||
Everything,
|
||||
}
|
||||
|
||||
impl Name {
|
||||
pub fn expand(&self) -> Vec<Name> {
|
||||
match self {
|
||||
Name::ObjectAccessedAll => vec![
|
||||
Name::ObjectAccessedGet,
|
||||
Name::ObjectAccessedHead,
|
||||
Name::ObjectAccessedGetRetention,
|
||||
Name::ObjectAccessedGetLegalHold,
|
||||
Name::ObjectAccessedAttributes,
|
||||
],
|
||||
Name::ObjectCreatedAll => vec![
|
||||
Name::ObjectCreatedCompleteMultipartUpload,
|
||||
Name::ObjectCreatedCopy,
|
||||
Name::ObjectCreatedPost,
|
||||
Name::ObjectCreatedPut,
|
||||
Name::ObjectCreatedPutRetention,
|
||||
Name::ObjectCreatedPutLegalHold,
|
||||
Name::ObjectCreatedPutTagging,
|
||||
Name::ObjectCreatedDeleteTagging,
|
||||
],
|
||||
Name::ObjectRemovedAll => vec![
|
||||
Name::ObjectRemovedDelete,
|
||||
Name::ObjectRemovedDeleteMarkerCreated,
|
||||
Name::ObjectRemovedNoOp,
|
||||
Name::ObjectRemovedDeleteAllVersions,
|
||||
],
|
||||
Name::ObjectReplicationAll => vec![
|
||||
Name::ObjectReplicationFailed,
|
||||
Name::ObjectReplicationComplete,
|
||||
Name::ObjectReplicationNotTracked,
|
||||
Name::ObjectReplicationMissedThreshold,
|
||||
Name::ObjectReplicationReplicatedAfterThreshold,
|
||||
],
|
||||
Name::ObjectRestoreAll => vec![Name::ObjectRestorePost, Name::ObjectRestoreCompleted],
|
||||
Name::ObjectTransitionAll => {
|
||||
vec![Name::ObjectTransitionFailed, Name::ObjectTransitionComplete]
|
||||
}
|
||||
Name::ObjectScannerAll => vec![Name::ObjectManyVersions, Name::ObjectLargeVersions, Name::PrefixManyFolders],
|
||||
Name::Everything => (1..=Name::IlmDelMarkerExpirationDelete as u32)
|
||||
.map(|i| Name::from_repr(i).unwrap())
|
||||
.collect(),
|
||||
_ => vec![*self],
|
||||
}
|
||||
}
|
||||
|
||||
pub fn mask(&self) -> u64 {
|
||||
if (*self as u32) < Name::ObjectAccessedAll as u32 {
|
||||
1 << (*self as u32 - 1)
|
||||
} else {
|
||||
self.expand().iter().fold(0, |acc, n| acc | (1 << (*n as u32 - 1)))
|
||||
}
|
||||
}
|
||||
|
||||
fn from_repr(discriminant: u32) -> Option<Self> {
|
||||
match discriminant {
|
||||
1 => Some(Name::ObjectAccessedGet),
|
||||
2 => Some(Name::ObjectAccessedGetRetention),
|
||||
3 => Some(Name::ObjectAccessedGetLegalHold),
|
||||
4 => Some(Name::ObjectAccessedHead),
|
||||
5 => Some(Name::ObjectAccessedAttributes),
|
||||
6 => Some(Name::ObjectCreatedCompleteMultipartUpload),
|
||||
7 => Some(Name::ObjectCreatedCopy),
|
||||
8 => Some(Name::ObjectCreatedPost),
|
||||
9 => Some(Name::ObjectCreatedPut),
|
||||
10 => Some(Name::ObjectCreatedPutRetention),
|
||||
11 => Some(Name::ObjectCreatedPutLegalHold),
|
||||
12 => Some(Name::ObjectCreatedPutTagging),
|
||||
13 => Some(Name::ObjectCreatedDeleteTagging),
|
||||
14 => Some(Name::ObjectRemovedDelete),
|
||||
15 => Some(Name::ObjectRemovedDeleteMarkerCreated),
|
||||
16 => Some(Name::ObjectRemovedDeleteAllVersions),
|
||||
17 => Some(Name::ObjectRemovedNoOp),
|
||||
18 => Some(Name::BucketCreated),
|
||||
19 => Some(Name::BucketRemoved),
|
||||
20 => Some(Name::ObjectReplicationFailed),
|
||||
21 => Some(Name::ObjectReplicationComplete),
|
||||
22 => Some(Name::ObjectReplicationMissedThreshold),
|
||||
23 => Some(Name::ObjectReplicationReplicatedAfterThreshold),
|
||||
24 => Some(Name::ObjectReplicationNotTracked),
|
||||
25 => Some(Name::ObjectRestorePost),
|
||||
26 => Some(Name::ObjectRestoreCompleted),
|
||||
27 => Some(Name::ObjectTransitionFailed),
|
||||
28 => Some(Name::ObjectTransitionComplete),
|
||||
29 => Some(Name::ObjectManyVersions),
|
||||
30 => Some(Name::ObjectLargeVersions),
|
||||
31 => Some(Name::PrefixManyFolders),
|
||||
32 => Some(Name::IlmDelMarkerExpirationDelete),
|
||||
33 => Some(Name::ObjectAccessedAll),
|
||||
34 => Some(Name::ObjectCreatedAll),
|
||||
35 => Some(Name::ObjectRemovedAll),
|
||||
36 => Some(Name::ObjectReplicationAll),
|
||||
37 => Some(Name::ObjectRestoreAll),
|
||||
38 => Some(Name::ObjectTransitionAll),
|
||||
39 => Some(Name::ObjectScannerAll),
|
||||
40 => Some(Name::Everything),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,142 +0,0 @@
|
||||
use crate::config::notifier::EventNotifierConfig;
|
||||
use crate::Event;
|
||||
use common::error::{Error, Result};
|
||||
use ecstore::store::ECStore;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::{broadcast, mpsc};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, instrument, warn};
|
||||
use tracing_subscriber::util::SubscriberInitExt;
|
||||
|
||||
/// 事件通知器
|
||||
pub struct EventNotifier {
|
||||
/// 事件发送通道
|
||||
sender: mpsc::Sender<Event>,
|
||||
/// 接收器任务句柄
|
||||
task_handle: Option<tokio::task::JoinHandle<()>>,
|
||||
/// 配置信息
|
||||
config: EventNotifierConfig,
|
||||
/// 关闭标记
|
||||
shutdown: CancellationToken,
|
||||
/// 关闭通知通道
|
||||
shutdown_complete_tx: Option<broadcast::Sender<()>>,
|
||||
}
|
||||
|
||||
impl EventNotifier {
|
||||
/// 创建新的事件通知器
|
||||
#[instrument(skip_all)]
|
||||
pub async fn new(store: Arc<ECStore>) -> Result<Self> {
|
||||
let manager = crate::store::manager::EventManager::new(store);
|
||||
|
||||
// 初始化配置
|
||||
let config = manager.init().await?;
|
||||
|
||||
// 创建适配器
|
||||
let adapters = manager.create_adapters().await?;
|
||||
info!("创建了 {} 个适配器", adapters.len());
|
||||
|
||||
// 创建关闭标记
|
||||
let shutdown = CancellationToken::new();
|
||||
let (shutdown_complete_tx, _) = broadcast::channel(1);
|
||||
|
||||
// 创建事件通道 - 使用默认容量，因为每个适配器都有自己的队列
|
||||
// 这里使用较小的通道容量，因为事件会被快速分发到适配器
|
||||
let (sender, mut receiver) = mpsc::channel(100);
|
||||
|
||||
let shutdown_clone = shutdown.clone();
|
||||
let shutdown_complete_tx_clone = shutdown_complete_tx.clone();
|
||||
let adapters_clone = adapters.clone();
|
||||
|
||||
// 启动事件处理任务
|
||||
let task_handle = tokio::spawn(async move {
|
||||
debug!("事件处理任务启动");
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
Some(event) = receiver.recv() => {
|
||||
debug!("收到事件：{}", event.id);
|
||||
|
||||
// 分发到所有适配器
|
||||
for adapter in &adapters_clone {
|
||||
let adapter_name = adapter.name();
|
||||
match adapter.send(&event).await {
|
||||
Ok(_) => {
|
||||
debug!("事件 {} 成功发送到适配器 {}", event.id, adapter_name);
|
||||
}
|
||||
Err(e) => {
|
||||
error!("事件 {} 发送到适配器 {} 失败：{}", event.id, adapter_name, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_ = shutdown_clone.cancelled() => {
|
||||
info!("接收到关闭信号，事件处理任务停止");
|
||||
let _ = shutdown_complete_tx_clone.send(());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
debug!("事件处理任务已停止");
|
||||
});
|
||||
|
||||
Ok(Self {
|
||||
sender,
|
||||
task_handle: Some(task_handle),
|
||||
config,
|
||||
shutdown,
|
||||
shutdown_complete_tx: Some(shutdown_complete_tx),
|
||||
})
|
||||
}
|
||||
|
||||
/// 关闭事件通知器
|
||||
pub async fn shutdown(&mut self) -> Result<()> {
|
||||
info!("关闭事件通知器");
|
||||
self.shutdown.cancel();
|
||||
|
||||
if let Some(shutdown_tx) = self.shutdown_complete_tx.take() {
|
||||
let mut rx = shutdown_tx.subscribe();
|
||||
|
||||
// 等待关闭完成信号或超时
|
||||
tokio::select! {
|
||||
_ = rx.recv() => {
|
||||
debug!("收到关闭完成信号");
|
||||
}
|
||||
_ = tokio::time::sleep(std::time::Duration::from_secs(10)) => {
|
||||
warn!("关闭超时，强制终止");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(handle) = self.task_handle.take() {
|
||||
handle.abort();
|
||||
match handle.await {
|
||||
Ok(_) => debug!("事件处理任务已正常终止"),
|
||||
Err(e) => {
|
||||
if e.is_cancelled() {
|
||||
debug!("事件处理任务已取消");
|
||||
} else {
|
||||
error!("等待事件处理任务终止时出错：{}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
info!("事件通知器已完全关闭");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 发送事件
|
||||
pub async fn send(&self, event: Event) -> Result<()> {
|
||||
self.sender
|
||||
.send(event)
|
||||
.await
|
||||
.map_err(|e| Error::msg(format!("发送事件到通道失败：{}", e)))
|
||||
}
|
||||
|
||||
/// 获取当前配置
|
||||
pub fn config(&self) -> &EventNotifierConfig {
|
||||
&self.config
|
||||
}
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
use crate::config::notifier::EventNotifierConfig;
|
||||
use crate::event_notifier::EventNotifier;
|
||||
use common::error::Result;
|
||||
use ecstore::store::ECStore;
|
||||
use once_cell::sync::OnceCell;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
/// 全局事件系统
|
||||
pub struct EventSystem {
|
||||
/// 事件通知器
|
||||
notifier: Mutex<Option<EventNotifier>>,
|
||||
}
|
||||
|
||||
impl EventSystem {
|
||||
/// 创建一个新的事件系统
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
notifier: Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// 初始化事件系统
|
||||
pub async fn init(&self, store: Arc<ECStore>) -> Result<EventNotifierConfig> {
|
||||
info!("初始化事件系统");
|
||||
let notifier = EventNotifier::new(store).await?;
|
||||
let config = notifier.config().clone();
|
||||
|
||||
let mut guard = self
|
||||
.notifier
|
||||
.lock()
|
||||
.map_err(|e| common::error::Error::msg(format!("获取锁失败：{}", e)))?;
|
||||
|
||||
*guard = Some(notifier);
|
||||
debug!("事件系统初始化完成");
|
||||
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
/// 发送事件
|
||||
pub async fn send_event(&self, event: crate::Event) -> Result<()> {
|
||||
let guard = self
|
||||
.notifier
|
||||
.lock()
|
||||
.map_err(|e| common::error::Error::msg(format!("获取锁失败：{}", e)))?;
|
||||
|
||||
if let Some(notifier) = &*guard {
|
||||
notifier.send(event).await
|
||||
} else {
|
||||
error!("事件系统未初始化");
|
||||
Err(common::error::Error::msg("事件系统未初始化"))
|
||||
}
|
||||
}
|
||||
|
||||
/// 关闭事件系统
|
||||
pub async fn shutdown(&self) -> Result<()> {
|
||||
info!("关闭事件系统");
|
||||
let mut guard = self
|
||||
.notifier
|
||||
.lock()
|
||||
.map_err(|e| common::error::Error::msg(format!("获取锁失败：{}", e)))?;
|
||||
|
||||
if let Some(ref mut notifier) = *guard {
|
||||
notifier.shutdown().await?;
|
||||
*guard = None;
|
||||
info!("事件系统已关闭");
|
||||
Ok(())
|
||||
} else {
|
||||
debug!("事件系统已经关闭");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 全局事件系统实例
|
||||
pub static GLOBAL_EVENT_SYS: OnceCell<EventSystem> = OnceCell::new();
|
||||
|
||||
/// 初始化全局事件系统
|
||||
pub fn init_global_event_system() -> &'static EventSystem {
|
||||
GLOBAL_EVENT_SYS.get_or_init(EventSystem::new)
|
||||
}
|
||||
@@ -1,234 +0,0 @@
|
||||
use crate::{create_adapters, Error, Event, NotifierConfig, NotifierSystem};
|
||||
use std::sync::{atomic, Arc};
|
||||
use tokio::sync::{Mutex, OnceCell};
|
||||
use tracing::instrument;
|
||||
|
||||
static GLOBAL_SYSTEM: OnceCell<Arc<Mutex<NotifierSystem>>> = OnceCell::const_new();
|
||||
static INITIALIZED: atomic::AtomicBool = atomic::AtomicBool::new(false);
|
||||
static READY: atomic::AtomicBool = atomic::AtomicBool::new(false);
|
||||
static INIT_LOCK: Mutex<()> = Mutex::const_new(());
|
||||
|
||||
/// Initializes the global notification system.
|
||||
///
|
||||
/// This function performs the following steps:
|
||||
/// 1. Checks if the system is already initialized.
|
||||
/// 2. Creates a new `NotificationSystem` instance.
|
||||
/// 3. Creates adapters based on the provided configuration.
|
||||
/// 4. Starts the notification system with the created adapters.
|
||||
/// 5. Sets the global system instance.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error if:
|
||||
/// - The system is already initialized.
|
||||
/// - Creating the `NotificationSystem` fails.
|
||||
/// - Creating adapters fails.
|
||||
/// - Starting the notification system fails.
|
||||
/// - Setting the global system instance fails.
|
||||
#[instrument]
|
||||
pub async fn initialize(config: &NotifierConfig) -> Result<(), Error> {
|
||||
let _lock = INIT_LOCK.lock().await;
|
||||
|
||||
// Check if the system is already initialized.
|
||||
if INITIALIZED.load(atomic::Ordering::SeqCst) {
|
||||
return Err(Error::custom("Notification system has already been initialized"));
|
||||
}
|
||||
|
||||
// Check if the system is already ready.
|
||||
if READY.load(atomic::Ordering::SeqCst) {
|
||||
return Err(Error::custom("Notification system is already ready"));
|
||||
}
|
||||
|
||||
// Check if the system is shutting down.
|
||||
if let Some(system) = GLOBAL_SYSTEM.get() {
|
||||
let system_guard = system.lock().await;
|
||||
if system_guard.shutdown_cancelled() {
|
||||
return Err(Error::custom("Notification system is shutting down"));
|
||||
}
|
||||
}
|
||||
|
||||
// check if config adapters len is than 0
|
||||
if config.adapters.is_empty() {
|
||||
return Err(Error::custom("No adapters configured"));
|
||||
}
|
||||
|
||||
// Attempt to initialize, and reset the INITIALIZED flag if it fails.
|
||||
let result: Result<(), Error> = async {
|
||||
let system = NotifierSystem::new(config.clone()).await.map_err(|e| {
|
||||
tracing::error!("Failed to create NotificationSystem: {:?}", e);
|
||||
e
|
||||
})?;
|
||||
let adapters = create_adapters(&config.adapters).map_err(|e| {
|
||||
tracing::error!("Failed to create adapters: {:?}", e);
|
||||
e
|
||||
})?;
|
||||
tracing::info!("adapters len:{:?}", adapters.len());
|
||||
let system_clone = Arc::new(Mutex::new(system));
|
||||
let adapters_clone = adapters.clone();
|
||||
|
||||
GLOBAL_SYSTEM.set(system_clone.clone()).map_err(|_| {
|
||||
let err = Error::custom("Unable to set up global notification system");
|
||||
tracing::error!("{:?}", err);
|
||||
err
|
||||
})?;
|
||||
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = system_clone.lock().await.start(adapters_clone).await {
|
||||
tracing::error!("Notification system failed to start: {}", e);
|
||||
}
|
||||
tracing::info!("Notification system started in background");
|
||||
});
|
||||
tracing::info!("system start success,start set READY value");
|
||||
|
||||
READY.store(true, atomic::Ordering::SeqCst);
|
||||
tracing::info!("Notification system is ready to process events");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
|
||||
if result.is_err() {
|
||||
INITIALIZED.store(false, atomic::Ordering::SeqCst);
|
||||
READY.store(false, atomic::Ordering::SeqCst);
|
||||
return result;
|
||||
}
|
||||
|
||||
INITIALIZED.store(true, atomic::Ordering::SeqCst);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Checks if the notification system is initialized.
|
||||
pub fn is_initialized() -> bool {
|
||||
INITIALIZED.load(atomic::Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// Checks if the notification system is ready.
|
||||
pub fn is_ready() -> bool {
|
||||
READY.load(atomic::Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// Sends an event to the notification system.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error if:
|
||||
/// - The system is not initialized.
|
||||
/// - The system is not ready.
|
||||
/// - Sending the event fails.
|
||||
#[instrument(fields(event))]
|
||||
pub async fn send_event(event: Event) -> Result<(), Error> {
|
||||
if !READY.load(atomic::Ordering::SeqCst) {
|
||||
return Err(Error::custom("Notification system not ready, please wait for initialization to complete"));
|
||||
}
|
||||
|
||||
let system = get_system().await?;
|
||||
let system_guard = system.lock().await;
|
||||
system_guard.send_event(event).await
|
||||
}
|
||||
|
||||
/// Shuts down the notification system.
|
||||
#[instrument]
|
||||
pub async fn shutdown() -> Result<(), Error> {
|
||||
if let Some(system) = GLOBAL_SYSTEM.get() {
|
||||
tracing::info!("Shutting down notification system start");
|
||||
let result = {
|
||||
let mut system_guard = system.lock().await;
|
||||
system_guard.shutdown().await
|
||||
};
|
||||
if let Err(e) = &result {
|
||||
tracing::error!("Notification system shutdown failed: {}", e);
|
||||
} else {
|
||||
tracing::info!("Event bus shutdown completed");
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
"Shutdown method called set static value start, READY: {}, INITIALIZED: {}",
|
||||
READY.load(atomic::Ordering::SeqCst),
|
||||
INITIALIZED.load(atomic::Ordering::SeqCst)
|
||||
);
|
||||
READY.store(false, atomic::Ordering::SeqCst);
|
||||
INITIALIZED.store(false, atomic::Ordering::SeqCst);
|
||||
tracing::info!(
|
||||
"Shutdown method called set static value end, READY: {}, INITIALIZED: {}",
|
||||
READY.load(atomic::Ordering::SeqCst),
|
||||
INITIALIZED.load(atomic::Ordering::SeqCst)
|
||||
);
|
||||
result
|
||||
} else {
|
||||
Err(Error::custom("Notification system not initialized"))
|
||||
}
|
||||
}
|
||||
|
||||
/// Retrieves the global notification system instance.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error if the system is not initialized.
|
||||
async fn get_system() -> Result<Arc<Mutex<NotifierSystem>>, Error> {
|
||||
GLOBAL_SYSTEM
|
||||
.get()
|
||||
.cloned()
|
||||
.ok_or_else(|| Error::custom("Notification system not initialized"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::{initialize, is_initialized, is_ready, NotifierConfig};
|
||||
|
||||
fn init_tracing() {
|
||||
// Use try_init to avoid panic if already initialized
|
||||
let _ = tracing_subscriber::fmt::try_init();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_initialize_success() {
|
||||
init_tracing();
|
||||
let config = NotifierConfig::default(); // assume there is a default configuration
|
||||
let result = initialize(&config).await;
|
||||
assert!(result.is_err(), "Initialization should not succeed");
|
||||
assert!(!is_initialized(), "System should not be marked as initialized");
|
||||
assert!(!is_ready(), "System should not be marked as ready");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_initialize_twice() {
|
||||
init_tracing();
|
||||
let config = NotifierConfig::default();
|
||||
let _ = initialize(&config.clone()).await; // first initialization
|
||||
let result = initialize(&config).await; // second initialization
|
||||
assert!(result.is_err(), "Initialization should succeed");
|
||||
assert!(result.is_err(), "Re-initialization should fail");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_initialize_failure_resets_state() {
|
||||
init_tracing();
|
||||
// Test with empty adapters to force failure
|
||||
let config = NotifierConfig {
|
||||
adapters: Vec::new(),
|
||||
..Default::default()
|
||||
};
|
||||
let result = initialize(&config).await;
|
||||
assert!(result.is_err(), "Initialization should fail with empty adapters");
|
||||
assert!(!is_initialized(), "System should not be marked as initialized after failure");
|
||||
assert!(!is_ready(), "System should not be marked as ready after failure");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_is_initialized_and_is_ready() {
|
||||
init_tracing();
|
||||
// Initially, the system should not be initialized or ready
|
||||
assert!(!is_initialized(), "System should not be initialized initially");
|
||||
assert!(!is_ready(), "System should not be ready initially");
|
||||
|
||||
// Test with empty adapters to ensure failure
|
||||
let config = NotifierConfig {
|
||||
adapters: Vec::new(),
|
||||
..Default::default()
|
||||
};
|
||||
let result = initialize(&config).await;
|
||||
assert!(result.is_err(), "Initialization should fail with empty adapters");
|
||||
assert!(!is_initialized(), "System should not be initialized after failed init");
|
||||
assert!(!is_ready(), "System should not be ready after failed init");
|
||||
}
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
mod adapter;
|
||||
mod config;
|
||||
mod error;
|
||||
mod event;
|
||||
mod event_notifier;
|
||||
mod event_system;
|
||||
mod global;
|
||||
mod notifier;
|
||||
mod store;
|
||||
|
||||
pub use adapter::create_adapters;
|
||||
#[cfg(all(feature = "kafka", target_os = "linux"))]
|
||||
pub use adapter::kafka::KafkaAdapter;
|
||||
#[cfg(feature = "mqtt")]
|
||||
pub use adapter::mqtt::MqttAdapter;
|
||||
#[cfg(feature = "webhook")]
|
||||
pub use adapter::webhook::WebhookAdapter;
|
||||
pub use adapter::ChannelAdapter;
|
||||
pub use adapter::ChannelAdapterType;
|
||||
pub use config::adapter::AdapterCommon;
|
||||
pub use config::adapter::AdapterConfig;
|
||||
#[cfg(all(feature = "kafka", target_os = "linux"))]
|
||||
pub use config::kafka::KafkaConfig;
|
||||
#[cfg(feature = "mqtt")]
|
||||
pub use config::mqtt::MqttConfig;
|
||||
pub use config::notifier::EventNotifierConfig;
|
||||
#[cfg(feature = "webhook")]
|
||||
pub use config::webhook::WebhookConfig;
|
||||
pub use config::{DEFAULT_MAX_RETRIES, DEFAULT_RETRY_INTERVAL};
|
||||
pub use error::Error;
|
||||
|
||||
pub use event::{Bucket, Event, EventBuilder, Identity, Log, Metadata, Name, Object, Source};
|
||||
pub use global::{initialize, is_initialized, is_ready, send_event, shutdown};
|
||||
pub use notifier::NotifierSystem;
|
||||
pub use store::queue::QueueStore;
|
||||
@@ -1,136 +0,0 @@
|
||||
use crate::{event_bus, ChannelAdapter, Error, Event, EventStore, NotifierConfig};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::instrument;
|
||||
|
||||
/// The `NotificationSystem` struct represents the notification system.
|
||||
/// It manages the event bus and the adapters.
|
||||
/// It is responsible for sending and receiving events.
|
||||
/// It also handles the shutdown process.
|
||||
pub struct NotifierSystem {
|
||||
tx: mpsc::Sender<Event>,
|
||||
rx: Option<mpsc::Receiver<Event>>,
|
||||
store: Arc<EventStore>,
|
||||
shutdown: CancellationToken,
|
||||
shutdown_complete: Option<tokio::sync::oneshot::Sender<()>>,
|
||||
shutdown_receiver: Option<tokio::sync::oneshot::Receiver<()>>,
|
||||
}
|
||||
|
||||
impl NotifierSystem {
|
||||
/// Creates a new `NotificationSystem` instance.
|
||||
#[instrument(skip(config))]
|
||||
pub async fn new(config: NotifierConfig) -> Result<Self, Error> {
|
||||
let (tx, rx) = mpsc::channel::<Event>(config.channel_capacity.try_into().unwrap());
|
||||
let store = Arc::new(EventStore::new(&config.store_path).await?);
|
||||
let shutdown = CancellationToken::new();
|
||||
|
||||
let restored_logs = store.load_logs().await?;
|
||||
for log in restored_logs {
|
||||
for event in log.records {
|
||||
// For example, where the send method may return a SendError when calling it
|
||||
tx.send(event).await.map_err(|e| Error::ChannelSend(Box::new(e)))?;
|
||||
}
|
||||
}
|
||||
// Initialize shutdown_complete to Some(tx)
|
||||
let (complete_tx, complete_rx) = tokio::sync::oneshot::channel();
|
||||
Ok(Self {
|
||||
tx,
|
||||
rx: Some(rx),
|
||||
store,
|
||||
shutdown,
|
||||
shutdown_complete: Some(complete_tx),
|
||||
shutdown_receiver: Some(complete_rx),
|
||||
})
|
||||
}
|
||||
|
||||
/// Starts the notification system.
|
||||
/// It initializes the event bus and the producer.
|
||||
#[instrument(skip_all)]
|
||||
pub async fn start(&mut self, adapters: Vec<Arc<dyn ChannelAdapter>>) -> Result<(), Error> {
|
||||
if self.shutdown.is_cancelled() {
|
||||
let error = Error::custom("System is shutting down");
|
||||
self.handle_error("start", &error);
|
||||
return Err(error);
|
||||
}
|
||||
self.log(tracing::Level::INFO, "start", "Starting the notification system");
|
||||
let rx = self.rx.take().ok_or_else(|| Error::EventBusStarted)?;
|
||||
let shutdown_clone = self.shutdown.clone();
|
||||
let store_clone = self.store.clone();
|
||||
let shutdown_complete = self.shutdown_complete.take();
|
||||
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = event_bus(rx, adapters, store_clone, shutdown_clone, shutdown_complete).await {
|
||||
tracing::error!("Event bus failed: {}", e);
|
||||
}
|
||||
});
|
||||
self.log(tracing::Level::INFO, "start", "Notification system started successfully");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Sends an event to the notification system.
|
||||
/// This method is used to send events to the event bus.
|
||||
#[instrument(skip(self))]
|
||||
pub async fn send_event(&self, event: Event) -> Result<(), Error> {
|
||||
self.log(tracing::Level::DEBUG, "send_event", &format!("Sending event: {:?}", event));
|
||||
if self.shutdown.is_cancelled() {
|
||||
let error = Error::custom("System is shutting down");
|
||||
self.handle_error("send_event", &error);
|
||||
return Err(error);
|
||||
}
|
||||
if let Err(e) = self.tx.send(event).await {
|
||||
let error = Error::ChannelSend(Box::new(e));
|
||||
self.handle_error("send_event", &error);
|
||||
return Err(error);
|
||||
}
|
||||
self.log(tracing::Level::INFO, "send_event", "Event sent successfully");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Shuts down the notification system.
|
||||
/// This method is used to cancel the event bus and producer tasks.
|
||||
#[instrument(skip(self))]
|
||||
pub async fn shutdown(&mut self) -> Result<(), Error> {
|
||||
tracing::info!("Shutting down the notification system");
|
||||
self.shutdown.cancel();
|
||||
// wait for the event bus to be completely closed
|
||||
if let Some(receiver) = self.shutdown_receiver.take() {
|
||||
match receiver.await {
|
||||
Ok(_) => {
|
||||
tracing::info!("Event bus shutdown completed successfully");
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
let error = Error::custom(format!("Failed to receive shutdown completion: {}", e).as_str());
|
||||
self.handle_error("shutdown", &error);
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
tracing::warn!("Shutdown receiver not available, the event bus might still be running");
|
||||
Err(Error::custom("Shutdown receiver not available"))
|
||||
}
|
||||
}
|
||||
|
||||
/// shutdown state
|
||||
pub fn shutdown_cancelled(&self) -> bool {
|
||||
self.shutdown.is_cancelled()
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
pub fn handle_error(&self, context: &str, error: &Error) {
|
||||
self.log(tracing::Level::ERROR, context, &format!("{:?}", error));
|
||||
// TODO Can be extended to record to files or send to monitoring systems
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
fn log(&self, level: tracing::Level, context: &str, message: &str) {
|
||||
match level {
|
||||
tracing::Level::ERROR => tracing::error!("[{}] {}", context, message),
|
||||
tracing::Level::WARN => tracing::warn!("[{}] {}", context, message),
|
||||
tracing::Level::INFO => tracing::info!("[{}] {}", context, message),
|
||||
tracing::Level::DEBUG => tracing::debug!("[{}] {}", context, message),
|
||||
tracing::Level::TRACE => tracing::trace!("[{}] {}", context, message),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,237 +0,0 @@
|
||||
use crate::{adapter, ChannelAdapter, EventNotifierConfig};
|
||||
use common::error::{Error, Result};
|
||||
use ecstore::config::com::{read_config, save_config, CONFIG_PREFIX};
|
||||
use ecstore::disk::RUSTFS_META_BUCKET;
|
||||
use ecstore::store::ECStore;
|
||||
use ecstore::store_api::ObjectOptions;
|
||||
use ecstore::utils::path::SLASH_SEPARATOR;
|
||||
use ecstore::StorageAPI;
|
||||
use once_cell::sync::Lazy;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Mutex;
|
||||
use tracing::instrument;
|
||||
|
||||
/// * config file
|
||||
const CONFIG_FILE: &str = "event.json";
|
||||
|
||||
/// event sys config
|
||||
const EVENT: &str = "event";
|
||||
|
||||
/// Global storage API access point
|
||||
pub static GLOBAL_STORE_API: Lazy<Mutex<Option<Arc<ECStore>>>> = Lazy::new(|| Mutex::new(None));
|
||||
|
||||
/// Global event system configuration
|
||||
pub static GLOBAL_EVENT_CONFIG: Lazy<Mutex<Option<EventNotifierConfig>>> = Lazy::new(|| Mutex::new(None));
|
||||
|
||||
/// EventManager Responsible for managing all operations of the event system
|
||||
#[derive(Debug)]
|
||||
pub struct EventManager {
|
||||
api: Arc<ECStore>,
|
||||
}
|
||||
|
||||
impl EventManager {
|
||||
/// Create a new Event Manager
|
||||
pub async fn new(api: Arc<ECStore>) -> Self {
|
||||
// Set the global storage API
|
||||
{
|
||||
let mut global_api = GLOBAL_STORE_API.lock().await;
|
||||
*global_api = Some(api.clone());
|
||||
}
|
||||
|
||||
Self { api }
|
||||
}
|
||||
|
||||
/// Initialize the Event Manager
|
||||
///
|
||||
/// # Returns
|
||||
/// If it succeeds, it returns configuration information, and if it fails, it returns an error
|
||||
#[instrument(skip_all)]
|
||||
pub async fn init(&self) -> Result<EventNotifierConfig> {
|
||||
tracing::info!("Event system configuration initialization begins");
|
||||
|
||||
let cfg = match read_config_without_migrate(self.api.clone()).await {
|
||||
Ok(cfg) => {
|
||||
tracing::info!("The event system configuration was successfully read");
|
||||
cfg
|
||||
}
|
||||
Err(err) => {
|
||||
tracing::error!("Failed to initialize the event system configuration:{:?}", err);
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
|
||||
*GLOBAL_EVENT_CONFIG.lock().await = Some(cfg.clone());
|
||||
|
||||
tracing::info!("The initialization of the event system configuration is complete");
|
||||
|
||||
Ok(cfg)
|
||||
}
|
||||
|
||||
/// Create a new configuration
|
||||
///
|
||||
/// # Parameters
|
||||
/// - `cfg`: The configuration to be created
|
||||
///
|
||||
/// # Returns
|
||||
/// The result of the operation
|
||||
pub async fn create_config(&self, cfg: &EventNotifierConfig) -> Result<()> {
|
||||
// Check whether the configuration already exists
|
||||
if read_event_config(self.api.clone()).await.is_ok() {
|
||||
return Err(Error::msg("The configuration already exists, use the update action"));
|
||||
}
|
||||
|
||||
save_event_config(self.api.clone(), cfg).await?;
|
||||
*GLOBAL_EVENT_CONFIG.lock().await = Some(cfg.clone());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Update the configuration
|
||||
///
|
||||
/// # Parameters
|
||||
/// - `cfg`: The configuration to be updated
|
||||
///
|
||||
/// # Returns
|
||||
/// The result of the operation
|
||||
pub async fn update_config(&self, cfg: &EventNotifierConfig) -> Result<()> {
|
||||
// Read the existing configuration first to merge
|
||||
let current_cfg = read_event_config(self.api.clone()).await.unwrap_or_default();
|
||||
|
||||
// This is where the merge logic can be implemented
|
||||
let merged_cfg = self.merge_configs(current_cfg, cfg.clone());
|
||||
|
||||
save_event_config(self.api.clone(), &merged_cfg).await?;
|
||||
*GLOBAL_EVENT_CONFIG.lock().await = Some(merged_cfg);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Merge the two configurations
|
||||
fn merge_configs(&self, current: EventNotifierConfig, new: EventNotifierConfig) -> EventNotifierConfig {
|
||||
let mut merged = current;
|
||||
|
||||
// Merge webhook configurations
|
||||
for (id, config) in new.webhook {
|
||||
merged.webhook.insert(id, config);
|
||||
}
|
||||
|
||||
// Merge Kafka configurations
|
||||
for (id, config) in new.kafka {
|
||||
merged.kafka.insert(id, config);
|
||||
}
|
||||
|
||||
// Merge MQTT configurations
|
||||
for (id, config) in new.mqtt {
|
||||
merged.mqtt.insert(id, config);
|
||||
}
|
||||
|
||||
merged
|
||||
}
|
||||
|
||||
/// Delete the configuration
|
||||
pub async fn delete_config(&self) -> Result<()> {
|
||||
let config_file = get_event_config_file();
|
||||
self.api
|
||||
.delete_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
&config_file,
|
||||
ObjectOptions {
|
||||
delete_prefix: true,
|
||||
delete_prefix_object: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Reset the global configuration to default
|
||||
// let _ = GLOBAL_EventSysConfig.set(self.read_config().await?);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read the configuration
|
||||
pub async fn read_config(&self) -> Result<EventNotifierConfig> {
|
||||
read_event_config(self.api.clone()).await
|
||||
}
|
||||
|
||||
/// Create all enabled adapters
|
||||
pub async fn create_adapters(&self) -> Result<Vec<Arc<dyn ChannelAdapter>>> {
|
||||
let config = match GLOBAL_EVENT_CONFIG.lock().await.clone() {
|
||||
Some(cfg) => cfg,
|
||||
None => return Err(Error::msg("The global configuration is not initialized")),
|
||||
};
|
||||
|
||||
let adapter_configs = config.to_adapter_configs();
|
||||
match adapter::create_adapters(adapter_configs) {
|
||||
Ok(adapters) => Ok(adapters),
|
||||
Err(err) => {
|
||||
tracing::error!("Failed to create adapters: {:?}", err);
|
||||
Err(Error::from(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the Global Storage API
|
||||
pub async fn get_global_store_api() -> Option<Arc<ECStore>> {
|
||||
GLOBAL_STORE_API.lock().await.clone()
|
||||
}
|
||||
|
||||
/// Get the Global Storage API
|
||||
pub async fn get_global_event_config() -> Option<EventNotifierConfig> {
|
||||
GLOBAL_EVENT_CONFIG.lock().await.clone()
|
||||
}
|
||||
|
||||
/// Read event configuration
|
||||
async fn read_event_config<S: StorageAPI>(api: Arc<S>) -> Result<EventNotifierConfig> {
|
||||
let config_file = get_event_config_file();
|
||||
let data = read_config(api, &config_file).await?;
|
||||
|
||||
EventNotifierConfig::unmarshal(&data)
|
||||
}
|
||||
|
||||
/// Save the event configuration
|
||||
async fn save_event_config<S: StorageAPI>(api: Arc<S>, config: &EventNotifierConfig) -> Result<()> {
|
||||
let config_file = get_event_config_file();
|
||||
let data = config.marshal()?;
|
||||
|
||||
save_config(api, &config_file, data).await
|
||||
}
|
||||
|
||||
/// Get the event profile path
|
||||
fn get_event_config_file() -> String {
|
||||
format!("{}{}{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, EVENT, SLASH_SEPARATOR, CONFIG_FILE)
|
||||
}
|
||||
|
||||
/// Read the configuration file and create a default configuration if it doesn't exist
|
||||
pub async fn read_config_without_migrate<S: StorageAPI>(api: Arc<S>) -> Result<EventNotifierConfig> {
|
||||
let config_file = get_event_config_file();
|
||||
let data = match read_config(api.clone(), &config_file).await {
|
||||
Ok(data) => {
|
||||
if data.is_empty() {
|
||||
return new_and_save_event_config(api).await;
|
||||
}
|
||||
data
|
||||
}
|
||||
Err(err) if ecstore::config::error::is_err_config_not_found(&err) => {
|
||||
tracing::warn!("If the configuration file does not exist, start initializing the default configuration");
|
||||
return new_and_save_event_config(api).await;
|
||||
}
|
||||
Err(err) => {
|
||||
tracing::error!("Read configuration file error: {:?}", err);
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
|
||||
// Parse configuration
|
||||
let cfg = EventNotifierConfig::unmarshal(&data)?;
|
||||
Ok(cfg)
|
||||
}
|
||||
|
||||
/// Create and save a new configuration
|
||||
async fn new_and_save_event_config<S: StorageAPI>(api: Arc<S>) -> Result<EventNotifierConfig> {
|
||||
let cfg = EventNotifierConfig::default();
|
||||
save_event_config(api, &cfg).await?;
|
||||
|
||||
Ok(cfg)
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
pub(crate) mod manager;
|
||||
pub(crate) mod queue;
|
||||
@@ -1,520 +0,0 @@
|
||||
use common::error::{Error, Result};
|
||||
use ecstore::utils::path::dir;
|
||||
use serde::{de::DeserializeOwned, Serialize};
|
||||
use snap::raw::{Decoder, Encoder};
|
||||
use std::collections::HashMap;
|
||||
use std::io::Read;
|
||||
use std::marker::PhantomData;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use std::{fs, io};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Keys in storage
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub struct Key {
|
||||
/// Key name
|
||||
pub name: String,
|
||||
/// Whether to compress
|
||||
pub compress: bool,
|
||||
/// filename extension
|
||||
pub extension: String,
|
||||
/// Number of items
|
||||
pub item_count: usize,
|
||||
}
|
||||
|
||||
impl Key {
|
||||
/// Create a new key
|
||||
pub fn new(name: impl Into<String>, extension: impl Into<String>, compress: bool) -> Self {
|
||||
Self {
|
||||
name: name.into(),
|
||||
compress,
|
||||
extension: extension.into(),
|
||||
item_count: 1,
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert to string form
|
||||
#[allow(clippy::inherent_to_string)]
|
||||
pub fn to_string(&self) -> String {
|
||||
let mut key_str = self.name.clone();
|
||||
if self.item_count > 1 {
|
||||
key_str = format!("{}:{}", self.item_count, self.name);
|
||||
}
|
||||
|
||||
let compress_ext = if self.compress { COMPRESS_EXT } else { "" };
|
||||
format!("{}{}{}", key_str, self.extension, compress_ext)
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse key from file name
|
||||
#[allow(clippy::redundant_closure)]
|
||||
pub fn parse_key(filename: &str) -> Key {
|
||||
let compress = filename.ends_with(COMPRESS_EXT);
|
||||
let filename = if compress {
|
||||
&filename[..filename.len() - 7] // 移除 ".snappy"
|
||||
} else {
|
||||
filename
|
||||
};
|
||||
|
||||
let mut parts = filename.splitn(2, '.');
|
||||
let name_part = parts.next().unwrap_or("");
|
||||
let extension = parts
|
||||
.next()
|
||||
.map_or_else(|| String::new(), |ext| format!(".{}", ext))
|
||||
.to_string();
|
||||
|
||||
let mut name = name_part.to_string();
|
||||
let mut item_count = 1;
|
||||
|
||||
if let Some(pos) = name_part.find(':') {
|
||||
if let Ok(count) = name_part[..pos].parse::<usize>() {
|
||||
item_count = count;
|
||||
name = name_part[pos + 1..].to_string();
|
||||
}
|
||||
}
|
||||
|
||||
Key {
|
||||
name,
|
||||
compress,
|
||||
extension,
|
||||
item_count,
|
||||
}
|
||||
}
|
||||
|
||||
/// Store the characteristics of the project
|
||||
pub trait Store<T>: Send + Sync
|
||||
where
|
||||
T: Serialize + DeserializeOwned + Clone + Send + Sync + 'static,
|
||||
{
|
||||
/// Store a single item
|
||||
fn put(&self, item: T) -> Result<Key>;
|
||||
|
||||
/// Store multiple projects
|
||||
fn put_multiple(&self, items: Vec<T>) -> Result<Key>;
|
||||
|
||||
/// Get a single item
|
||||
fn get(&self, key: &Key) -> Result<T>;
|
||||
|
||||
/// Get multiple items
|
||||
fn get_multiple(&self, key: &Key) -> Result<Vec<T>>;
|
||||
|
||||
/// Get the raw bytes
|
||||
fn get_raw(&self, key: &Key) -> Result<Vec<u8>>;
|
||||
|
||||
/// Stores raw bytes
|
||||
fn put_raw(&self, data: &[u8]) -> Result<Key>;
|
||||
|
||||
/// Gets the number of items in storage
|
||||
fn len(&self) -> usize;
|
||||
|
||||
/// Whether it is empty or not
|
||||
fn is_empty(&self) -> bool {
|
||||
self.len() == 0
|
||||
}
|
||||
|
||||
/// Lists all keys
|
||||
fn list(&self) -> Vec<Key>;
|
||||
|
||||
/// Delete the key
|
||||
fn del(&self, key: &Key) -> Result<()>;
|
||||
|
||||
/// Open Storage
|
||||
fn open(&self) -> Result<()>;
|
||||
|
||||
/// Delete the storage
|
||||
fn delete(&self) -> Result<()>;
|
||||
}
|
||||
|
||||
const DEFAULT_LIMIT: u64 = 100000;
|
||||
const DEFAULT_EXT: &str = ".unknown";
|
||||
const COMPRESS_EXT: &str = ".snappy";
|
||||
|
||||
/// Queue storage implementation
|
||||
pub struct QueueStore<T> {
|
||||
/// Project Limitations
|
||||
entry_limit: u64,
|
||||
/// Storage directory
|
||||
directory: PathBuf,
|
||||
/// filename extension
|
||||
file_ext: String,
|
||||
/// Item mapping: key -> modified time (Unix nanoseconds)
|
||||
entries: Arc<RwLock<HashMap<String, i64>>>,
|
||||
/// Type tags
|
||||
_phantom: PhantomData<T>,
|
||||
/// Whether to compress
|
||||
compress: bool,
|
||||
/// Store name
|
||||
name: String,
|
||||
}
|
||||
|
||||
impl<T> QueueStore<T>
|
||||
where
|
||||
T: Serialize + DeserializeOwned + Clone + Send + Sync + 'static,
|
||||
{
|
||||
/// Create a new queue store
|
||||
pub fn new<P: AsRef<Path>>(directory: P, name: String, limit: u64, ext: Option<String>) -> Self {
|
||||
let limit = if limit == 0 { DEFAULT_LIMIT } else { limit };
|
||||
let ext = ext.unwrap_or_else(|| DEFAULT_EXT.to_string());
|
||||
let mut path = PathBuf::from(directory.as_ref());
|
||||
path.push(&name);
|
||||
|
||||
// Create a directory (if it doesn't exist)
|
||||
if !path.exists() {
|
||||
if let Err(e) = fs::create_dir_all(&path) {
|
||||
tracing::error!("创建存储目录失败 {}: {}", path.display(), e);
|
||||
}
|
||||
}
|
||||
|
||||
Self {
|
||||
directory: directory.into(),
|
||||
name,
|
||||
entry_limit: limit,
|
||||
file_ext: ext,
|
||||
compress: true, // Default to compressing
|
||||
entries: Arc::new(RwLock::new(HashMap::with_capacity(limit as usize))),
|
||||
_phantom: PhantomData,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the file extension
|
||||
pub fn with_file_ext(mut self, file_ext: &str) -> Self {
|
||||
self.file_ext = file_ext.to_string();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set whether to compress or not
|
||||
pub fn with_compression(mut self, compress: bool) -> Self {
|
||||
self.compress = compress;
|
||||
self
|
||||
}
|
||||
|
||||
/// Get the file path
|
||||
fn get_file_path(&self, key: &Key) -> PathBuf {
|
||||
let mut filename = key.to_string();
|
||||
filename.push_str(if self.compress { COMPRESS_EXT } else { &self.file_ext });
|
||||
self.directory.join(filename)
|
||||
}
|
||||
|
||||
/// Serialize the project
|
||||
fn serialize_item(&self, item: &T) -> Result<Vec<u8>> {
|
||||
let data = serde_json::to_vec(item).map_err(|e| Error::msg(format!("Serialization failed: {}", e)))?;
|
||||
|
||||
if self.compress {
|
||||
let mut encoder = Encoder::new();
|
||||
Ok(encoder
|
||||
.compress_vec(&data)
|
||||
.map_err(|e| Error::msg(format!("Compression failed: {}", e)))?)
|
||||
} else {
|
||||
Ok(data)
|
||||
}
|
||||
}
|
||||
|
||||
/// Deserialize the project
|
||||
fn deserialize_item(&self, data: &[u8], is_compressed: bool) -> Result<T> {
|
||||
let data = if is_compressed {
|
||||
let mut decoder = Decoder::new();
|
||||
decoder
|
||||
.decompress_vec(data)
|
||||
.map_err(|e| Error::msg(format!("Unzipping failed: {}", e)))?
|
||||
} else {
|
||||
data.to_vec()
|
||||
};
|
||||
|
||||
serde_json::from_slice(&data).map_err(|e| Error::msg(format!("Deserialization failed: {}", e)))
|
||||
}
|
||||
|
||||
/// Lists all files in the directory, sorted by modification time (oldest takes precedence.)）
|
||||
fn list_files(&self) -> Result<Vec<fs::DirEntry>> {
|
||||
let mut files = Vec::new();
|
||||
|
||||
for entry in fs::read_dir(&self.directory)? {
|
||||
let entry = entry?;
|
||||
let metadata = entry.metadata()?;
|
||||
if metadata.is_file() {
|
||||
files.push(entry);
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by modification time
|
||||
files.sort_by(|a, b| {
|
||||
let a_time = a
|
||||
.metadata()
|
||||
.map(|m| m.modified())
|
||||
.unwrap_or(Ok(UNIX_EPOCH))
|
||||
.unwrap_or(UNIX_EPOCH);
|
||||
let b_time = b
|
||||
.metadata()
|
||||
.map(|m| m.modified())
|
||||
.unwrap_or(Ok(UNIX_EPOCH))
|
||||
.unwrap_or(UNIX_EPOCH);
|
||||
a_time.cmp(&b_time)
|
||||
});
|
||||
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
/// Write the object to a file
|
||||
fn write_object(&self, key: &Key, item: &T) -> Result<()> {
|
||||
// Serialize the object
|
||||
let data = serde_json::to_vec(item)?;
|
||||
self.write_bytes(key, &data)
|
||||
}
|
||||
|
||||
/// Write multiple objects to a file
|
||||
fn write_multiple_objects(&self, key: &Key, items: &[T]) -> Result<()> {
|
||||
let mut data = Vec::new();
|
||||
for item in items {
|
||||
let item_data = serde_json::to_vec(item)?;
|
||||
data.extend_from_slice(&item_data);
|
||||
data.push(b'\n');
|
||||
}
|
||||
self.write_bytes(key, &data)
|
||||
}
|
||||
|
||||
/// Write bytes to a file
|
||||
fn write_bytes(&self, key: &Key, data: &[u8]) -> Result<()> {
|
||||
let path = self.directory.join(key.to_string());
|
||||
|
||||
let file_data = if key.compress {
|
||||
// Use snap to compress data
|
||||
let mut encoder = Encoder::new();
|
||||
encoder
|
||||
.compress_vec(data)
|
||||
.map_err(|e| Error::msg(format!("Compression failed:{}", e)))?
|
||||
} else {
|
||||
data.to_vec()
|
||||
};
|
||||
|
||||
// Make sure the directory exists
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
|
||||
// Write to the file
|
||||
fs::write(&path, &file_data)?;
|
||||
|
||||
// Update the item mapping
|
||||
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_nanos() as i64;
|
||||
|
||||
let mut entries = self.entries.write().map_err(|_| Error::msg("获取写锁失败"))?;
|
||||
entries.insert(key.to_string(), now);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read bytes from a file
|
||||
fn read_bytes(&self, key: &Key) -> Result<Vec<u8>> {
|
||||
let path = self.directory.join(key.to_string());
|
||||
let data = fs::read(&path)?;
|
||||
|
||||
if data.is_empty() {
|
||||
return Err(Error::msg("The file is empty"));
|
||||
}
|
||||
|
||||
if key.compress {
|
||||
// Use Snap to extract the data
|
||||
let mut decoder = Decoder::new();
|
||||
decoder
|
||||
.decompress_vec(&data)
|
||||
.map_err(|e| Error::msg(format!("Failed to decompress:{}", e)))
|
||||
} else {
|
||||
Ok(data)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Store<T> for QueueStore<T>
|
||||
where
|
||||
T: Serialize + DeserializeOwned + Clone + Send + Sync + 'static,
|
||||
{
|
||||
fn open(&self) -> Result<()> {
|
||||
// Create a directory (if it doesn't exist)
|
||||
fs::create_dir_all(&self.directory)?;
|
||||
|
||||
// Read existing files
|
||||
let files = self.list_files()?;
|
||||
|
||||
let mut entries = self
|
||||
.entries
|
||||
.write()
|
||||
.map_err(|_| Error::msg("Failed to obtain a write lock"))?;
|
||||
|
||||
for file in files {
|
||||
if let Ok(meta) = file.metadata() {
|
||||
if let Ok(modified) = meta.modified() {
|
||||
if let Ok(since_epoch) = modified.duration_since(UNIX_EPOCH) {
|
||||
entries.insert(file.file_name().to_string_lossy().to_string(), since_epoch.as_nanos() as i64);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn delete(&self) -> Result<()> {
|
||||
fs::remove_dir_all(&self.directory)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn put(&self, item: T) -> Result<Key> {
|
||||
{
|
||||
let entries = self.entries.read().map_err(|_| Error::msg("Failed to obtain a read lock"))?;
|
||||
if entries.len() as u64 >= self.entry_limit {
|
||||
return Err(Error::msg("The storage limit has been reached"));
|
||||
}
|
||||
}
|
||||
|
||||
// generate a new uuid
|
||||
let uuid = Uuid::new_v4();
|
||||
let key = Key::new(uuid.to_string(), &self.file_ext, true);
|
||||
|
||||
self.write_object(&key, &item)?;
|
||||
|
||||
Ok(key)
|
||||
}
|
||||
|
||||
fn put_multiple(&self, items: Vec<T>) -> Result<Key> {
|
||||
if items.is_empty() {
|
||||
return Err(Error::msg("The list of items is empty"));
|
||||
}
|
||||
|
||||
{
|
||||
let entries = self.entries.read().map_err(|_| Error::msg("Failed to obtain a read lock"))?;
|
||||
if entries.len() as u64 >= self.entry_limit {
|
||||
return Err(Error::msg("The storage limit has been reached"));
|
||||
}
|
||||
}
|
||||
|
||||
// Generate a new UUID
|
||||
let uuid = Uuid::new_v4();
|
||||
let mut key = Key::new(uuid.to_string(), &self.file_ext, true);
|
||||
key.item_count = items.len();
|
||||
|
||||
self.write_multiple_objects(&key, &items)?;
|
||||
|
||||
Ok(key)
|
||||
}
|
||||
|
||||
fn get(&self, key: &Key) -> Result<T> {
|
||||
let items = self.get_multiple(key)?;
|
||||
if items.is_empty() {
|
||||
return Err(Error::msg("Item not found"));
|
||||
}
|
||||
|
||||
Ok(items[0].clone())
|
||||
}
|
||||
|
||||
fn get_multiple(&self, key: &Key) -> Result<Vec<T>> {
|
||||
let data = self.get_raw(key)?;
|
||||
|
||||
let mut items = Vec::with_capacity(key.item_count);
|
||||
let mut reader = io::Cursor::new(&data);
|
||||
|
||||
// Try to read each JSON object
|
||||
let mut buffer = Vec::new();
|
||||
|
||||
// if the read fails try parsing it once
|
||||
if reader.read_to_end(&mut buffer).is_err() {
|
||||
// Try to parse the entire data as a single object
|
||||
match serde_json::from_slice::<T>(&data) {
|
||||
Ok(item) => {
|
||||
items.push(item);
|
||||
return Ok(items);
|
||||
}
|
||||
Err(_) => {
|
||||
// An attempt was made to resolve to an array of objects
|
||||
match serde_json::from_slice::<Vec<T>>(&data) {
|
||||
Ok(array_items) => return Ok(array_items),
|
||||
Err(e) => return Err(Error::msg(format!("Failed to parse the data:{}", e))),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Read JSON objects by row
|
||||
for line in buffer.split(|&b| b == b'\n') {
|
||||
if !line.is_empty() {
|
||||
match serde_json::from_slice::<T>(line) {
|
||||
Ok(item) => items.push(item),
|
||||
Err(e) => tracing::warn!("Failed to parse row data:{}", e),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if items.is_empty() {
|
||||
return Err(Error::msg("Failed to resolve any items"));
|
||||
}
|
||||
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
fn get_raw(&self, key: &Key) -> Result<Vec<u8>> {
|
||||
let data = self.read_bytes(key)?;
|
||||
|
||||
// Delete the wrong file
|
||||
if data.is_empty() {
|
||||
let _ = self.del(key);
|
||||
return Err(Error::msg("the file is empty"));
|
||||
}
|
||||
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
fn put_raw(&self, data: &[u8]) -> Result<Key> {
|
||||
{
|
||||
let entries = self.entries.read().map_err(|_| Error::msg("Failed to obtain a read lock"))?;
|
||||
if entries.len() as u64 >= self.entry_limit {
|
||||
return Err(Error::msg("the storage limit has been reached"));
|
||||
}
|
||||
}
|
||||
|
||||
// Generate a new UUID
|
||||
let uuid = Uuid::new_v4();
|
||||
let key = Key::new(uuid.to_string(), &self.file_ext, true);
|
||||
|
||||
self.write_bytes(&key, data)?;
|
||||
|
||||
Ok(key)
|
||||
}
|
||||
|
||||
fn len(&self) -> usize {
|
||||
self.entries.read().map(|e| e.len()).unwrap_or(0)
|
||||
}
|
||||
|
||||
fn list(&self) -> Vec<Key> {
|
||||
let entries = match self.entries.read() {
|
||||
Ok(guard) => guard,
|
||||
Err(_) => return Vec::new(),
|
||||
};
|
||||
|
||||
// Convert entries to vectors and sort by timestamp
|
||||
let mut entries_vec: Vec<_> = entries.iter().collect();
|
||||
entries_vec.sort_by(|a, b| a.1.cmp(b.1));
|
||||
|
||||
// Parsing key
|
||||
entries_vec.iter().map(|(filename, _)| parse_key(filename)).collect()
|
||||
}
|
||||
|
||||
fn del(&self, key: &Key) -> Result<()> {
|
||||
let path = self.directory.join(key.to_string());
|
||||
|
||||
// Delete the file
|
||||
if let Err(e) = fs::remove_file(&path) {
|
||||
if e.kind() != io::ErrorKind::NotFound {
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
|
||||
// Remove the entry from the map
|
||||
let mut entries = self
|
||||
.entries
|
||||
.write()
|
||||
.map_err(|_| Error::msg("Failed to obtain a write lock"))?;
|
||||
entries.remove(&key.to_string());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1,177 +0,0 @@
|
||||
use rustfs_event::{AdapterCommon, AdapterConfig, ChannelAdapterType, NotifierSystem, WebhookConfig};
|
||||
use rustfs_event::{Bucket, Event, EventBuilder, Identity, Metadata, Name, Object, Source};
|
||||
use rustfs_event::{ChannelAdapter, WebhookAdapter};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_webhook_adapter() {
|
||||
let adapter = WebhookAdapter::new(WebhookConfig {
|
||||
common: AdapterCommon {
|
||||
identifier: "webhook".to_string(),
|
||||
comment: "webhook".to_string(),
|
||||
enable: true,
|
||||
queue_dir: "./deploy/logs/event_queue".to_string(),
|
||||
queue_limit: 100,
|
||||
},
|
||||
endpoint: "http://localhost:8080/webhook".to_string(),
|
||||
auth_token: None,
|
||||
custom_headers: None,
|
||||
max_retries: 1,
|
||||
timeout: Some(5),
|
||||
retry_interval: Some(5),
|
||||
client_cert: None,
|
||||
client_key: None,
|
||||
});
|
||||
|
||||
// create an s3 metadata object
|
||||
let metadata = Metadata {
|
||||
schema_version: "1.0".to_string(),
|
||||
configuration_id: "test-config".to_string(),
|
||||
bucket: Bucket {
|
||||
name: "my-bucket".to_string(),
|
||||
owner_identity: Identity {
|
||||
principal_id: "owner123".to_string(),
|
||||
},
|
||||
arn: "arn:aws:s3:::my-bucket".to_string(),
|
||||
},
|
||||
object: Object {
|
||||
key: "test.txt".to_string(),
|
||||
size: Some(1024),
|
||||
etag: Some("abc123".to_string()),
|
||||
content_type: Some("text/plain".to_string()),
|
||||
user_metadata: None,
|
||||
version_id: None,
|
||||
sequencer: "1234567890".to_string(),
|
||||
},
|
||||
};
|
||||
|
||||
// create source object
|
||||
let source = Source {
|
||||
host: "localhost".to_string(),
|
||||
port: "80".to_string(),
|
||||
user_agent: "curl/7.68.0".to_string(),
|
||||
};
|
||||
|
||||
// Create events using builder mode
|
||||
let event = Event::builder()
|
||||
.event_version("2.0")
|
||||
.event_source("aws:s3")
|
||||
.aws_region("us-east-1")
|
||||
.event_time("2023-10-01T12:00:00.000Z")
|
||||
.event_name(Name::ObjectCreatedPut)
|
||||
.user_identity(Identity {
|
||||
principal_id: "user123".to_string(),
|
||||
})
|
||||
.request_parameters(HashMap::new())
|
||||
.response_elements(HashMap::new())
|
||||
.s3(metadata)
|
||||
.source(source)
|
||||
.channels(vec![ChannelAdapterType::Webhook.to_string()])
|
||||
.build()
|
||||
.expect("failed to create event");
|
||||
|
||||
let result = adapter.send(&event).await;
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_notification_system() {
|
||||
let config = rustfs_event::NotifierConfig {
|
||||
store_path: "./test_events".to_string(),
|
||||
channel_capacity: 100,
|
||||
adapters: vec![AdapterConfig::Webhook(WebhookConfig {
|
||||
common: Default::default(),
|
||||
endpoint: "http://localhost:8080/webhook".to_string(),
|
||||
auth_token: None,
|
||||
custom_headers: None,
|
||||
max_retries: 1,
|
||||
timeout: Some(5),
|
||||
retry_interval: Some(5),
|
||||
client_cert: None,
|
||||
client_key: None,
|
||||
})],
|
||||
};
|
||||
let system = Arc::new(tokio::sync::Mutex::new(NotifierSystem::new(config.clone()).await.unwrap()));
|
||||
let adapters: Vec<Arc<dyn ChannelAdapter>> = vec![Arc::new(WebhookAdapter::new(WebhookConfig {
|
||||
common: Default::default(),
|
||||
endpoint: "http://localhost:8080/webhook".to_string(),
|
||||
auth_token: None,
|
||||
custom_headers: None,
|
||||
max_retries: 1,
|
||||
timeout: Some(5),
|
||||
retry_interval: Some(5),
|
||||
client_cert: None,
|
||||
client_key: None,
|
||||
}))];
|
||||
|
||||
// create an s3 metadata object
|
||||
let metadata = Metadata {
|
||||
schema_version: "1.0".to_string(),
|
||||
configuration_id: "test-config".to_string(),
|
||||
bucket: Bucket {
|
||||
name: "my-bucket".to_string(),
|
||||
owner_identity: Identity {
|
||||
principal_id: "owner123".to_string(),
|
||||
},
|
||||
arn: "arn:aws:s3:::my-bucket".to_string(),
|
||||
},
|
||||
object: Object {
|
||||
key: "test.txt".to_string(),
|
||||
size: Some(1024),
|
||||
etag: Some("abc123".to_string()),
|
||||
content_type: Some("text/plain".to_string()),
|
||||
user_metadata: None,
|
||||
version_id: None,
|
||||
sequencer: "1234567890".to_string(),
|
||||
},
|
||||
};
|
||||
|
||||
// create source object
|
||||
let source = Source {
|
||||
host: "localhost".to_string(),
|
||||
port: "80".to_string(),
|
||||
user_agent: "curl/7.68.0".to_string(),
|
||||
};
|
||||
|
||||
// create a preconfigured builder with objects
|
||||
let event = EventBuilder::for_object_creation(metadata, source)
|
||||
.user_identity(Identity {
|
||||
principal_id: "user123".to_string(),
|
||||
})
|
||||
.event_time("2023-10-01T12:00:00.000Z")
|
||||
.channels(vec![ChannelAdapterType::Webhook.to_string()])
|
||||
.build()
|
||||
.expect("failed to create event");
|
||||
|
||||
{
|
||||
let system_lock = system.lock().await;
|
||||
system_lock.send_event(event).await.unwrap();
|
||||
}
|
||||
|
||||
let system_clone = Arc::clone(&system);
|
||||
let system_handle = tokio::spawn(async move {
|
||||
let mut system = system_clone.lock().await;
|
||||
system.start(adapters).await
|
||||
});
|
||||
|
||||
// set 10 seconds timeout
|
||||
match tokio::time::timeout(std::time::Duration::from_secs(10), system_handle).await {
|
||||
Ok(result) => {
|
||||
println!("System started successfully");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
Err(_) => {
|
||||
println!("System operation timed out, forcing shutdown");
|
||||
// create a new task to handle the timeout
|
||||
let system = Arc::clone(&system);
|
||||
tokio::spawn(async move {
|
||||
if let Ok(mut guard) = system.try_lock() {
|
||||
guard.shutdown().await.unwrap();
|
||||
}
|
||||
});
|
||||
// give the system some time to clean up resources
|
||||
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user