gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-11 15:46:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

improve channel adapter

Browse Source
This commit is contained in:
houseme
2025-05-21 16:44:59 +08:00
parent 1c088546b3
commit 0f22b21c8d
21 changed files with 278 additions and 372 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/event/src/adapter/kafka.rs
+2 -2
View File
@@ -1,7 +1,7 @@
use crate::ChannelAdapter;
use crate::Error;
use crate::Event;
use crate::KafkaConfig;
use crate::{ChannelAdapter, ChannelAdapterType};
use async_trait::async_trait;
use rdkafka::error::KafkaError;
use rdkafka::producer::{FutureProducer, FutureRecord};
@@ -60,7 +60,7 @@ impl KafkaAdapter {
#[async_trait]
impl ChannelAdapter for KafkaAdapter {
fn name(&self) -> String {
"kafka".to_string()
ChannelAdapterType::Kafka.to_string()
}
async fn send(&self, event: &Event) -> Result<(), Error> {
crates/event/src/adapter/mod.rs
+47
View File
@@ -11,6 +11,53 @@ pub(crate) mod mqtt;
#[cfg(feature = "webhook")]
pub(crate) mod webhook;
/// The `ChannelAdapterType` enum represents the different types of channel adapters.
///
/// It is used to identify the type of adapter being used in the system.
///
/// # Variants
///
/// - `Webhook`: Represents a webhook adapter.
/// - `Kafka`: Represents a Kafka adapter.
/// - `Mqtt`: Represents an MQTT adapter.
///
/// # Example
///
/// ```
/// use rustfs_event::ChannelAdapterType;
///
/// let adapter_type = ChannelAdapterType::Webhook;
/// match adapter_type {
/// ChannelAdapterType::Webhook => println!("Using webhook adapter"),
/// ChannelAdapterType::Kafka => println!("Using Kafka adapter"),
/// ChannelAdapterType::Mqtt => println!("Using MQTT adapter"),
/// }
pub enum ChannelAdapterType {
Webhook,
Kafka,
Mqtt,
}
impl ChannelAdapterType {
pub fn as_str(&self) -> &'static str {
match self {
ChannelAdapterType::Webhook => "webhook",
ChannelAdapterType::Kafka => "kafka",
ChannelAdapterType::Mqtt => "mqtt",
}
}
}
impl std::fmt::Display for ChannelAdapterType {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
ChannelAdapterType::Webhook => write!(f, "webhook"),
ChannelAdapterType::Kafka => write!(f, "kafka"),
ChannelAdapterType::Mqtt => write!(f, "mqtt"),
}
}
}
/// The `ChannelAdapter` trait defines the interface for all channel adapters.
#[async_trait]
pub trait ChannelAdapter: Send + Sync + 'static {
crates/event/src/adapter/mqtt.rs
+2 -2
View File
@@ -1,7 +1,7 @@
use crate::ChannelAdapter;
use crate::Error;
use crate::Event;
use crate::MqttConfig;
use crate::{ChannelAdapter, ChannelAdapterType};
use async_trait::async_trait;
use rumqttc::{AsyncClient, MqttOptions, QoS};
use std::time::Duration;
@@ -33,7 +33,7 @@ impl MqttAdapter {
#[async_trait]
impl ChannelAdapter for MqttAdapter {
fn name(&self) -> String {
"mqtt".to_string()
ChannelAdapterType::Mqtt.to_string()
}
async fn send(&self, event: &Event) -> Result<(), Error> {
crates/event/src/adapter/webhook.rs
+7 -6
View File
@@ -1,7 +1,7 @@
use crate::ChannelAdapter;
use crate::Error;
use crate::Event;
use crate::WebhookConfig;
use crate::{ChannelAdapter, ChannelAdapterType};
use async_trait::async_trait;
use reqwest::{Client, RequestBuilder};
use std::time::Duration;
@@ -16,10 +16,11 @@ pub struct WebhookAdapter {
impl WebhookAdapter {
/// Creates a new Webhook adapter.
pub fn new(config: WebhookConfig) -> Self {
let client = Client::builder()
.timeout(Duration::from_secs(config.timeout))
.build()
.expect("Failed to build reqwest client");
let mut builder = Client::builder();
if config.timeout > 0 {
builder = builder.timeout(Duration::from_secs(config.timeout));
}
let client = builder.build().expect("Failed to build reqwest client");
Self { config, client }
}
/// Builds the request to send the event.
@@ -40,7 +41,7 @@ impl WebhookAdapter {
#[async_trait]
impl ChannelAdapter for WebhookAdapter {
fn name(&self) -> String {
"webhook".to_string()
ChannelAdapterType::Webhook.to_string()
}
async fn send(&self, event: &Event) -> Result<(), Error> {
crates/event/src/bus.rs
-10
View File
@@ -21,17 +21,10 @@ pub async fn event_bus(
shutdown: CancellationToken,
shutdown_complete: Option<tokio::sync::oneshot::Sender<()>>,
) -> Result<(), Error> {
let mut current_log = Log {
event_name: crate::event::Name::Everything,
key: SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs().to_string(),
records: Vec::new(),
};
let mut unprocessed_events = Vec::new();
loop {
tokio::select! {
Some(event) = rx.recv() => {
current_log.records.push(event.clone());
let mut send_tasks = Vec::new();
for adapter in &adapters {
if event.channels.contains(&adapter.name()) {
@@ -54,9 +47,6 @@ pub async fn event_bus(
unprocessed_events.push(failed_event);
}
}
// Clear the current log because we only care about unprocessed events
current_log.records.clear();
}
_ = shutdown.cancelled() => {
tracing::info!("Shutting down event bus, saving pending logs...");
crates/event/src/config.rs
+7 -4
View File
@@ -3,6 +3,8 @@ use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::env;
const DEFAULT_CONFIG_FILE: &str = "event";
/// Configuration for the webhook adapter.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WebhookConfig {
@@ -206,17 +208,18 @@ impl NotifierConfig {
}
}
const DEFAULT_CONFIG_FILE: &str = "event";
/// Provide temporary directories as default storage paths
fn default_store_path() -> String {
env::var("EVENT_STORE_PATH").unwrap_or_else(|e| {
tracing::error!("Failed to get EVENT_STORE_PATH: {}", e);
tracing::info!("Failed to get `EVENT_STORE_PATH` failed err: {}", e.to_string());
env::temp_dir().join(DEFAULT_CONFIG_FILE).to_string_lossy().to_string()
})
}
/// Provides the recommended default channel capacity for high concurrency systems
fn default_channel_capacity() -> usize {
10000 // Reasonable default values for high concurrency systems
env::var("EVENT_CHANNEL_CAPACITY")
.unwrap_or_else(|_| "10000".to_string())
.parse()
.unwrap_or(10000) // Default to 10000 if parsing fails
}
crates/event/src/event_sys/mod.rs
-267
View File
@@ -1,267 +0,0 @@
use crate::NotifierConfig;
use common::error::{Error, Result};
use ecstore::config::com::CONFIG_PREFIX;
use ecstore::config::error::{is_err_config_not_found, ConfigError};
use ecstore::disk::RUSTFS_META_BUCKET;
use ecstore::store::ECStore;
use ecstore::store_api::{ObjectInfo, ObjectOptions, PutObjReader};
use ecstore::store_err::is_err_object_not_found;
use ecstore::utils::path::SLASH_SEPARATOR;
use ecstore::StorageAPI;
use http::HeaderMap;
use lazy_static::lazy_static;
use std::io::Cursor;
use std::sync::{Arc, OnceLock};
use tracing::{error, instrument, warn};
lazy_static! {
pub static ref GLOBAL_EventSys: EventSys = EventSys::new();
pub static ref GLOBAL_EventSysConfig: OnceLock<NotifierConfig> = OnceLock::new();
}
/// * config file
const CONFIG_FILE: &str = "event.json";
/// event sys config
const EVENT: &str = "event";
#[derive(Debug)]
pub struct EventSys {}
impl Default for EventSys {
fn default() -> Self {
Self::new()
}
}
impl EventSys {
pub fn new() -> Self {
Self {}
}
#[instrument(skip_all)]
pub async fn init(&self, api: Arc<ECStore>) -> Result<()> {
tracing::info!("event sys config init start");
let cfg = read_config_without_migrate(api.clone().clone()).await?;
let _ = GLOBAL_EventSysConfig.set(cfg);
tracing::info!("event sys config init done");
Ok(())
}
}
/// get event sys config file
///
/// # Returns
/// NotifierConfig
pub fn get_event_notifier_config() -> &'static NotifierConfig {
GLOBAL_EventSysConfig.get_or_init(NotifierConfig::default)
}
fn get_event_sys_file() -> String {
format!("{}{}{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, EVENT, SLASH_SEPARATOR, CONFIG_FILE)
}
/// read config without migrate
///
/// # Parameters
/// - `api`: StorageAPI
///
/// # Returns
/// Configuration information
pub async fn read_config_without_migrate<S: StorageAPI>(api: Arc<S>) -> Result<NotifierConfig> {
let config_file = get_event_sys_file();
let data = match read_config(api.clone(), config_file.as_str()).await {
Ok(res) => res,
Err(err) => {
return if is_err_config_not_found(&err) {
warn!("config not found, start to init");
let cfg = new_and_save_server_config(api).await?;
warn!("config init done");
Ok(cfg)
} else {
error!("read config err {:?}", &err);
Err(err)
}
}
};
read_server_config(api, data.as_slice()).await
}
/// save config with options
///
/// # Parameters
/// - `api`: StorageAPI
/// - `file`: file name
/// - `data`: data to save
/// - `opts`: object options
///
/// # Returns
/// Result
pub async fn save_config_with_opts<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()> {
let size = data.len();
let _ = api
.put_object(RUSTFS_META_BUCKET, file, &mut PutObjReader::new(Box::new(Cursor::new(data)), size), opts)
.await?;
Ok(())
}
/// new server config
///
/// # Returns
/// NotifierConfig
fn new_server_config() -> NotifierConfig {
NotifierConfig::new()
}
async fn new_and_save_server_config<S: StorageAPI>(api: Arc<S>) -> Result<NotifierConfig> {
let cfg = new_server_config();
save_server_config(api, &cfg).await?;
Ok(cfg)
}
async fn read_server_config<S: StorageAPI>(api: Arc<S>, data: &[u8]) -> Result<NotifierConfig> {
let cfg = {
if data.is_empty() {
let config_file = get_event_sys_file();
let cfg_data = match read_config(api.clone(), config_file.as_str()).await {
Ok(res) => res,
Err(err) => {
return if is_err_config_not_found(&err) {
warn!("config not found init start");
let cfg = new_and_save_server_config(api).await?;
warn!("config not found init done");
Ok(cfg)
} else {
error!("read config err {:?}", &err);
Err(err)
}
}
};
// TODO: decrypt
NotifierConfig::unmarshal(cfg_data.as_slice())?
} else {
NotifierConfig::unmarshal(data)?
}
};
Ok(cfg.merge())
}
/// save server config
///
/// # Parameters
/// - `api`: StorageAPI
/// - `cfg`: configuration to save
///
/// # Returns
/// Result
async fn save_server_config<S: StorageAPI>(api: Arc<S>, cfg: &NotifierConfig) -> Result<()> {
let data = cfg.marshal()?;
let config_file = get_event_sys_file();
save_config(api, &config_file, data).await
}
/// save config
///
/// # Parameters
/// - `api`: StorageAPI
/// - `file`: file name
/// - `data`: data to save
///
/// # Returns
/// Result
pub async fn save_config<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()> {
save_config_with_opts(
api,
file,
data,
&ObjectOptions {
max_parity: true,
..Default::default()
},
)
.await
}
/// delete config
///
/// # Parameters
/// - `api`: StorageAPI
/// - `file`: file name
///
/// # Returns
/// Result
pub async fn delete_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<()> {
match api
.delete_object(
RUSTFS_META_BUCKET,
file,
ObjectOptions {
delete_prefix: true,
delete_prefix_object: true,
..Default::default()
},
)
.await
{
Ok(_) => Ok(()),
Err(err) => {
if is_err_object_not_found(&err) {
Err(Error::new(ConfigError::NotFound))
} else {
Err(err)
}
}
}
}
/// read config
///
/// # Parameters
/// - `api`: StorageAPI
/// - `file`: file name
///
/// # Returns
/// Configuration data
pub async fn read_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
let (data, _obj) = read_config_with_metadata(api, file, &ObjectOptions::default()).await?;
Ok(data)
}
/// read config with metadata
///
/// # Parameters
/// - `api`: StorageAPI
/// - `file`: file name
/// - `opts`: object options
///
/// # Returns
/// Configuration data and object info
pub async fn read_config_with_metadata<S: StorageAPI>(
api: Arc<S>,
file: &str,
opts: &ObjectOptions,
) -> Result<(Vec<u8>, ObjectInfo)> {
let h = HeaderMap::new();
let mut rd = api
.get_object_reader(RUSTFS_META_BUCKET, file, None, h, opts)
.await
.map_err(|err| {
if is_err_object_not_found(&err) {
Error::new(ConfigError::NotFound)
} else {
err
}
})?;
let data = rd.read_all().await?;
if data.is_empty() {
return Err(Error::new(ConfigError::NotFound));
}
Ok((data, rd.object_info))
}
crates/event/src/lib.rs
+5 -8
View File
@@ -3,7 +3,6 @@ mod bus;
mod config;
mod error;
mod event;
mod event_sys;
mod global;
mod notifier;
mod store;
@@ -16,6 +15,7 @@ pub use adapter::mqtt::MqttAdapter;
#[cfg(feature = "webhook")]
pub use adapter::webhook::WebhookAdapter;
pub use adapter::ChannelAdapter;
pub use adapter::ChannelAdapterType;
pub use bus::event_bus;
#[cfg(all(feature = "kafka", target_os = "linux"))]
pub use config::KafkaConfig;
@@ -29,12 +29,9 @@ pub use error::Error;
pub use event::{Bucket, Event, EventBuilder, Identity, Log, Metadata, Name, Object, Source};
pub use global::{initialize, is_initialized, is_ready, send_event, shutdown};
pub use notifier::NotifierSystem;
pub use store::EventStore;
pub use store::event::EventStore;
pub use event_sys::delete_config;
pub use event_sys::get_event_notifier_config;
pub use event_sys::read_config;
pub use event_sys::save_config;
pub use store::get_event_notifier_config;
pub use event_sys::EventSys;
pub use event_sys::GLOBAL_EventSys;
pub use store::EventSys;
pub use store::GLOBAL_EventSys;
crates/event/src/store.rs → crates/event/src/store/event.rs
View File
crates/event/src/store/mod.rs
+119
View File
@@ -0,0 +1,119 @@
pub(crate) mod event;
use crate::NotifierConfig;
use common::error::Result;
use ecstore::config::com::{read_config, save_config, CONFIG_PREFIX};
use ecstore::config::error::is_err_config_not_found;
use ecstore::store::ECStore;
use ecstore::utils::path::SLASH_SEPARATOR;
use ecstore::StorageAPI;
use lazy_static::lazy_static;
use std::sync::{Arc, OnceLock};
use tracing::{error, instrument, warn};
lazy_static! {
pub static ref GLOBAL_EventSys: EventSys = EventSys::new();
pub static ref GLOBAL_EventSysConfig: OnceLock<NotifierConfig> = OnceLock::new();
}
/// * config file
const CONFIG_FILE: &str = "event.json";
/// event sys config
const EVENT: &str = "event";
#[derive(Debug)]
pub struct EventSys {}
impl Default for EventSys {
fn default() -> Self {
Self::new()
}
}
impl EventSys {
pub fn new() -> Self {
Self {}
}
#[instrument(skip_all)]
pub async fn init(&self, api: Arc<ECStore>) -> Result<()> {
tracing::info!("event sys config init start");
let cfg = read_config_without_migrate(api.clone().clone()).await?;
let _ = GLOBAL_EventSysConfig.set(cfg);
tracing::info!("event sys config init done");
Ok(())
}
}
/// get event sys config file
///
/// # Returns
/// NotifierConfig
pub fn get_event_notifier_config() -> &'static NotifierConfig {
GLOBAL_EventSysConfig.get_or_init(NotifierConfig::default)
}
fn get_event_sys_file() -> String {
format!("{}{}{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, EVENT, SLASH_SEPARATOR, CONFIG_FILE)
}
/// new server config
///
/// # Returns
/// NotifierConfig
fn new_server_config() -> NotifierConfig {
NotifierConfig::new()
}
async fn new_and_save_server_config<S: StorageAPI>(api: Arc<S>) -> Result<NotifierConfig> {
let cfg = new_server_config();
save_server_config(api, &cfg).await?;
Ok(cfg)
}
/// save server config
///
/// # Parameters
/// - `api`: StorageAPI
/// - `cfg`: configuration to save
///
/// # Returns
/// Result
async fn save_server_config<S: StorageAPI>(api: Arc<S>, cfg: &NotifierConfig) -> Result<()> {
let data = cfg.marshal()?;
let config_file = get_event_sys_file();
save_config(api, &config_file, data).await
}
/// read config without migrate
///
/// # Parameters
/// - `api`: StorageAPI
///
/// # Returns
/// Configuration information
pub async fn read_config_without_migrate<S: StorageAPI>(api: Arc<S>) -> Result<NotifierConfig> {
let config_file = get_event_sys_file();
let data = match read_config(api.clone(), config_file.as_str()).await {
Ok(data) => {
if data.is_empty() {
return new_and_save_server_config(api).await;
}
data
}
Err(err) if is_err_config_not_found(&err) => {
warn!("config not found, start to init");
return new_and_save_server_config(api).await;
}
Err(err) => {
error!("read config error: {:?}", err);
return Err(err);
}
};
// TODO: decrypt if needed
let cfg = NotifierConfig::unmarshal(data.as_slice())?;
Ok(cfg.merge())
}