gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-18 18:46:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
houseme
2025-06-18 23:46:55 +08:00
parent 9339093638
commit 09e8bc8f02
19 changed files with 997 additions and 428 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/notify/src/adapter/mod.rs
+2 -2
View File
@@ -85,7 +85,7 @@ pub trait ChannelAdapter: Send + Sync + 'static {
}
/// Creates channel adapters based on the provided configuration.
pub fn create_adapters(configs: Vec<AdapterConfig>) -> Result<Vec<Arc<dyn ChannelAdapter>>, Error> {
pub async fn create_adapters(configs: Vec<AdapterConfig>) -> Result<Vec<Arc<dyn ChannelAdapter>>, Error> {
let mut adapters: Vec<Arc<dyn ChannelAdapter>> = Vec::new();
for config in configs {
@@ -93,7 +93,7 @@ pub fn create_adapters(configs: Vec<AdapterConfig>) -> Result<Vec<Arc<dyn Channe
#[cfg(feature = "webhook")]
AdapterConfig::Webhook(webhook_config) => {
webhook_config.validate().map_err(Error::ConfigError)?;
adapters.push(Arc::new(webhook::WebhookAdapter::new(webhook_config.clone())));
adapters.push(Arc::new(webhook::WebhookAdapter::new(webhook_config.clone()).await));
}
#[cfg(feature = "mqtt")]
AdapterConfig::Mqtt(mqtt_config) => {
crates/notify/src/adapter/webhook.rs
+29 -38
View File
@@ -1,9 +1,12 @@
use crate::config::STORE_PREFIX;
use crate::{ChannelAdapter, ChannelAdapterType};
use crate::error::Error;
use crate::store::Store;
use crate::{ChannelAdapter, ChannelAdapterType, QueueStore};
use crate::{Event, DEFAULT_RETRY_INTERVAL};
use async_trait::async_trait;
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
use reqwest::{self, Client, Identity, RequestBuilder};
use rustfs_config::notify::webhook::WebhookArgs;
use std::fs;
use std::path::PathBuf;
use std::sync::Arc;
@@ -30,7 +33,7 @@ pub const ENV_WEBHOOK_CLIENT_KEY: &str = "RUSTFS_NOTIFY_WEBHOOK_CLIENT_KEY";
/// Webhook adapter for sending events to a webhook endpoint.
pub struct WebhookAdapter {
/// Configuration information
config: WebhookConfig,
config: WebhookArgs,
/// Event storage queues
store: Option<Arc<QueueStore<Event>>>,
/// HTTP client
@@ -39,16 +42,9 @@ pub struct WebhookAdapter {
impl WebhookAdapter {
/// Creates a new Webhook adapter.
pub fn new(config: WebhookConfig) -> Self {
pub async fn new(config: WebhookArgs) -> Self {
let mut builder = Client::builder();
if config.timeout.is_some() {
// Set the timeout for the client
match config.timeout {
Some(t) => builder = builder.timeout(Duration::from_secs(t)),
None => tracing::warn!("Timeout is not set, using default timeout"),
}
}
let client = if let (Some(cert_path), Some(key_path)) = (&config.client_cert, &config.client_key) {
let client = if let (cert_path, key_path) = (&config.client_cert, &config.client_key) {
let cert_path = PathBuf::from(cert_path);
let key_path = PathBuf::from(key_path);
@@ -90,21 +86,20 @@ impl WebhookAdapter {
});
// create a queue store if enabled
let store = if !config.common.queue_dir.len() > 0 {
let store_path = PathBuf::from(&config.common.queue_dir).join(format!(
let store = if !config.queue_dir.len() > 0 {
let store_path = PathBuf::from(&config.queue_dir).join(format!(
"{}-{}-{}",
STORE_PREFIX,
Webhook.as_str(),
config.common.identifier
"identifier".to_string()
));
let queue_limit = if config.common.queue_limit > 0 {
config.common.queue_limit
let queue_limit = if config.queue_limit > 0 {
config.queue_limit
} else {
crate::config::default_queue_limit()
};
let name = config.common.identifier.clone();
let store = QueueStore::new(store_path, name, queue_limit, Some(".event".to_string()));
if let Err(e) = store.open() {
let store = QueueStore::new(store_path, queue_limit, Some(".event"));
if let Err(e) = store.open().await {
tracing::error!("Unable to open queue storage: {}", e);
None
} else {
@@ -120,9 +115,10 @@ impl WebhookAdapter {
/// Handle backlog events in storage
pub async fn process_backlog(&self) -> Result<(), Error> {
if let Some(store) = &self.store {
let keys = store.list();
let keys = store.list().await;
for key in keys {
match store.get_multiple(&key) {
let key_clone = key.clone();
match store.get_multiple(key).await {
Ok(events) => {
for event in events {
if let Err(e) = self.send_with_retry(&event).await {
@@ -132,7 +128,7 @@ impl WebhookAdapter {
}
}
// Deleted after successful processing
if let Err(e) = store.del(&key) {
if let Err(e) = store.del(key_clone).await {
tracing::error!("Failed to delete a handled event: {}", e);
}
}
@@ -140,7 +136,7 @@ impl WebhookAdapter {
tracing::error!("Failed to read events from storage: {}", e);
// delete the broken entries
// If the event cannot be read, it may be corrupted, delete it
if let Err(del_err) = store.del(&key) {
if let Err(del_err) = store.del(key_clone).await {
tracing::error!("Failed to delete a corrupted event: {}", del_err);
}
}
@@ -153,10 +149,7 @@ impl WebhookAdapter {
///Send events to the webhook endpoint with retry logic
async fn send_with_retry(&self, event: &Event) -> Result<(), Error> {
let retry_interval = match self.config.retry_interval {
Some(t) => Duration::from_secs(t),
None => Duration::from_secs(DEFAULT_RETRY_INTERVAL), // Default to 3 seconds if not set
};
let retry_interval = Duration::from_secs(DEFAULT_RETRY_INTERVAL);
let mut attempts = 0;
loop {
@@ -164,18 +157,15 @@ impl WebhookAdapter {
match self.send_request(event).await {
Ok(_) => return Ok(()),
Err(e) => {
if attempts <= self.config.max_retries {
tracing::warn!("Send to webhook fails and will be retried after 3 seconds:{}", e);
sleep(retry_interval).await;
} else if let Some(store) = &self.store {
tracing::warn!("Send to webhook fails and will be retried after 3 seconds:{}", e);
sleep(retry_interval).await;
if let Some(store) = &self.store {
// store in a queue for later processing
tracing::warn!("The maximum number of retries is reached, and the event is stored in a queue:{}", e);
if let Err(store_err) = store.put(event.clone()) {
if let Err(store_err) = store.put(event.clone()).await {
tracing::error!("Events cannot be stored to a queue:{}", store_err);
}
return Err(e);
} else {
return Err(e);
}
}
}
@@ -211,7 +201,7 @@ impl WebhookAdapter {
.post(&self.config.endpoint)
.json(event)
.header("Content-Type", "application/json");
if let Some(token) = &self.config.auth_token {
if let token = &self.config.auth_token {
let tokens: Vec<&str> = token.split_whitespace().collect();
match tokens.len() {
2 => request = request.header("Authorization", token),
@@ -234,9 +224,10 @@ impl WebhookAdapter {
/// Save the event to the queue
async fn save_to_queue(&self, event: &Event) -> Result<(), Error> {
if let Some(store) = &self.store {
store
.put(event.clone())
.map_err(|e| Error::Custom(format!("Saving events to queue failed: {}", e)))?;
store.put(event.clone()).await.map_err(|e| {
tracing::error!("Failed to save event to queue: {}", e);
Error::Custom(format!("Failed to save event to queue: {}", e))
})?;
}
Ok(())
}
crates/notify/src/lib.rs
+4 -1
View File
@@ -3,7 +3,7 @@ mod config;
mod error;
mod event;
mod notifier;
mod store;
pub mod store;
mod system;
pub use adapter::create_adapters;
@@ -17,3 +17,6 @@ pub use adapter::ChannelAdapterType;
pub use config::{AdapterConfig, EventNotifierConfig, DEFAULT_MAX_RETRIES, DEFAULT_RETRY_INTERVAL};
pub use error::Error;
pub use event::{Bucket, Event, EventBuilder, Identity, Log, Metadata, Name, Object, Source};
pub use store::manager;
pub use store::queue;
pub use store::queue::QueueStore;
crates/notify/src/store/manager.rs
+1 -1
View File
@@ -157,7 +157,7 @@ impl EventManager {
};
let adapter_configs = config.to_adapter_configs();
match adapter::create_adapters(adapter_configs) {
match adapter::create_adapters(adapter_configs).await {
Ok(adapters) => Ok(adapters),
Err(err) => {
tracing::error!("Failed to create adapters: {:?}", err);
crates/notify/src/store/mod.rs
+12 -7
View File
@@ -8,8 +8,8 @@ use std::time::Duration;
use tokio::sync::mpsc;
use tokio::time;
pub(crate) mod manager;
pub(crate) mod queue;
pub mod manager;
pub mod queue;
// 常量定义
pub const RETRY_INTERVAL: Duration = Duration::from_secs(3);
@@ -231,11 +231,16 @@ where
}
// 发送项目辅助函数
pub async fn send_items(target: &dyn Target, mut key_ch: mpsc::Receiver<Key>, mut done_ch: mpsc::Receiver<()>, logger: Logger) {
pub async fn send_items(
target: Arc<dyn Target>,
mut key_ch: mpsc::Receiver<Key>,
mut done_ch: mpsc::Receiver<()>,
logger: Logger,
) {
let mut retry_interval = time::interval(RETRY_INTERVAL);
let target_clone = target.clone();
async fn try_send(
target: &dyn Target,
target: Arc<dyn Target>,
key: Key,
retry_interval: &mut time::Interval,
done_ch: &mut mpsc::Receiver<()>,
@@ -265,7 +270,7 @@ pub async fn send_items(target: &dyn Target, mut key_ch: mpsc::Receiver<Key>, mu
maybe_key = key_ch.recv() => {
match maybe_key {
Some(key) => {
if !try_send(target, key, &mut retry_interval, &mut done_ch, logger).await {
if !try_send(target_clone.clone(), key, &mut retry_interval, &mut done_ch, logger).await {
return;
}
}
@@ -280,7 +285,7 @@ pub async fn send_items(target: &dyn Target, mut key_ch: mpsc::Receiver<Key>, mu
}
// 流式传输项目
pub async fn stream_items<T>(store: Arc<dyn Store<T>>, target: &dyn Target, done_ch: mpsc::Receiver<()>, logger: Logger)
pub async fn stream_items<T>(store: Arc<dyn Store<T>>, target: Arc<dyn Target>, done_ch: mpsc::Receiver<()>, logger: Logger)
where
T: Serialize + DeserializeOwned + Send + Sync + 'static,
{