gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-09 14:49:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
houseme
2025-06-20 10:51:36 +08:00
parent c658d88d25
commit d3cc36f6e0
22 changed files with 353 additions and 544 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/notify/src/event.rs
+46 -84
View File
@@ -3,23 +3,23 @@ use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::fmt;
/// 当解析事件名称字符串失败时返回的错误。
/// Error returned when parsing event name string fails。
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ParseEventNameError(String);
impl fmt::Display for ParseEventNameError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "无效的事件名称：{}", self.0)
write!(f, "Invalid event name:{}", self.0)
}
}
impl std::error::Error for ParseEventNameError {}
/// 表示对象上发生的事件类型。
/// 基于 AWS S3 事件类型，并包含 RustFS 扩展。
/// Represents the type of event that occurs on the object.
/// Based on AWS S3 event type and includes RustFS extension.
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)]
pub enum EventName {
// 单一事件类型 (值为 1-32 以兼容掩码逻辑)
// Single event type (values are 1-32 for compatible mask logic)
ObjectAccessedGet = 1,
ObjectAccessedGetRetention = 2,
ObjectAccessedGetLegalHold = 3,
@@ -48,23 +48,23 @@ pub enum EventName {
ObjectRestoreCompleted = 26,
ObjectTransitionFailed = 27,
ObjectTransitionComplete = 28,
ScannerManyVersions = 29, // 对应 Go 的 ObjectManyVersions
ScannerLargeVersions = 30, // 对应 Go 的 ObjectLargeVersions
ScannerBigPrefix = 31, // 对应 Go 的 PrefixManyFolders
LifecycleDelMarkerExpirationDelete = 32, // 对应 Go 的 ILMDelMarkerExpirationDelete
ScannerManyVersions = 29, // ObjectManyVersions corresponding to Go
ScannerLargeVersions = 30, // ObjectLargeVersions corresponding to Go
ScannerBigPrefix = 31, // PrefixManyFolders corresponding to Go
LifecycleDelMarkerExpirationDelete = 32, // ILMDelMarkerExpirationDelete corresponding to Go
// 复合 "All" 事件类型 (没有用于掩码的顺序值)
// Compound "All" event type (no sequential value for mask)
ObjectAccessedAll,
ObjectCreatedAll,
ObjectRemovedAll,
ObjectReplicationAll,
ObjectRestoreAll,
ObjectTransitionAll,
ObjectScannerAll, // 新增，来自 Go
Everything, // 新增，来自 Go
ObjectScannerAll, // New, from Go
Everything, // New, from Go
}
// 用于 Everything.expand() 的单一事件类型顺序数组
// Single event type sequential array for Everything.expand()
const SINGLE_EVENT_NAMES_IN_ORDER: [EventName; 32] = [
EventName::ObjectAccessedGet,
EventName::ObjectAccessedGetRetention,
@@ -103,7 +103,7 @@ const SINGLE_EVENT_NAMES_IN_ORDER: [EventName; 32] = [
const LAST_SINGLE_TYPE_VALUE: u32 = EventName::LifecycleDelMarkerExpirationDelete as u32;
impl EventName {
/// 解析字符串为 EventName。
/// The parsed string is EventName.
pub fn parse(s: &str) -> Result<Self, ParseEventNameError> {
match s {
"s3:BucketCreated:*" => Ok(EventName::BucketCreated),
@@ -115,9 +115,7 @@ impl EventName {
"s3:ObjectAccessed:Head" => Ok(EventName::ObjectAccessedHead),
"s3:ObjectAccessed:Attributes" => Ok(EventName::ObjectAccessedAttributes),
"s3:ObjectCreated:*" => Ok(EventName::ObjectCreatedAll),
"s3:ObjectCreated:CompleteMultipartUpload" => {
Ok(EventName::ObjectCreatedCompleteMultipartUpload)
}
"s3:ObjectCreated:CompleteMultipartUpload" => Ok(EventName::ObjectCreatedCompleteMultipartUpload),
"s3:ObjectCreated:Copy" => Ok(EventName::ObjectCreatedCopy),
"s3:ObjectCreated:Post" => Ok(EventName::ObjectCreatedPost),
"s3:ObjectCreated:Put" => Ok(EventName::ObjectCreatedPut),
@@ -127,25 +125,15 @@ impl EventName {
"s3:ObjectCreated:DeleteTagging" => Ok(EventName::ObjectCreatedDeleteTagging),
"s3:ObjectRemoved:*" => Ok(EventName::ObjectRemovedAll),
"s3:ObjectRemoved:Delete" => Ok(EventName::ObjectRemovedDelete),
"s3:ObjectRemoved:DeleteMarkerCreated" => {
Ok(EventName::ObjectRemovedDeleteMarkerCreated)
}
"s3:ObjectRemoved:DeleteMarkerCreated" => Ok(EventName::ObjectRemovedDeleteMarkerCreated),
"s3:ObjectRemoved:NoOP" => Ok(EventName::ObjectRemovedNoOP),
"s3:ObjectRemoved:DeleteAllVersions" => Ok(EventName::ObjectRemovedDeleteAllVersions),
"s3:LifecycleDelMarkerExpiration:Delete" => {
Ok(EventName::LifecycleDelMarkerExpirationDelete)
}
"s3:LifecycleDelMarkerExpiration:Delete" => Ok(EventName::LifecycleDelMarkerExpirationDelete),
"s3:Replication:*" => Ok(EventName::ObjectReplicationAll),
"s3:Replication:OperationFailedReplication" => Ok(EventName::ObjectReplicationFailed),
"s3:Replication:OperationCompletedReplication" => {
Ok(EventName::ObjectReplicationComplete)
}
"s3:Replication:OperationMissedThreshold" => {
Ok(EventName::ObjectReplicationMissedThreshold)
}
"s3:Replication:OperationReplicatedAfterThreshold" => {
Ok(EventName::ObjectReplicationReplicatedAfterThreshold)
}
"s3:Replication:OperationCompletedReplication" => Ok(EventName::ObjectReplicationComplete),
"s3:Replication:OperationMissedThreshold" => Ok(EventName::ObjectReplicationMissedThreshold),
"s3:Replication:OperationReplicatedAfterThreshold" => Ok(EventName::ObjectReplicationReplicatedAfterThreshold),
"s3:Replication:OperationNotTracked" => Ok(EventName::ObjectReplicationNotTracked),
"s3:ObjectRestore:*" => Ok(EventName::ObjectRestoreAll),
"s3:ObjectRestore:Post" => Ok(EventName::ObjectRestorePost),
@@ -156,12 +144,12 @@ impl EventName {
"s3:Scanner:ManyVersions" => Ok(EventName::ScannerManyVersions),
"s3:Scanner:LargeVersions" => Ok(EventName::ScannerLargeVersions),
"s3:Scanner:BigPrefix" => Ok(EventName::ScannerBigPrefix),
// ObjectScannerAll 和 Everything 不能从字符串解析，因为 Go 版本也没有定义它们的字符串表示
// ObjectScannerAll and Everything cannot be parsed from strings, because the Go version also does not define their string representation.
_ => Err(ParseEventNameError(s.to_string())),
}
}
/// 返回事件类型的字符串表示。
/// Returns a string representation of the event type.
pub fn as_str(&self) -> &'static str {
match self {
EventName::BucketCreated => "s3:BucketCreated:*",
@@ -173,9 +161,7 @@ impl EventName {
EventName::ObjectAccessedHead => "s3:ObjectAccessed:Head",
EventName::ObjectAccessedAttributes => "s3:ObjectAccessed:Attributes",
EventName::ObjectCreatedAll => "s3:ObjectCreated:*",
EventName::ObjectCreatedCompleteMultipartUpload => {
"s3:ObjectCreated:CompleteMultipartUpload"
}
EventName::ObjectCreatedCompleteMultipartUpload => "s3:ObjectCreated:CompleteMultipartUpload",
EventName::ObjectCreatedCopy => "s3:ObjectCreated:Copy",
EventName::ObjectCreatedPost => "s3:ObjectCreated:Post",
EventName::ObjectCreatedPut => "s3:ObjectCreated:Put",
@@ -188,19 +174,13 @@ impl EventName {
EventName::ObjectRemovedDeleteMarkerCreated => "s3:ObjectRemoved:DeleteMarkerCreated",
EventName::ObjectRemovedNoOP => "s3:ObjectRemoved:NoOP",
EventName::ObjectRemovedDeleteAllVersions => "s3:ObjectRemoved:DeleteAllVersions",
EventName::LifecycleDelMarkerExpirationDelete => {
"s3:LifecycleDelMarkerExpiration:Delete"
}
EventName::LifecycleDelMarkerExpirationDelete => "s3:LifecycleDelMarkerExpiration:Delete",
EventName::ObjectReplicationAll => "s3:Replication:*",
EventName::ObjectReplicationFailed => "s3:Replication:OperationFailedReplication",
EventName::ObjectReplicationComplete => "s3:Replication:OperationCompletedReplication",
EventName::ObjectReplicationNotTracked => "s3:Replication:OperationNotTracked",
EventName::ObjectReplicationMissedThreshold => {
"s3:Replication:OperationMissedThreshold"
}
EventName::ObjectReplicationReplicatedAfterThreshold => {
"s3:Replication:OperationReplicatedAfterThreshold"
}
EventName::ObjectReplicationMissedThreshold => "s3:Replication:OperationMissedThreshold",
EventName::ObjectReplicationReplicatedAfterThreshold => "s3:Replication:OperationReplicatedAfterThreshold",
EventName::ObjectRestoreAll => "s3:ObjectRestore:*",
EventName::ObjectRestorePost => "s3:ObjectRestore:Post",
EventName::ObjectRestoreCompleted => "s3:ObjectRestore:Completed",
@@ -210,13 +190,13 @@ impl EventName {
EventName::ScannerManyVersions => "s3:Scanner:ManyVersions",
EventName::ScannerLargeVersions => "s3:Scanner:LargeVersions",
EventName::ScannerBigPrefix => "s3:Scanner:BigPrefix",
// Go 的 String() 对 ObjectScannerAll 和 Everything 返回 ""
EventName::ObjectScannerAll => "s3:Scanner:*", // 遵循 Go Expand 中的模式
EventName::Everything => "", // Go String() 对未处理的返回 ""
// Go's String() returns "" for ObjectScannerAll and Everything
EventName::ObjectScannerAll => "s3:Scanner:*", // Follow the pattern in Go Expand
EventName::Everything => "", // Go String() returns "" to unprocessed
}
}
/// 返回缩写事件类型的扩展值。
/// Returns the extended value of the abbreviation event type.
pub fn expand(&self) -> Vec<Self> {
match self {
EventName::ObjectAccessedAll => vec![
@@ -249,41 +229,35 @@ impl EventName {
EventName::ObjectReplicationMissedThreshold,
EventName::ObjectReplicationReplicatedAfterThreshold,
],
EventName::ObjectRestoreAll => vec![
EventName::ObjectRestorePost,
EventName::ObjectRestoreCompleted,
],
EventName::ObjectTransitionAll => vec![
EventName::ObjectTransitionFailed,
EventName::ObjectTransitionComplete,
],
EventName::ObjectRestoreAll => vec![EventName::ObjectRestorePost, EventName::ObjectRestoreCompleted],
EventName::ObjectTransitionAll => vec![EventName::ObjectTransitionFailed, EventName::ObjectTransitionComplete],
EventName::ObjectScannerAll => vec![
// 新增
// New
EventName::ScannerManyVersions,
EventName::ScannerLargeVersions,
EventName::ScannerBigPrefix,
],
EventName::Everything => {
// 新增
// New
SINGLE_EVENT_NAMES_IN_ORDER.to_vec()
}
// 单一类型直接返回自身
// A single type returns to itself directly
_ => vec![*self],
}
}
/// 返回类型的掩码。
/// 复合 "All" 类型会被展开。
/// Returns the mask of type.
/// The compound "All" type will be expanded.
pub fn mask(&self) -> u64 {
let value = *self as u32;
if value > 0 && value <= LAST_SINGLE_TYPE_VALUE {
// 是单一类型
// It's a single type
1u64 << (value - 1)
} else {
// 是复合类型
// It's a compound type
let mut mask = 0u64;
for n in self.expand() {
mask |= n.mask(); // 递归调用 mask
mask |= n.mask(); // Recursively call mask
}
mask
}
@@ -296,7 +270,7 @@ impl fmt::Display for EventName {
}
}
/// 根据字符串转换为 `EventName`
/// Convert to `EventName` according to string
impl From<&str> for EventName {
fn from(event_str: &str) -> Self {
EventName::parse(event_str).unwrap_or_else(|e| panic!("{}", e))
@@ -399,28 +373,16 @@ impl Event {
pub fn new_test_event(bucket: &str, key: &str, event_name: EventName) -> Self {
let mut user_metadata = HashMap::new();
user_metadata.insert("x-amz-meta-test".to_string(), "value".to_string());
user_metadata.insert(
"x-amz-storage-storage-options".to_string(),
"value".to_string(),
);
user_metadata.insert("x-amz-storage-storage-options".to_string(), "value".to_string());
user_metadata.insert("x-amz-meta-".to_string(), "value".to_string());
user_metadata.insert("x-rustfs-meta-".to_string(), "rustfs-value".to_string());
user_metadata.insert("x-request-id".to_string(), "request-id-123".to_string());
user_metadata.insert("x-bucket".to_string(), "bucket".to_string());
user_metadata.insert("x-object".to_string(), "object".to_string());
user_metadata.insert(
"x-rustfs-origin-endpoint".to_string(),
"http://127.0.0.1".to_string(),
);
user_metadata.insert("x-rustfs-origin-endpoint".to_string(), "http://127.0.0.1".to_string());
user_metadata.insert("x-rustfs-user-metadata".to_string(), "metadata".to_string());
user_metadata.insert(
"x-rustfs-deployment-id".to_string(),
"deployment-id-123".to_string(),
);
user_metadata.insert(
"x-rustfs-origin-endpoint-code".to_string(),
"http://127.0.0.1".to_string(),
);
user_metadata.insert("x-rustfs-deployment-id".to_string(), "deployment-id-123".to_string());
user_metadata.insert("x-rustfs-origin-endpoint-code".to_string(), "http://127.0.0.1".to_string());
user_metadata.insert("x-rustfs-bucket-name".to_string(), "bucket".to_string());
user_metadata.insert("x-rustfs-object-key".to_string(), key.to_string());
user_metadata.insert("x-rustfs-object-size".to_string(), "1024".to_string());
@@ -466,7 +428,7 @@ impl Event {
},
}
}
/// 返回事件掩码
/// Return event mask
pub fn mask(&self) -> u64 {
self.event_name.mask()
}
crates/notify/src/lib.rs
+2 -2
View File
@@ -1,4 +1,4 @@
//! RustFs Notify - A flexible and extensible event notification system for object storage.
//! RustFS Notify - A flexible and extensible event notification system for object storage.
//!
//! This library provides a Rust implementation of a storage bucket notification system,
//! similar to RustFS's notification system. It supports sending events to various targets
@@ -35,7 +35,7 @@ use tracing_subscriber::{fmt, prelude::*, util::SubscriberInitExt, EnvFilter};
///
/// # Example
/// ```
/// notify::init_logger(notify::LogLevel::Info);
/// rustfs_notify::init_logger(rustfs_notify::LogLevel::Info);
/// ```
pub fn init_logger(level: LogLevel) {
let filter = EnvFilter::default().add_directive(level.into());
crates/notify/src/rules/config.rs
+3 -9
View File
@@ -39,7 +39,7 @@ impl BucketNotificationConfig {
/// Parses notification configuration from XML.
/// `arn_list` is a list of valid ARN strings for validation.
pub fn from_xml<R: Read>(
pub fn from_xml<R: Read + std::io::BufRead>(
reader: R,
current_region: &str,
arn_list: &[String],
@@ -72,11 +72,7 @@ impl BucketNotificationConfig {
/// However, Go's Config has a Validate method.
/// The primary validation now happens during `from_xml` via `NotificationConfiguration::validate`.
/// This method could re-check against an updated arn_list or region if needed.
pub fn validate(
&self,
current_region: &str,
arn_list: &[String],
) -> Result<(), BucketNotificationConfigError> {
pub fn validate(&self, current_region: &str, arn_list: &[String]) -> Result<(), BucketNotificationConfigError> {
if self.region != current_region {
return Err(BucketNotificationConfigError::RegionMismatch {
config_region: self.region.clone(),
@@ -93,9 +89,7 @@ impl BucketNotificationConfig {
// Construct the ARN string for this target_id and self.region
let arn_to_check = target_id.to_arn(&self.region); // Assuming TargetID has to_arn
if !arn_list.contains(&arn_to_check.to_arn_string()) {
return Err(BucketNotificationConfigError::ArnNotFound(
arn_to_check.to_arn_string(),
));
return Err(BucketNotificationConfigError::ArnNotFound(arn_to_check.to_arn_string()));
}
}
}
crates/notify/src/rules/rules_map.rs
+14 -20
View File
@@ -22,12 +22,7 @@ impl RulesMap {
/// target_id: Notify the target.
///
/// This method expands the composite event name.
pub fn add_rule_config(
&mut self,
event_names: &[EventName],
pattern: String,
target_id: TargetID,
) {
pub fn add_rule_config(&mut self, event_names: &[EventName], pattern: String, target_id: TargetID) {
let mut effective_pattern = pattern;
if effective_pattern.is_empty() {
effective_pattern = "*".to_string(); // Match all by default
@@ -55,8 +50,7 @@ impl RulesMap {
}
}
/// 从当前 RulesMap 中移除另一个 RulesMap 中定义的规则。
/// 对应 Go 的 `RulesMap.Remove(rulesMap2 RulesMap)`
/// Remove another rule defined in the RulesMap from the current RulesMap.
pub fn remove_map(&mut self, other_map: &Self) {
let mut events_to_remove = Vec::new();
for (event_name, self_pattern_rules) in &mut self.map {
@@ -72,24 +66,24 @@ impl RulesMap {
}
}
/// 匹配给定事件名称和对象键的规则，返回所有匹配的 TargetID。
///Rules matching the given event name and object key, returning all matching TargetIDs.
pub fn match_rules(&self, event_name: EventName, object_key: &str) -> TargetIdSet {
// 首先尝试直接匹配事件名称
// First try to directly match the event name
if let Some(pattern_rules) = self.map.get(&event_name) {
let targets = pattern_rules.match_targets(object_key);
if !targets.is_empty() {
return targets;
}
}
// Go 的 RulesMap[eventName] 直接获取，如果不存在则为空 Rules。
// Rust 的 HashMap::get 返回 Option。如果事件名不存在，则没有规则。
// 复合事件（如 ObjectCreatedAll）在 add_rule_config 时已展开为单一事件。
// 因此，查询时应使用单一事件名称。
// 如果 event_name 本身就是单一类型，则直接查找。
// 如果 event_name 是复合类型，Go 的逻辑是在添加时展开。
// 这里的 match_rules 应该接收已经可能是单一的事件。
// 如果调用者传入的是复合事件，它应该先自行展开或此函数处理。
// 假设 event_name 已经是具体的、可用于查找的事件。
// Go's RulesMap[eventName] is directly retrieved, and if it does not exist, it is empty Rules.
// Rust's HashMap::get returns Option. If the event name does not exist, there is no rule.
// Compound events (such as ObjectCreatedAll) have been expanded as a single event when add_rule_config.
// Therefore, a single event name should be used when querying.
// If event_name itself is a single type, look it up directly.
// If event_name is a compound type, Go's logic is expanded when added.
// Here match_rules should receive events that may already be single.
// If the caller passes in a compound event, it should expand itself or handle this function first.
// Assume that event_name is already a specific event that can be used for searching.
self.map
.get(&event_name)
.map_or_else(TargetIdSet::new, |pr| pr.match_targets(object_key))
@@ -99,7 +93,7 @@ impl RulesMap {
self.map.is_empty()
}
/// 返回内部规则的克隆，用于 BucketNotificationConfig::validate 等场景。
/// Returns a clone of internal rules for use in scenarios such as BucketNotificationConfig::validate.
pub fn inner(&self) -> &HashMap<EventName, PatternRules> {
&self.map
}
crates/notify/src/rules/xml_config.rs
+10 -10
View File
@@ -9,7 +9,7 @@ use thiserror::Error;
#[derive(Debug, Error)]
pub enum ParseConfigError {
#[error("XML parsing error:{0}")]
XmlError(#[from] quick_xml::errors::Error),
XmlError(#[from] quick_xml::errors::serialize::DeError),
#[error("Invalid filter value:{0}")]
InvalidFilterValue(String),
#[error("Invalid filter name: {0}, only 'prefix' or 'suffix' is allowed")]
@@ -193,10 +193,10 @@ impl QueueConfig {
pub struct LambdaConfigDetail {
#[serde(rename = "CloudFunction")]
pub arn: String,
// 根据 AWS S3 文档，<CloudFunctionConfiguration> 通常还包含 Id, Event, Filter
// 但为了严格对应提供的 Go `lambda` 结构体，这里只包含 ARN。
// 如果需要完整支持，可以添加其他字段。
// 例如：
// According to AWS S3 documentation, <CloudFunctionConfiguration> usually also contains Id, Event, Filter
// But in order to strictly correspond to the Go `lambda` structure provided, only ARN is included here.
// If full support is required, additional fields can be added.
// For example:
// #[serde(rename = "Id", skip_serializing_if = "Option::is_none")]
// pub id: Option<String>,
// #[serde(rename = "Event", default, skip_serializing_if = "Vec::is_empty")]
@@ -211,7 +211,7 @@ pub struct LambdaConfigDetail {
pub struct TopicConfigDetail {
#[serde(rename = "Topic")]
pub arn: String,
// 类似于 LambdaConfigDetail，可以根据需要扩展以包含 Id, Event, Filter 等字段。
// Similar to LambdaConfigDetail, it can be extended to include fields such as Id, Event, Filter, etc.
}
#[derive(Debug, Serialize, Deserialize, Clone, Default, PartialEq, Eq)]
@@ -236,8 +236,8 @@ pub struct NotificationConfiguration {
}
impl NotificationConfiguration {
pub fn from_reader<R: Read>(reader: R) -> Result<Self, ParseConfigError> {
let config: NotificationConfiguration = quick_xml::reader::Reader::from_reader(reader)?;
pub fn from_reader<R: Read + std::io::BufRead>(reader: R) -> Result<Self, ParseConfigError> {
let config: NotificationConfiguration = quick_xml::de::from_reader(reader)?;
Ok(config)
}
@@ -268,7 +268,7 @@ impl NotificationConfiguration {
if self.xmlns.is_none() {
self.xmlns = Some("http://s3.amazonaws.com/doc/2006-03-01/".to_string());
}
// 注意：如果 LambdaConfigDetail 和 TopicConfigDetail 将来包含区域等信息，
// 也可能需要在这里设置默认值。但根据当前定义，它们只包含 ARN 字符串。
// Note: If LambdaConfigDetail and TopicConfigDetail contain information such as regions in the future,
// You may also need to set the default value here. But according to the current definition, they only contain ARN strings.
}
}
crates/notify/src/stream.rs
+40 -107
View File
@@ -13,7 +13,7 @@ use tracing::{debug, error, info, warn};
/// Streams events from the store to the target
pub async fn stream_events(
store: &mut (dyn Store<crate::event::Event, Error = StoreError, Key = Key> + Send),
store: &mut (dyn Store<Event, Error = StoreError, Key = Key> + Send),
target: &dyn Target,
mut cancel_rx: mpsc::Receiver<()>,
) {
@@ -42,10 +42,7 @@ pub async fn stream_events(
for key in keys {
// Check for cancellation before processing each event
if cancel_rx.try_recv().is_ok() {
info!(
"Cancellation received during processing for target: {}",
target.name()
);
info!("Cancellation received during processing for target: {}", target.name());
return;
}
@@ -70,7 +67,7 @@ pub async fn stream_events(
TargetError::Timeout(_) => {
warn!("Timeout for target {}, retrying...", target.name());
retry_count += 1;
sleep(Duration::from_secs((retry_count * 5) as u64)).await; // 指数退避
sleep(Duration::from_secs((retry_count * 5) as u64)).await; // Exponential backoff
}
_ => {
// Permanent error, skip this event
@@ -84,11 +81,7 @@ pub async fn stream_events(
// Remove event from store if successfully sent
if retry_count >= MAX_RETRIES && !success {
warn!(
"Max retries exceeded for event {}, target: {}, skipping",
key.to_string(),
target.name()
);
warn!("Max retries exceeded for event {}, target: {}, skipping", key.to_string(), target.name());
}
}
@@ -120,10 +113,7 @@ pub fn start_event_stream_with_batching(
semaphore: Arc<Semaphore>,
) -> mpsc::Sender<()> {
let (cancel_tx, cancel_rx) = mpsc::channel(1);
debug!(
"Starting event stream with batching for target: {}",
target.name()
);
debug!("Starting event stream with batching for target: {}", target.name());
tokio::spawn(async move {
stream_events_with_batching(&mut *store, &*target, cancel_rx, metrics, semaphore).await;
info!("Event stream stopped for target: {}", target.name());
@@ -132,7 +122,7 @@ pub fn start_event_stream_with_batching(
cancel_tx
}
/// 带批处理的事件流处理
/// Event stream processing with batch processing
pub async fn stream_events_with_batching(
store: &mut (dyn Store<Event, Error = StoreError, Key = Key> + Send),
target: &dyn Target,
@@ -140,10 +130,7 @@ pub async fn stream_events_with_batching(
metrics: Arc<NotificationMetrics>,
semaphore: Arc<Semaphore>,
) {
info!(
"Starting event stream with batching for target: {}",
target.name()
);
info!("Starting event stream with batching for target: {}", target.name());
// Configuration parameters
const DEFAULT_BATCH_SIZE: usize = 1;
@@ -160,106 +147,64 @@ pub async fn stream_events_with_batching(
let mut last_flush = Instant::now();
loop {
// 检查取消信号
// Check the cancel signal
if cancel_rx.try_recv().is_ok() {
info!("Cancellation received for target: {}", target.name());
return;
}
// 获取存储中的事件列表
// Get a list of events in storage
let keys = store.list();
debug!(
"Found {} keys in store for target: {}",
keys.len(),
target.name()
);
debug!("Found {} keys in store for target: {}", keys.len(), target.name());
if keys.is_empty() {
// 如果批处理中有数据且超时，则刷新批处理
// If there is data in the batch and timeout, refresh the batch
if !batch.is_empty() && last_flush.elapsed() >= BATCH_TIMEOUT {
process_batch(
&mut batch,
&mut batch_keys,
target,
MAX_RETRIES,
BASE_RETRY_DELAY,
&metrics,
&semaphore,
)
.await;
process_batch(&mut batch, &mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
last_flush = Instant::now();
}
// 无事件，等待后再检查
// No event, wait before checking
tokio::time::sleep(Duration::from_millis(500)).await;
continue;
}
// 处理每个事件
// Handle each event
for key in keys {
// 再次检查取消信号
// Check the cancel signal again
if cancel_rx.try_recv().is_ok() {
info!(
"Cancellation received during processing for target: {}",
target.name()
);
info!("Cancellation received during processing for target: {}", target.name());
// 在退出前处理已收集的批次
// Processing collected batches before exiting
if !batch.is_empty() {
process_batch(
&mut batch,
&mut batch_keys,
target,
MAX_RETRIES,
BASE_RETRY_DELAY,
&metrics,
&semaphore,
)
.await;
process_batch(&mut batch, &mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
}
return;
}
// 尝试从存储中获取事件
// Try to get events from storage
match store.get(&key) {
Ok(event) => {
// 添加到批处理
// Add to batch
batch.push(event);
batch_keys.push(key);
metrics.increment_processing();
// 如果批次已满或距离上次刷新已经过了足够时间，则处理批次
// If the batch is full or enough time has passed since the last refresh, the batch will be processed
if batch.len() >= batch_size || last_flush.elapsed() >= BATCH_TIMEOUT {
process_batch(
&mut batch,
&mut batch_keys,
target,
MAX_RETRIES,
BASE_RETRY_DELAY,
&metrics,
&semaphore,
)
.await;
process_batch(&mut batch, &mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore)
.await;
last_flush = Instant::now();
}
}
Err(e) => {
error!(
"Failed to target: {}, get event {} from store: {}",
target.name(),
key.to_string(),
e
);
// 可以考虑删除无法读取的事件，防止无限循环尝试读取
error!("Failed to target: {}, get event {} from store: {}", target.name(), key.to_string(), e);
// Consider deleting unreadable events to prevent infinite loops from trying to read
match store.del(&key) {
Ok(_) => {
info!("Deleted corrupted event {} from store", key.to_string());
}
Err(del_err) => {
error!(
"Failed to delete corrupted event {}: {}",
key.to_string(),
del_err
);
error!("Failed to delete corrupted event {}: {}", key.to_string(), del_err);
}
}
@@ -268,12 +213,12 @@ pub async fn stream_events_with_batching(
}
}
// 小延迟再进行下一轮检查
// A small delay will be conducted to check the next round
tokio::time::sleep(Duration::from_millis(100)).await;
}
}
/// 处理事件批次
/// Processing event batches
async fn process_batch(
batch: &mut Vec<Event>,
batch_keys: &mut Vec<Key>,
@@ -283,16 +228,12 @@ async fn process_batch(
metrics: &Arc<NotificationMetrics>,
semaphore: &Arc<Semaphore>,
) {
debug!(
"Processing batch of {} events for target: {}",
batch.len(),
target.name()
);
debug!("Processing batch of {} events for target: {}", batch.len(), target.name());
if batch.is_empty() {
return;
}
// 获取信号量许可，限制并发
// Obtain semaphore permission to limit concurrency
let permit = match semaphore.clone().acquire_owned().await {
Ok(permit) => permit,
Err(e) => {
@@ -301,30 +242,26 @@ async fn process_batch(
}
};
// 处理批次中的每个事件
// Handle every event in the batch
for (_event, key) in batch.iter().zip(batch_keys.iter()) {
let mut retry_count = 0;
let mut success = false;
// 重试逻辑
// Retry logic
while retry_count < max_retries && !success {
match target.send_from_store(key.clone()).await {
Ok(_) => {
info!(
"Successfully sent event for target: {}, Key: {}",
target.name(),
key.to_string()
);
info!("Successfully sent event for target: {}, Key: {}", target.name(), key.to_string());
success = true;
metrics.increment_processed();
}
Err(e) => {
// 根据错误类型采用不同的重试策略
// Different retry strategies are adopted according to the error type
match &e {
TargetError::NotConnected => {
warn!("Target {} not connected, retrying...", target.name());
retry_count += 1;
tokio::time::sleep(base_delay * (1 << retry_count)).await; // 指数退避
tokio::time::sleep(base_delay * (1 << retry_count)).await; // Exponential backoff
}
TargetError::Timeout(_) => {
warn!("Timeout for target {}, retrying...", target.name());
@@ -332,7 +269,7 @@ async fn process_batch(
tokio::time::sleep(base_delay * (1 << retry_count)).await;
}
_ => {
// 永久性错误，跳过此事件
// Permanent error, skip this event
error!("Permanent error for target {}: {}", target.name(), e);
metrics.increment_failed();
break;
@@ -342,21 +279,17 @@ async fn process_batch(
}
}
// 处理最大重试次数耗尽的情况
// Handle the situation where the maximum number of retry exhaustion is exhausted
if retry_count >= max_retries && !success {
warn!(
"Max retries exceeded for event {}, target: {}, skipping",
key.to_string(),
target.name()
);
warn!("Max retries exceeded for event {}, target: {}, skipping", key.to_string(), target.name());
metrics.increment_failed();
}
}
// 清空已处理的批次
// Clear processed batches
batch.clear();
batch_keys.clear();
// 释放信号量许可（通过 drop）
// Release semaphore permission (via drop)
drop(permit);
}
crates/notify/src/target/constants.rs
+10 -10
View File
@@ -1,22 +1,22 @@
#[allow(dead_code)]
const NOTIFY_KAFKA_SUB_SYS: &str = "notify_kafka";
pub const NOTIFY_KAFKA_SUB_SYS: &str = "notify_kafka";
#[allow(dead_code)]
const NOTIFY_MQTT_SUB_SYS: &str = "notify_mqtt";
pub const NOTIFY_MQTT_SUB_SYS: &str = "notify_mqtt";
#[allow(dead_code)]
const NOTIFY_MY_SQL_SUB_SYS: &str = "notify_mysql";
pub const NOTIFY_MY_SQL_SUB_SYS: &str = "notify_mysql";
#[allow(dead_code)]
const NOTIFY_NATS_SUB_SYS: &str = "notify_nats";
pub const NOTIFY_NATS_SUB_SYS: &str = "notify_nats";
#[allow(dead_code)]
const NOTIFY_NSQ_SUB_SYS: &str = "notify_nsq";
pub const NOTIFY_NSQ_SUB_SYS: &str = "notify_nsq";
#[allow(dead_code)]
const NOTIFY_ES_SUB_SYS: &str = "notify_elasticsearch";
pub const NOTIFY_ES_SUB_SYS: &str = "notify_elasticsearch";
#[allow(dead_code)]
const NOTIFY_AMQP_SUB_SYS: &str = "notify_amqp";
pub const NOTIFY_AMQP_SUB_SYS: &str = "notify_amqp";
#[allow(dead_code)]
const NOTIFY_POSTGRES_SUB_SYS: &str = "notify_postgres";
pub const NOTIFY_POSTGRES_SUB_SYS: &str = "notify_postgres";
#[allow(dead_code)]
const NOTIFY_REDIS_SUB_SYS: &str = "notify_redis";
const NOTIFY_WEBHOOK_SUB_SYS: &str = "notify_webhook";
pub const NOTIFY_REDIS_SUB_SYS: &str = "notify_redis";
pub const NOTIFY_WEBHOOK_SUB_SYS: &str = "notify_webhook";
// Webhook constants
pub const WEBHOOK_ENDPOINT: &str = "endpoint";
crates/notify/src/utils.rs
+54 -50
View File
@@ -1,17 +1,14 @@
use std::env;
use std::fmt;
#[cfg(unix)]
use libc::uname;
#[cfg(unix)]
use std::ffi::CStr;
use std::os::unix::process::ExitStatusExt;
#[cfg(windows)]
use std::process::Command;
use std::os::windows::process::ExitStatusExt;
use std::{env, process};
// 定义 Rustfs 版本
// Define Rustfs version
const RUSTFS_VERSION: &str = "1.0.0";
// 业务类型枚举
// Business Type Enumeration
#[derive(Debug, Clone, PartialEq)]
pub enum ServiceType {
Basis,
@@ -33,7 +30,7 @@ impl ServiceType {
}
}
// UserAgent 结构体
// UserAgent structure
struct UserAgent {
os_platform: String,
arch: String,
@@ -42,7 +39,7 @@ struct UserAgent {
}
impl UserAgent {
// 创建新的 UserAgent 实例，接受业务类型参数
// Create a new UserAgent instance and accept business type parameters
fn new(service: ServiceType) -> Self {
let os_platform = Self::get_os_platform();
let arch = env::consts::ARCH.to_string();
@@ -56,7 +53,7 @@ impl UserAgent {
}
}
// 获取操作系统平台信息
// Obtain operating system platform information
fn get_os_platform() -> String {
if cfg!(target_os = "windows") {
Self::get_windows_platform()
@@ -69,14 +66,18 @@ impl UserAgent {
}
}
// 获取 Windows 平台信息
// Get Windows platform information
#[cfg(windows)]
fn get_windows_platform() -> String {
// 使用 cmd /c ver 获取版本
let output = Command::new("cmd")
// Use cmd /c ver to get the version
let output = process::Command::new("cmd")
.args(&["/C", "ver"])
.output()
.unwrap_or_default();
.unwrap_or_else(|_| process::Output {
status: process::ExitStatus::from_raw(0),
stdout: Vec::new(),
stderr: Vec::new(),
});
let version = String::from_utf8_lossy(&output.stdout);
let version = version
.lines()
@@ -92,27 +93,29 @@ impl UserAgent {
"N/A".to_string()
}
// 获取 macOS 平台信息
// Get macOS platform information
#[cfg(target_os = "macos")]
fn get_macos_platform() -> String {
unsafe {
let mut name = std::mem::zeroed();
if uname(&mut name) == 0 {
let release = CStr::from_ptr(name.release.as_ptr()).to_string_lossy();
// 映射内核版本（如 23.5.0）到 User-Agent 格式（如 14_5_0）
let major = release
.split('.')
.next()
.unwrap_or("14")
.parse::<i32>()
.unwrap_or(14);
let minor = if major >= 20 { major - 9 } else { 14 };
let patch = release.split('.').nth(1).unwrap_or("0");
format!("Macintosh; Intel Mac OS X {}_{}_{}", minor, patch, 0)
} else {
"Macintosh; Intel Mac OS X 14_5_0".to_string()
}
}
let output = process::Command::new("sw_vers")
.args(&["-productVersion"])
.output()
.unwrap_or_else(|_| process::Output {
status: process::ExitStatus::from_raw(0),
stdout: Vec::new(),
stderr: Vec::new(),
});
let version = String::from_utf8_lossy(&output.stdout).trim().to_string();
let parts: Vec<&str> = version.split('.').collect();
let major = parts.get(0).unwrap_or(&"10").parse::<i32>().unwrap_or(10);
let minor = parts.get(1).map_or("15", |&m| m);
let patch = parts.get(2).map_or("0", |&p| p);
// Detect whether it is an Apple Silicon chip
let arch = env::consts::ARCH;
let cpu_info = if arch == "aarch64" { "Apple" } else { "Intel" };
// Convert to User-Agent format
format!("Macintosh; {} Mac OS X {}_{}_{}", cpu_info, major, minor, patch)
}
#[cfg(not(target_os = "macos"))]
@@ -120,17 +123,22 @@ impl UserAgent {
"N/A".to_string()
}
// 获取 Linux 平台信息
// Get Linux platform information
#[cfg(target_os = "linux")]
fn get_linux_platform() -> String {
unsafe {
let mut name = std::mem::zeroed();
if uname(&mut name) == 0 {
let release = CStr::from_ptr(name.release.as_ptr()).to_string_lossy();
format!("X11; Linux {}", release)
} else {
"X11; Linux Unknown".to_string()
}
let output = process::Command::new("uname")
.arg("-r")
.output()
.unwrap_or_else(|_| process::Output {
status: process::ExitStatus::from_raw(0),
stdout: Vec::new(),
stderr: Vec::new(),
});
if output.status.success() {
let release = String::from_utf8_lossy(&output.stdout).trim().to_string();
format!("X11; Linux {}", release)
} else {
"X11; Linux Unknown".to_string()
}
}
@@ -140,15 +148,11 @@ impl UserAgent {
}
}
// 实现 Display trait 以格式化 User-Agent
// Implement Display trait to format User-Agent
impl fmt::Display for UserAgent {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
if self.service == ServiceType::Basis {
return write!(
f,
"Mozilla/5.0 ({}; {}) Rustfs/{}",
self.os_platform, self.arch, self.version
);
return write!(f, "Mozilla/5.0 ({}; {}) Rustfs/{}", self.os_platform, self.arch, self.version);
}
write!(
f,
@@ -161,7 +165,7 @@ impl fmt::Display for UserAgent {
}
}
// 获取 User-Agent 字符串，接受业务类型参数
// Get the User-Agent string and accept business type parameters
pub fn get_user_agent(service: ServiceType) -> String {
UserAgent::new(service).to_string()
}