gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-27 15:37:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix notification event stream cleanup, add bounded send concurrency, and reduce overhead (#1224)

Browse Source
This commit is contained in:
houseme
2025-12-22 00:57:05 +08:00
committed by GitHub
parent 1c51e204ab
commit 08f1a31f3f
20 changed files with 921 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/notify/src/rules/rules_map.rs
+81 -26
View File
@@ -12,8 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::pattern_rules::PatternRules;
use super::target_id_set::TargetIdSet;
use crate::rules::{PatternRules, TargetIdSet};
use hashbrown::HashMap;
use rustfs_targets::EventName;
use rustfs_targets::arn::TargetID;
@@ -31,6 +30,9 @@ pub struct RulesMap {
impl RulesMap {
/// Create a new, empty RulesMap.
///
/// # Returns
/// A new instance of RulesMap with an empty map and a total_events_mask set to 0.
pub fn new() -> Self {
Default::default()
}
@@ -67,12 +69,12 @@ impl RulesMap {
/// Merge another RulesMap.
/// `RulesMap.Add(rulesMap2 RulesMap) corresponding to Go
///
/// # Parameters
/// * `other_map` - The other RulesMap to be merged into the current one.
pub fn add_map(&mut self, other_map: &Self) {
for (event_name, other_pattern_rules) in &other_map.map {
let self_pattern_rules = self.map.entry(*event_name).or_default();
// PatternRules::union Returns the new PatternRules, we need to modify the existing ones
let merged_rules = self_pattern_rules.union(other_pattern_rules);
*self_pattern_rules = merged_rules;
self.map.entry(*event_name).or_default().union_in_place(other_pattern_rules);
}
// Directly merge two masks.
self.total_events_mask |= other_map.total_events_mask;
@@ -81,11 +83,14 @@ impl RulesMap {
/// Remove another rule defined in the RulesMap from the current RulesMap.
///
/// After the rule is removed, `total_events_mask` is recalculated to ensure its accuracy.
///
/// # Parameters
/// * `other_map` - The other RulesMap containing rules to be removed from the current one.
pub fn remove_map(&mut self, other_map: &Self) {
let mut events_to_remove = Vec::new();
for (event_name, self_pattern_rules) in &mut self.map {
if let Some(other_pattern_rules) = other_map.map.get(event_name) {
*self_pattern_rules = self_pattern_rules.difference(other_pattern_rules);
self_pattern_rules.difference_in_place(other_pattern_rules);
if self_pattern_rules.is_empty() {
events_to_remove.push(*event_name);
}
@@ -102,6 +107,9 @@ impl RulesMap {
///
/// This method uses a bitmask for a quick check of O(1) complexity.
/// `event_name` can be a compound type, such as `ObjectCreatedAll`.
///
/// # Parameters
/// * `event_name` - The event name to check for subscribers.
pub fn has_subscriber(&self, event_name: &EventName) -> bool {
// event_name.mask() will handle compound events correctly
(self.total_events_mask & event_name.mask()) != 0
@@ -112,39 +120,54 @@ impl RulesMap {
/// # Notice
/// The `event_name` parameter should be a specific, non-compound event type.
/// Because this is taken from the `Event` object that actually occurs.
///
/// # Parameters
/// * `event_name` - The specific event name to match against.
/// * `object_key` - The object key to match against the patterns in the rules.
///
/// # Returns
/// * A set of TargetIDs that match the given event and object key.
pub fn match_rules(&self, event_name: EventName, object_key: &str) -> TargetIdSet {
// Use bitmask to quickly determine whether there is a matching rule
if (self.total_events_mask & event_name.mask()) == 0 {
return TargetIdSet::new(); // No matching rules
}
// First try to directly match the event name
if let Some(pattern_rules) = self.map.get(&event_name) {
let targets = pattern_rules.match_targets(object_key);
if !targets.is_empty() {
return targets;
}
}
// Go's RulesMap[eventName] is directly retrieved, and if it does not exist, it is empty Rules.
// Rust's HashMap::get returns Option. If the event name does not exist, there is no rule.
// Compound events (such as ObjectCreatedAll) have been expanded as a single event when add_rule_config.
// Therefore, a single event name should be used when querying.
// If event_name itself is a single type, look it up directly.
// If event_name is a compound type, Go's logic is expanded when added.
// Here match_rules should receive events that may already be single.
// If the caller passes in a compound event, it should expand itself or handle this function first.
// Assume that event_name is already a specific event that can be used for searching.
// In Go, RulesMap[eventName] returns empty rules if the key doesn't exist.
// Rust's HashMap::get returns Option, so missing key means no rules.
// Compound events like ObjectCreatedAll are expanded into specific events during add_rule_config.
// Thus, queries should use specific event names.
// If event_name is compound, expansion happens at addition time.
// match_rules assumes event_name is already a specific event for lookup.
// Callers should expand compound events before calling this method.
self.map
.get(&event_name)
.map_or_else(TargetIdSet::new, |pr| pr.match_targets(object_key))
}
/// Check if RulesMap is empty.
///
/// # Returns
/// * `true` if there are no rules in the map; `false` otherwise
pub fn is_empty(&self) -> bool {
self.map.is_empty()
}
/// Determine whether the current RulesMap contains the specified TargetID (referenced by any event / pattern).
///
/// # Parameters
/// * `target_id` - The TargetID to check for existence within the RulesMap
///
/// # Returns
/// * `true` if the TargetID exists in any of the PatternRules; `false` otherwise.
pub fn contains_target_id(&self, target_id: &TargetID) -> bool {
self.map.values().any(|pr| pr.contains_target_id(target_id))
}
/// Returns a clone of internal rules for use in scenarios such as BucketNotificationConfig::validate.
///
/// # Returns
/// A reference to the internal HashMap of EventName to PatternRules.
pub fn inner(&self) -> &HashMap<EventName, PatternRules> {
&self.map
}
@@ -160,18 +183,32 @@ impl RulesMap {
}
/// Remove rules and optimize performance
///
/// # Parameters
/// * `event_name` - The EventName from which to remove the rule.
/// * `pattern` - The pattern of the rule to be removed.
#[allow(dead_code)]
pub fn remove_rule(&mut self, event_name: &EventName, pattern: &str) {
let mut remove_event = false;
if let Some(pattern_rules) = self.map.get_mut(event_name) {
pattern_rules.rules.remove(pattern);
pattern_rules.remove_pattern(pattern);
if pattern_rules.is_empty() {
self.map.remove(event_name);
remove_event = true;
}
}
if remove_event {
self.map.remove(event_name);
}
self.recalculate_mask(); // Delay calculation mask
}
/// Batch Delete Rules
/// Batch Delete Rules and Optimize Performance
///
/// # Parameters
/// * `event_names` - A slice of EventNames to be removed.
#[allow(dead_code)]
pub fn remove_rules(&mut self, event_names: &[EventName]) {
for event_name in event_names {
@@ -181,9 +218,27 @@ impl RulesMap {
}
/// Update rules and optimize performance
///
/// # Parameters
/// * `event_name` - The EventName to update.
/// * `pattern` - The pattern of the rule to be updated.
/// * `target_id` - The TargetID to be added.
#[allow(dead_code)]
pub fn update_rule(&mut self, event_name: EventName, pattern: String, target_id: TargetID) {
self.map.entry(event_name).or_default().add(pattern, target_id);
self.total_events_mask |= event_name.mask(); // Update only the relevant bitmask
}
/// Iterate all EventName keys contained in this RulesMap.
///
/// Used by snapshot compilation to compute bucket event_mask.
///
/// # Returns
/// An iterator over all EventName keys in the RulesMap.
#[inline]
pub fn iter_events(&self) -> impl Iterator<Item = EventName> + '_ {
// `inner()` is already used by config.rs, so we reuse it here.
// If the key type is `EventName`, `.copied()` is the cheapest way to return values.
self.inner().keys().copied()
}
}