refactor(targets): unify queue/connectivity handling and coverage (#2953)

Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
Co-authored-by: marshawcoco <marshawcoco@gmail.com>

crates/notify/src/bucket_config_manager.rs

@@ -0,0 +1,125 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::{
+    BucketNotificationConfig, NotificationError, config_manager::notify_configuration_hint,
+    notification_system_subscriber::NotificationSystemSubscriberView, notifier::EventNotifier, rule_engine::NotifyRuleEngine,
+    rules::ParseConfigError,
+};
+use rustfs_s3_common::EventName;
+use std::sync::Arc;
+use tracing::{debug, info, warn};
+
+#[derive(Clone)]
+pub struct NotifyBucketConfigManager {
+    notifier: Arc<EventNotifier>,
+    rule_engine: NotifyRuleEngine,
+    subscriber_view: Arc<NotificationSystemSubscriberView>,
+}
+
+impl NotifyBucketConfigManager {
+    pub fn new(
+        notifier: Arc<EventNotifier>,
+        rule_engine: NotifyRuleEngine,
+        subscriber_view: Arc<NotificationSystemSubscriberView>,
+    ) -> Self {
+        Self {
+            notifier,
+            rule_engine,
+            subscriber_view,
+        }
+    }
+
+    pub async fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool {
+        if !self.subscriber_view.has_subscriber(bucket, event) {
+            return false;
+        }
+        self.rule_engine.has_subscriber(bucket, event).await
+    }
+
+    pub async fn load_bucket_notification_config(
+        &self,
+        bucket: &str,
+        cfg: &BucketNotificationConfig,
+    ) -> Result<(), NotificationError> {
+        let arn_list = self.notifier.get_arn_list(&cfg.region).await;
+        if arn_list.is_empty() {
+            return Err(NotificationError::Configuration(notify_configuration_hint()));
+        }
+        info!("Available ARNs: {:?}", arn_list);
+
+        if let Err(e) = cfg.validate(&cfg.region, &arn_list) {
+            debug!("Bucket notification config validation region:{} failed: {}", &cfg.region, e);
+            if !matches!(e, ParseConfigError::ArnNotFound(_)) {
+                return Err(NotificationError::BucketNotification(e.to_string()));
+            }
+            warn!(
+                bucket = %bucket,
+                region = %cfg.region,
+                error = %e,
+                "Bucket notification config references missing target ARN; keeping compatibility and loading remaining rules"
+            );
+        }
+
+        self.subscriber_view.apply_bucket_config(bucket, cfg);
+        self.rule_engine.set_bucket_rules(bucket, cfg.get_rules_map().clone()).await;
+        info!("Loaded notification config for bucket: {}", bucket);
+        Ok(())
+    }
+
+    pub async fn remove_bucket_notification_config(&self, bucket: &str) {
+        self.subscriber_view.clear_bucket(bucket);
+        self.rule_engine.clear_bucket_rules(bucket).await;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::NotifyBucketConfigManager;
+    use crate::{
+        BucketNotificationConfig, integration::NotificationMetrics,
+        notification_system_subscriber::NotificationSystemSubscriberView, notifier::EventNotifier, rule_engine::NotifyRuleEngine,
+    };
+    use rustfs_s3_common::EventName;
+    use rustfs_targets::arn::TargetID;
+    use std::sync::Arc;
+
+    fn build_manager() -> NotifyBucketConfigManager {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier::new(metrics, rule_engine.clone()));
+        let subscriber_view = Arc::new(NotificationSystemSubscriberView::new());
+        NotifyBucketConfigManager::new(notifier, rule_engine, subscriber_view)
+    }
+
+    #[tokio::test]
+    async fn bucket_config_manager_reports_no_subscriber_for_empty_state() {
+        let manager = build_manager();
+        assert!(!manager.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
+    }
+
+    #[tokio::test]
+    async fn bucket_config_manager_clears_bucket_snapshot() {
+        let manager = build_manager();
+        let target_id = TargetID::new("primary".to_string(), "webhook".to_string());
+        let mut cfg = BucketNotificationConfig::new("us-east-1");
+        cfg.add_rule(&[EventName::ObjectCreatedPut], "*".to_string(), target_id);
+
+        manager.subscriber_view.apply_bucket_config("bucket", &cfg);
+        assert!(manager.subscriber_view.has_subscriber("bucket", &EventName::ObjectCreatedPut));
+
+        manager.remove_bucket_notification_config("bucket").await;
+        assert!(!manager.subscriber_view.has_subscriber("bucket", &EventName::ObjectCreatedPut));
+    }
+}

crates/notify/src/config_manager.rs

@@ -0,0 +1,338 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::{
+    Event, NotificationError, registry::TargetRegistry, rule_engine::NotifyRuleEngine, runtime_facade::NotifyRuntimeFacade,
+};
+use rustfs_config::notify::{
+    NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS,
+    NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
+};
+use rustfs_ecstore::config::{Config, KVS};
+use rustfs_targets::{Target, arn::TargetID};
+use std::sync::Arc;
+use tokio::sync::RwLock;
+use tracing::{debug, info, warn};
+
+pub(crate) fn notify_configuration_hint() -> String {
+    let webhook_enable_primary = format!("{}_PRIMARY", rustfs_config::notify::ENV_NOTIFY_WEBHOOK_ENABLE);
+    let webhook_endpoint_primary = format!("{}_PRIMARY", rustfs_config::notify::ENV_NOTIFY_WEBHOOK_ENDPOINT);
+    format!(
+        "No notify targets configured. Check {}=true and instance-scoped target env vars (for example {webhook_enable_primary} + {webhook_endpoint_primary} for arn:rustfs:sqs::primary:webhook). If using default queue_dir, ensure {} is writable.",
+        rustfs_config::ENV_NOTIFY_ENABLE,
+        rustfs_config::EVENT_DEFAULT_DIR,
+    )
+}
+
+fn subsystem_target_type(target_type: &str) -> &str {
+    match target_type {
+        NOTIFY_AMQP_SUB_SYS => "amqp",
+        NOTIFY_WEBHOOK_SUB_SYS => "webhook",
+        NOTIFY_KAFKA_SUB_SYS => "kafka",
+        NOTIFY_MQTT_SUB_SYS => "mqtt",
+        NOTIFY_MYSQL_SUB_SYS => "mysql",
+        NOTIFY_NATS_SUB_SYS => "nats",
+        NOTIFY_POSTGRES_SUB_SYS => "postgres",
+        NOTIFY_PULSAR_SUB_SYS => "pulsar",
+        NOTIFY_REDIS_SUB_SYS => "redis",
+        _ => target_type,
+    }
+}
+
+pub fn runtime_target_id_for_subsystem(target_type: &str, target_name: &str) -> TargetID {
+    TargetID {
+        id: target_name.to_lowercase(),
+        name: subsystem_target_type(target_type).to_string(),
+    }
+}
+
+#[derive(Clone)]
+pub struct NotifyConfigManager {
+    config: Arc<RwLock<Config>>,
+    registry: Arc<TargetRegistry>,
+    rule_engine: NotifyRuleEngine,
+    runtime_facade: NotifyRuntimeFacade,
+}
+
+impl NotifyConfigManager {
+    pub fn new(
+        config: Arc<RwLock<Config>>,
+        registry: Arc<TargetRegistry>,
+        rule_engine: NotifyRuleEngine,
+        runtime_facade: NotifyRuntimeFacade,
+    ) -> Self {
+        Self {
+            config,
+            registry,
+            rule_engine,
+            runtime_facade,
+        }
+    }
+
+    pub async fn init(&self) -> Result<(), NotificationError> {
+        info!("Initialize notification system...");
+
+        let config = {
+            let guard = self.config.read().await;
+            debug!(
+                subsystem_count = guard.0.len(),
+                "Initializing notification system with configuration summary"
+            );
+            guard.clone()
+        };
+
+        let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self.registry.create_targets_from_config(&config).await?;
+
+        info!("{} notification targets were created", targets.len());
+        if targets.is_empty() {
+            warn!("{}", notify_configuration_hint());
+        }
+
+        let activation = self.runtime_facade.activate_targets_with_replay(targets).await;
+        self.runtime_facade.replace_targets(activation).await?;
+        info!("Notification system initialized");
+        Ok(())
+    }
+
+    pub async fn remove_target(&self, target_id: &TargetID, target_type: &str) -> Result<(), NotificationError> {
+        info!("Attempting to remove target: {}", target_id);
+
+        let ttype = target_type.to_lowercase();
+        let tname = target_id.id.to_lowercase();
+
+        self.update_config_and_reload(|config| {
+            let mut changed = false;
+            if let Some(targets_of_type) = config.0.get_mut(&ttype) {
+                if targets_of_type.remove(&tname).is_some() {
+                    info!("Removed target {} from configuration", target_id);
+                    changed = true;
+                }
+                if targets_of_type.is_empty() {
+                    config.0.remove(&ttype);
+                }
+            }
+            if !changed {
+                warn!("Target {} not found in configuration", target_id);
+            }
+            changed
+        })
+        .await
+    }
+
+    pub async fn set_target_config(&self, target_type: &str, target_name: &str, kvs: KVS) -> Result<(), NotificationError> {
+        info!("Setting config for target {} of type {}", target_name, target_type);
+        let ttype = target_type.to_lowercase();
+        let tname = target_name.to_lowercase();
+        self.update_config_and_reload(|config| {
+            config.0.entry(ttype.clone()).or_default().insert(tname.clone(), kvs.clone());
+            true
+        })
+        .await
+    }
+
+    pub async fn remove_target_config(&self, target_type: &str, target_name: &str) -> Result<(), NotificationError> {
+        info!("Removing config for target {} of type {}", target_name, target_type);
+
+        let ttype = target_type.to_lowercase();
+        let tname = target_name.to_lowercase();
+        let target_id = runtime_target_id_for_subsystem(&ttype, &tname);
+
+        if self.rule_engine.is_target_bound_to_any_bucket(&target_id).await {
+            return Err(NotificationError::Configuration(format!(
+                "Target is still bound to bucket rules and deletion is prohibited: type={} name={}",
+                ttype, tname
+            )));
+        }
+
+        self.update_config_and_reload(|config| {
+            let mut changed = false;
+            if let Some(targets) = config.0.get_mut(&ttype) {
+                if targets.remove(&tname).is_some() {
+                    changed = true;
+                }
+                if targets.is_empty() {
+                    config.0.remove(&ttype);
+                }
+            }
+            if !changed {
+                info!("Target {} of type {} not found, no changes made.", target_name, target_type);
+            }
+            debug!(
+                subsystem_count = config.0.len(),
+                "Target config removal processed and configuration summary updated"
+            );
+            changed
+        })
+        .await
+    }
+
+    pub async fn reload_config(&self, new_config: Config) -> Result<(), NotificationError> {
+        info!("Reload notification configuration starts");
+
+        self.update_config(new_config.clone()).await;
+
+        let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self
+            .registry
+            .create_targets_from_config(&new_config)
+            .await
+            .map_err(NotificationError::Target)?;
+
+        info!("{} notification targets were created from the new configuration", targets.len());
+        if targets.is_empty() {
+            warn!("{}", notify_configuration_hint());
+        }
+
+        let activation = self.runtime_facade.activate_targets_with_replay(targets).await;
+        self.runtime_facade.replace_targets(activation).await?;
+        info!("Configuration reloaded end");
+        Ok(())
+    }
+
+    async fn update_config(&self, new_config: Config) {
+        let mut config = self.config.write().await;
+        *config = new_config;
+    }
+
+    async fn update_config_and_reload<F>(&self, mut modifier: F) -> Result<(), NotificationError>
+    where
+        F: FnMut(&mut Config) -> bool,
+    {
+        let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
+            return Err(NotificationError::StorageNotAvailable(
+                "Failed to save target configuration: server storage not initialized".to_string(),
+            ));
+        };
+
+        let mut new_config = rustfs_ecstore::config::com::read_config_without_migrate(store.clone())
+            .await
+            .map_err(|e| NotificationError::ReadConfig(e.to_string()))?;
+
+        if !modifier(&mut new_config) {
+            info!("Configuration not changed, skipping save and reload.");
+            return Ok(());
+        }
+
+        rustfs_ecstore::config::com::save_server_config(store, &new_config)
+            .await
+            .map_err(|e| NotificationError::SaveConfig(e.to_string()))?;
+
+        info!("Configuration updated. Reloading system...");
+        self.reload_config(new_config).await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{NotifyConfigManager, runtime_target_id_for_subsystem};
+    use crate::{
+        integration::NotificationMetrics, notifier::EventNotifier, registry::TargetRegistry, rule_engine::NotifyRuleEngine,
+        runtime_facade::NotifyRuntimeFacade,
+    };
+    use rustfs_config::notify::{
+        NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_SUB_SYS,
+        NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
+    };
+    use rustfs_ecstore::config::Config;
+    use rustfs_targets::ReplayWorkerManager;
+    use std::sync::Arc;
+    use tokio::sync::{RwLock, Semaphore};
+
+    fn build_manager() -> NotifyConfigManager {
+        let config = Arc::new(RwLock::new(Config::default()));
+        let registry = Arc::new(TargetRegistry::new());
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
+        let target_list = notifier.target_list();
+        let runtime_facade = NotifyRuntimeFacade::new(
+            target_list,
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            Arc::new(Semaphore::new(4)),
+            metrics,
+        );
+
+        NotifyConfigManager::new(config, registry, rule_engine, runtime_facade)
+    }
+
+    #[tokio::test]
+    async fn config_manager_init_accepts_empty_target_set() {
+        let manager = build_manager();
+        manager.init().await.expect("init should succeed for empty targets");
+    }
+
+    #[tokio::test]
+    async fn config_manager_reload_accepts_empty_target_set() {
+        let manager = build_manager();
+        manager
+            .reload_config(Config::default())
+            .await
+            .expect("reload_config should succeed for empty targets");
+    }
+
+    #[test]
+    fn runtime_target_id_for_subsystem_maps_notify_webhook_to_runtime_type() {
+        let target_id = runtime_target_id_for_subsystem(NOTIFY_WEBHOOK_SUB_SYS, "Primary");
+        assert_eq!(target_id.id, "primary");
+        assert_eq!(target_id.name, "webhook");
+    }
+
+    #[test]
+    fn runtime_target_id_for_subsystem_maps_notify_amqp_to_runtime_type() {
+        let target_id = runtime_target_id_for_subsystem(NOTIFY_AMQP_SUB_SYS, "Primary");
+        assert_eq!(target_id.id, "primary");
+        assert_eq!(target_id.name, "amqp");
+    }
+
+    #[test]
+    fn runtime_target_id_for_subsystem_maps_notify_mqtt_to_runtime_type() {
+        let target_id = runtime_target_id_for_subsystem(NOTIFY_MQTT_SUB_SYS, "Analytics");
+        assert_eq!(target_id.id, "analytics");
+        assert_eq!(target_id.name, "mqtt");
+    }
+
+    #[test]
+    fn runtime_target_id_for_subsystem_maps_notify_kafka_to_runtime_type() {
+        let target_id = runtime_target_id_for_subsystem(NOTIFY_KAFKA_SUB_SYS, "EventBus");
+        assert_eq!(target_id.id, "eventbus");
+        assert_eq!(target_id.name, "kafka");
+    }
+
+    #[test]
+    fn runtime_target_id_for_subsystem_maps_notify_nats_to_runtime_type() {
+        let target_id = runtime_target_id_for_subsystem(NOTIFY_NATS_SUB_SYS, "Bus");
+        assert_eq!(target_id.id, "bus");
+        assert_eq!(target_id.name, "nats");
+    }
+
+    #[test]
+    fn runtime_target_id_for_subsystem_maps_notify_pulsar_to_runtime_type() {
+        let target_id = runtime_target_id_for_subsystem(NOTIFY_PULSAR_SUB_SYS, "Ledger");
+        assert_eq!(target_id.id, "ledger");
+        assert_eq!(target_id.name, "pulsar");
+    }
+
+    #[test]
+    fn runtime_target_id_for_subsystem_maps_notify_redis_to_runtime_type() {
+        let target_id = runtime_target_id_for_subsystem(NOTIFY_REDIS_SUB_SYS, "Primary");
+        assert_eq!(target_id.id, "primary");
+        assert_eq!(target_id.name, "redis");
+    }
+
+    #[test]
+    fn runtime_target_id_for_subsystem_maps_notify_postgres_to_runtime_type() {
+        let target_id = runtime_target_id_for_subsystem(NOTIFY_POSTGRES_SUB_SYS, "AuditTrail");
+        assert_eq!(target_id.id, "audittrail");
+        assert_eq!(target_id.name, "postgres");
+    }
+}

crates/notify/src/event_bridge.rs

@@ -0,0 +1,15 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub use crate::pipeline::{LiveEventHistory, NotifyEventBridge};

crates/notify/src/factory.rs

@@ -13,146 +13,11 @@
 // limitations under the License.
 
 use crate::Event;
-use rustfs_config::EVENT_DEFAULT_DIR;
-use rustfs_config::notify::{
-    NOTIFY_AMQP_KEYS, NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS,
-    NOTIFY_MYSQL_KEYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_KEYS,
-    NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_DEFAULT_CHANNEL, NOTIFY_REDIS_KEYS,
-    NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
-};
-use rustfs_targets::config::{
-    build_amqp_args, build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args,
-    build_pulsar_args, build_redis_args, build_webhook_args, validate_amqp_config, validate_kafka_config, validate_mqtt_config,
-    validate_mysql_config, validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config,
-    validate_webhook_config,
-};
-use rustfs_targets::target::{ChannelTargetType, TargetType};
-use rustfs_targets::{BuiltinTargetDescriptor, TargetPluginDescriptor, TargetRequestValidator, boxed_target};
+use rustfs_targets::catalog::builtin::builtin_notify_target_descriptors;
+use rustfs_targets::{BuiltinTargetDescriptor, TargetPluginDescriptor};
 
 pub fn builtin_target_descriptors() -> Vec<BuiltinTargetDescriptor<Event>> {
-    vec![
-        BuiltinTargetDescriptor::new(
-            NOTIFY_WEBHOOK_SUB_SYS,
-            TargetRequestValidator::Webhook,
-            TargetPluginDescriptor::new(
-                ChannelTargetType::Webhook.as_str(),
-                NOTIFY_WEBHOOK_KEYS,
-                |config| validate_webhook_config(config, EVENT_DEFAULT_DIR),
-                |id, config| {
-                    let args = build_webhook_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
-                    Ok(boxed_target(rustfs_targets::target::webhook::WebhookTarget::new(id, args)?))
-                },
-            ),
-        ),
-        BuiltinTargetDescriptor::new(
-            NOTIFY_AMQP_SUB_SYS,
-            TargetRequestValidator::Amqp(TargetType::NotifyEvent),
-            TargetPluginDescriptor::new(
-                ChannelTargetType::Amqp.as_str(),
-                NOTIFY_AMQP_KEYS,
-                |config| validate_amqp_config(config, EVENT_DEFAULT_DIR),
-                |id, config| {
-                    let args = build_amqp_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
-                    Ok(boxed_target(rustfs_targets::target::amqp::AMQPTarget::new(id, args)?))
-                },
-            ),
-        ),
-        BuiltinTargetDescriptor::new(
-            NOTIFY_KAFKA_SUB_SYS,
-            TargetRequestValidator::Kafka(TargetType::NotifyEvent),
-            TargetPluginDescriptor::new(
-                ChannelTargetType::Kafka.as_str(),
-                NOTIFY_KAFKA_KEYS,
-                |config| validate_kafka_config(config, EVENT_DEFAULT_DIR),
-                |id, config| {
-                    let args = build_kafka_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
-                    Ok(boxed_target(rustfs_targets::target::kafka::KafkaTarget::new(id, args)?))
-                },
-            ),
-        ),
-        BuiltinTargetDescriptor::new(
-            NOTIFY_MQTT_SUB_SYS,
-            TargetRequestValidator::Mqtt,
-            TargetPluginDescriptor::new(
-                ChannelTargetType::Mqtt.as_str(),
-                NOTIFY_MQTT_KEYS,
-                validate_mqtt_config,
-                |id, config| {
-                    let args = build_mqtt_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
-                    Ok(boxed_target(rustfs_targets::target::mqtt::MQTTTarget::new(id, args)?))
-                },
-            ),
-        ),
-        BuiltinTargetDescriptor::new(
-            NOTIFY_MYSQL_SUB_SYS,
-            TargetRequestValidator::MySql(TargetType::NotifyEvent),
-            TargetPluginDescriptor::new(
-                ChannelTargetType::MySql.as_str(),
-                NOTIFY_MYSQL_KEYS,
-                |config| validate_mysql_config(config, EVENT_DEFAULT_DIR),
-                |id, config| {
-                    let args = build_mysql_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
-                    Ok(boxed_target(rustfs_targets::target::mysql::MySqlTarget::new(id, args)?))
-                },
-            ),
-        ),
-        BuiltinTargetDescriptor::new(
-            NOTIFY_NATS_SUB_SYS,
-            TargetRequestValidator::Nats(TargetType::NotifyEvent),
-            TargetPluginDescriptor::new(
-                ChannelTargetType::Nats.as_str(),
-                NOTIFY_NATS_KEYS,
-                |config| validate_nats_config(config, EVENT_DEFAULT_DIR),
-                |id, config| {
-                    let args = build_nats_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
-                    Ok(boxed_target(rustfs_targets::target::nats::NATSTarget::new(id, args)?))
-                },
-            ),
-        ),
-        BuiltinTargetDescriptor::new(
-            NOTIFY_POSTGRES_SUB_SYS,
-            TargetRequestValidator::Postgres(TargetType::NotifyEvent),
-            TargetPluginDescriptor::new(
-                ChannelTargetType::Postgres.as_str(),
-                NOTIFY_POSTGRES_KEYS,
-                |config| validate_postgres_config(config, EVENT_DEFAULT_DIR),
-                |id, config| {
-                    let args = build_postgres_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
-                    Ok(boxed_target(rustfs_targets::target::postgres::PostgresTarget::new(id, args)?))
-                },
-            ),
-        ),
-        BuiltinTargetDescriptor::new(
-            NOTIFY_REDIS_SUB_SYS,
-            TargetRequestValidator::Redis {
-                default_channel: NOTIFY_REDIS_DEFAULT_CHANNEL,
-                target_type: TargetType::NotifyEvent,
-            },
-            TargetPluginDescriptor::new(
-                ChannelTargetType::Redis.as_str(),
-                NOTIFY_REDIS_KEYS,
-                |config| validate_redis_config(config, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL),
-                |id, config| {
-                    let args =
-                        build_redis_args(config, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL, TargetType::NotifyEvent)?;
-                    Ok(boxed_target(rustfs_targets::target::redis::RedisTarget::new(id, args)?))
-                },
-            ),
-        ),
-        BuiltinTargetDescriptor::new(
-            NOTIFY_PULSAR_SUB_SYS,
-            TargetRequestValidator::Pulsar(TargetType::NotifyEvent),
-            TargetPluginDescriptor::new(
-                ChannelTargetType::Pulsar.as_str(),
-                NOTIFY_PULSAR_KEYS,
-                |config| validate_pulsar_config(config, EVENT_DEFAULT_DIR),
-                |id, config| {
-                    let args = build_pulsar_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
-                    Ok(boxed_target(rustfs_targets::target::pulsar::PulsarTarget::new(id, args)?))
-                },
-            ),
-        ),
-    ]
+    builtin_notify_target_descriptors::<Event>()
 }
 
 pub fn builtin_target_plugins() -> Vec<TargetPluginDescriptor<Event>> {

crates/notify/src/integration.rs

@@ -13,66 +13,29 @@
 // limitations under the License.
 
 use crate::notification_system_subscriber::NotificationSystemSubscriberView;
-use crate::notifier::TargetList;
+use crate::notifier::{EventNotifier, TargetList};
+use crate::services::NotifyServices;
 use crate::{
-    Event,
-    error::NotificationError,
-    notifier::EventNotifier,
-    registry::TargetRegistry,
-    rules::{BucketNotificationConfig, ParseConfigError},
-    stream,
+    Event, error::NotificationError, pipeline::LiveEventHistory, registry::TargetRegistry, rule_engine::NotifyRuleEngine,
+    rules::BucketNotificationConfig,
 };
 use hashbrown::HashMap;
-use rustfs_config::notify::{
-    DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_WEBHOOK_ENABLE,
-    ENV_NOTIFY_WEBHOOK_ENDPOINT, NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS,
-    NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
-};
-use rustfs_config::{ENV_NOTIFY_ENABLE, EVENT_DEFAULT_DIR};
+use metrics::{counter, gauge};
+use rustfs_config::notify::{DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY};
 use rustfs_ecstore::config::{Config, KVS};
 use rustfs_s3_common::EventName;
 use rustfs_targets::arn::TargetID;
-use rustfs_targets::store::{Key, Store};
-use rustfs_targets::target::QueuedPayload;
-use rustfs_targets::{StoreError, Target};
-use std::collections::VecDeque;
+use rustfs_targets::{ReplayWorkerManager, RuntimeTargetHealthSnapshot, SharedTarget};
 use std::sync::Arc;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::time::{Duration, Instant};
-use tokio::sync::{RwLock, Semaphore, broadcast, mpsc};
-use tracing::{debug, info, warn};
+use tokio::sync::{RwLock, Semaphore, broadcast};
+use tracing::info;
 
-const MAX_RECENT_LIVE_EVENTS: usize = 1024;
-
-fn notify_configuration_hint() -> String {
-    let webhook_enable_primary = format!("{ENV_NOTIFY_WEBHOOK_ENABLE}_PRIMARY");
-    let webhook_endpoint_primary = format!("{ENV_NOTIFY_WEBHOOK_ENDPOINT}_PRIMARY");
-    format!(
-        "No notify targets configured. Check {ENV_NOTIFY_ENABLE}=true and instance-scoped target env vars (for example {webhook_enable_primary} + {webhook_endpoint_primary} for arn:rustfs:sqs::primary:webhook). If using default queue_dir, ensure {EVENT_DEFAULT_DIR} is writable."
-    )
-}
-
-fn subsystem_target_type(target_type: &str) -> &str {
-    match target_type {
-        NOTIFY_AMQP_SUB_SYS => "amqp",
-        NOTIFY_WEBHOOK_SUB_SYS => "webhook",
-        NOTIFY_KAFKA_SUB_SYS => "kafka",
-        NOTIFY_MQTT_SUB_SYS => "mqtt",
-        NOTIFY_MYSQL_SUB_SYS => "mysql",
-        NOTIFY_NATS_SUB_SYS => "nats",
-        NOTIFY_POSTGRES_SUB_SYS => "postgres",
-        NOTIFY_PULSAR_SUB_SYS => "pulsar",
-        NOTIFY_REDIS_SUB_SYS => "redis",
-        _ => target_type,
-    }
-}
-
-fn runtime_target_id_for_subsystem(target_type: &str, target_name: &str) -> TargetID {
-    TargetID {
-        id: target_name.to_lowercase(),
-        name: subsystem_target_type(target_type).to_string(),
-    }
-}
+const METRIC_NOTIFICATION_CURRENT_SEND_IN_PROGRESS: &str = "rustfs_notification_current_send_in_progress";
+const METRIC_NOTIFICATION_EVENTS_ERRORS_TOTAL: &str = "rustfs_notification_events_errors_total";
+const METRIC_NOTIFICATION_EVENTS_SENT_TOTAL: &str = "rustfs_notification_events_sent_total";
+const METRIC_NOTIFICATION_EVENTS_SKIPPED_TOTAL: &str = "rustfs_notification_events_skipped_total";
 
 #[derive(Clone)]
 pub struct LiveEventBatch {
@@ -81,46 +44,6 @@ pub struct LiveEventBatch {
     pub truncated: bool,
 }
 
-#[derive(Default)]
-struct LiveEventHistory {
-    next_sequence: u64,
-    events: VecDeque<(u64, Arc<Event>)>,
-}
-
-impl LiveEventHistory {
-    fn record(&mut self, event: Arc<Event>) {
-        self.next_sequence = self.next_sequence.saturating_add(1);
-        self.events.push_back((self.next_sequence, event));
-        while self.events.len() > MAX_RECENT_LIVE_EVENTS {
-            self.events.pop_front();
-        }
-    }
-
-    fn snapshot_since(&self, after_sequence: u64, limit: usize) -> LiveEventBatch {
-        let mut events = Vec::new();
-        let mut next_sequence = after_sequence;
-        let mut truncated = false;
-
-        for (sequence, event) in self.events.iter() {
-            if *sequence <= after_sequence {
-                continue;
-            }
-            if events.len() >= limit {
-                truncated = true;
-                break;
-            }
-            next_sequence = *sequence;
-            events.push(event.clone());
-        }
-
-        LiveEventBatch {
-            events,
-            next_sequence,
-            truncated,
-        }
-    }
-}
-
 /// Notify the system of monitoring indicators
 pub struct NotificationMetrics {
     /// The number of events currently being processed
@@ -231,18 +154,7 @@ pub struct NotificationSystem {
     pub registry: Arc<TargetRegistry>,
     /// The current configuration
     pub config: Arc<RwLock<Config>>,
-    /// Cancel sender for managing stream processing tasks
-    stream_cancellers: Arc<RwLock<HashMap<TargetID, mpsc::Sender<()>>>>,
-    /// Concurrent control signal quantity
-    concurrency_limiter: Arc<Semaphore>,
-    /// Monitoring indicators
-    metrics: Arc<NotificationMetrics>,
-    /// Subscriber view
-    subscriber_view: NotificationSystemSubscriberView,
-    /// Live event fan-out for in-process streaming consumers.
-    live_event_sender: broadcast::Sender<Arc<Event>>,
-    /// Recent live event history for peer fan-in consumers.
-    live_event_history: Arc<RwLock<LiveEventHistory>>,
+    services: NotifyServices,
 }
 
 impl NotificationSystem {
@@ -252,107 +164,40 @@ impl NotificationSystem {
             rustfs_utils::get_env_usize(ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY);
         let (live_event_sender, _) = broadcast::channel(1024);
         let metrics = Arc::new(NotificationMetrics::new());
-        NotificationSystem {
-            subscriber_view: NotificationSystemSubscriberView::new(),
-            notifier: Arc::new(EventNotifier::new(metrics.clone())),
-            registry: Arc::new(TargetRegistry::new()),
-            config: Arc::new(RwLock::new(config)),
-            stream_cancellers: Arc::new(RwLock::new(HashMap::new())),
-            concurrency_limiter: Arc::new(Semaphore::new(concurrency_limiter)), // Limit the maximum number of concurrent processing events to 20
+        let subscriber_view = Arc::new(NotificationSystemSubscriberView::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
+        let target_list = notifier.target_list();
+        let registry = Arc::new(TargetRegistry::new());
+        let config = Arc::new(RwLock::new(config));
+        let stream_cancellers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
+        let concurrency_limiter = Arc::new(Semaphore::new(concurrency_limiter)); // Limit the maximum number of concurrent processing events to 20
+        let live_event_history = Arc::new(RwLock::new(LiveEventHistory::default()));
+        let services = NotifyServices::new(
+            notifier.clone(),
+            rule_engine,
+            target_list,
+            registry.clone(),
+            config.clone(),
+            stream_cancellers,
+            concurrency_limiter,
             metrics,
+            subscriber_view,
             live_event_sender,
-            live_event_history: Arc::new(RwLock::new(LiveEventHistory::default())),
+            live_event_history,
+        );
+
+        NotificationSystem {
+            notifier,
+            registry,
+            config,
+            services,
         }
     }
 
-    /// Initializes targets and starts event streams for those with stores.
-    /// Returns a map of (target_id -> cancel_sender) for streams that were started.
-    async fn init_targets_and_start_streams(
-        &self,
-        targets: &[Box<dyn Target<Event> + Send + Sync>],
-    ) -> HashMap<TargetID, mpsc::Sender<()>> {
-        let mut cancellers = HashMap::new();
-        for target in targets {
-            let target_id = target.id();
-            info!("Initializing target: {}", target_id);
-
-            let has_store = target.store().is_some();
-
-            if let Err(e) = target.init().await {
-                warn!("Target {} Initialization failed: {}", target_id, e);
-                // For targets without a store, init failure is fatal — skip.
-                // For store-backed targets, still start the stream so queued events
-                // can be drained when connectivity recovers (send_from_store retries).
-                if !has_store {
-                    continue;
-                }
-                warn!(
-                    "Target {} has a store, starting stream despite init failure — \
-                     connectivity will be retried by send_from_store",
-                    target_id
-                );
-            } else {
-                debug!("Target {} initialized successfully, enabled: {}", target_id, target.is_enabled());
-            }
-
-            if !target.is_enabled() {
-                info!("Target {} is not enabled, event stream processing is skipped", target_id);
-                continue;
-            }
-
-            if let Some(store) = target.store() {
-                info!("Start event stream processing for target {}", target_id);
-
-                let store_clone = store.boxed_clone();
-                let target_arc = Arc::from(target.clone_dyn());
-
-                let cancel_tx = self.enhanced_start_event_stream(
-                    store_clone,
-                    target_arc,
-                    self.metrics.clone(),
-                    self.concurrency_limiter.clone(),
-                );
-
-                let target_id_clone = target_id.clone();
-                cancellers.insert(target_id, cancel_tx);
-                info!("Event stream processing for target {} is started successfully", target_id_clone);
-            } else {
-                info!("Target {} No storage is configured, event stream processing is skipped", target_id);
-            }
-        }
-        cancellers
-    }
-
     /// Initializes the notification system
     pub async fn init(&self) -> Result<(), NotificationError> {
-        info!("Initialize notification system...");
-
-        let config = {
-            let guard = self.config.read().await;
-            debug!(
-                subsystem_count = guard.0.len(),
-                "Initializing notification system with configuration summary"
-            );
-            guard.clone()
-        };
-
-        let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self.registry.create_targets_from_config(&config).await?;
-
-        info!("{} notification targets were created", targets.len());
-        if targets.is_empty() {
-            warn!("{}", notify_configuration_hint());
-        }
-
-        // Initialize targets and start event streams
-        let cancellers = self.init_targets_and_start_streams(&targets).await;
-
-        // Update canceller collection
-        *self.stream_cancellers.write().await = cancellers;
-
-        // Initialize the bucket target
-        self.notifier.init_bucket_targets(targets).await?;
-        info!("Notification system initialized");
-        Ok(())
+        self.services.config_manager.init().await
     }
 
     /// Gets a list of Targets for all currently active (initialized).
@@ -360,7 +205,7 @@ impl NotificationSystem {
     /// # Return
     /// A Vec containing all active Targets `TargetID`.
     pub async fn get_active_targets(&self) -> Vec<TargetID> {
-        self.notifier.target_list().read().await.keys()
+        self.services.runtime_view.get_active_targets().await
     }
 
     /// Gets the complete Target list, including both active and inactive Targets.
@@ -368,67 +213,34 @@ impl NotificationSystem {
     /// # Return
     /// An `Arc<RwLock<TargetList>>` containing all Targets.
     pub async fn get_all_targets(&self) -> Arc<RwLock<TargetList>> {
-        self.notifier.target_list()
+        self.services.runtime_view.get_all_targets()
     }
 
     /// Gets all Target values, including both active and inactive Targets.
     ///
     /// # Return
     /// A Vec containing all Targets.
-    pub async fn get_target_values(&self) -> Vec<Arc<dyn Target<Event> + Send + Sync>> {
-        self.notifier.target_list().read().await.values()
+    pub async fn get_target_values(&self) -> Vec<SharedTarget<Event>> {
+        self.services.runtime_view.get_target_values().await
     }
 
     /// Checks if there are active subscribers for the given bucket and event name.
     pub async fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool {
-        if !self.subscriber_view.has_subscriber(bucket, event) {
-            return false;
-        }
-        self.notifier.has_subscriber(bucket, event).await
+        self.services.bucket_config_manager.has_subscriber(bucket, event).await
     }
 
     /// Returns true when at least one in-process consumer is subscribed to live events.
     pub fn has_live_listeners(&self) -> bool {
-        self.live_event_sender.receiver_count() > 0
+        self.services.pipeline.has_live_listeners()
     }
 
     /// Subscribes to the in-process live event stream.
     pub fn subscribe_live_events(&self) -> broadcast::Receiver<Arc<Event>> {
-        self.live_event_sender.subscribe()
+        self.services.pipeline.subscribe_live_events()
     }
 
     pub async fn recent_live_events_since(&self, after_sequence: u64, limit: usize) -> LiveEventBatch {
-        let history = self.live_event_history.read().await;
-        history.snapshot_since(after_sequence, limit.max(1))
-    }
-
-    async fn update_config_and_reload<F>(&self, mut modifier: F) -> Result<(), NotificationError>
-    where
-        F: FnMut(&mut Config) -> bool, // The closure returns a boolean value indicating whether the configuration has been changed
-    {
-        let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
-            return Err(NotificationError::StorageNotAvailable(
-                "Failed to save target configuration: server storage not initialized".to_string(),
-            ));
-        };
-
-        let mut new_config = rustfs_ecstore::config::com::read_config_without_migrate(store.clone())
-            .await
-            .map_err(|e| NotificationError::ReadConfig(e.to_string()))?;
-
-        if !modifier(&mut new_config) {
-            // If the closure indication has not changed, return in advance
-            info!("Configuration not changed, skipping save and reload.");
-            return Ok(());
-        }
-
-        // Save the modified configuration to storage
-        rustfs_ecstore::config::com::save_server_config(store, &new_config)
-            .await
-            .map_err(|e| NotificationError::SaveConfig(e.to_string()))?;
-
-        info!("Configuration updated. Reloading system...");
-        self.reload_config(new_config).await
+        self.services.pipeline.recent_live_events_since(after_sequence, limit).await
     }
 
     /// Accurately remove a Target and its related resources through TargetID.
@@ -444,28 +256,7 @@ impl NotificationSystem {
     /// # return
     /// If successful, return `Ok(())`.
     pub async fn remove_target(&self, target_id: &TargetID, target_type: &str) -> Result<(), NotificationError> {
-        info!("Attempting to remove target: {}", target_id);
-
-        let ttype = target_type.to_lowercase();
-        let tname = target_id.id.to_lowercase();
-
-        self.update_config_and_reload(|config| {
-            let mut changed = false;
-            if let Some(targets_of_type) = config.0.get_mut(&ttype) {
-                if targets_of_type.remove(&tname).is_some() {
-                    info!("Removed target {} from configuration", target_id);
-                    changed = true;
-                }
-                if targets_of_type.is_empty() {
-                    config.0.remove(&ttype);
-                }
-            }
-            if !changed {
-                warn!("Target {} not found in configuration", target_id);
-            }
-            changed
-        })
-        .await
+        self.services.config_manager.remove_target(target_id, target_type).await
     }
 
     /// Set or update a Target configuration.
@@ -481,14 +272,10 @@ impl NotificationSystem {
     /// If the target configuration is successfully set, it returns Ok(()).
     /// If the target configuration is invalid, it returns Err(NotificationError::Configuration).
     pub async fn set_target_config(&self, target_type: &str, target_name: &str, kvs: KVS) -> Result<(), NotificationError> {
-        info!("Setting config for target {} of type {}", target_name, target_type);
-        let ttype = target_type.to_lowercase();
-        let tname = target_name.to_lowercase();
-        self.update_config_and_reload(|config| {
-            config.0.entry(ttype.clone()).or_default().insert(tname.clone(), kvs.clone());
-            true // The configuration is always modified
-        })
-        .await
+        self.services
+            .config_manager
+            .set_target_config(target_type, target_name, kvs)
+            .await
     }
 
     /// Removes all notification configurations for a bucket.
@@ -498,8 +285,10 @@ impl NotificationSystem {
     /// * `bucket` - The name of the bucket whose notification configuration is to be removed.
     ///
     pub async fn remove_bucket_notification_config(&self, bucket: &str) {
-        self.subscriber_view.clear_bucket(bucket);
-        self.notifier.remove_rules_map(bucket).await;
+        self.services
+            .bucket_config_manager
+            .remove_bucket_notification_config(bucket)
+            .await;
     }
 
     /// Removes a Target configuration.
@@ -515,108 +304,15 @@ impl NotificationSystem {
     /// If the target configuration is successfully removed, it returns Ok(()).
     /// If the target configuration does not exist, it returns Ok(()) without making any changes.
     pub async fn remove_target_config(&self, target_type: &str, target_name: &str) -> Result<(), NotificationError> {
-        info!("Removing config for target {} of type {}", target_name, target_type);
-
-        let ttype = target_type.to_lowercase();
-        let tname = target_name.to_lowercase();
-        let target_id = runtime_target_id_for_subsystem(&ttype, &tname);
-
-        // Deletion is prohibited if bucket rules refer to it
-        if self.notifier.is_target_bound_to_any_bucket(&target_id).await {
-            return Err(NotificationError::Configuration(format!(
-                "Target is still bound to bucket rules and deletion is prohibited: type={} name={}",
-                ttype, tname
-            )));
-        }
-
-        let config_result = self
-            .update_config_and_reload(|config| {
-                let mut changed = false;
-                if let Some(targets) = config.0.get_mut(&ttype) {
-                    if targets.remove(&tname).is_some() {
-                        changed = true;
-                    }
-                    if targets.is_empty() {
-                        config.0.remove(&ttype);
-                    }
-                }
-                if !changed {
-                    info!("Target {} of type {} not found, no changes made.", target_name, target_type);
-                }
-                debug!(
-                    subsystem_count = config.0.len(),
-                    "Target config removal processed and configuration summary updated"
-                );
-                changed
-            })
-            .await;
-
-        if config_result.is_ok() {
-            // Remove from target list
-            let target_list = self.notifier.target_list();
-            let mut target_list_guard = target_list.write().await;
-            let _ = target_list_guard.remove_target_only(&target_id).await;
-        }
-
-        config_result
-    }
-
-    /// Enhanced event stream startup function, including monitoring and concurrency control
-    fn enhanced_start_event_stream(
-        &self,
-        store: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>,
-        target: Arc<dyn Target<Event> + Send + Sync>,
-        metrics: Arc<NotificationMetrics>,
-        semaphore: Arc<Semaphore>,
-    ) -> mpsc::Sender<()> {
-        stream::start_event_stream_with_batching(store, target, metrics, semaphore)
-    }
-
-    /// Update configuration
-    async fn update_config(&self, new_config: Config) {
-        let mut config = self.config.write().await;
-        *config = new_config;
+        self.services
+            .config_manager
+            .remove_target_config(target_type, target_name)
+            .await
     }
 
     /// Reloads the configuration
     pub async fn reload_config(&self, new_config: Config) -> Result<(), NotificationError> {
-        info!("Reload notification configuration starts");
-
-        // Stop all existing streaming services
-        let mut cancellers = self.stream_cancellers.write().await;
-        for (target_id, cancel_tx) in cancellers.drain() {
-            info!("Stop event stream processing for target {}", target_id);
-            let _ = cancel_tx.send(()).await;
-        }
-
-        // Clear the target_list and ensure that reload is a replacement reconstruction
-        self.notifier.remove_all_bucket_targets().await;
-
-        // Update the config
-        self.update_config(new_config.clone()).await;
-
-        // Create new targets from configuration
-        let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self
-            .registry
-            .create_targets_from_config(&new_config)
-            .await
-            .map_err(NotificationError::Target)?;
-
-        info!("{} notification targets were created from the new configuration", targets.len());
-        if targets.is_empty() {
-            warn!("{}", notify_configuration_hint());
-        }
-
-        // Initialize targets and start event streams using shared helper
-        let new_cancellers = self.init_targets_and_start_streams(&targets).await;
-
-        // Update canceler collection
-        *cancellers = new_cancellers;
-
-        // Initialize the bucket target
-        self.notifier.init_bucket_targets(targets).await?;
-        info!("Configuration reloaded end");
-        Ok(())
+        self.services.config_manager.reload_config(new_config).await
     }
 
     /// Loads the bucket notification configuration
@@ -625,93 +321,41 @@ impl NotificationSystem {
         bucket: &str,
         cfg: &BucketNotificationConfig,
     ) -> Result<(), NotificationError> {
-        let arn_list = self.notifier.get_arn_list(&cfg.region).await;
-        if arn_list.is_empty() {
-            return Err(NotificationError::Configuration(notify_configuration_hint()));
-        }
-        info!("Available ARNs: {:?}", arn_list);
-        // Validate the configuration against the available ARNs
-        if let Err(e) = cfg.validate(&cfg.region, &arn_list) {
-            debug!("Bucket notification config validation region:{} failed: {}", &cfg.region, e);
-            if !matches!(e, ParseConfigError::ArnNotFound(_)) {
-                return Err(NotificationError::BucketNotification(e.to_string()));
-            }
-            warn!(
-                bucket = %bucket,
-                region = %cfg.region,
-                error = %e,
-                "Bucket notification config references missing target ARN; keeping compatibility and loading remaining rules"
-            );
-        }
-
-        self.subscriber_view.apply_bucket_config(bucket, cfg);
-        let rules_map = cfg.get_rules_map();
-        self.notifier.add_rules_map(bucket, rules_map.clone()).await;
-        info!("Loaded notification config for bucket: {}", bucket);
-        Ok(())
+        self.services
+            .bucket_config_manager
+            .load_bucket_notification_config(bucket, cfg)
+            .await
     }
 
     /// Sends an event
     pub async fn send_event(&self, event: Arc<Event>) {
-        self.live_event_history.write().await.record(event.clone());
-        let _ = self.live_event_sender.send(event.clone());
-        self.notifier.send(event).await;
+        self.services.pipeline.send_event(event).await;
     }
 
     /// Obtain system status information
     pub fn get_status(&self) -> HashMap<String, String> {
-        let mut status = HashMap::new();
-
-        status.insert("uptime_seconds".to_string(), self.metrics.uptime().as_secs().to_string());
-        status.insert("processing_events".to_string(), self.metrics.processing_count().to_string());
-        status.insert("processed_events".to_string(), self.metrics.processed_count().to_string());
-        status.insert("failed_events".to_string(), self.metrics.failed_count().to_string());
-        status.insert("skipped_events".to_string(), self.metrics.skipped_count().to_string());
-
-        status
+        self.services.status_view.get_status()
     }
 
     pub fn snapshot_metrics(&self) -> NotificationMetricSnapshot {
-        self.metrics.snapshot()
+        self.services.status_view.snapshot_metrics()
     }
 
     pub async fn snapshot_target_metrics(&self) -> Vec<NotificationTargetMetricSnapshot> {
-        let targets = self.notifier.target_list().read().await.values();
-        let mut snapshots = Vec::with_capacity(targets.len());
+        self.services.runtime_view.snapshot_target_metrics().await
+    }
 
-        for target in targets {
-            let delivery = target.delivery_snapshot();
-            let target_id = target.id();
-            snapshots.push(NotificationTargetMetricSnapshot {
-                failed_messages: delivery.failed_messages,
-                queue_length: delivery.queue_length,
-                target_id: target_id.to_string(),
-                target_type: target_id.name,
-                total_messages: delivery.total_messages,
-            });
-        }
+    pub async fn snapshot_target_health(&self) -> Vec<RuntimeTargetHealthSnapshot> {
+        self.services.runtime_view.snapshot_target_health().await
+    }
 
-        snapshots.sort_by(|a, b| a.target_id.cmp(&b.target_id));
-        snapshots
+    pub async fn runtime_status_snapshot(&self) -> rustfs_targets::RuntimeStatusSnapshot {
+        self.services.runtime_view.runtime_status_snapshot().await
     }
 
     // Add a method to shut down the system
     pub async fn shutdown(&self) {
-        info!("Turn off the notification system");
-
-        // Get the number of active targets
-        let active_targets = self.stream_cancellers.read().await.len();
-        info!("Stops {} active event stream processing tasks", active_targets);
-
-        let mut cancellers = self.stream_cancellers.write().await;
-        for (target_id, cancel_tx) in cancellers.drain() {
-            info!("Stop event stream processing for target {}", target_id);
-            let _ = cancel_tx.send(()).await;
-        }
-        // Wait for a short while to make sure the task has a chance to complete
-        tokio::time::sleep(Duration::from_millis(500)).await;
-
-        info!("Notify the system to be shut down completed");
+        self.services.runtime_facade.shutdown().await;
     }
 }
 
@@ -719,6 +363,22 @@ impl Drop for NotificationSystem {
     fn drop(&mut self) {
         // Asynchronous operation cannot be used here, but logs can be recorded.
         info!("Notify the system instance to be destroyed");
+
+        let snapshot = self.snapshot_metrics();
+        for (name, value, is_gauge) in [
+            (METRIC_NOTIFICATION_CURRENT_SEND_IN_PROGRESS, snapshot.current_send_in_progress, true),
+            (METRIC_NOTIFICATION_EVENTS_ERRORS_TOTAL, snapshot.events_errors_total, false),
+            (METRIC_NOTIFICATION_EVENTS_SENT_TOTAL, snapshot.events_sent_total, false),
+            (METRIC_NOTIFICATION_EVENTS_SKIPPED_TOTAL, snapshot.events_skipped_total, false),
+        ] {
+            if is_gauge {
+                gauge!(name).set(value as f64);
+            } else {
+                counter!(name).absolute(value);
+            }
+            info!("shutdown metric {}={}", name, value);
+        }
+
         let status = self.get_status();
         for (key, value) in status {
             info!("key:{}, value:{}", key, value);
@@ -772,58 +432,22 @@ mod tests {
         assert_eq!(batch.events[0].s3.object.key, "one");
     }
 
-    #[test]
-    fn runtime_target_id_for_subsystem_maps_notify_webhook_to_runtime_type() {
-        let target_id = runtime_target_id_for_subsystem(NOTIFY_WEBHOOK_SUB_SYS, "Primary");
-        assert_eq!(target_id.id, "primary");
-        assert_eq!(target_id.name, "webhook");
-    }
+    #[tokio::test]
+    async fn notification_system_exposes_live_event_pipeline() {
+        let system = NotificationSystem::new(Config::default());
+        assert!(!system.has_live_listeners());
 
-    #[test]
-    fn runtime_target_id_for_subsystem_maps_notify_amqp_to_runtime_type() {
-        let target_id = runtime_target_id_for_subsystem(NOTIFY_AMQP_SUB_SYS, "Primary");
-        assert_eq!(target_id.id, "primary");
-        assert_eq!(target_id.name, "amqp");
-    }
+        let _rx = system.subscribe_live_events();
+        assert!(system.has_live_listeners());
 
-    #[test]
-    fn runtime_target_id_for_subsystem_maps_notify_mqtt_to_runtime_type() {
-        let target_id = runtime_target_id_for_subsystem(NOTIFY_MQTT_SUB_SYS, "Analytics");
-        assert_eq!(target_id.id, "analytics");
-        assert_eq!(target_id.name, "mqtt");
-    }
+        system
+            .send_event(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
+            .await;
 
-    #[test]
-    fn runtime_target_id_for_subsystem_maps_notify_kafka_to_runtime_type() {
-        let target_id = runtime_target_id_for_subsystem(NOTIFY_KAFKA_SUB_SYS, "EventBus");
-        assert_eq!(target_id.id, "eventbus");
-        assert_eq!(target_id.name, "kafka");
-    }
-
-    #[test]
-    fn runtime_target_id_for_subsystem_maps_notify_nats_to_runtime_type() {
-        let target_id = runtime_target_id_for_subsystem(NOTIFY_NATS_SUB_SYS, "Bus");
-        assert_eq!(target_id.id, "bus");
-        assert_eq!(target_id.name, "nats");
-    }
-
-    #[test]
-    fn runtime_target_id_for_subsystem_maps_notify_pulsar_to_runtime_type() {
-        let target_id = runtime_target_id_for_subsystem(NOTIFY_PULSAR_SUB_SYS, "Ledger");
-        assert_eq!(target_id.id, "ledger");
-        assert_eq!(target_id.name, "pulsar");
-    }
-
-    #[test]
-    fn runtime_target_id_for_subsystem_maps_notify_redis_to_runtime_type() {
-        let target_id = runtime_target_id_for_subsystem(NOTIFY_REDIS_SUB_SYS, "Primary");
-        assert_eq!(target_id.id, "primary");
-        assert_eq!(target_id.name, "redis");
-    }
-    #[test]
-    fn runtime_target_id_for_subsystem_maps_notify_postgres_to_runtime_type() {
-        let target_id = runtime_target_id_for_subsystem(NOTIFY_POSTGRES_SUB_SYS, "AuditTrail");
-        assert_eq!(target_id.id, "audittrail");
-        assert_eq!(target_id.name, "postgres");
+        let batch = system.recent_live_events_since(0, 16).await;
+        assert_eq!(batch.events.len(), 1);
+        assert_eq!(batch.events[0].s3.object.key, "object");
+        assert_eq!(batch.next_sequence, 1);
+        assert!(!batch.truncated);
     }
 }

crates/notify/src/lib.rs

@@ -18,22 +18,39 @@
 //! It supports sending events to various targets
 //! (like Webhook and MQTT) and includes features like event persistence and retry on failure.
 
+mod bucket_config_manager;
+mod config_manager;
 mod error;
 mod event;
+mod event_bridge;
 pub mod factory;
 mod global;
 pub mod integration;
 mod notification_system_subscriber;
 pub mod notifier;
+mod pipeline;
 pub mod registry;
+mod rule_engine;
 pub mod rules;
-pub mod stream;
+mod runtime_facade;
+mod runtime_view;
+mod services;
+mod status_view;
 
+pub use bucket_config_manager::NotifyBucketConfigManager;
+pub use config_manager::{NotifyConfigManager, runtime_target_id_for_subsystem};
 pub use error::{LifecycleError, NotificationError};
 pub use event::{Event, EventArgs, EventArgsBuilder};
+pub use event_bridge::{LiveEventHistory, NotifyEventBridge};
 pub use global::{
     initialize, initialize_live_events, is_notification_system_initialized, notification_metrics_snapshot, notification_system,
     notification_target_metrics, notifier_global,
 };
 pub use integration::{NotificationMetricSnapshot, NotificationSystem, NotificationTargetMetricSnapshot};
+pub use pipeline::NotifyPipeline;
+pub use rule_engine::NotifyRuleEngine;
 pub use rules::BucketNotificationConfig;
+pub use runtime_facade::NotifyRuntimeFacade;
+pub use runtime_view::NotifyRuntimeView;
+pub use services::NotifyServices;
+pub use status_view::NotifyStatusView;

crates/notify/src/notifier.rs

@@ -12,52 +12,28 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::{
-    error::NotificationError,
-    event::Event,
-    integration::NotificationMetrics,
-    rules::{RulesMap, TargetIdSet},
-};
-use hashbrown::HashMap;
-use percent_encoding::percent_decode_str;
+use crate::{error::NotificationError, event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine};
 use rustfs_config::notify::{DEFAULT_NOTIFY_SEND_CONCURRENCY, ENV_NOTIFY_SEND_CONCURRENCY};
-use rustfs_s3_common::EventName;
-use rustfs_targets::Target;
 use rustfs_targets::arn::TargetID;
 use rustfs_targets::target::EntityTarget;
-use starshard::AsyncShardedHashMap;
+use rustfs_targets::{SharedTarget, Target, TargetRuntimeManager};
 use std::sync::Arc;
 use tokio::sync::{RwLock, Semaphore};
 use tracing::{debug, error, info, instrument, warn};
 
-fn decoded_object_key_for_matching(object_key: &str) -> Option<String> {
-    if !object_key.contains('%') {
-        return None;
-    }
-
-    let decoded = percent_decode_str(object_key).decode_utf8().ok()?;
-    (decoded != object_key).then(|| decoded.into_owned())
-}
-
-fn match_event_targets(rules: &RulesMap, event_name: EventName, object_key: &str) -> TargetIdSet {
-    let mut target_ids = rules.match_rules(event_name, object_key);
-    if let Some(decoded_key) = decoded_object_key_for_matching(object_key) {
-        target_ids.extend(rules.match_rules(event_name, &decoded_key));
-    }
-    target_ids
-}
+pub type SharedNotifyTargetList = Arc<RwLock<TargetList>>;
 
 /// Manages event notification to targets based on rules
 pub struct EventNotifier {
     metrics: Arc<NotificationMetrics>,
-    target_list: Arc<RwLock<TargetList>>,
-    bucket_rules_map: Arc<AsyncShardedHashMap<String, RulesMap, rustc_hash::FxBuildHasher>>,
+    rule_engine: NotifyRuleEngine,
+    target_list: SharedNotifyTargetList,
     send_limiter: Arc<Semaphore>,
 }
 
 impl Default for EventNotifier {
     fn default() -> Self {
-        Self::new(Arc::new(NotificationMetrics::new()))
+        Self::new(Arc::new(NotificationMetrics::new()), NotifyRuleEngine::new())
     }
 }
 
@@ -66,56 +42,25 @@ impl EventNotifier {
     ///
     /// # Returns
     /// Returns a new instance of EventNotifier.
-    pub fn new(metrics: Arc<NotificationMetrics>) -> Self {
+    pub fn new(metrics: Arc<NotificationMetrics>, rule_engine: NotifyRuleEngine) -> Self {
         let max_inflight = rustfs_utils::get_env_usize(ENV_NOTIFY_SEND_CONCURRENCY, DEFAULT_NOTIFY_SEND_CONCURRENCY);
         EventNotifier {
             metrics,
+            rule_engine,
             target_list: Arc::new(RwLock::new(TargetList::new())),
-            bucket_rules_map: Arc::new(AsyncShardedHashMap::new(0)),
             send_limiter: Arc::new(Semaphore::new(max_inflight)),
         }
     }
 
-    /// Checks whether a TargetID is still referenced by any bucket's rules.
-    ///
-    /// # Arguments
-    /// * `target_id` - The TargetID to check.
-    ///
-    /// # Returns
-    /// Returns `true` if the TargetID is bound to any bucket, otherwise `false`.
-    pub async fn is_target_bound_to_any_bucket(&self, target_id: &TargetID) -> bool {
-        // `AsyncShardedHashMap::iter()`: Traverse (bucket_name, rules_map)
-        let items = self.bucket_rules_map.iter().await;
-        for (_bucket, rules_map) in items {
-            if rules_map.contains_target_id(target_id) {
-                return true;
-            }
-        }
-        false
-    }
-
     /// Returns a reference to the target list
     /// This method provides access to the target list for external use.
     ///
     /// # Returns
     /// Returns an `Arc<RwLock<TargetList>>` representing the target list.
-    pub fn target_list(&self) -> Arc<RwLock<TargetList>> {
+    pub fn target_list(&self) -> SharedNotifyTargetList {
         Arc::clone(&self.target_list)
     }
 
-    /// Removes all notification rules for a bucket
-    ///
-    /// # Arguments
-    /// * `bucket` - The name of the bucket for which to remove rules
-    ///
-    /// This method removes all rules associated with the specified bucket name.
-    /// It will log a message indicating the removal of rules.
-    pub async fn remove_rules_map(&self, bucket: &str) {
-        if self.bucket_rules_map.remove(&bucket.to_string()).await.is_some() {
-            info!("Removed all notification rules for bucket: {}", bucket);
-        }
-    }
-
     /// Returns a list of ARNs for the registered targets
     ///
     /// # Arguments
@@ -132,40 +77,6 @@ impl EventNotifier {
             .collect()
     }
 
-    /// Adds a rules map for a bucket
-    ///
-    /// # Arguments
-    /// * `bucket` - The name of the bucket for which to add the rules map
-    /// * `rules_map` - The rules map to add for the bucket
-    pub async fn add_rules_map(&self, bucket: &str, rules_map: RulesMap) {
-        if rules_map.is_empty() {
-            self.bucket_rules_map.remove(&bucket.to_string()).await;
-        } else {
-            self.bucket_rules_map.insert(bucket.to_string(), rules_map).await;
-        }
-        info!("Added rules for bucket: {}", bucket);
-    }
-
-    /// Gets the rules map for a specific bucket.
-    ///
-    /// # Arguments
-    /// * `bucket` - The name of the bucket for which to get the rules map
-    ///
-    /// # Returns
-    /// Returns `Some(RulesMap)` if rules exist for the bucket, otherwise returns `None`.
-    pub async fn get_rules_map(&self, bucket: &str) -> Option<RulesMap> {
-        self.bucket_rules_map.get(&bucket.to_string()).await
-    }
-
-    /// Removes notification rules for a bucket
-    ///
-    /// # Arguments
-    /// * `bucket` - The name of the bucket for which to remove notification rules
-    pub async fn remove_notification(&self, bucket: &str) {
-        self.bucket_rules_map.remove(&bucket.to_string()).await;
-        info!("Removed notification rules for bucket: {}", bucket);
-    }
-
     /// Removes all targets
     pub async fn remove_all_bucket_targets(&self) {
         let mut target_list_guard = self.target_list.write().await;
@@ -175,26 +86,6 @@ impl EventNotifier {
         info!("Removed all targets and their streams");
     }
 
-    /// Checks if there are active subscribers for the given bucket and event name.
-    ///
-    /// # Parameters
-    /// * `bucket_name` - bucket name.
-    /// * `event_name` - Event name.
-    ///
-    /// # Return value
-    /// Return `true` if at least one matching notification rule exists.
-    pub async fn has_subscriber(&self, bucket_name: &str, event_name: &EventName) -> bool {
-        // Rules to check if the bucket exists
-        if let Some(rules_map) = self.bucket_rules_map.get(&bucket_name.to_string()).await {
-            // A composite event (such as ObjectCreatedAll) is expanded to multiple single events.
-            // We need to check whether any of these single events have the rules configured.
-            rules_map.has_subscriber(event_name)
-        } else {
-            // If no bucket is found, no subscribers
-            false
-        }
-    }
-
     /// Sends an event to the appropriate targets based on the bucket rules
     ///
     /// # Arguments
@@ -205,13 +96,7 @@ impl EventNotifier {
         let object_key = &event.s3.object.key;
         let event_name = event.event_name;
 
-        let Some(rules) = self.bucket_rules_map.get(bucket_name).await else {
-            debug!("No rules found for bucket: {}", bucket_name);
-            self.metrics.increment_skipped();
-            return;
-        };
-
-        let target_ids = match_event_targets(&rules, event_name, object_key);
+        let target_ids = self.rule_engine.match_targets(bucket_name, event_name, object_key).await;
         if target_ids.is_empty() {
             debug!("No matching targets for event in bucket: {}", bucket_name);
             self.metrics.increment_skipped();
@@ -287,45 +172,30 @@ impl EventNotifier {
         info!("Event processing initiated for {} targets for bucket: {}", target_ids_len, bucket_name);
     }
 
-    /// Initializes the targets for buckets
-    ///
-    /// # Arguments
-    /// * `targets_to_init` - A vector of boxed targets to initialize
-    ///
-    /// # Returns
-    /// Returns `Ok(())` if initialization is successful, otherwise returns a `NotificationError`.
+    /// Initializes the targets for buckets from shared target handles.
     #[instrument(skip(self, targets_to_init))]
-    pub async fn init_bucket_targets(
-        &self,
-        targets_to_init: Vec<Box<dyn Target<Event> + Send + Sync>>,
-    ) -> Result<(), NotificationError> {
-        // Currently active, simpler logic
-        let mut target_list_guard = self.target_list.write().await; //Gets a write lock for the TargetList
-
-        // Clear existing targets first - rebuild from scratch to ensure consistency with new configuration
+    pub async fn init_bucket_targets_shared(&self, targets_to_init: Vec<SharedTarget<Event>>) -> Result<(), NotificationError> {
+        let mut target_list_guard = self.target_list.write().await;
         target_list_guard.clear();
 
-        for target_boxed in targets_to_init {
-            // Traverse the incoming Box<dyn Target >
-            debug!("init bucket target: {}", target_boxed.name());
-            // TargetList::add method expectations Arc<dyn Target + Send + Sync>
-            // Therefore, you need to convert Box<dyn Target + Send + Sync> to Arc<dyn Target + Send + Sync>
-            let target_arc: Arc<dyn Target<Event> + Send + Sync> = Arc::from(target_boxed);
-            target_list_guard.add(target_arc)?; // Add Arc<dyn Target> to the list
+        for target in targets_to_init {
+            debug!("init bucket target: {}", target.name());
+            target_list_guard.add(target)?;
         }
+
         info!(
-            "Initialized {} targets, list size: {}", // Clearer logs
+            "Initialized {} shared targets, list size: {}",
             target_list_guard.len(),
             target_list_guard.len()
         );
-        Ok(()) // Make sure to return a Result
+        Ok(())
     }
 }
 
 /// A thread-safe list of targets
 pub struct TargetList {
     /// Map of TargetID to Target
-    targets: HashMap<TargetID, Arc<dyn Target<Event> + Send + Sync>>,
+    runtime: TargetRuntimeManager<Event>,
 }
 
 impl Default for TargetList {
@@ -337,7 +207,9 @@ impl Default for TargetList {
 impl TargetList {
     /// Creates a new TargetList
     pub fn new() -> Self {
-        TargetList { targets: HashMap::new() }
+        TargetList {
+            runtime: TargetRuntimeManager::new(),
+        }
     }
 
     /// Adds a target to the list
@@ -349,17 +221,17 @@ impl TargetList {
     /// Returns `Ok(())` if the target was added successfully, or a `NotificationError` if an error occurred.
     pub fn add(&mut self, target: Arc<dyn Target<Event> + Send + Sync>) -> Result<(), NotificationError> {
         let id = target.id();
-        if self.targets.contains_key(&id) {
+        if self.runtime.get_by_target_id(&id).is_some() {
             // Potentially update or log a warning/error if replacing an existing target.
             warn!("Target with ID {} already exists in TargetList. It will be overwritten.", id);
         }
-        self.targets.insert(id, target);
+        self.runtime.add_arc(target);
         Ok(())
     }
 
     /// Clears all targets from the list
     pub fn clear(&mut self) {
-        self.targets.clear();
+        self.runtime.clear();
     }
 
     /// Removes a target by ID. Note: This does not stop its associated event stream.
@@ -370,30 +242,14 @@ impl TargetList {
     ///
     /// # Returns
     /// Returns the removed target if it existed, otherwise `None`.
-    pub async fn remove_target_only(&mut self, id: &TargetID) -> Option<Arc<dyn Target<Event> + Send + Sync>> {
-        if let Some(target_arc) = self.targets.remove(id) {
-            if let Err(e) = target_arc.close().await {
-                // Target's own close logic
-                error!("Failed to close target {} during removal: {}", id, e);
-            }
-            Some(target_arc)
-        } else {
-            None
-        }
+    pub async fn remove_target_only(&mut self, id: &TargetID) -> Option<SharedTarget<Event>> {
+        self.runtime.remove_by_target_id_and_close(id).await
     }
 
     /// Clears all targets from the list. Note: This does not stop their associated event streams.
     /// Stream cancellation should be handled by EventNotifier.
     pub async fn clear_targets_only(&mut self) {
-        let target_ids_to_clear: Vec<TargetID> = self.targets.keys().cloned().collect();
-        for id in target_ids_to_clear {
-            if let Some(target_arc) = self.targets.remove(&id)
-                && let Err(e) = target_arc.close().await
-            {
-                error!("Failed to close target {} during clear: {}", id, e);
-            }
-        }
-        self.targets.clear();
+        self.runtime.clear_and_close().await;
     }
 
     /// Returns a target by ID
@@ -403,34 +259,54 @@ impl TargetList {
     ///
     /// # Returns
     /// Returns the target if it exists, otherwise `None`.
-    pub fn get(&self, id: &TargetID) -> Option<Arc<dyn Target<Event> + Send + Sync>> {
-        self.targets.get(id).cloned()
+    pub fn get(&self, id: &TargetID) -> Option<SharedTarget<Event>> {
+        self.runtime.get_by_target_id(id)
     }
 
     /// Returns all target IDs
     pub fn keys(&self) -> Vec<TargetID> {
-        self.targets.keys().cloned().collect()
+        self.runtime.target_ids()
     }
 
     /// Returns all targets in the list
-    pub fn values(&self) -> Vec<Arc<dyn Target<Event> + Send + Sync>> {
-        self.targets.values().cloned().collect()
+    pub fn values(&self) -> Vec<SharedTarget<Event>> {
+        self.runtime.values()
+    }
+
+    pub fn runtime_snapshots(&self) -> Vec<rustfs_targets::RuntimeTargetSnapshot> {
+        self.runtime.snapshots()
+    }
+
+    pub async fn runtime_health_snapshots(&self) -> Vec<rustfs_targets::RuntimeTargetHealthSnapshot> {
+        self.runtime.health_snapshots().await
+    }
+
+    pub fn runtime_status_snapshot(
+        &self,
+        replay_workers: &rustfs_targets::ReplayWorkerManager,
+    ) -> rustfs_targets::RuntimeStatusSnapshot {
+        self.runtime.status_snapshot(replay_workers)
+    }
+
+    pub fn runtime_mut(&mut self) -> &mut TargetRuntimeManager<Event> {
+        &mut self.runtime
     }
 
     /// Returns the number of targets
     pub fn len(&self) -> usize {
-        self.targets.len()
+        self.runtime.len()
     }
 
     /// is_empty can be derived from len()
     pub fn is_empty(&self) -> bool {
-        self.targets.is_empty()
+        self.runtime.is_empty()
     }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::{rule_engine::NotifyRuleEngine, rules::RulesMap};
     use async_trait::async_trait;
     use rustfs_s3_common::EventName;
     use rustfs_targets::StoreError;
@@ -445,40 +321,57 @@ mod tests {
         atomic::{AtomicUsize, Ordering},
     };
 
-    #[test]
-    fn encoded_event_key_matches_raw_prefix_suffix_filter() {
+    #[tokio::test]
+    async fn encoded_event_key_matches_raw_prefix_suffix_filter() {
         let target_id = TargetID::new("primary".to_string(), "webhook".to_string());
         let mut rules_map = RulesMap::new();
         rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "uploads/*.csv".to_string(), target_id.clone());
 
-        let targets = match_event_targets(&rules_map, EventName::ObjectCreatedPut, "uploads%2Freport.csv");
+        let rule_engine = NotifyRuleEngine::new();
+        rule_engine.set_bucket_rules("test-bucket", rules_map).await;
+
+        let targets = rule_engine
+            .match_targets("test-bucket", EventName::ObjectCreatedPut, "uploads%2Freport.csv")
+            .await;
 
         assert!(targets.contains(&target_id));
     }
 
-    #[test]
-    fn encoded_event_key_matches_raw_and_decoded_rule_targets() {
+    #[tokio::test]
+    async fn encoded_event_key_matches_raw_and_decoded_rule_targets() {
         let raw_target = TargetID::new("raw".to_string(), "webhook".to_string());
         let decoded_target = TargetID::new("decoded".to_string(), "webhook".to_string());
         let mut rules_map = RulesMap::new();
         rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "uploads%2F*.csv".to_string(), raw_target.clone());
         rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "uploads/*.csv".to_string(), decoded_target.clone());
 
-        let targets = match_event_targets(&rules_map, EventName::ObjectCreatedPut, "uploads%2Freport.csv");
+        let rule_engine = NotifyRuleEngine::new();
+        rule_engine.set_bucket_rules("test-bucket", rules_map).await;
+
+        let targets = rule_engine
+            .match_targets("test-bucket", EventName::ObjectCreatedPut, "uploads%2Freport.csv")
+            .await;
 
         assert_eq!(targets.len(), 2);
         assert!(targets.contains(&raw_target));
         assert!(targets.contains(&decoded_target));
     }
 
-    #[test]
-    fn encoded_event_key_does_not_bypass_suffix_filter() {
+    #[tokio::test]
+    async fn encoded_event_key_does_not_bypass_suffix_filter() {
         let target_id = TargetID::new("primary".to_string(), "webhook".to_string());
         let mut rules_map = RulesMap::new();
         rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "uploads/*.csv".to_string(), target_id);
 
-        let root_targets = match_event_targets(&rules_map, EventName::ObjectCreatedPut, "report.csv");
-        let suffix_targets = match_event_targets(&rules_map, EventName::ObjectCreatedPut, "uploads%2Freport.txt");
+        let rule_engine = NotifyRuleEngine::new();
+        rule_engine.set_bucket_rules("test-bucket", rules_map).await;
+
+        let root_targets = rule_engine
+            .match_targets("test-bucket", EventName::ObjectCreatedPut, "report.csv")
+            .await;
+        let suffix_targets = rule_engine
+            .match_targets("test-bucket", EventName::ObjectCreatedPut, "uploads%2Freport.txt")
+            .await;
 
         assert!(root_targets.is_empty());
         assert!(suffix_targets.is_empty());
@@ -547,7 +440,8 @@ mod tests {
 
     #[tokio::test]
     async fn test_send_event_skips_disabled_target() {
-        let notifier = EventNotifier::new(Arc::new(NotificationMetrics::new()));
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = EventNotifier::new(Arc::new(NotificationMetrics::new()), rule_engine.clone());
 
         let enabled_target = TestTarget::new("enabled-target", "webhook", true);
         let disabled_target = TestTarget::new("disabled-target", "webhook", false);
@@ -556,7 +450,7 @@ mod tests {
         rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), enabled_target.id.clone());
         rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), disabled_target.id.clone());
 
-        notifier.add_rules_map("bucket", rules_map).await;
+        rule_engine.set_bucket_rules("bucket", rules_map).await;
         notifier
             .target_list()
             .write()

crates/notify/src/pipeline.rs

@@ -0,0 +1,141 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::{Event, integration::LiveEventBatch, notifier::EventNotifier};
+use std::collections::VecDeque;
+use std::sync::Arc;
+use tokio::sync::{RwLock, broadcast};
+
+const MAX_RECENT_LIVE_EVENTS: usize = 1024;
+
+#[derive(Default)]
+pub struct LiveEventHistory {
+    next_sequence: u64,
+    events: VecDeque<(u64, Arc<Event>)>,
+}
+
+impl LiveEventHistory {
+    pub fn record(&mut self, event: Arc<Event>) {
+        self.next_sequence = self.next_sequence.saturating_add(1);
+        self.events.push_back((self.next_sequence, event));
+        while self.events.len() > MAX_RECENT_LIVE_EVENTS {
+            self.events.pop_front();
+        }
+    }
+
+    pub fn snapshot_since(&self, after_sequence: u64, limit: usize) -> LiveEventBatch {
+        let mut events = Vec::new();
+        let mut next_sequence = after_sequence;
+        let mut truncated = false;
+
+        for (sequence, event) in self.events.iter() {
+            if *sequence <= after_sequence {
+                continue;
+            }
+            if events.len() >= limit {
+                truncated = true;
+                break;
+            }
+            next_sequence = *sequence;
+            events.push(event.clone());
+        }
+
+        LiveEventBatch {
+            events,
+            next_sequence,
+            truncated,
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct NotifyPipeline {
+    notifier: Arc<EventNotifier>,
+    live_event_sender: broadcast::Sender<Arc<Event>>,
+    live_event_history: Arc<RwLock<LiveEventHistory>>,
+}
+
+impl NotifyPipeline {
+    pub fn new(
+        notifier: Arc<EventNotifier>,
+        live_event_sender: broadcast::Sender<Arc<Event>>,
+        live_event_history: Arc<RwLock<LiveEventHistory>>,
+    ) -> Self {
+        Self {
+            notifier,
+            live_event_sender,
+            live_event_history,
+        }
+    }
+
+    pub fn has_live_listeners(&self) -> bool {
+        self.live_event_sender.receiver_count() > 0
+    }
+
+    pub fn subscribe_live_events(&self) -> broadcast::Receiver<Arc<Event>> {
+        self.live_event_sender.subscribe()
+    }
+
+    pub async fn recent_live_events_since(&self, after_sequence: u64, limit: usize) -> LiveEventBatch {
+        let history = self.live_event_history.read().await;
+        history.snapshot_since(after_sequence, limit.max(1))
+    }
+
+    pub async fn send_event(&self, event: Arc<Event>) {
+        self.live_event_history.write().await.record(event.clone());
+        let _ = self.live_event_sender.send(event.clone());
+        self.notifier.send(event).await;
+    }
+}
+
+pub type NotifyEventBridge = NotifyPipeline;
+
+#[cfg(test)]
+mod tests {
+    use super::{LiveEventHistory, NotifyPipeline};
+    use crate::{Event, integration::NotificationMetrics, notifier::EventNotifier, rule_engine::NotifyRuleEngine};
+    use rustfs_s3_common::EventName;
+    use std::sync::Arc;
+    use tokio::sync::{RwLock, broadcast};
+
+    fn build_pipeline() -> NotifyPipeline {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let notifier = Arc::new(EventNotifier::new(metrics, NotifyRuleEngine::new()));
+        let (live_event_sender, _) = broadcast::channel(16);
+        NotifyPipeline::new(notifier, live_event_sender, Arc::new(RwLock::new(LiveEventHistory::default())))
+    }
+
+    #[tokio::test]
+    async fn pipeline_reports_live_listener_subscription_state() {
+        let pipeline = build_pipeline();
+        assert!(!pipeline.has_live_listeners());
+
+        let _rx = pipeline.subscribe_live_events();
+        assert!(pipeline.has_live_listeners());
+    }
+
+    #[tokio::test]
+    async fn pipeline_records_recent_live_events() {
+        let pipeline = build_pipeline();
+        let event = Arc::new(Event::new_test_event("bucket", "one", EventName::ObjectCreatedPut));
+
+        pipeline.send_event(event).await;
+
+        let batch = pipeline.recent_live_events_since(0, 16).await;
+        assert_eq!(batch.next_sequence, 1);
+        assert!(!batch.truncated);
+        assert_eq!(batch.events.len(), 1);
+        assert_eq!(batch.events[0].s3.object.key, "one");
+    }
+}

crates/notify/src/rule_engine.rs

@@ -0,0 +1,133 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::rules::{RulesMap, TargetIdSet};
+use percent_encoding::percent_decode_str;
+use rustfs_s3_common::EventName;
+use rustfs_targets::arn::TargetID;
+use starshard::AsyncShardedHashMap;
+use std::sync::Arc;
+use tracing::info;
+
+fn decoded_object_key_for_matching(object_key: &str) -> Option<String> {
+    if !object_key.contains('%') {
+        return None;
+    }
+
+    let decoded = percent_decode_str(object_key).decode_utf8().ok()?;
+    (decoded != object_key).then(|| decoded.into_owned())
+}
+
+#[derive(Clone)]
+pub struct NotifyRuleEngine {
+    bucket_rules_map: Arc<AsyncShardedHashMap<String, RulesMap, rustc_hash::FxBuildHasher>>,
+}
+
+impl NotifyRuleEngine {
+    pub fn new() -> Self {
+        Self {
+            bucket_rules_map: Arc::new(AsyncShardedHashMap::new(0)),
+        }
+    }
+
+    pub async fn is_target_bound_to_any_bucket(&self, target_id: &TargetID) -> bool {
+        let items = self.bucket_rules_map.iter().await;
+        for (_bucket, rules_map) in items {
+            if rules_map.contains_target_id(target_id) {
+                return true;
+            }
+        }
+        false
+    }
+
+    pub async fn set_bucket_rules(&self, bucket: &str, rules_map: RulesMap) {
+        if rules_map.is_empty() {
+            self.bucket_rules_map.remove(&bucket.to_string()).await;
+        } else {
+            self.bucket_rules_map.insert(bucket.to_string(), rules_map).await;
+        }
+        info!("Updated notification rules for bucket: {}", bucket);
+    }
+
+    pub async fn get_bucket_rules(&self, bucket: &str) -> Option<RulesMap> {
+        self.bucket_rules_map.get(&bucket.to_string()).await
+    }
+
+    pub async fn clear_bucket_rules(&self, bucket: &str) {
+        if self.bucket_rules_map.remove(&bucket.to_string()).await.is_some() {
+            info!("Removed all notification rules for bucket: {}", bucket);
+        }
+    }
+
+    pub async fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool {
+        self.get_bucket_rules(bucket)
+            .await
+            .is_some_and(|rules_map| rules_map.has_subscriber(event))
+    }
+
+    pub async fn match_targets(&self, bucket: &str, event_name: EventName, object_key: &str) -> TargetIdSet {
+        self.get_bucket_rules(bucket)
+            .await
+            .map_or_else(TargetIdSet::new, |rules_map| {
+                let mut target_ids = rules_map.match_rules(event_name, object_key);
+                if let Some(decoded_key) = decoded_object_key_for_matching(object_key) {
+                    target_ids.extend(rules_map.match_rules(event_name, &decoded_key));
+                }
+                target_ids
+            })
+    }
+}
+
+impl Default for NotifyRuleEngine {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::NotifyRuleEngine;
+    use crate::rules::RulesMap;
+    use rustfs_s3_common::EventName;
+    use rustfs_targets::arn::TargetID;
+
+    #[tokio::test]
+    async fn rule_engine_tracks_bucket_rule_lifecycle() {
+        let engine = NotifyRuleEngine::new();
+        let target_id = TargetID::new("primary".to_string(), "webhook".to_string());
+        let mut rules_map = RulesMap::new();
+        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target_id.clone());
+
+        assert!(!engine.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
+        assert!(!engine.is_target_bound_to_any_bucket(&target_id).await);
+
+        engine.set_bucket_rules("bucket", rules_map).await;
+
+        assert!(engine.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
+        assert!(engine.is_target_bound_to_any_bucket(&target_id).await);
+        assert_eq!(
+            engine
+                .match_targets("bucket", EventName::ObjectCreatedPut, "object")
+                .await
+                .into_iter()
+                .collect::<Vec<_>>(),
+            vec![target_id.clone()]
+        );
+
+        engine.clear_bucket_rules("bucket").await;
+
+        assert!(!engine.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
+        assert!(!engine.is_target_bound_to_any_bucket(&target_id).await);
+    }
+}

crates/notify/src/runtime_facade.rs

@@ -0,0 +1,239 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::{Event, NotificationError, integration::NotificationMetrics, notifier::SharedNotifyTargetList};
+use rustfs_targets::{
+    BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter, ReplayEvent, ReplayWorkerManager, RuntimeActivation, Target,
+};
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::{RwLock, Semaphore};
+use tracing::info;
+
+#[derive(Clone)]
+pub struct NotifyRuntimeFacade {
+    target_list: SharedNotifyTargetList,
+    replay_workers: Arc<RwLock<ReplayWorkerManager>>,
+    runtime_adapter: Arc<dyn PluginRuntimeAdapter<Event>>,
+}
+
+impl NotifyRuntimeFacade {
+    pub fn new(
+        target_list: SharedNotifyTargetList,
+        replay_workers: Arc<RwLock<ReplayWorkerManager>>,
+        concurrency_limiter: Arc<Semaphore>,
+        metrics: Arc<NotificationMetrics>,
+    ) -> Self {
+        let replay_metrics = metrics;
+        let runtime_adapter = BuiltinPluginRuntimeAdapter::new(
+            Arc::new(move |event: ReplayEvent<Event>| {
+                let metrics = replay_metrics.clone();
+                Box::pin(async move {
+                    match event {
+                        ReplayEvent::Delivered { .. } => metrics.increment_processed(),
+                        ReplayEvent::RetryableError { .. } => {}
+                        ReplayEvent::Dropped { target, .. }
+                        | ReplayEvent::PermanentFailure { target, .. }
+                        | ReplayEvent::RetryExhausted { target, .. } => {
+                            target.record_final_failure();
+                            metrics.increment_failed();
+                        }
+                        ReplayEvent::UnreadableEntry { .. } => {}
+                    }
+                })
+            }),
+            Arc::new(|target_id, has_replay| {
+                if has_replay {
+                    info!("Event stream processing for target {} is started successfully", target_id);
+                } else {
+                    info!("Target {} has no replay worker to start", target_id);
+                }
+            }),
+            Some(concurrency_limiter),
+            Duration::from_secs(5),
+            Duration::from_millis(500),
+            "Stop event stream processing for target",
+        );
+
+        Self {
+            target_list,
+            replay_workers,
+            runtime_adapter: Arc::new(runtime_adapter),
+        }
+    }
+
+    pub async fn activate_targets_with_replay(
+        &self,
+        targets: Vec<Box<dyn Target<Event> + Send + Sync>>,
+    ) -> RuntimeActivation<Event> {
+        self.runtime_adapter.activate_with_replay(targets).await
+    }
+
+    pub async fn replace_targets(&self, activation: RuntimeActivation<Event>) -> Result<(), NotificationError> {
+        let mut target_list = self.target_list.write().await;
+        let mut replay_workers = self.replay_workers.write().await;
+        self.runtime_adapter
+            .replace_runtime_targets(target_list.runtime_mut(), &mut replay_workers, activation)
+            .await
+            .map_err(NotificationError::Target)?;
+        Ok(())
+    }
+
+    pub async fn stop_replay_workers(&self) {
+        let mut replay_workers = self.replay_workers.write().await;
+        self.runtime_adapter.stop_replay_workers(&mut replay_workers).await;
+    }
+
+    pub async fn shutdown(&self) {
+        info!("Turn off the notification system");
+
+        let active_targets = self.replay_workers.read().await.len();
+        info!("Stops {} active event stream processing tasks", active_targets);
+
+        {
+            let mut target_list = self.target_list.write().await;
+            let mut replay_workers = self.replay_workers.write().await;
+            if let Err(err) = self
+                .runtime_adapter
+                .shutdown(target_list.runtime_mut(), &mut replay_workers)
+                .await
+            {
+                tracing::error!(error = %err, "Failed to shutdown notify runtime cleanly");
+            }
+        }
+        tokio::time::sleep(Duration::from_millis(500)).await;
+
+        info!("Notify the system to be shut down completed");
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::NotifyRuntimeFacade;
+    use crate::{
+        Event, integration::NotificationMetrics, notifier::EventNotifier, rule_engine::NotifyRuleEngine,
+        runtime_view::NotifyRuntimeView,
+    };
+    use async_trait::async_trait;
+    use rustfs_targets::arn::TargetID;
+    use rustfs_targets::store::{Key, Store};
+    use rustfs_targets::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
+    use rustfs_targets::{ReplayWorkerManager, SharedTarget, StoreError, Target, TargetError};
+    use serde::{Serialize, de::DeserializeOwned};
+    use std::sync::Arc;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use tokio::sync::{RwLock, Semaphore};
+
+    #[derive(Clone)]
+    struct TestTarget {
+        close_calls: Arc<AtomicUsize>,
+        id: TargetID,
+    }
+
+    impl TestTarget {
+        fn new(id: &str, name: &str) -> Self {
+            Self {
+                close_calls: Arc::new(AtomicUsize::new(0)),
+                id: TargetID::new(id.to_string(), name.to_string()),
+            }
+        }
+    }
+
+    #[async_trait]
+    impl<E> Target<E> for TestTarget
+    where
+        E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+    {
+        fn id(&self) -> TargetID {
+            self.id.clone()
+        }
+
+        async fn is_active(&self) -> Result<bool, TargetError> {
+            Ok(true)
+        }
+
+        async fn save(&self, _event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        async fn send_raw_from_store(&self, _key: Key, _body: Vec<u8>, _meta: QueuedPayloadMeta) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        async fn close(&self) -> Result<(), TargetError> {
+            self.close_calls.fetch_add(1, Ordering::SeqCst);
+            Ok(())
+        }
+
+        fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
+            None
+        }
+
+        fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
+            Box::new(self.clone())
+        }
+
+        async fn init(&self) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        fn is_enabled(&self) -> bool {
+            true
+        }
+    }
+
+    fn build_facade() -> (NotifyRuntimeFacade, Arc<EventNotifier>, Arc<RwLock<ReplayWorkerManager>>) {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let notifier = Arc::new(EventNotifier::new(metrics.clone(), NotifyRuleEngine::new()));
+        let target_list = notifier.target_list();
+        let replay_workers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
+        let facade = NotifyRuntimeFacade::new(target_list, replay_workers.clone(), Arc::new(Semaphore::new(4)), metrics);
+        (facade, notifier, replay_workers)
+    }
+
+    #[tokio::test]
+    async fn runtime_facade_stops_empty_replay_workers() {
+        let (facade, _, _) = build_facade();
+        facade.stop_replay_workers().await;
+    }
+
+    #[tokio::test]
+    async fn runtime_facade_activates_empty_target_list() {
+        let (facade, _, _) = build_facade();
+        let activation = facade.activate_targets_with_replay(Vec::new()).await;
+
+        assert!(activation.targets.is_empty());
+        assert_eq!(activation.replay_workers.len(), 0);
+    }
+
+    #[tokio::test]
+    async fn runtime_facade_replace_targets_commits_runtime_state() {
+        let (facade, notifier, replay_workers) = build_facade();
+        let target = TestTarget::new("primary", "webhook");
+        let activation = rustfs_targets::RuntimeActivation {
+            replay_workers: ReplayWorkerManager::new(),
+            targets: vec![Arc::new(target) as SharedTarget<Event>],
+        };
+
+        facade
+            .replace_targets(activation)
+            .await
+            .expect("replace_targets should succeed");
+
+        let runtime_view = NotifyRuntimeView::new(notifier.target_list(), replay_workers.clone());
+        let active_targets = runtime_view.get_active_targets().await;
+        assert_eq!(active_targets, vec![TargetID::new("primary".to_string(), "webhook".to_string())]);
+        assert_eq!(replay_workers.read().await.len(), 0);
+    }
+}

crates/notify/src/runtime_view.rs

@@ -0,0 +1,261 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::{Event, NotificationTargetMetricSnapshot, notifier::SharedNotifyTargetList};
+use rustfs_targets::{ReplayWorkerManager, RuntimeTargetHealthSnapshot, SharedTarget, arn::TargetID};
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+#[derive(Clone)]
+pub struct NotifyRuntimeView {
+    target_list: SharedNotifyTargetList,
+    stream_cancellers: Arc<RwLock<ReplayWorkerManager>>,
+}
+
+impl NotifyRuntimeView {
+    pub fn new(target_list: SharedNotifyTargetList, stream_cancellers: Arc<RwLock<ReplayWorkerManager>>) -> Self {
+        Self {
+            target_list,
+            stream_cancellers,
+        }
+    }
+
+    pub async fn get_active_targets(&self) -> Vec<TargetID> {
+        self.target_list.read().await.keys()
+    }
+
+    pub fn get_all_targets(&self) -> SharedNotifyTargetList {
+        self.target_list.clone()
+    }
+
+    pub async fn get_target_values(&self) -> Vec<SharedTarget<Event>> {
+        self.target_list.read().await.values()
+    }
+
+    pub async fn snapshot_target_metrics(&self) -> Vec<NotificationTargetMetricSnapshot> {
+        self.target_list
+            .read()
+            .await
+            .runtime_snapshots()
+            .into_iter()
+            .map(|snapshot| NotificationTargetMetricSnapshot {
+                failed_messages: snapshot.failed_messages,
+                queue_length: snapshot.queue_length,
+                target_id: snapshot.target_id,
+                target_type: snapshot.target_type,
+                total_messages: snapshot.total_messages,
+            })
+            .collect()
+    }
+
+    pub async fn snapshot_target_health(&self) -> Vec<RuntimeTargetHealthSnapshot> {
+        self.target_list.read().await.runtime_health_snapshots().await
+    }
+
+    pub async fn runtime_status_snapshot(&self) -> rustfs_targets::RuntimeStatusSnapshot {
+        let replay_workers = self.stream_cancellers.read().await;
+        let target_list = self.target_list.read().await;
+        target_list.runtime_status_snapshot(&replay_workers)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::NotifyRuntimeView;
+    use crate::{Event, notifier::TargetList};
+    use async_trait::async_trait;
+    use rustfs_targets::arn::TargetID;
+    use rustfs_targets::store::{Key, Store};
+    use rustfs_targets::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliverySnapshot};
+    use rustfs_targets::{ReplayWorkerManager, StoreError, Target, TargetError};
+    use serde::{Serialize, de::DeserializeOwned};
+    use std::sync::{
+        Arc,
+        atomic::{AtomicU64, Ordering},
+    };
+    use tokio::sync::RwLock;
+
+    #[derive(Clone)]
+    struct TestTarget {
+        active: bool,
+        enabled: bool,
+        failed_messages: Arc<AtomicU64>,
+        id: TargetID,
+        total_messages: Arc<AtomicU64>,
+    }
+
+    impl TestTarget {
+        fn new(id: &str, name: &str) -> Self {
+            Self {
+                active: true,
+                enabled: true,
+                failed_messages: Arc::new(AtomicU64::new(0)),
+                id: TargetID::new(id.to_string(), name.to_string()),
+                total_messages: Arc::new(AtomicU64::new(0)),
+            }
+        }
+
+        fn with_active(mut self, active: bool) -> Self {
+            self.active = active;
+            self
+        }
+
+        fn with_enabled(mut self, enabled: bool) -> Self {
+            self.enabled = enabled;
+            self
+        }
+
+        fn record_successes(&self, count: u64) {
+            self.total_messages.store(count, Ordering::Relaxed);
+        }
+
+        fn record_failures(&self, count: u64) {
+            self.failed_messages.store(count, Ordering::Relaxed);
+        }
+    }
+
+    #[async_trait]
+    impl<E> Target<E> for TestTarget
+    where
+        E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+    {
+        fn id(&self) -> TargetID {
+            self.id.clone()
+        }
+
+        async fn is_active(&self) -> Result<bool, TargetError> {
+            Ok(self.active)
+        }
+
+        async fn save(&self, _event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        async fn send_raw_from_store(&self, _key: Key, _body: Vec<u8>, _meta: QueuedPayloadMeta) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        async fn close(&self) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
+            None
+        }
+
+        fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
+            Box::new(self.clone())
+        }
+
+        async fn init(&self) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        fn is_enabled(&self) -> bool {
+            self.enabled
+        }
+
+        fn delivery_snapshot(&self) -> TargetDeliverySnapshot {
+            TargetDeliverySnapshot {
+                failed_messages: self.failed_messages.load(Ordering::Relaxed),
+                queue_length: 0,
+                total_messages: self.total_messages.load(Ordering::Relaxed),
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn runtime_view_reports_empty_runtime_queries() {
+        let runtime_view = NotifyRuntimeView::new(
+            Arc::new(RwLock::new(TargetList::new())),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+        );
+
+        assert!(runtime_view.get_active_targets().await.is_empty());
+        assert!(runtime_view.get_target_values().await.is_empty());
+        assert!(runtime_view.get_all_targets().read().await.is_empty());
+    }
+
+    #[tokio::test]
+    async fn runtime_view_reports_empty_runtime_snapshots() {
+        let runtime_view = NotifyRuntimeView::new(
+            Arc::new(RwLock::new(TargetList::new())),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+        );
+
+        assert!(runtime_view.snapshot_target_metrics().await.is_empty());
+        assert!(runtime_view.snapshot_target_health().await.is_empty());
+
+        let status = runtime_view.runtime_status_snapshot().await;
+        assert_eq!(status.target_count, 0);
+        assert_eq!(status.replay_worker_count, 0);
+    }
+
+    #[tokio::test]
+    async fn runtime_view_reports_non_empty_runtime_queries_and_snapshots() {
+        let target_list = Arc::new(RwLock::new(TargetList::new()));
+        let replay_workers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
+
+        let online = Arc::new(TestTarget::new("primary", "webhook"));
+        online.record_successes(3);
+        online.record_failures(1);
+
+        let disabled = Arc::new(TestTarget::new("backup", "mqtt").with_enabled(false).with_active(false));
+        disabled.record_successes(2);
+
+        {
+            let mut targets = target_list.write().await;
+            targets.add(online.clone() as Arc<dyn Target<Event> + Send + Sync>).unwrap();
+            targets.add(disabled.clone() as Arc<dyn Target<Event> + Send + Sync>).unwrap();
+        }
+
+        let runtime_view = NotifyRuntimeView::new(target_list.clone(), replay_workers.clone());
+
+        let mut active_targets = runtime_view.get_active_targets().await;
+        active_targets.sort();
+        assert_eq!(
+            active_targets,
+            vec![
+                TargetID::new("backup".to_string(), "mqtt".to_string()),
+                TargetID::new("primary".to_string(), "webhook".to_string())
+            ]
+        );
+
+        let target_values = runtime_view.get_target_values().await;
+        assert_eq!(target_values.len(), 2);
+        assert_eq!(runtime_view.get_all_targets().read().await.len(), 2);
+
+        let metric_snapshots = runtime_view.snapshot_target_metrics().await;
+        assert_eq!(metric_snapshots.len(), 2);
+        assert_eq!(metric_snapshots[0].target_id, "backup:mqtt");
+        assert_eq!(metric_snapshots[0].failed_messages, 0);
+        assert_eq!(metric_snapshots[0].total_messages, 2);
+        assert_eq!(metric_snapshots[1].target_id, "primary:webhook");
+        assert_eq!(metric_snapshots[1].failed_messages, 1);
+        assert_eq!(metric_snapshots[1].total_messages, 3);
+
+        let health_snapshots = runtime_view.snapshot_target_health().await;
+        assert_eq!(health_snapshots.len(), 2);
+        assert_eq!(health_snapshots[0].target_id, "backup:mqtt");
+        assert!(!health_snapshots[0].enabled);
+        assert_eq!(health_snapshots[0].state, rustfs_targets::RuntimeTargetHealthState::Disabled);
+        assert_eq!(health_snapshots[1].target_id, "primary:webhook");
+        assert!(health_snapshots[1].enabled);
+        assert_eq!(health_snapshots[1].state, rustfs_targets::RuntimeTargetHealthState::Online);
+
+        let status = runtime_view.runtime_status_snapshot().await;
+        assert_eq!(status.target_count, 2);
+        assert_eq!(status.replay_worker_count, 0);
+    }
+}

crates/notify/src/services.rs

@@ -0,0 +1,120 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::{
+    Event,
+    bucket_config_manager::NotifyBucketConfigManager,
+    config_manager::NotifyConfigManager,
+    integration::NotificationMetrics,
+    notification_system_subscriber::NotificationSystemSubscriberView,
+    notifier::{EventNotifier, SharedNotifyTargetList},
+    pipeline::{LiveEventHistory, NotifyPipeline},
+    registry::TargetRegistry,
+    rule_engine::NotifyRuleEngine,
+    runtime_facade::NotifyRuntimeFacade,
+    runtime_view::NotifyRuntimeView,
+    status_view::NotifyStatusView,
+};
+use rustfs_ecstore::config::Config;
+use rustfs_targets::ReplayWorkerManager;
+use std::sync::Arc;
+use tokio::sync::{RwLock, Semaphore, broadcast};
+
+#[derive(Clone)]
+pub struct NotifyServices {
+    pub bucket_config_manager: NotifyBucketConfigManager,
+    pub config_manager: NotifyConfigManager,
+    pub pipeline: NotifyPipeline,
+    pub runtime_facade: NotifyRuntimeFacade,
+    pub runtime_view: NotifyRuntimeView,
+    pub status_view: NotifyStatusView,
+}
+
+impl NotifyServices {
+    #[allow(clippy::too_many_arguments)]
+    pub fn new(
+        notifier: Arc<EventNotifier>,
+        rule_engine: NotifyRuleEngine,
+        target_list: SharedNotifyTargetList,
+        registry: Arc<TargetRegistry>,
+        config: Arc<RwLock<Config>>,
+        stream_cancellers: Arc<RwLock<ReplayWorkerManager>>,
+        concurrency_limiter: Arc<Semaphore>,
+        metrics: Arc<NotificationMetrics>,
+        subscriber_view: Arc<NotificationSystemSubscriberView>,
+        live_event_sender: broadcast::Sender<Arc<Event>>,
+        live_event_history: Arc<RwLock<LiveEventHistory>>,
+    ) -> Self {
+        let runtime_view = NotifyRuntimeView::new(target_list.clone(), stream_cancellers.clone());
+        let runtime_facade = NotifyRuntimeFacade::new(target_list, stream_cancellers, concurrency_limiter, metrics.clone());
+        let config_manager = NotifyConfigManager::new(config, registry, rule_engine.clone(), runtime_facade.clone());
+        let bucket_config_manager = NotifyBucketConfigManager::new(notifier.clone(), rule_engine, subscriber_view);
+        let pipeline = NotifyPipeline::new(notifier, live_event_sender, live_event_history);
+        let status_view = NotifyStatusView::new(metrics);
+
+        Self {
+            bucket_config_manager,
+            config_manager,
+            pipeline,
+            runtime_facade,
+            runtime_view,
+            status_view,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::NotifyServices;
+    use crate::{
+        integration::NotificationMetrics, notification_system_subscriber::NotificationSystemSubscriberView,
+        notifier::EventNotifier, pipeline::LiveEventHistory, registry::TargetRegistry, rule_engine::NotifyRuleEngine,
+    };
+    use rustfs_ecstore::config::Config;
+    use rustfs_targets::ReplayWorkerManager;
+    use std::sync::Arc;
+    use tokio::sync::{RwLock, Semaphore, broadcast};
+
+    #[tokio::test]
+    async fn services_build_empty_runtime_views() {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
+        let target_list = notifier.target_list();
+        let registry = Arc::new(TargetRegistry::new());
+        let config = Arc::new(RwLock::new(Config::default()));
+        let stream_cancellers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
+        let concurrency_limiter = Arc::new(Semaphore::new(4));
+        let subscriber_view = Arc::new(NotificationSystemSubscriberView::new());
+        let (live_event_sender, _) = broadcast::channel(16);
+        let live_event_history = Arc::new(RwLock::new(LiveEventHistory::default()));
+
+        let services = NotifyServices::new(
+            notifier,
+            rule_engine,
+            target_list,
+            registry,
+            config,
+            stream_cancellers,
+            concurrency_limiter,
+            metrics,
+            subscriber_view,
+            live_event_sender,
+            live_event_history,
+        );
+
+        assert!(services.runtime_view.get_active_targets().await.is_empty());
+        assert_eq!(services.status_view.snapshot_metrics().events_sent_total, 0);
+    }
+}

crates/notify/src/status_view.rs

@@ -0,0 +1,74 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::integration::{NotificationMetricSnapshot, NotificationMetrics};
+use hashbrown::HashMap;
+use std::sync::Arc;
+
+#[derive(Clone)]
+pub struct NotifyStatusView {
+    metrics: Arc<NotificationMetrics>,
+}
+
+impl NotifyStatusView {
+    pub fn new(metrics: Arc<NotificationMetrics>) -> Self {
+        Self { metrics }
+    }
+
+    pub fn get_status(&self) -> HashMap<String, String> {
+        let mut status = HashMap::new();
+
+        status.insert("uptime_seconds".to_string(), self.metrics.uptime().as_secs().to_string());
+        status.insert("processing_events".to_string(), self.metrics.processing_count().to_string());
+        status.insert("processed_events".to_string(), self.metrics.processed_count().to_string());
+        status.insert("failed_events".to_string(), self.metrics.failed_count().to_string());
+        status.insert("skipped_events".to_string(), self.metrics.skipped_count().to_string());
+
+        status
+    }
+
+    pub fn snapshot_metrics(&self) -> NotificationMetricSnapshot {
+        self.metrics.snapshot()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::NotifyStatusView;
+    use crate::integration::NotificationMetrics;
+    use std::sync::Arc;
+
+    #[test]
+    fn status_view_reports_empty_metrics_snapshot() {
+        let status_view = NotifyStatusView::new(Arc::new(NotificationMetrics::new()));
+
+        let snapshot = status_view.snapshot_metrics();
+        assert_eq!(snapshot.current_send_in_progress, 0);
+        assert_eq!(snapshot.events_errors_total, 0);
+        assert_eq!(snapshot.events_sent_total, 0);
+        assert_eq!(snapshot.events_skipped_total, 0);
+    }
+
+    #[test]
+    fn status_view_exposes_status_map_keys() {
+        let status_view = NotifyStatusView::new(Arc::new(NotificationMetrics::new()));
+        let status = status_view.get_status();
+
+        assert!(status.contains_key("uptime_seconds"));
+        assert!(status.contains_key("processing_events"));
+        assert!(status.contains_key("processed_events"));
+        assert!(status.contains_key("failed_events"));
+        assert!(status.contains_key("skipped_events"));
+    }
+}

crates/notify/src/stream.rs

@@ -1,341 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use crate::{Event, integration::NotificationMetrics};
-use rustfs_targets::{
-    StoreError, Target, TargetError,
-    store::{Key, Store, ensure_store_entry_raw_readable},
-    target::QueuedPayload,
-};
-use rustfs_utils::get_env_usize;
-use std::sync::Arc;
-use std::time::{Duration, Instant};
-use tokio::sync::{Semaphore, mpsc};
-use tokio::time::sleep;
-use tracing::{debug, error, info, warn};
-
-/// Streams events from the store to the target with retry logic
-///
-/// # Arguments
-/// - `store`: The event store
-/// - `target`: The target to send events to
-/// - `cancel_rx`: Receiver to listen for cancellation signals
-pub async fn stream_events(
-    store: &mut (dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send),
-    target: &dyn Target<Event>,
-    mut cancel_rx: mpsc::Receiver<()>,
-) {
-    info!("Starting event stream for target: {}", target.name());
-
-    // Retry configuration
-    const MAX_RETRIES: usize = 5;
-    const RETRY_DELAY: Duration = Duration::from_secs(5);
-
-    loop {
-        // Check for cancellation signal
-        if cancel_rx.try_recv().is_ok() {
-            info!("Cancellation received for target: {}", target.name());
-            return;
-        }
-
-        // Get list of events in the store
-        let keys = store.list();
-        if keys.is_empty() {
-            // No events, wait before checking again
-            sleep(Duration::from_secs(1)).await;
-            continue;
-        }
-
-        // Process each event
-        for key in keys {
-            // Check for cancellation before processing each event
-            if cancel_rx.try_recv().is_ok() {
-                info!("Cancellation received during processing for target: {}", target.name());
-                return;
-            }
-
-            let mut retry_count = 0;
-            let mut success = false;
-
-            // Retry logic
-            while retry_count < MAX_RETRIES && !success {
-                match target.send_from_store(key.clone()).await {
-                    Ok(_) => {
-                        info!("Successfully sent event for target: {}", target.name());
-                        // send_from_store deletes the event from store on success
-                        success = true;
-                    }
-                    Err(e) => {
-                        // Handle specific errors
-                        match &e {
-                            TargetError::NotConnected => {
-                                warn!("Target {} not connected, retrying...", target.name());
-                                retry_count += 1;
-                                sleep(RETRY_DELAY).await;
-                            }
-                            TargetError::Timeout(_) => {
-                                warn!("Timeout for target {}, retrying...", target.name());
-                                retry_count += 1;
-                                sleep(Duration::from_secs((retry_count * 5) as u64)).await; // Exponential backoff
-                            }
-                            _ => {
-                                // Permanent error, skip this event
-                                error!("Permanent error for target {}: {}", target.name(), e);
-                                break;
-                            }
-                        }
-                    }
-                }
-            }
-
-            // Remove event from store if successfully sent
-            if retry_count >= MAX_RETRIES && !success {
-                warn!("Max retries exceeded for event {}, target: {}, skipping", key.to_string(), target.name());
-            }
-        }
-
-        // Small delay before next iteration
-        sleep(Duration::from_millis(100)).await;
-    }
-}
-
-/// Starts the event streaming process for a target
-///
-/// # Arguments
-/// - `store`: The event store
-/// - `target`: The target to send events to
-///
-/// # Returns
-/// A sender to signal cancellation of the event stream
-pub fn start_event_stream(
-    mut store: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>,
-    target: Arc<dyn Target<Event> + Send + Sync>,
-) -> mpsc::Sender<()> {
-    let (cancel_tx, cancel_rx) = mpsc::channel(1);
-
-    tokio::spawn(async move {
-        stream_events(&mut *store, &*target, cancel_rx).await;
-        info!("Event stream stopped for target: {}", target.name());
-    });
-
-    cancel_tx
-}
-
-/// Start event stream with batch processing
-///
-/// # Arguments
-/// - `store`: The event store
-/// - `target`: The target to send events to clients
-/// - `metrics`: Metrics for monitoring
-/// - `semaphore`: Semaphore to limit concurrency
-///
-/// # Returns
-/// A sender to signal cancellation of the event stream
-pub fn start_event_stream_with_batching(
-    mut store: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>,
-    target: Arc<dyn Target<Event> + Send + Sync>,
-    metrics: Arc<NotificationMetrics>,
-    semaphore: Arc<Semaphore>,
-) -> mpsc::Sender<()> {
-    let (cancel_tx, cancel_rx) = mpsc::channel(1);
-    debug!("Starting event stream with batching for target: {}", target.name());
-    tokio::spawn(async move {
-        stream_events_with_batching(&mut *store, &*target, cancel_rx, metrics, semaphore).await;
-        info!("Event stream stopped for target: {}", target.name());
-    });
-
-    cancel_tx
-}
-
-/// Event stream processing with batch processing
-///
-/// # Arguments
-/// - `store`: The event store
-/// - `target`: The target to send events to clients
-/// - `cancel_rx`: Receiver to listen for cancellation signals
-/// - `metrics`: Metrics for monitoring
-/// - `semaphore`: Semaphore to limit concurrency
-///
-/// # Notes
-/// This function processes events in batches to improve efficiency.
-pub async fn stream_events_with_batching(
-    store: &mut (dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send),
-    target: &dyn Target<Event>,
-    mut cancel_rx: mpsc::Receiver<()>,
-    metrics: Arc<NotificationMetrics>,
-    semaphore: Arc<Semaphore>,
-) {
-    info!("Starting event stream with batching for target: {}", target.name());
-
-    // Configuration parameters
-    const DEFAULT_BATCH_SIZE: usize = 1;
-    let batch_size = get_env_usize("RUSTFS_EVENT_BATCH_SIZE", DEFAULT_BATCH_SIZE);
-    const BATCH_TIMEOUT: Duration = Duration::from_secs(5);
-    const MAX_RETRIES: usize = 5;
-    const BASE_RETRY_DELAY: Duration = Duration::from_secs(2);
-
-    let mut batch_keys = Vec::with_capacity(batch_size);
-    let mut last_flush = Instant::now();
-
-    loop {
-        // Check the cancel signal
-        if cancel_rx.try_recv().is_ok() {
-            info!("Cancellation received for target: {}", target.name());
-            return;
-        }
-
-        // Get a list of events in storage
-        let keys = store.list();
-        debug!("Found {} keys in store for target: {}", keys.len(), target.name());
-        if keys.is_empty() {
-            // If there is data in the batch and timeout, refresh the batch
-            if !batch_keys.is_empty() && last_flush.elapsed() >= BATCH_TIMEOUT {
-                process_batch(&mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
-                last_flush = Instant::now();
-            }
-
-            // No event, wait before checking
-            tokio::time::sleep(Duration::from_millis(500)).await;
-            continue;
-        }
-
-        // Handle each event
-        for key in keys {
-            // Check the cancel signal again
-            if cancel_rx.try_recv().is_ok() {
-                info!("Cancellation received during processing for target: {}", target.name());
-
-                // Processing collected batches before exiting
-                if !batch_keys.is_empty() {
-                    process_batch(&mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
-                }
-                return;
-            }
-
-            // Skip unreadable entries so a single corrupt file cannot stall the stream.
-            // ensure_store_entry_raw_readable attempts get_raw; on I/O error it calls del() to
-            // remove the corrupt entry before returning Err, so no cleanup is needed here.
-            match ensure_store_entry_raw_readable(&*store, &key) {
-                Ok(true) => {}         // entry is readable, proceed
-                Ok(false) => continue, // entry not found (already removed), skip
-                Err(err) => {
-                    warn!("Skipping unreadable store entry {} for target {}: {}", key, target.name(), err);
-                    continue; // corrupt entry was already deleted by ensure_store_entry_raw_readable
-                }
-            }
-
-            batch_keys.push(key);
-            metrics.increment_processing();
-
-            // If the batch is full or enough time has passed since the last refresh, the batch will be processed
-            if batch_keys.len() >= batch_size || last_flush.elapsed() >= BATCH_TIMEOUT {
-                process_batch(&mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
-                last_flush = Instant::now();
-            }
-        }
-
-        // A small delay will be conducted to check the next round
-        tokio::time::sleep(Duration::from_millis(100)).await;
-    }
-}
-
-/// Processing event batches for targets
-/// # Arguments
-/// - `batch`: The batch of events to process
-/// - `batch_keys`: The corresponding keys of the events in the batch
-/// - `target`: The target to send events to clients
-/// - `max_retries`: Maximum number of retries for sending an event
-/// - `base_delay`: Base delay duration for retries
-/// - `metrics`: Metrics for monitoring
-/// - `semaphore`: Semaphore to limit concurrency
-/// # Notes
-/// This function processes a batch of events, sending each event to the target with retry
-async fn process_batch(
-    batch_keys: &mut Vec<Key>,
-    target: &dyn Target<Event>,
-    max_retries: usize,
-    base_delay: Duration,
-    metrics: &Arc<NotificationMetrics>,
-    semaphore: &Arc<Semaphore>,
-) {
-    debug!("Processing batch of {} events for target: {}", batch_keys.len(), target.name());
-    if batch_keys.is_empty() {
-        return;
-    }
-
-    // Obtain semaphore permission to limit concurrency
-    let permit = match semaphore.clone().acquire_owned().await {
-        Ok(permit) => permit,
-        Err(e) => {
-            error!("Failed to acquire semaphore permit: {}", e);
-            return;
-        }
-    };
-
-    // Handle every event in the batch
-    for key in batch_keys.iter() {
-        let mut retry_count = 0;
-        let mut success = false;
-
-        // Retry logic
-        while retry_count < max_retries && !success {
-            match target.send_from_store(key.clone()).await {
-                Ok(_) => {
-                    debug!("Successfully sent event for target: {}, Key: {}", target.name(), key.to_string());
-                    success = true;
-                    metrics.increment_processed();
-                }
-                Err(e) => match &e {
-                    TargetError::NotConnected => {
-                        warn!("Target {} not connected, retrying...", target.name());
-                        retry_count += 1;
-                        let jitter = Duration::from_millis(key.to_string().len() as u64 % 500);
-                        let backoff = 1u32 << retry_count as u32;
-                        tokio::time::sleep(base_delay * backoff + jitter).await;
-                    }
-                    TargetError::Timeout(_) => {
-                        warn!("Timeout for target {}, retrying...", target.name());
-                        retry_count += 1;
-                        let jitter = Duration::from_millis(key.to_string().len() as u64 % 500);
-                        let backoff = 1u32 << retry_count as u32;
-                        tokio::time::sleep(base_delay * backoff + jitter).await;
-                    }
-                    TargetError::Dropped(reason) => {
-                        warn!("Dropped queued payload for target {}: {}", target.name(), reason);
-                        metrics.increment_failed();
-                        break;
-                    }
-                    _ => {
-                        error!("Permanent error for target {}: {}", target.name(), e);
-                        target.record_final_failure();
-                        metrics.increment_failed();
-                        break;
-                    }
-                },
-            }
-        }
-
-        if retry_count >= max_retries && !success {
-            warn!("Max retries exceeded for event {}, target: {}, skipping", key.to_string(), target.name());
-            target.record_final_failure();
-            metrics.increment_failed();
-        }
-    }
-
-    // Clear processed batches
-    batch_keys.clear();
-
-    // Release semaphore permission (via drop)
-    drop(permit);
-}