refactor(targets): unify queue/connectivity handling and coverage (#2953)

Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
Co-authored-by: marshawcoco <marshawcoco@gmail.com>

Cargo.lock

@@ -9381,6 +9381,7 @@ dependencies = [
 name = "rustfs-audit"
 version = "1.0.0-beta.3"
 dependencies = [
+ "async-trait",
  "chrono",
  "const-str",
  "futures",
@@ -9836,6 +9837,7 @@ dependencies = [
  "chrono",
  "form_urlencoded",
  "hashbrown 0.17.1",
+ "metrics",
  "percent-encoding",
  "quick-xml 0.39.4",
  "rayon",

crates/audit/AGENTS.md

@@ -0,0 +1,53 @@
+# Audit Crate Instructions
+
+Applies to `crates/audit/`.
+
+`rustfs-audit` is the domain layer for audit event fan-out and observability.
+It composes shared plugin/runtime abstractions from `rustfs-targets` and keeps
+audit-specific dispatch semantics, state transitions, and metrics in this
+crate.
+
+## Domain Boundaries
+
+- Keep audit-specific behavior here:
+  - audit event shaping and fan-out pipeline
+  - audit system lifecycle/state transitions
+  - audit metrics and reporting
+- Keep shared plugin/runtime mechanics in `rustfs-targets`:
+  - no duplicated replay worker orchestration
+  - no duplicated runtime manager primitives
+  - no plugin install/control-plane modeling in this crate
+
+## Runtime Layering Rules
+
+- `pipeline.rs` hosts:
+  - `AuditPipeline` (dispatch and snapshot access)
+  - `AuditRuntimeFacade` (runtime mutation path)
+  - `AuditRuntimeView` (runtime read path)
+- `registry.rs` should remain the single owner of runtime target container and
+  plugin registry composition for audit.
+- `system.rs` should coordinate lifecycle by calling facade/view/registry
+  boundaries rather than embedding low-level runtime logic.
+
+## Change Style
+
+- Preserve audit delivery semantics and error handling behavior unless the task
+  explicitly changes them.
+- Prefer extending shared abstractions in `rustfs-targets` over patching
+  one-off audit-only runtime flows.
+- Keep logging and observability machine-meaningful; avoid noisy churn in hot
+  dispatch paths.
+
+## Testing
+
+- Keep unit tests close to changed modules.
+- Keep pipeline-layer regressions in `tests/pipeline_layer_test.rs`.
+- Add regression tests for:
+  - runtime facade activation/replace/stop/shutdown behavior
+  - runtime view target/snapshot access
+  - system reload and runtime commit/clear boundaries
+- Suggested validation:
+  - `cargo test -p rustfs-audit`
+  - Focused: `cargo test -p rustfs-audit --test pipeline_layer_test`
+  - Focused: `cargo test -p rustfs-audit pipeline`
+- Full gate before commit: `make pre-commit`

crates/audit/Cargo.toml

@@ -42,6 +42,7 @@ tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "
 tracing = { workspace = true, features = ["std", "attributes"] }
 
 [dev-dependencies]
+async-trait = { workspace = true }
 temp-env = { workspace = true }
 url = { workspace = true }
 
@@ -49,5 +50,4 @@ url = { workspace = true }
 workspace = true
 
 [lib]
-test = false
 doctest = false

crates/audit/src/factory.rs

@@ -13,138 +13,11 @@
 // limitations under the License.
 
 use crate::AuditEntry;
-use rustfs_config::AUDIT_DEFAULT_DIR;
-use rustfs_config::audit::{
-    AUDIT_AMQP_KEYS, AUDIT_KAFKA_KEYS, AUDIT_MQTT_KEYS, AUDIT_MYSQL_KEYS, AUDIT_NATS_KEYS, AUDIT_POSTGRES_KEYS,
-    AUDIT_PULSAR_KEYS, AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_REDIS_KEYS, AUDIT_WEBHOOK_KEYS,
-};
-use rustfs_targets::config::{
-    build_amqp_args, build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args,
-    build_pulsar_args, build_redis_args, build_webhook_args, validate_amqp_config, validate_kafka_config, validate_mqtt_config,
-    validate_mysql_config, validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config,
-    validate_webhook_config,
-};
-use rustfs_targets::target::{ChannelTargetType, TargetType};
-use rustfs_targets::{BuiltinTargetDescriptor, TargetPluginDescriptor, TargetRequestValidator, boxed_target};
+use rustfs_targets::catalog::builtin::builtin_audit_target_descriptors;
+use rustfs_targets::{BuiltinTargetDescriptor, TargetPluginDescriptor};
 
 pub fn builtin_target_descriptors() -> Vec<BuiltinTargetDescriptor<AuditEntry>> {
-    vec![
-        BuiltinTargetDescriptor::new(
-            rustfs_config::audit::AUDIT_AMQP_SUB_SYS,
-            TargetRequestValidator::Amqp(TargetType::AuditLog),
-            TargetPluginDescriptor::new(
-                ChannelTargetType::Amqp.as_str(),
-                AUDIT_AMQP_KEYS,
-                |config| validate_amqp_config(config, AUDIT_DEFAULT_DIR),
-                |id, config| {
-                    let args = build_amqp_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
-                    Ok(boxed_target(rustfs_targets::target::amqp::AMQPTarget::new(id, args)?))
-                },
-            ),
-        ),
-        BuiltinTargetDescriptor::new(
-            rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS,
-            TargetRequestValidator::Webhook,
-            TargetPluginDescriptor::new(
-                ChannelTargetType::Webhook.as_str(),
-                AUDIT_WEBHOOK_KEYS,
-                |config| validate_webhook_config(config, AUDIT_DEFAULT_DIR),
-                |id, config| {
-                    let args = build_webhook_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
-                    Ok(boxed_target(rustfs_targets::target::webhook::WebhookTarget::new(id, args)?))
-                },
-            ),
-        ),
-        BuiltinTargetDescriptor::new(
-            rustfs_config::audit::AUDIT_MQTT_SUB_SYS,
-            TargetRequestValidator::Mqtt,
-            TargetPluginDescriptor::new(ChannelTargetType::Mqtt.as_str(), AUDIT_MQTT_KEYS, validate_mqtt_config, |id, config| {
-                let args = build_mqtt_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
-                Ok(boxed_target(rustfs_targets::target::mqtt::MQTTTarget::new(id, args)?))
-            }),
-        ),
-        BuiltinTargetDescriptor::new(
-            rustfs_config::audit::AUDIT_NATS_SUB_SYS,
-            TargetRequestValidator::Nats(TargetType::AuditLog),
-            TargetPluginDescriptor::new(
-                ChannelTargetType::Nats.as_str(),
-                AUDIT_NATS_KEYS,
-                |config| validate_nats_config(config, AUDIT_DEFAULT_DIR),
-                |id, config| {
-                    let args = build_nats_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
-                    Ok(boxed_target(rustfs_targets::target::nats::NATSTarget::new(id, args)?))
-                },
-            ),
-        ),
-        BuiltinTargetDescriptor::new(
-            rustfs_config::audit::AUDIT_PULSAR_SUB_SYS,
-            TargetRequestValidator::Pulsar(TargetType::AuditLog),
-            TargetPluginDescriptor::new(
-                ChannelTargetType::Pulsar.as_str(),
-                AUDIT_PULSAR_KEYS,
-                |config| validate_pulsar_config(config, AUDIT_DEFAULT_DIR),
-                |id, config| {
-                    let args = build_pulsar_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
-                    Ok(boxed_target(rustfs_targets::target::pulsar::PulsarTarget::new(id, args)?))
-                },
-            ),
-        ),
-        BuiltinTargetDescriptor::new(
-            rustfs_config::audit::AUDIT_KAFKA_SUB_SYS,
-            TargetRequestValidator::Kafka(TargetType::AuditLog),
-            TargetPluginDescriptor::new(
-                ChannelTargetType::Kafka.as_str(),
-                AUDIT_KAFKA_KEYS,
-                |config| validate_kafka_config(config, AUDIT_DEFAULT_DIR),
-                |id, config| {
-                    let args = build_kafka_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
-                    Ok(boxed_target(rustfs_targets::target::kafka::KafkaTarget::new(id, args)?))
-                },
-            ),
-        ),
-        BuiltinTargetDescriptor::new(
-            rustfs_config::audit::AUDIT_REDIS_SUB_SYS,
-            TargetRequestValidator::Redis {
-                default_channel: AUDIT_REDIS_DEFAULT_CHANNEL,
-                target_type: TargetType::AuditLog,
-            },
-            TargetPluginDescriptor::new(
-                ChannelTargetType::Redis.as_str(),
-                AUDIT_REDIS_KEYS,
-                |config| validate_redis_config(config, AUDIT_DEFAULT_DIR, AUDIT_REDIS_DEFAULT_CHANNEL),
-                |id, config| {
-                    let args = build_redis_args(config, AUDIT_DEFAULT_DIR, AUDIT_REDIS_DEFAULT_CHANNEL, TargetType::AuditLog)?;
-                    Ok(boxed_target(rustfs_targets::target::redis::RedisTarget::new(id, args)?))
-                },
-            ),
-        ),
-        BuiltinTargetDescriptor::new(
-            rustfs_config::audit::AUDIT_MYSQL_SUB_SYS,
-            TargetRequestValidator::MySql(TargetType::AuditLog),
-            TargetPluginDescriptor::new(
-                ChannelTargetType::MySql.as_str(),
-                AUDIT_MYSQL_KEYS,
-                |config| validate_mysql_config(config, AUDIT_DEFAULT_DIR),
-                |id, config| {
-                    let args = build_mysql_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
-                    Ok(boxed_target(rustfs_targets::target::mysql::MySqlTarget::new(id, args)?))
-                },
-            ),
-        ),
-        BuiltinTargetDescriptor::new(
-            rustfs_config::audit::AUDIT_POSTGRES_SUB_SYS,
-            TargetRequestValidator::Postgres(TargetType::AuditLog),
-            TargetPluginDescriptor::new(
-                ChannelTargetType::Postgres.as_str(),
-                AUDIT_POSTGRES_KEYS,
-                |config| validate_postgres_config(config, AUDIT_DEFAULT_DIR),
-                |id, config| {
-                    let args = build_postgres_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
-                    Ok(boxed_target(rustfs_targets::target::postgres::PostgresTarget::new(id, args)?))
-                },
-            ),
-        ),
-    ]
+    builtin_audit_target_descriptors::<AuditEntry>()
 }
 
 pub fn builtin_target_plugins() -> Vec<TargetPluginDescriptor<AuditEntry>> {

crates/audit/src/lib.rs

@@ -23,6 +23,7 @@ pub mod error;
 pub mod factory;
 pub mod global;
 pub mod observability;
+pub mod pipeline;
 pub mod registry;
 pub mod system;
 
@@ -30,5 +31,6 @@ pub use entity::{ApiDetails, AuditEntry, ObjectVersion};
 pub use error::{AuditError, AuditResult};
 pub use global::*;
 pub use observability::{AuditMetrics, AuditMetricsReport, PerformanceValidation};
+pub use pipeline::{AuditPipeline, AuditRuntimeFacade, AuditRuntimeView};
 pub use registry::AuditRegistry;
 pub use system::{AuditSystem, AuditTargetMetricSnapshot};

crates/audit/src/pipeline.rs

@@ -0,0 +1,355 @@
+//  Copyright 2024 RustFS Team
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use crate::{AuditEntry, AuditResult, observability, system::AuditTargetMetricSnapshot};
+use rustfs_targets::{
+    BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter, ReplayEvent, ReplayWorkerManager, RuntimeActivation, SharedTarget, Target,
+    target::EntityTarget,
+};
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::{Mutex, RwLock};
+use tracing::{error, info, warn};
+
+#[derive(Clone)]
+pub struct AuditPipeline {
+    registry: Arc<Mutex<crate::AuditRegistry>>,
+}
+
+impl AuditPipeline {
+    pub fn new(registry: Arc<Mutex<crate::AuditRegistry>>) -> Self {
+        Self { registry }
+    }
+
+    pub async fn dispatch(&self, entry: Arc<AuditEntry>) -> AuditResult<()> {
+        let start_time = std::time::Instant::now();
+
+        let targets: Vec<SharedTarget<AuditEntry>> = {
+            let registry = self.registry.lock().await;
+            let targets = registry.list_target_values();
+
+            if targets.is_empty() {
+                warn!("No audit targets configured for dispatch");
+                return Ok(());
+            }
+
+            targets
+        };
+
+        let mut tasks = Vec::new();
+
+        for target in targets {
+            let entity_target = EntityTarget {
+                object_name: entry.api.name.clone().unwrap_or_default(),
+                bucket_name: entry.api.bucket.clone().unwrap_or_default(),
+                event_name: entry.event,
+                data: (*entry).clone(),
+            };
+
+            let task = async move {
+                let result = target.save(Arc::new(entity_target)).await;
+                (target.id().to_string(), result)
+            };
+
+            tasks.push(task);
+        }
+
+        let results = futures::future::join_all(tasks).await;
+
+        let mut errors = Vec::new();
+        let mut success_count = 0;
+
+        for (target_key, result) in results {
+            match result {
+                Ok(_) => {
+                    success_count += 1;
+                    observability::record_target_success();
+                }
+                Err(e) => {
+                    error!(target_id = %target_key, error = %e, "Failed to dispatch audit log to target");
+                    errors.push(e);
+                    observability::record_target_failure();
+                }
+            }
+        }
+
+        let dispatch_time = start_time.elapsed();
+
+        if errors.is_empty() {
+            observability::record_audit_success(dispatch_time);
+        } else {
+            observability::record_audit_failure(dispatch_time);
+            warn!(
+                error_count = errors.len(),
+                success_count = success_count,
+                "Some audit targets failed to receive log entry"
+            );
+        }
+
+        Ok(())
+    }
+
+    pub async fn dispatch_batch(&self, entries: Vec<Arc<AuditEntry>>) -> AuditResult<()> {
+        let start_time = std::time::Instant::now();
+
+        let targets: Vec<SharedTarget<AuditEntry>> = {
+            let registry = self.registry.lock().await;
+            let targets = registry.list_target_values();
+
+            if targets.is_empty() {
+                warn!("No audit targets configured for batch dispatch");
+                return Ok(());
+            }
+
+            targets
+        };
+
+        let mut tasks = Vec::new();
+        for target in targets {
+            let entries_clone: Vec<_> = entries.iter().map(Arc::clone).collect();
+
+            let task = async move {
+                let mut success_count = 0;
+                let mut errors = Vec::new();
+                for entry in entries_clone {
+                    let entity_target = EntityTarget {
+                        object_name: entry.api.name.clone().unwrap_or_default(),
+                        bucket_name: entry.api.bucket.clone().unwrap_or_default(),
+                        event_name: entry.event,
+                        data: (*entry).clone(),
+                    };
+                    match target.save(Arc::new(entity_target)).await {
+                        Ok(_) => success_count += 1,
+                        Err(e) => errors.push(e),
+                    }
+                }
+                (target.id().to_string(), success_count, errors)
+            };
+            tasks.push(task);
+        }
+
+        let results = futures::future::join_all(tasks).await;
+        let mut total_success = 0;
+        let mut total_errors = 0;
+        for (_target_id, success_count, errors) in results {
+            total_success += success_count;
+            total_errors += errors.len();
+            for e in errors {
+                error!("Batch dispatch error: {:?}", e);
+            }
+        }
+
+        let dispatch_time = start_time.elapsed();
+        info!(
+            "Batch dispatched {} entries, success: {}, errors: {}, time: {:?}",
+            entries.len(),
+            total_success,
+            total_errors,
+            dispatch_time
+        );
+
+        Ok(())
+    }
+
+    pub async fn snapshot_target_metrics(&self) -> Vec<AuditTargetMetricSnapshot> {
+        let registry = self.registry.lock().await;
+        registry
+            .list_target_values()
+            .into_iter()
+            .map(|target| {
+                let delivery = target.delivery_snapshot();
+                AuditTargetMetricSnapshot {
+                    failed_messages: delivery.failed_messages,
+                    queue_length: delivery.queue_length,
+                    target_id: target.id().to_string(),
+                    total_messages: delivery.total_messages,
+                }
+            })
+            .collect()
+    }
+
+    pub async fn snapshot_target_health(&self) -> Vec<rustfs_targets::RuntimeTargetHealthSnapshot> {
+        let registry = self.registry.lock().await;
+        registry.runtime_manager().health_snapshots().await
+    }
+}
+
+#[derive(Clone)]
+pub struct AuditRuntimeView {
+    registry: Arc<Mutex<crate::AuditRegistry>>,
+}
+
+impl AuditRuntimeView {
+    pub fn new(registry: Arc<Mutex<crate::AuditRegistry>>) -> Self {
+        Self { registry }
+    }
+
+    pub async fn list_targets(&self) -> Vec<String> {
+        let registry = self.registry.lock().await;
+        registry.list_targets()
+    }
+
+    pub async fn get_target_values(&self) -> Vec<rustfs_targets::SharedTarget<AuditEntry>> {
+        let registry = self.registry.lock().await;
+        registry.list_target_values()
+    }
+
+    pub async fn get_target(&self, target_id: &str) -> Option<String> {
+        let registry = self.registry.lock().await;
+        registry.get_target(target_id).map(|target| target.id().to_string())
+    }
+
+    pub async fn enable_target(&self, target_id: &str) -> AuditResult<()> {
+        let registry = self.registry.lock().await;
+        if registry.get_target(target_id).is_some() {
+            info!(target_id = %target_id, "Target enabled");
+            Ok(())
+        } else {
+            Err(crate::AuditError::Configuration(format!("Target not found: {target_id}"), None))
+        }
+    }
+
+    pub async fn disable_target(&self, target_id: &str) -> AuditResult<()> {
+        let registry = self.registry.lock().await;
+        if registry.get_target(target_id).is_some() {
+            info!(target_id = %target_id, "Target disabled");
+            Ok(())
+        } else {
+            Err(crate::AuditError::Configuration(format!("Target not found: {target_id}"), None))
+        }
+    }
+
+    pub async fn remove_target(&self, target_id: &str) -> AuditResult<()> {
+        let mut registry = self.registry.lock().await;
+        if registry.remove_target(target_id).await.is_some() {
+            info!(target_id = %target_id, "Target removed");
+            Ok(())
+        } else {
+            Err(crate::AuditError::Configuration(format!("Target not found: {target_id}"), None))
+        }
+    }
+
+    pub async fn upsert_target(&self, target_id: String, target: Box<dyn Target<AuditEntry> + Send + Sync>) -> AuditResult<()> {
+        if let Err(err) = target.init().await {
+            return Err(crate::AuditError::Target(err));
+        }
+
+        let shared_target: SharedTarget<AuditEntry> = Arc::from(target);
+        let mut registry = self.registry.lock().await;
+        let _ = registry.remove_target(&target_id).await;
+        registry.add_shared_target(target_id.clone(), shared_target);
+        info!(target_id = %target_id, "Target upserted");
+        Ok(())
+    }
+}
+
+#[derive(Clone)]
+pub struct AuditRuntimeFacade {
+    registry: Arc<Mutex<crate::AuditRegistry>>,
+    replay_workers: Arc<RwLock<ReplayWorkerManager>>,
+    runtime_adapter: Arc<dyn PluginRuntimeAdapter<AuditEntry>>,
+}
+
+impl AuditRuntimeFacade {
+    pub fn new(registry: Arc<Mutex<crate::AuditRegistry>>, replay_workers: Arc<RwLock<ReplayWorkerManager>>) -> Self {
+        let runtime_adapter = BuiltinPluginRuntimeAdapter::new(
+            Arc::new(move |event: ReplayEvent<AuditEntry>| {
+                Box::pin(async move {
+                    match event {
+                        ReplayEvent::Delivered { key, target } => {
+                            info!("Successfully sent audit entry, target: {}, key: {}", target.id(), key.to_string());
+                            observability::record_target_success();
+                        }
+                        ReplayEvent::RetryableError { error, target, .. } => match error {
+                            rustfs_targets::TargetError::NotConnected => {
+                                warn!("Target {} not connected, retrying...", target.id());
+                            }
+                            rustfs_targets::TargetError::Timeout(_) => {
+                                warn!("Timeout sending to target {}, retrying...", target.id());
+                            }
+                            _ => {}
+                        },
+                        ReplayEvent::Dropped { reason, target, .. } => {
+                            warn!("Dropped queued payload for target {}: {}", target.id(), reason);
+                            observability::record_target_failure();
+                        }
+                        ReplayEvent::PermanentFailure { error, target, .. } => {
+                            error!("Permanent error for target {}: {}", target.id(), error);
+                            target.record_final_failure();
+                            observability::record_target_failure();
+                        }
+                        ReplayEvent::RetryExhausted { key, target } => {
+                            warn!("Max retries exceeded for key {}, target: {}, skipping", key.to_string(), target.id());
+                            target.record_final_failure();
+                            observability::record_target_failure();
+                        }
+                        ReplayEvent::UnreadableEntry { key, error, target } => {
+                            warn!("Skipping unreadable audit store entry {} for target {}: {}", key, target.id(), error);
+                        }
+                    }
+                })
+            }),
+            Arc::new(|target_id, has_replay| {
+                if has_replay {
+                    info!(target_id = %target_id, "Audit stream processing started");
+                } else {
+                    info!(target_id = %target_id, "No store configured, skip audit stream processing");
+                }
+            }),
+            None,
+            Duration::from_millis(500),
+            Duration::from_millis(500),
+            "Stopping audit stream",
+        );
+
+        Self {
+            registry,
+            replay_workers,
+            runtime_adapter: Arc::new(runtime_adapter),
+        }
+    }
+
+    pub async fn replace_targets(&self, activation: RuntimeActivation<AuditEntry>) -> AuditResult<()> {
+        let mut registry = self.registry.lock().await;
+        let mut replay_workers = self.replay_workers.write().await;
+        self.runtime_adapter
+            .replace_runtime_targets(registry.runtime_manager_mut(), &mut replay_workers, activation)
+            .await
+            .map_err(crate::AuditError::Target)?;
+        Ok(())
+    }
+
+    pub async fn shutdown_runtime(
+        &self,
+        registry: &mut crate::AuditRegistry,
+        replay_workers: &mut ReplayWorkerManager,
+    ) -> AuditResult<()> {
+        self.runtime_adapter
+            .shutdown(registry.runtime_manager_mut(), replay_workers)
+            .await
+            .map_err(crate::AuditError::Target)
+    }
+
+    pub async fn activate_targets_with_replay(
+        &self,
+        targets: Vec<Box<dyn Target<AuditEntry> + Send + Sync>>,
+    ) -> RuntimeActivation<AuditEntry> {
+        self.runtime_adapter.activate_with_replay(targets).await
+    }
+
+    pub async fn stop_replay_workers(&self) {
+        let mut replay_workers = self.replay_workers.write().await;
+        self.runtime_adapter.stop_replay_workers(&mut replay_workers).await;
+    }
+}

crates/audit/src/registry.rs

@@ -13,17 +13,16 @@
 //  limitations under the License.
 
 use crate::{AuditEntry, AuditError, AuditResult, factory::builtin_target_plugins};
-use hashbrown::HashMap;
 use rustfs_config::audit::AUDIT_ROUTE_PREFIX;
 use rustfs_ecstore::config::{Config, KVS};
 use rustfs_targets::arn::TargetID;
-use rustfs_targets::{Target, TargetError, TargetPluginRegistry};
-use tracing::{error, info};
+use rustfs_targets::{SharedTarget, Target, TargetError, TargetPluginRegistry, TargetRuntimeManager};
+use tracing::info;
 
 /// Registry for managing audit targets
 pub struct AuditRegistry {
     /// Storage for created targets
-    targets: HashMap<String, Box<dyn Target<AuditEntry> + Send + Sync>>,
+    targets: TargetRuntimeManager<AuditEntry>,
     /// Registered plugins for creating targets
     plugins: TargetPluginRegistry<AuditEntry>,
 }
@@ -41,7 +40,7 @@ impl AuditRegistry {
         plugins.register_all(builtin_target_plugins());
 
         AuditRegistry {
-            targets: HashMap::new(),
+            targets: TargetRuntimeManager::new(),
             plugins,
         }
     }
@@ -92,8 +91,14 @@ impl AuditRegistry {
     /// # Arguments
     /// * `id` - The identifier for the target.
     /// * `target` - The target instance to be added.
-    pub fn add_target(&mut self, id: String, target: Box<dyn Target<AuditEntry> + Send + Sync>) {
-        self.targets.insert(id, target);
+    pub fn add_target(&mut self, _id: String, target: Box<dyn Target<AuditEntry> + Send + Sync>) {
+        debug_assert_eq!(_id, target.id().to_string());
+        self.targets.add_boxed(target);
+    }
+
+    pub fn add_shared_target(&mut self, _id: String, target: SharedTarget<AuditEntry>) {
+        debug_assert_eq!(_id, target.id().to_string());
+        self.targets.add_arc(target);
     }
 
     /// Removes a target from the registry
@@ -103,8 +108,8 @@ impl AuditRegistry {
     ///
     /// # Returns
     /// * `Option<Box<dyn Target<AuditEntry> + Send + Sync>>` - The removed target if it existed.
-    pub fn remove_target(&mut self, id: &str) -> Option<Box<dyn Target<AuditEntry> + Send + Sync>> {
-        self.targets.remove(id)
+    pub async fn remove_target(&mut self, id: &str) -> Option<rustfs_targets::SharedTarget<AuditEntry>> {
+        self.targets.remove_and_close(id).await
     }
 
     /// Gets a target from the registry
@@ -114,13 +119,21 @@ impl AuditRegistry {
     ///
     /// # Returns
     /// * `Option<&(dyn Target<AuditEntry> + Send + Sync)>` - The target if it exists.
-    pub fn get_target(&self, id: &str) -> Option<&(dyn Target<AuditEntry> + Send + Sync)> {
-        self.targets.get(id).map(|t| t.as_ref())
+    pub fn get_target(&self, id: &str) -> Option<rustfs_targets::SharedTarget<AuditEntry>> {
+        self.targets.get(id)
     }
 
     /// Lists cloned target values for runtime inspection without exposing mutable registry access.
-    pub fn list_target_values(&self) -> Vec<Box<dyn Target<AuditEntry> + Send + Sync>> {
-        self.targets.values().map(|target| target.clone_dyn()).collect()
+    pub fn list_target_values(&self) -> Vec<rustfs_targets::SharedTarget<AuditEntry>> {
+        self.targets.values()
+    }
+
+    pub fn runtime_manager(&self) -> &TargetRuntimeManager<AuditEntry> {
+        &self.targets
+    }
+
+    pub fn runtime_manager_mut(&mut self) -> &mut TargetRuntimeManager<AuditEntry> {
+        &mut self.targets
     }
 
     /// Lists all target IDs
@@ -128,7 +141,7 @@ impl AuditRegistry {
     /// # Returns
     /// * `Vec<String>` - A vector of all target IDs in the registry.
     pub fn list_targets(&self) -> Vec<String> {
-        self.targets.keys().cloned().collect()
+        self.targets.keys()
     }
 
     /// Closes all targets and clears the registry
@@ -136,20 +149,23 @@ impl AuditRegistry {
     /// # Returns
     /// * `AuditResult<()>` - Result indicating success or failure.
     pub async fn close_all(&mut self) -> AuditResult<()> {
-        let mut errors = Vec::new();
+        let mut first_error = None;
 
-        for (id, target) in self.targets.drain() {
-            if let Err(e) = target.close().await {
-                error!(target_id = %id, error = %e, "Failed to close audit target");
-                errors.push(e);
+        for target_id in self.targets.keys() {
+            if let Some(target) = self.targets.remove(&target_id)
+                && let Err(err) = target.close().await
+            {
+                tracing::error!(target_id = %target_id, error = %err, "Failed to close target during shutdown");
+                if first_error.is_none() {
+                    first_error = Some(err);
+                }
             }
         }
 
-        if let Some(error) = errors.into_iter().next() {
-            return Err(AuditError::Target(error));
+        match first_error {
+            Some(err) => Err(AuditError::Target(err)),
+            None => Ok(()),
         }
-
-        Ok(())
     }
 
     /// Creates a unique key for a target based on its type and ID
@@ -224,7 +240,8 @@ impl AuditRegistry {
         target: Box<dyn Target<AuditEntry> + Send + Sync>,
     ) -> AuditResult<()> {
         let key = self.create_key(target_type, target_id);
-        self.targets.insert(key, target);
+        debug_assert_eq!(key, target.id().to_string());
+        self.targets.add_boxed(target);
         Ok(())
     }
 }
@@ -232,7 +249,70 @@ impl AuditRegistry {
 #[cfg(test)]
 mod tests {
     use super::AuditRegistry;
-    use rustfs_targets::target::ChannelTargetType;
+    use crate::{AuditEntry, AuditError};
+    use rustfs_targets::arn::TargetID;
+    use rustfs_targets::store::{Key, Store};
+    use rustfs_targets::target::{ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta};
+    use rustfs_targets::{StoreError, Target, TargetError};
+    use std::sync::Arc;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+
+    #[derive(Clone)]
+    struct CloseTestTarget {
+        id: TargetID,
+        close_calls: Arc<AtomicUsize>,
+        fail_on_close: bool,
+    }
+
+    impl CloseTestTarget {
+        fn new(id: TargetID, close_calls: Arc<AtomicUsize>, fail_on_close: bool) -> Self {
+            Self {
+                id,
+                close_calls,
+                fail_on_close,
+            }
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl Target<AuditEntry> for CloseTestTarget {
+        fn id(&self) -> TargetID {
+            self.id.clone()
+        }
+
+        async fn is_active(&self) -> Result<bool, TargetError> {
+            Ok(true)
+        }
+
+        async fn save(&self, _event: Arc<EntityTarget<AuditEntry>>) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        async fn send_raw_from_store(&self, _key: Key, _body: Vec<u8>, _meta: QueuedPayloadMeta) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        async fn close(&self) -> Result<(), TargetError> {
+            self.close_calls.fetch_add(1, Ordering::SeqCst);
+            if self.fail_on_close {
+                Err(TargetError::Unknown("close failed".to_string()))
+            } else {
+                Ok(())
+            }
+        }
+
+        fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
+            None
+        }
+
+        fn clone_dyn(&self) -> Box<dyn Target<AuditEntry> + Send + Sync> {
+            Box::new(self.clone())
+        }
+
+        fn is_enabled(&self) -> bool {
+            true
+        }
+    }
 
     #[test]
     fn registry_registers_amqp_factory() {
@@ -240,4 +320,27 @@ mod tests {
 
         assert!(registry.supports_target_type(ChannelTargetType::Amqp.as_str()));
     }
+
+    #[tokio::test]
+    async fn close_all_returns_first_error_and_clears_targets() {
+        let mut registry = AuditRegistry::new();
+        let ok_calls = Arc::new(AtomicUsize::new(0));
+        let fail_calls = Arc::new(AtomicUsize::new(0));
+
+        let ok_id = TargetID::new("ok".to_string(), "webhook".to_string());
+        let fail_id = TargetID::new("fail".to_string(), "webhook".to_string());
+
+        registry.add_target(ok_id.to_string(), Box::new(CloseTestTarget::new(ok_id, Arc::clone(&ok_calls), false)));
+        registry.add_target(
+            fail_id.to_string(),
+            Box::new(CloseTestTarget::new(fail_id, Arc::clone(&fail_calls), true)),
+        );
+
+        let result = registry.close_all().await;
+
+        assert!(matches!(result, Err(AuditError::Target(TargetError::Unknown(_)))));
+        assert_eq!(ok_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(fail_calls.load(Ordering::SeqCst), 1);
+        assert!(registry.list_targets().is_empty());
+    }
 }

crates/audit/src/system.rs

@@ -12,16 +12,14 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use crate::{AuditEntry, AuditError, AuditRegistry, AuditResult, observability};
-use hashbrown::HashMap;
-use rustfs_ecstore::config::Config;
-use rustfs_targets::{
-    StoreError, Target, TargetError,
-    store::{Key, Store},
-    target::{EntityTarget, QueuedPayload},
+use crate::{
+    AuditEntry, AuditError, AuditRegistry, AuditResult, observability,
+    pipeline::{AuditPipeline, AuditRuntimeFacade, AuditRuntimeView},
 };
+use rustfs_ecstore::config::Config;
+use rustfs_targets::{ReplayWorkerManager, Target};
 use std::sync::Arc;
-use tokio::sync::{Mutex, RwLock, mpsc};
+use tokio::sync::{Mutex, RwLock};
 use tracing::{error, info, warn};
 
 #[derive(Debug, Clone, Default, PartialEq, Eq)]
@@ -49,7 +47,7 @@ pub struct AuditSystem {
     state: Arc<RwLock<AuditSystemState>>,
     config: Arc<RwLock<Option<Config>>>,
     /// Cancellation senders for active audit stream tasks (target_id -> cancel tx)
-    stream_cancellers: Arc<RwLock<HashMap<String, mpsc::Sender<()>>>>,
+    stream_cancellers: Arc<RwLock<ReplayWorkerManager>>,
 }
 
 impl Default for AuditSystem {
@@ -59,16 +57,68 @@ impl Default for AuditSystem {
 }
 
 impl AuditSystem {
+    fn pipeline(&self) -> AuditPipeline {
+        AuditPipeline::new(self.registry.clone())
+    }
+
+    fn runtime_view(&self) -> AuditRuntimeView {
+        AuditRuntimeView::new(self.registry.clone())
+    }
+
+    fn runtime_facade(&self) -> AuditRuntimeFacade {
+        AuditRuntimeFacade::new(self.registry.clone(), self.stream_cancellers.clone())
+    }
+
     /// Creates a new audit system
     pub fn new() -> Self {
         Self {
             registry: Arc::new(Mutex::new(AuditRegistry::new())),
             state: Arc::new(RwLock::new(AuditSystemState::Stopped)),
             config: Arc::new(RwLock::new(None)),
-            stream_cancellers: Arc::new(RwLock::new(HashMap::new())),
+            stream_cancellers: Arc::new(RwLock::new(ReplayWorkerManager::new())),
         }
     }
 
+    async fn create_targets_from_config(&self, config: &Config) -> AuditResult<Vec<Box<dyn Target<AuditEntry> + Send + Sync>>> {
+        let registry = self.registry.lock().await;
+        registry.create_audit_targets_from_config(config).await
+    }
+
+    async fn clear_runtime_targets(&self) -> AuditResult<()> {
+        {
+            let mut registry = self.registry.lock().await;
+            let mut replay_workers = self.stream_cancellers.write().await;
+            self.runtime_facade()
+                .shutdown_runtime(&mut registry, &mut replay_workers)
+                .await?;
+        }
+
+        let mut state = self.state.write().await;
+        *state = AuditSystemState::Stopped;
+        Ok(())
+    }
+
+    async fn commit_runtime_targets(
+        &self,
+        targets: Vec<Box<dyn Target<AuditEntry> + Send + Sync>>,
+        final_state: AuditSystemState,
+    ) -> AuditResult<()> {
+        if targets.is_empty() {
+            info!("No enabled audit targets found, keeping audit system stopped");
+            self.clear_runtime_targets().await?;
+            return Ok(());
+        }
+
+        info!(target_count = targets.len(), "Created audit targets successfully");
+
+        let activation = self.runtime_facade().activate_targets_with_replay(targets).await;
+        self.runtime_facade().replace_targets(activation).await?;
+
+        let mut state = self.state.write().await;
+        *state = final_state;
+        Ok(())
+    }
+
     /// Starts the audit system with the given configuration
     ///
     /// # Arguments
@@ -103,31 +153,14 @@ impl AuditSystem {
             *config_guard = Some(config.clone());
         }
 
-        // Create targets from configuration
-        let mut registry = self.registry.lock().await;
-        match registry.create_audit_targets_from_config(&config).await {
+        match self.create_targets_from_config(&config).await {
             Ok(targets) => {
-                if targets.is_empty() {
-                    info!("No enabled audit targets found, keeping audit system stopped");
-                    drop(registry);
-                    return Ok(());
-                }
-
                 {
                     let mut state = self.state.write().await;
                     *state = AuditSystemState::Starting;
                 }
 
-                info!(target_count = targets.len(), "Created audit targets successfully");
-
-                // Initialize all targets
-                for target in targets {
-                    self.init_and_register_target(target, &mut registry).await;
-                }
-
-                // Update state to running
-                let mut state = self.state.write().await;
-                *state = AuditSystemState::Running;
+                self.commit_runtime_targets(targets, AuditSystemState::Running).await?;
                 info!("Audit system started successfully");
                 Ok(())
             }
@@ -207,18 +240,10 @@ impl AuditSystem {
         info!("Stopping audit system");
 
         // Stop all stream tasks first
-        self.stop_all_streams().await;
-
-        // Close all targets
-        let mut registry = self.registry.lock().await;
-        if let Err(e) = registry.close_all().await {
+        if let Err(e) = self.clear_runtime_targets().await {
             error!(error = %e, "Failed to close some audit targets");
         }
 
-        // Update state to stopped
-        let mut state = self.state.write().await;
-        *state = AuditSystemState::Stopped;
-
         // Clear configuration
         let mut config_guard = self.config.write().await;
         *config_guard = None;
@@ -248,8 +273,6 @@ impl AuditSystem {
     /// # Returns
     /// * `AuditResult<()>` - Result indicating success or failure
     pub async fn dispatch(&self, entry: Arc<AuditEntry>) -> AuditResult<()> {
-        let start_time = std::time::Instant::now();
-
         let state = self.state.read().await;
 
         match *state {
@@ -262,77 +285,7 @@ impl AuditSystem {
             }
         }
         drop(state);
-
-        // Collect cloned targets under lock, then dispatch without holding it
-        let targets: Vec<(String, Box<dyn Target<AuditEntry> + Send + Sync>)> = {
-            let registry = self.registry.lock().await;
-            let target_keys = registry.list_targets();
-
-            if target_keys.is_empty() {
-                warn!("No audit targets configured for dispatch");
-                return Ok(());
-            }
-
-            target_keys
-                .into_iter()
-                .filter_map(|key| registry.get_target(&key).map(|t| (key, t.clone_dyn())))
-                .collect()
-        };
-
-        // Dispatch to all targets concurrently (no lock held)
-        let mut tasks = Vec::new();
-
-        for (target_key, target) in targets {
-            let entity_target = EntityTarget {
-                object_name: entry.api.name.clone().unwrap_or_default(),
-                bucket_name: entry.api.bucket.clone().unwrap_or_default(),
-                event_name: entry.event,
-                data: (*entry).clone(),
-            };
-
-            let task = async move {
-                let result = target.save(Arc::new(entity_target)).await;
-                (target_key, result)
-            };
-
-            tasks.push(task);
-        }
-
-        // Execute all dispatch tasks
-        let results = futures::future::join_all(tasks).await;
-
-        let mut errors = Vec::new();
-        let mut success_count = 0;
-
-        for (target_key, result) in results {
-            match result {
-                Ok(_) => {
-                    success_count += 1;
-                    observability::record_target_success();
-                }
-                Err(e) => {
-                    error!(target_id = %target_key, error = %e, "Failed to dispatch audit log to target");
-                    errors.push(e);
-                    observability::record_target_failure();
-                }
-            }
-        }
-
-        let dispatch_time = start_time.elapsed();
-
-        if errors.is_empty() {
-            observability::record_audit_success(dispatch_time);
-        } else {
-            observability::record_audit_failure(dispatch_time);
-            // Log errors but don't fail the entire dispatch
-            warn!(
-                error_count = errors.len(),
-                success_count = success_count,
-                "Some audit targets failed to receive log entry"
-            );
-        }
-
-        Ok(())
+        self.pipeline().dispatch(entry).await
     }
 
     /// Dispatches a batch of audit log entries to all active targets
@@ -343,239 +296,12 @@ impl AuditSystem {
     /// # Returns
     /// * `AuditResult<()>` - Result indicating success or failure
     pub async fn dispatch_batch(&self, entries: Vec<Arc<AuditEntry>>) -> AuditResult<()> {
-        let start_time = std::time::Instant::now();
-
         let state = self.state.read().await;
         if *state != AuditSystemState::Running {
             return Err(AuditError::NotInitialized("Audit system is not running".to_string()));
         }
         drop(state);
-
-        // Collect targets under lock, then dispatch without holding it
-        let targets: Vec<(String, Box<dyn Target<AuditEntry> + Send + Sync>)> = {
-            let registry = self.registry.lock().await;
-            let target_keys = registry.list_targets();
-
-            if target_keys.is_empty() {
-                warn!("No audit targets configured for batch dispatch");
-                return Ok(());
-            }
-
-            target_keys
-                .into_iter()
-                .filter_map(|key| registry.get_target(&key).map(|t| (key, t.clone_dyn())))
-                .collect()
-        };
-
-        let mut tasks = Vec::new();
-        for (target_key, target) in targets {
-            let entries_clone: Vec<_> = entries.iter().map(Arc::clone).collect();
-            let target_key_clone = target_key.clone();
-
-            let task = async move {
-                let mut success_count = 0;
-                let mut errors = Vec::new();
-                for entry in entries_clone {
-                    let entity_target = EntityTarget {
-                        object_name: entry.api.name.clone().unwrap_or_default(),
-                        bucket_name: entry.api.bucket.clone().unwrap_or_default(),
-                        event_name: entry.event,
-                        data: (*entry).clone(),
-                    };
-                    match target.save(Arc::new(entity_target)).await {
-                        Ok(_) => success_count += 1,
-                        Err(e) => errors.push(e),
-                    }
-                }
-                (target_key_clone, success_count, errors)
-            };
-            tasks.push(task);
-        }
-
-        let results = futures::future::join_all(tasks).await;
-        let mut total_success = 0;
-        let mut total_errors = 0;
-        for (_target_id, success_count, errors) in results {
-            total_success += success_count;
-            total_errors += errors.len();
-            for e in errors {
-                error!("Batch dispatch error: {:?}", e);
-            }
-        }
-
-        let dispatch_time = start_time.elapsed();
-        info!(
-            "Batch dispatched {} entries, success: {}, errors: {}, time: {:?}",
-            entries.len(),
-            total_success,
-            total_errors,
-            dispatch_time
-        );
-
-        Ok(())
-    }
-
-    /// Stops all active audit stream tasks by sending cancellation signals.
-    async fn stop_all_streams(&self) {
-        let mut cancellers = self.stream_cancellers.write().await;
-        for (target_id, cancel_tx) in cancellers.drain() {
-            info!(target_id = %target_id, "Stopping audit stream");
-            let _ = cancel_tx.send(()).await;
-        }
-    }
-
-    /// Initializes a single target: runs init(), starts stream if store is present,
-    /// and adds it to the registry. For store-backed targets, registration and stream
-    /// startup proceed even if init() fails so queued entries can be drained later.
-    async fn init_and_register_target(
-        &self,
-        target: Box<dyn Target<AuditEntry> + Send + Sync>,
-        registry: &mut AuditRegistry,
-    ) -> Option<String> {
-        let target_id = target.id().to_string();
-        let has_store = target.store().is_some();
-
-        if let Err(e) = target.init().await {
-            error!(target_id = %target_id, error = %e, "Failed to initialize audit target");
-            // Non-store targets: init failure is fatal.
-            if !has_store {
-                return None;
-            }
-            // Store-backed targets: still register and start the stream so queued
-            // entries can be drained when connectivity recovers.
-            warn!(
-                target_id = %target_id,
-                "Proceeding with store-backed audit target despite init failure"
-            );
-        }
-
-        if target.is_enabled() {
-            if let Some(store) = target.store() {
-                info!(target_id = %target_id, "Start audit stream processing for target");
-                let store_clone: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send> = store.boxed_clone();
-                let target_arc: Arc<dyn Target<AuditEntry> + Send + Sync> = Arc::from(target.clone_dyn());
-                let cancel_tx = self.start_audit_stream_with_batching(store_clone, target_arc);
-
-                self.stream_cancellers.write().await.insert(target_id.clone(), cancel_tx);
-                info!(target_id = %target_id, "Audit stream processing started");
-            } else {
-                info!(target_id = %target_id, "No store configured, skip audit stream processing");
-            }
-        } else {
-            info!(target_id = %target_id, "Target disabled, skip audit stream processing");
-        }
-
-        registry.add_target(target_id.clone(), target);
-        Some(target_id)
-    }
-
-    /// Starts the audit stream processing for a target with batching and retry logic
-    ///
-    /// # Arguments
-    /// * `store` - The store from which to read audit entries
-    /// * `target` - The target to which audit entries will be sent
-    ///
-    /// This function spawns a background task that continuously reads audit entries from the provided store
-    /// and attempts to send them to the specified target. It implements retry logic with exponential backoff
-    fn start_audit_stream_with_batching(
-        &self,
-        store: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>,
-        target: Arc<dyn Target<AuditEntry> + Send + Sync>,
-    ) -> mpsc::Sender<()> {
-        let (cancel_tx, mut cancel_rx) = mpsc::channel(1);
-        let state = self.state.clone();
-
-        tokio::spawn(async move {
-            use std::time::Duration;
-            use tokio::time::sleep;
-
-            info!("Starting audit stream for target: {}", target.id());
-
-            const MAX_RETRIES: usize = 5;
-            const BASE_RETRY_DELAY: Duration = Duration::from_secs(2);
-
-            loop {
-                // Check for cancellation signal
-                if cancel_rx.try_recv().is_ok() {
-                    info!("Audit stream cancelled for target: {}", target.id());
-                    break;
-                }
-
-                match *state.read().await {
-                    AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => {}
-                    _ => {
-                        info!("Audit stream stopped for target: {}", target.id());
-                        break;
-                    }
-                }
-
-                let keys: Vec<Key> = store.list();
-                if keys.is_empty() {
-                    tokio::select! {
-                        _ = sleep(Duration::from_millis(500)) => {},
-                        _ = cancel_rx.recv() => {
-                            info!("Audit stream cancelled during idle for target: {}", target.id());
-                            return;
-                        }
-                    }
-                    continue;
-                }
-
-                for key in keys {
-                    if cancel_rx.try_recv().is_ok() {
-                        info!("Audit stream cancelled during processing for target: {}", target.id());
-                        return;
-                    }
-
-                    let mut retries = 0usize;
-                    let mut success = false;
-
-                    while retries < MAX_RETRIES && !success {
-                        match target.send_from_store(key.clone()).await {
-                            Ok(_) => {
-                                info!("Successfully sent audit entry, target: {}, key: {}", target.id(), key.to_string());
-                                observability::record_target_success();
-                                success = true;
-                            }
-                            Err(e) => {
-                                match &e {
-                                    TargetError::NotConnected => {
-                                        warn!("Target {} not connected, retrying...", target.id());
-                                    }
-                                    TargetError::Timeout(_) => {
-                                        warn!("Timeout sending to target {}, retrying...", target.id());
-                                    }
-                                    TargetError::Dropped(reason) => {
-                                        warn!("Dropped queued payload for target {}: {}", target.id(), reason);
-                                        observability::record_target_failure();
-                                        break;
-                                    }
-                                    _ => {
-                                        error!("Permanent error for target {}: {}", target.id(), e);
-                                        target.record_final_failure();
-                                        observability::record_target_failure();
-                                        break;
-                                    }
-                                }
-                                retries += 1;
-                                let backoff = BASE_RETRY_DELAY * (1 << retries);
-                                sleep(backoff).await;
-                            }
-                        }
-                    }
-
-                    if retries >= MAX_RETRIES && !success {
-                        warn!("Max retries exceeded for key {}, target: {}, skipping", key.to_string(), target.id());
-                        target.record_final_failure();
-                        observability::record_target_failure();
-                    }
-                }
-
-                sleep(Duration::from_millis(100)).await;
-            }
-        });
-
-        cancel_tx
+        self.pipeline().dispatch_batch(entries).await
     }
 
     /// Enables a specific target
@@ -586,15 +312,7 @@ impl AuditSystem {
     /// # Returns
     /// * `AuditResult<()>` - Result indicating success or failure
     pub async fn enable_target(&self, target_id: &str) -> AuditResult<()> {
-        // This would require storing enabled/disabled state per target
-        // For now, just check if target exists
-        let registry = self.registry.lock().await;
-        if registry.get_target(target_id).is_some() {
-            info!(target_id = %target_id, "Target enabled");
-            Ok(())
-        } else {
-            Err(AuditError::Configuration(format!("Target not found: {target_id}"), None))
-        }
+        self.runtime_view().enable_target(target_id).await
     }
 
     /// Disables a specific target
@@ -605,15 +323,7 @@ impl AuditSystem {
     /// # Returns
     /// * `AuditResult<()>` - Result indicating success or failure
     pub async fn disable_target(&self, target_id: &str) -> AuditResult<()> {
-        // This would require storing enabled/disabled state per target
-        // For now, just check if target exists
-        let registry = self.registry.lock().await;
-        if registry.get_target(target_id).is_some() {
-            info!(target_id = %target_id, "Target disabled");
-            Ok(())
-        } else {
-            Err(AuditError::Configuration(format!("Target not found: {target_id}"), None))
-        }
+        self.runtime_view().disable_target(target_id).await
     }
 
     /// Removes a target from the system
@@ -624,16 +334,7 @@ impl AuditSystem {
     /// # Returns
     /// * `AuditResult<()>` - Result indicating success or failure
     pub async fn remove_target(&self, target_id: &str) -> AuditResult<()> {
-        let mut registry = self.registry.lock().await;
-        if let Some(target) = registry.remove_target(target_id) {
-            if let Err(e) = target.close().await {
-                error!(target_id = %target_id, error = %e, "Failed to close removed target");
-            }
-            info!(target_id = %target_id, "Target removed");
-            Ok(())
-        } else {
-            Err(AuditError::Configuration(format!("Target not found: {target_id}"), None))
-        }
+        self.runtime_view().remove_target(target_id).await
     }
 
     /// Updates or inserts a target
@@ -645,23 +346,7 @@ impl AuditSystem {
     /// # Returns
     /// * `AuditResult<()>` - Result indicating success or failure
     pub async fn upsert_target(&self, target_id: String, target: Box<dyn Target<AuditEntry> + Send + Sync>) -> AuditResult<()> {
-        let mut registry = self.registry.lock().await;
-
-        // Initialize the target
-        if let Err(e) = target.init().await {
-            return Err(AuditError::Target(e));
-        }
-
-        // Remove existing target if present
-        if let Some(old_target) = registry.remove_target(&target_id)
-            && let Err(e) = old_target.close().await
-        {
-            error!(target_id = %target_id, error = %e, "Failed to close old target during upsert");
-        }
-
-        registry.add_target(target_id.clone(), target);
-        info!(target_id = %target_id, "Target upserted");
-        Ok(())
+        self.runtime_view().upsert_target(target_id, target).await
     }
 
     /// Lists all targets
@@ -669,33 +354,27 @@ impl AuditSystem {
     /// # Returns
     /// * `Vec<String>` - List of target IDs
     pub async fn list_targets(&self) -> Vec<String> {
-        let registry = self.registry.lock().await;
-        registry.list_targets()
+        self.runtime_view().list_targets().await
     }
 
     /// Returns cloned target values for read-only runtime inspection.
-    pub async fn get_target_values(&self) -> Vec<Box<dyn Target<AuditEntry> + Send + Sync>> {
-        let registry = self.registry.lock().await;
-        registry.list_target_values()
+    pub async fn get_target_values(&self) -> Vec<rustfs_targets::SharedTarget<AuditEntry>> {
+        self.runtime_view().get_target_values().await
     }
 
     /// Returns per-target delivery metrics for Prometheus collection.
     pub async fn snapshot_target_metrics(&self) -> Vec<AuditTargetMetricSnapshot> {
-        let targets = self.get_target_values().await;
-        let mut snapshots = Vec::with_capacity(targets.len());
+        self.pipeline().snapshot_target_metrics().await
+    }
 
-        for target in targets {
-            let delivery = target.delivery_snapshot();
-            snapshots.push(AuditTargetMetricSnapshot {
-                failed_messages: delivery.failed_messages,
-                queue_length: delivery.queue_length,
-                target_id: target.id().to_string(),
-                total_messages: delivery.total_messages,
-            });
-        }
+    pub async fn snapshot_target_health(&self) -> Vec<rustfs_targets::RuntimeTargetHealthSnapshot> {
+        self.pipeline().snapshot_target_health().await
+    }
 
-        snapshots.sort_by(|a, b| a.target_id.cmp(&b.target_id));
-        snapshots
+    pub async fn runtime_status_snapshot(&self) -> rustfs_targets::RuntimeStatusSnapshot {
+        let replay_workers = self.stream_cancellers.read().await;
+        let registry = self.registry.lock().await;
+        registry.runtime_manager().status_snapshot(&replay_workers)
     }
 
     /// Gets information about a specific target
@@ -706,8 +385,7 @@ impl AuditSystem {
     /// # Returns
     /// * `Option<String>` - Target ID if found
     pub async fn get_target(&self, target_id: &str) -> Option<String> {
-        let registry = self.registry.lock().await;
-        registry.get_target(target_id).map(|target| target.id().to_string())
+        self.runtime_view().get_target(target_id).await
     }
 
     /// Reloads configuration and updates targets
@@ -722,32 +400,20 @@ impl AuditSystem {
 
         observability::record_config_reload();
 
-        // Stop all existing stream tasks first
-        self.stop_all_streams().await;
-
         // Store new configuration
         {
             let mut config_guard = self.config.write().await;
             *config_guard = Some(new_config.clone());
         }
 
-        // Close all existing targets
-        let mut registry = self.registry.lock().await;
-        if let Err(e) = registry.close_all().await {
-            error!(error = %e, "Failed to close existing targets during reload");
-        }
+        let final_state = match self.get_state().await {
+            AuditSystemState::Paused => AuditSystemState::Paused,
+            _ => AuditSystemState::Running,
+        };
 
-        // Create new targets from updated configuration
-        match registry.create_audit_targets_from_config(&new_config).await {
+        match self.create_targets_from_config(&new_config).await {
             Ok(targets) => {
-                info!(target_count = targets.len(), "Reloaded audit targets successfully");
-
-                for target in targets {
-                    if let Some(target_id) = self.init_and_register_target(target, &mut registry).await {
-                        info!(target_id = %target_id, "Target initialized (reload)");
-                    }
-                }
-
+                self.commit_runtime_targets(targets, final_state).await?;
                 info!("Audit configuration reloaded successfully");
                 Ok(())
             }
@@ -779,3 +445,106 @@ impl AuditSystem {
         observability::reset_metrics().await;
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::{AuditSystem, AuditSystemState};
+    use async_trait::async_trait;
+    use rustfs_targets::ReplayWorkerManager;
+    use rustfs_targets::arn::TargetID;
+    use rustfs_targets::store::{Key, Store};
+    use rustfs_targets::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
+    use rustfs_targets::{StoreError, Target, TargetError};
+    use serde::{Serialize, de::DeserializeOwned};
+    use std::collections::HashMap;
+    use std::sync::Arc;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use tokio::sync::mpsc;
+
+    #[derive(Clone)]
+    struct TestTarget {
+        close_calls: Arc<AtomicUsize>,
+        id: TargetID,
+    }
+
+    impl TestTarget {
+        fn new(id: &str, name: &str) -> Self {
+            Self {
+                close_calls: Arc::new(AtomicUsize::new(0)),
+                id: TargetID::new(id.to_string(), name.to_string()),
+            }
+        }
+    }
+
+    #[async_trait]
+    impl<E> Target<E> for TestTarget
+    where
+        E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+    {
+        fn id(&self) -> TargetID {
+            self.id.clone()
+        }
+
+        async fn is_active(&self) -> Result<bool, TargetError> {
+            Ok(true)
+        }
+
+        async fn save(&self, _event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        async fn send_raw_from_store(&self, _key: Key, _body: Vec<u8>, _meta: QueuedPayloadMeta) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        async fn close(&self) -> Result<(), TargetError> {
+            self.close_calls.fetch_add(1, Ordering::SeqCst);
+            Ok(())
+        }
+
+        fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
+            None
+        }
+
+        fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
+            Box::new(self.clone())
+        }
+
+        fn is_enabled(&self) -> bool {
+            true
+        }
+    }
+
+    #[tokio::test]
+    async fn reload_with_empty_config_stops_existing_runtime() {
+        let system = AuditSystem::new();
+        let target = TestTarget::new("primary", "webhook");
+        let close_calls = Arc::clone(&target.close_calls);
+
+        {
+            let mut registry = system.registry.lock().await;
+            registry.add_target("primary:webhook".to_string(), Box::new(target));
+        }
+        {
+            let mut state = system.state.write().await;
+            *state = AuditSystemState::Running;
+        }
+        {
+            let mut replay_workers = system.stream_cancellers.write().await;
+            let (cancel_tx, _cancel_rx) = mpsc::channel(1);
+            replay_workers.insert("primary:webhook".to_string(), cancel_tx);
+            assert_eq!(replay_workers.len(), 1);
+        }
+
+        system
+            .reload_config(rustfs_ecstore::config::Config(HashMap::new()))
+            .await
+            .expect("reload with empty config should succeed");
+
+        assert_eq!(system.get_state().await, AuditSystemState::Stopped);
+        assert!(system.list_targets().await.is_empty());
+        assert_eq!(system.runtime_status_snapshot().await, ReplayWorkerManager::new().snapshot(0));
+        assert_eq!(close_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(*system.config.read().await, Some(rustfs_ecstore::config::Config(HashMap::new())));
+    }
+}

crates/audit/tests/pipeline_layer_test.rs

@@ -0,0 +1,170 @@
+//  Copyright 2024 RustFS Team
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use async_trait::async_trait;
+use rustfs_audit::{AuditPipeline, AuditRegistry, AuditRuntimeFacade, AuditRuntimeView};
+use rustfs_targets::arn::TargetID;
+use rustfs_targets::store::{Key, Store};
+use rustfs_targets::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
+use rustfs_targets::{StoreError, Target, TargetError};
+use serde::{Serialize, de::DeserializeOwned};
+use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use tokio::sync::{Mutex, RwLock};
+
+#[derive(Clone)]
+struct TestTarget {
+    close_calls: Arc<AtomicUsize>,
+    id: TargetID,
+    init_calls: Arc<AtomicUsize>,
+}
+
+impl TestTarget {
+    fn new(id: &str, name: &str) -> Self {
+        Self {
+            close_calls: Arc::new(AtomicUsize::new(0)),
+            id: TargetID::new(id.to_string(), name.to_string()),
+            init_calls: Arc::new(AtomicUsize::new(0)),
+        }
+    }
+}
+
+#[async_trait]
+impl<E> Target<E> for TestTarget
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    fn id(&self) -> TargetID {
+        self.id.clone()
+    }
+
+    async fn is_active(&self) -> Result<bool, TargetError> {
+        Ok(true)
+    }
+
+    async fn save(&self, _event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
+        Ok(())
+    }
+
+    async fn send_raw_from_store(&self, _key: Key, _body: Vec<u8>, _meta: QueuedPayloadMeta) -> Result<(), TargetError> {
+        Ok(())
+    }
+
+    async fn close(&self) -> Result<(), TargetError> {
+        self.close_calls.fetch_add(1, Ordering::SeqCst);
+        Ok(())
+    }
+
+    fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
+        None
+    }
+
+    fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
+        Box::new(self.clone())
+    }
+
+    async fn init(&self) -> Result<(), TargetError> {
+        self.init_calls.fetch_add(1, Ordering::SeqCst);
+        Ok(())
+    }
+
+    fn is_enabled(&self) -> bool {
+        true
+    }
+}
+
+#[tokio::test]
+async fn audit_runtime_view_lists_empty_targets() {
+    let registry = Arc::new(Mutex::new(AuditRegistry::new()));
+    let runtime_view = AuditRuntimeView::new(registry);
+
+    assert!(runtime_view.list_targets().await.is_empty());
+    assert!(runtime_view.get_target_values().await.is_empty());
+    assert!(runtime_view.get_target("missing").await.is_none());
+}
+
+#[tokio::test]
+async fn audit_pipeline_reports_empty_runtime_snapshots() {
+    let registry = Arc::new(Mutex::new(AuditRegistry::new()));
+    let pipeline = AuditPipeline::new(registry);
+
+    assert!(pipeline.snapshot_target_metrics().await.is_empty());
+    assert!(pipeline.snapshot_target_health().await.is_empty());
+}
+
+#[tokio::test]
+async fn audit_runtime_facade_stops_empty_replay_workers() {
+    let registry = Arc::new(Mutex::new(AuditRegistry::new()));
+    let replay_workers = Arc::new(RwLock::new(rustfs_targets::ReplayWorkerManager::new()));
+    let facade = AuditRuntimeFacade::new(registry, replay_workers);
+
+    facade.stop_replay_workers().await;
+}
+
+#[tokio::test]
+async fn audit_runtime_facade_activates_empty_target_list() {
+    let registry = Arc::new(Mutex::new(AuditRegistry::new()));
+    let replay_workers = Arc::new(RwLock::new(rustfs_targets::ReplayWorkerManager::new()));
+    let facade = AuditRuntimeFacade::new(registry, replay_workers);
+
+    let activation = facade.activate_targets_with_replay(Vec::new()).await;
+    assert!(activation.targets.is_empty());
+    assert_eq!(activation.replay_workers.len(), 0);
+}
+
+#[tokio::test]
+async fn audit_runtime_view_upsert_and_remove_target() {
+    let registry = Arc::new(Mutex::new(AuditRegistry::new()));
+    let runtime_view = AuditRuntimeView::new(registry.clone());
+    let target = TestTarget::new("primary", "webhook");
+    let init_calls = Arc::clone(&target.init_calls);
+    let close_calls = Arc::clone(&target.close_calls);
+
+    runtime_view
+        .upsert_target("primary:webhook".to_string(), Box::new(target))
+        .await
+        .expect("upsert should succeed");
+
+    assert_eq!(runtime_view.list_targets().await, vec!["primary:webhook".to_string()]);
+    assert_eq!(init_calls.load(Ordering::SeqCst), 1);
+
+    runtime_view
+        .remove_target("primary:webhook")
+        .await
+        .expect("remove should succeed");
+
+    assert!(runtime_view.list_targets().await.is_empty());
+    assert_eq!(close_calls.load(Ordering::SeqCst), 1);
+}
+
+#[tokio::test]
+async fn audit_runtime_facade_replace_targets_commits_runtime_state() {
+    let registry = Arc::new(Mutex::new(AuditRegistry::new()));
+    let replay_workers = Arc::new(RwLock::new(rustfs_targets::ReplayWorkerManager::new()));
+    let facade = AuditRuntimeFacade::new(registry.clone(), replay_workers.clone());
+    let target = TestTarget::new("primary", "webhook");
+    let activation = rustfs_targets::RuntimeActivation {
+        replay_workers: rustfs_targets::ReplayWorkerManager::new(),
+        targets: vec![Arc::new(target) as rustfs_targets::SharedTarget<rustfs_audit::AuditEntry>],
+    };
+
+    facade
+        .replace_targets(activation)
+        .await
+        .expect("replace_targets should succeed");
+
+    let runtime_view = AuditRuntimeView::new(registry);
+    assert_eq!(runtime_view.list_targets().await, vec!["primary:webhook".to_string()]);
+    assert_eq!(replay_workers.read().await.len(), 0);
+}

crates/notify/AGENTS.md

@@ -0,0 +1,51 @@
+# Notify Crate Instructions
+
+Applies to `crates/notify/`.
+
+`rustfs-notify` is the domain layer for bucket notification semantics. It
+builds rules, event dispatch flow, and config/runtime orchestration on top of
+shared plugin/runtime primitives from `rustfs-targets`.
+
+## Domain Boundaries
+
+- Keep notify-specific business logic here:
+  - bucket/rule evaluation
+  - event bridge and pipeline dispatch
+  - notify config reload orchestration
+- Keep shared runtime/plugin mechanics in `rustfs-targets`:
+  - do not duplicate replay worker lifecycle logic
+  - do not reimplement plugin descriptor/registry/catalog semantics
+  - do not move install/control-plane state into this crate
+
+## Runtime Layering Rules
+
+- `runtime_facade.rs` is the mutation/orchestration boundary:
+  activation, replace, stop workers, shutdown.
+- `runtime_view.rs` is read-only runtime observation:
+  active targets, metrics/health snapshots, runtime status snapshots.
+- `config_manager.rs` should map config to runtime updates through facade/view
+  and `runtime_target_id_for_subsystem`; avoid bypassing these boundaries.
+- `stream.rs` is a compatibility shim; new replay/runtime work should prefer
+  shared helpers in `rustfs-targets::runtime`.
+
+## Change Style
+
+- Preserve best-effort dispatch semantics and observability signals unless the
+  task explicitly requests behavior changes.
+- Reuse existing notify constants and subsystem mappings from `rustfs_config`.
+- Keep changes local and avoid cross-crate refactors from this crate unless
+  required by the task.
+
+## Testing
+
+- Keep unit tests close to changed modules.
+- Add regression tests for:
+  - rules to runtime target resolution
+  - runtime facade replace/shutdown behavior
+  - runtime view health/status/metrics snapshots
+- Suggested validation:
+  - `cargo test -p rustfs-notify`
+  - Focused: `cargo test -p rustfs-notify runtime_facade`
+  - Focused: `cargo test -p rustfs-notify runtime_view`
+  - Focused: `cargo test -p rustfs-notify config_manager`
+- Full gate before commit: `make pre-commit`

crates/notify/Cargo.toml

@@ -46,6 +46,7 @@ tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time"] }
 tracing = { workspace = true }
 url = { workspace = true }
 wildmatch = { workspace = true, features = ["serde"] }
+metrics = { workspace = true }
 
 # quick-xml dependencies for custom S3KeyFilter XML deserialization
 # Custom deserializer implemented for S3KeyFilter to handle both XML structures:

crates/notify/src/bucket_config_manager.rs

@@ -0,0 +1,125 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::{
+    BucketNotificationConfig, NotificationError, config_manager::notify_configuration_hint,
+    notification_system_subscriber::NotificationSystemSubscriberView, notifier::EventNotifier, rule_engine::NotifyRuleEngine,
+    rules::ParseConfigError,
+};
+use rustfs_s3_common::EventName;
+use std::sync::Arc;
+use tracing::{debug, info, warn};
+
+#[derive(Clone)]
+pub struct NotifyBucketConfigManager {
+    notifier: Arc<EventNotifier>,
+    rule_engine: NotifyRuleEngine,
+    subscriber_view: Arc<NotificationSystemSubscriberView>,
+}
+
+impl NotifyBucketConfigManager {
+    pub fn new(
+        notifier: Arc<EventNotifier>,
+        rule_engine: NotifyRuleEngine,
+        subscriber_view: Arc<NotificationSystemSubscriberView>,
+    ) -> Self {
+        Self {
+            notifier,
+            rule_engine,
+            subscriber_view,
+        }
+    }
+
+    pub async fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool {
+        if !self.subscriber_view.has_subscriber(bucket, event) {
+            return false;
+        }
+        self.rule_engine.has_subscriber(bucket, event).await
+    }
+
+    pub async fn load_bucket_notification_config(
+        &self,
+        bucket: &str,
+        cfg: &BucketNotificationConfig,
+    ) -> Result<(), NotificationError> {
+        let arn_list = self.notifier.get_arn_list(&cfg.region).await;
+        if arn_list.is_empty() {
+            return Err(NotificationError::Configuration(notify_configuration_hint()));
+        }
+        info!("Available ARNs: {:?}", arn_list);
+
+        if let Err(e) = cfg.validate(&cfg.region, &arn_list) {
+            debug!("Bucket notification config validation region:{} failed: {}", &cfg.region, e);
+            if !matches!(e, ParseConfigError::ArnNotFound(_)) {
+                return Err(NotificationError::BucketNotification(e.to_string()));
+            }
+            warn!(
+                bucket = %bucket,
+                region = %cfg.region,
+                error = %e,
+                "Bucket notification config references missing target ARN; keeping compatibility and loading remaining rules"
+            );
+        }
+
+        self.subscriber_view.apply_bucket_config(bucket, cfg);
+        self.rule_engine.set_bucket_rules(bucket, cfg.get_rules_map().clone()).await;
+        info!("Loaded notification config for bucket: {}", bucket);
+        Ok(())
+    }
+
+    pub async fn remove_bucket_notification_config(&self, bucket: &str) {
+        self.subscriber_view.clear_bucket(bucket);
+        self.rule_engine.clear_bucket_rules(bucket).await;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::NotifyBucketConfigManager;
+    use crate::{
+        BucketNotificationConfig, integration::NotificationMetrics,
+        notification_system_subscriber::NotificationSystemSubscriberView, notifier::EventNotifier, rule_engine::NotifyRuleEngine,
+    };
+    use rustfs_s3_common::EventName;
+    use rustfs_targets::arn::TargetID;
+    use std::sync::Arc;
+
+    fn build_manager() -> NotifyBucketConfigManager {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier::new(metrics, rule_engine.clone()));
+        let subscriber_view = Arc::new(NotificationSystemSubscriberView::new());
+        NotifyBucketConfigManager::new(notifier, rule_engine, subscriber_view)
+    }
+
+    #[tokio::test]
+    async fn bucket_config_manager_reports_no_subscriber_for_empty_state() {
+        let manager = build_manager();
+        assert!(!manager.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
+    }
+
+    #[tokio::test]
+    async fn bucket_config_manager_clears_bucket_snapshot() {
+        let manager = build_manager();
+        let target_id = TargetID::new("primary".to_string(), "webhook".to_string());
+        let mut cfg = BucketNotificationConfig::new("us-east-1");
+        cfg.add_rule(&[EventName::ObjectCreatedPut], "*".to_string(), target_id);
+
+        manager.subscriber_view.apply_bucket_config("bucket", &cfg);
+        assert!(manager.subscriber_view.has_subscriber("bucket", &EventName::ObjectCreatedPut));
+
+        manager.remove_bucket_notification_config("bucket").await;
+        assert!(!manager.subscriber_view.has_subscriber("bucket", &EventName::ObjectCreatedPut));
+    }
+}

crates/notify/src/config_manager.rs

@@ -0,0 +1,338 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::{
+    Event, NotificationError, registry::TargetRegistry, rule_engine::NotifyRuleEngine, runtime_facade::NotifyRuntimeFacade,
+};
+use rustfs_config::notify::{
+    NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS,
+    NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
+};
+use rustfs_ecstore::config::{Config, KVS};
+use rustfs_targets::{Target, arn::TargetID};
+use std::sync::Arc;
+use tokio::sync::RwLock;
+use tracing::{debug, info, warn};
+
+pub(crate) fn notify_configuration_hint() -> String {
+    let webhook_enable_primary = format!("{}_PRIMARY", rustfs_config::notify::ENV_NOTIFY_WEBHOOK_ENABLE);
+    let webhook_endpoint_primary = format!("{}_PRIMARY", rustfs_config::notify::ENV_NOTIFY_WEBHOOK_ENDPOINT);
+    format!(
+        "No notify targets configured. Check {}=true and instance-scoped target env vars (for example {webhook_enable_primary} + {webhook_endpoint_primary} for arn:rustfs:sqs::primary:webhook). If using default queue_dir, ensure {} is writable.",
+        rustfs_config::ENV_NOTIFY_ENABLE,
+        rustfs_config::EVENT_DEFAULT_DIR,
+    )
+}
+
+fn subsystem_target_type(target_type: &str) -> &str {
+    match target_type {
+        NOTIFY_AMQP_SUB_SYS => "amqp",
+        NOTIFY_WEBHOOK_SUB_SYS => "webhook",
+        NOTIFY_KAFKA_SUB_SYS => "kafka",
+        NOTIFY_MQTT_SUB_SYS => "mqtt",
+        NOTIFY_MYSQL_SUB_SYS => "mysql",
+        NOTIFY_NATS_SUB_SYS => "nats",
+        NOTIFY_POSTGRES_SUB_SYS => "postgres",
+        NOTIFY_PULSAR_SUB_SYS => "pulsar",
+        NOTIFY_REDIS_SUB_SYS => "redis",
+        _ => target_type,
+    }
+}
+
+pub fn runtime_target_id_for_subsystem(target_type: &str, target_name: &str) -> TargetID {
+    TargetID {
+        id: target_name.to_lowercase(),
+        name: subsystem_target_type(target_type).to_string(),
+    }
+}
+
+#[derive(Clone)]
+pub struct NotifyConfigManager {
+    config: Arc<RwLock<Config>>,
+    registry: Arc<TargetRegistry>,
+    rule_engine: NotifyRuleEngine,
+    runtime_facade: NotifyRuntimeFacade,
+}
+
+impl NotifyConfigManager {
+    pub fn new(
+        config: Arc<RwLock<Config>>,
+        registry: Arc<TargetRegistry>,
+        rule_engine: NotifyRuleEngine,
+        runtime_facade: NotifyRuntimeFacade,
+    ) -> Self {
+        Self {
+            config,
+            registry,
+            rule_engine,
+            runtime_facade,
+        }
+    }
+
+    pub async fn init(&self) -> Result<(), NotificationError> {
+        info!("Initialize notification system...");
+
+        let config = {
+            let guard = self.config.read().await;
+            debug!(
+                subsystem_count = guard.0.len(),
+                "Initializing notification system with configuration summary"
+            );
+            guard.clone()
+        };
+
+        let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self.registry.create_targets_from_config(&config).await?;
+
+        info!("{} notification targets were created", targets.len());
+        if targets.is_empty() {
+            warn!("{}", notify_configuration_hint());
+        }
+
+        let activation = self.runtime_facade.activate_targets_with_replay(targets).await;
+        self.runtime_facade.replace_targets(activation).await?;
+        info!("Notification system initialized");
+        Ok(())
+    }
+
+    pub async fn remove_target(&self, target_id: &TargetID, target_type: &str) -> Result<(), NotificationError> {
+        info!("Attempting to remove target: {}", target_id);
+
+        let ttype = target_type.to_lowercase();
+        let tname = target_id.id.to_lowercase();
+
+        self.update_config_and_reload(|config| {
+            let mut changed = false;
+            if let Some(targets_of_type) = config.0.get_mut(&ttype) {
+                if targets_of_type.remove(&tname).is_some() {
+                    info!("Removed target {} from configuration", target_id);
+                    changed = true;
+                }
+                if targets_of_type.is_empty() {
+                    config.0.remove(&ttype);
+                }
+            }
+            if !changed {
+                warn!("Target {} not found in configuration", target_id);
+            }
+            changed
+        })
+        .await
+    }
+
+    pub async fn set_target_config(&self, target_type: &str, target_name: &str, kvs: KVS) -> Result<(), NotificationError> {
+        info!("Setting config for target {} of type {}", target_name, target_type);
+        let ttype = target_type.to_lowercase();
+        let tname = target_name.to_lowercase();
+        self.update_config_and_reload(|config| {
+            config.0.entry(ttype.clone()).or_default().insert(tname.clone(), kvs.clone());
+            true
+        })
+        .await
+    }
+
+    pub async fn remove_target_config(&self, target_type: &str, target_name: &str) -> Result<(), NotificationError> {
+        info!("Removing config for target {} of type {}", target_name, target_type);
+
+        let ttype = target_type.to_lowercase();
+        let tname = target_name.to_lowercase();
+        let target_id = runtime_target_id_for_subsystem(&ttype, &tname);
+
+        if self.rule_engine.is_target_bound_to_any_bucket(&target_id).await {
+            return Err(NotificationError::Configuration(format!(
+                "Target is still bound to bucket rules and deletion is prohibited: type={} name={}",
+                ttype, tname
+            )));
+        }
+
+        self.update_config_and_reload(|config| {
+            let mut changed = false;
+            if let Some(targets) = config.0.get_mut(&ttype) {
+                if targets.remove(&tname).is_some() {
+                    changed = true;
+                }
+                if targets.is_empty() {
+                    config.0.remove(&ttype);
+                }
+            }
+            if !changed {
+                info!("Target {} of type {} not found, no changes made.", target_name, target_type);
+            }
+            debug!(
+                subsystem_count = config.0.len(),
+                "Target config removal processed and configuration summary updated"
+            );
+            changed
+        })
+        .await
+    }
+
+    pub async fn reload_config(&self, new_config: Config) -> Result<(), NotificationError> {
+        info!("Reload notification configuration starts");
+
+        self.update_config(new_config.clone()).await;
+
+        let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self
+            .registry
+            .create_targets_from_config(&new_config)
+            .await
+            .map_err(NotificationError::Target)?;
+
+        info!("{} notification targets were created from the new configuration", targets.len());
+        if targets.is_empty() {
+            warn!("{}", notify_configuration_hint());
+        }
+
+        let activation = self.runtime_facade.activate_targets_with_replay(targets).await;
+        self.runtime_facade.replace_targets(activation).await?;
+        info!("Configuration reloaded end");
+        Ok(())
+    }
+
+    async fn update_config(&self, new_config: Config) {
+        let mut config = self.config.write().await;
+        *config = new_config;
+    }
+
+    async fn update_config_and_reload<F>(&self, mut modifier: F) -> Result<(), NotificationError>
+    where
+        F: FnMut(&mut Config) -> bool,
+    {
+        let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
+            return Err(NotificationError::StorageNotAvailable(
+                "Failed to save target configuration: server storage not initialized".to_string(),
+            ));
+        };
+
+        let mut new_config = rustfs_ecstore::config::com::read_config_without_migrate(store.clone())
+            .await
+            .map_err(|e| NotificationError::ReadConfig(e.to_string()))?;
+
+        if !modifier(&mut new_config) {
+            info!("Configuration not changed, skipping save and reload.");
+            return Ok(());
+        }
+
+        rustfs_ecstore::config::com::save_server_config(store, &new_config)
+            .await
+            .map_err(|e| NotificationError::SaveConfig(e.to_string()))?;
+
+        info!("Configuration updated. Reloading system...");
+        self.reload_config(new_config).await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{NotifyConfigManager, runtime_target_id_for_subsystem};
+    use crate::{
+        integration::NotificationMetrics, notifier::EventNotifier, registry::TargetRegistry, rule_engine::NotifyRuleEngine,
+        runtime_facade::NotifyRuntimeFacade,
+    };
+    use rustfs_config::notify::{
+        NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_SUB_SYS,
+        NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
+    };
+    use rustfs_ecstore::config::Config;
+    use rustfs_targets::ReplayWorkerManager;
+    use std::sync::Arc;
+    use tokio::sync::{RwLock, Semaphore};
+
+    fn build_manager() -> NotifyConfigManager {
+        let config = Arc::new(RwLock::new(Config::default()));
+        let registry = Arc::new(TargetRegistry::new());
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
+        let target_list = notifier.target_list();
+        let runtime_facade = NotifyRuntimeFacade::new(
+            target_list,
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            Arc::new(Semaphore::new(4)),
+            metrics,
+        );
+
+        NotifyConfigManager::new(config, registry, rule_engine, runtime_facade)
+    }
+
+    #[tokio::test]
+    async fn config_manager_init_accepts_empty_target_set() {
+        let manager = build_manager();
+        manager.init().await.expect("init should succeed for empty targets");
+    }
+
+    #[tokio::test]
+    async fn config_manager_reload_accepts_empty_target_set() {
+        let manager = build_manager();
+        manager
+            .reload_config(Config::default())
+            .await
+            .expect("reload_config should succeed for empty targets");
+    }
+
+    #[test]
+    fn runtime_target_id_for_subsystem_maps_notify_webhook_to_runtime_type() {
+        let target_id = runtime_target_id_for_subsystem(NOTIFY_WEBHOOK_SUB_SYS, "Primary");
+        assert_eq!(target_id.id, "primary");
+        assert_eq!(target_id.name, "webhook");
+    }
+
+    #[test]
+    fn runtime_target_id_for_subsystem_maps_notify_amqp_to_runtime_type() {
+        let target_id = runtime_target_id_for_subsystem(NOTIFY_AMQP_SUB_SYS, "Primary");
+        assert_eq!(target_id.id, "primary");
+        assert_eq!(target_id.name, "amqp");
+    }
+
+    #[test]
+    fn runtime_target_id_for_subsystem_maps_notify_mqtt_to_runtime_type() {
+        let target_id = runtime_target_id_for_subsystem(NOTIFY_MQTT_SUB_SYS, "Analytics");
+        assert_eq!(target_id.id, "analytics");
+        assert_eq!(target_id.name, "mqtt");
+    }
+
+    #[test]
+    fn runtime_target_id_for_subsystem_maps_notify_kafka_to_runtime_type() {
+        let target_id = runtime_target_id_for_subsystem(NOTIFY_KAFKA_SUB_SYS, "EventBus");
+        assert_eq!(target_id.id, "eventbus");
+        assert_eq!(target_id.name, "kafka");
+    }
+
+    #[test]
+    fn runtime_target_id_for_subsystem_maps_notify_nats_to_runtime_type() {
+        let target_id = runtime_target_id_for_subsystem(NOTIFY_NATS_SUB_SYS, "Bus");
+        assert_eq!(target_id.id, "bus");
+        assert_eq!(target_id.name, "nats");
+    }
+
+    #[test]
+    fn runtime_target_id_for_subsystem_maps_notify_pulsar_to_runtime_type() {
+        let target_id = runtime_target_id_for_subsystem(NOTIFY_PULSAR_SUB_SYS, "Ledger");
+        assert_eq!(target_id.id, "ledger");
+        assert_eq!(target_id.name, "pulsar");
+    }
+
+    #[test]
+    fn runtime_target_id_for_subsystem_maps_notify_redis_to_runtime_type() {
+        let target_id = runtime_target_id_for_subsystem(NOTIFY_REDIS_SUB_SYS, "Primary");
+        assert_eq!(target_id.id, "primary");
+        assert_eq!(target_id.name, "redis");
+    }
+
+    #[test]
+    fn runtime_target_id_for_subsystem_maps_notify_postgres_to_runtime_type() {
+        let target_id = runtime_target_id_for_subsystem(NOTIFY_POSTGRES_SUB_SYS, "AuditTrail");
+        assert_eq!(target_id.id, "audittrail");
+        assert_eq!(target_id.name, "postgres");
+    }
+}

crates/notify/src/event_bridge.rs

@@ -0,0 +1,15 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub use crate::pipeline::{LiveEventHistory, NotifyEventBridge};

crates/notify/src/factory.rs

@@ -13,146 +13,11 @@
 // limitations under the License.
 
 use crate::Event;
-use rustfs_config::EVENT_DEFAULT_DIR;
-use rustfs_config::notify::{
-    NOTIFY_AMQP_KEYS, NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS,
-    NOTIFY_MYSQL_KEYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_KEYS,
-    NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_DEFAULT_CHANNEL, NOTIFY_REDIS_KEYS,
-    NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
-};
-use rustfs_targets::config::{
-    build_amqp_args, build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args,
-    build_pulsar_args, build_redis_args, build_webhook_args, validate_amqp_config, validate_kafka_config, validate_mqtt_config,
-    validate_mysql_config, validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config,
-    validate_webhook_config,
-};
-use rustfs_targets::target::{ChannelTargetType, TargetType};
-use rustfs_targets::{BuiltinTargetDescriptor, TargetPluginDescriptor, TargetRequestValidator, boxed_target};
+use rustfs_targets::catalog::builtin::builtin_notify_target_descriptors;
+use rustfs_targets::{BuiltinTargetDescriptor, TargetPluginDescriptor};
 
 pub fn builtin_target_descriptors() -> Vec<BuiltinTargetDescriptor<Event>> {
-    vec![
-        BuiltinTargetDescriptor::new(
-            NOTIFY_WEBHOOK_SUB_SYS,
-            TargetRequestValidator::Webhook,
-            TargetPluginDescriptor::new(
-                ChannelTargetType::Webhook.as_str(),
-                NOTIFY_WEBHOOK_KEYS,
-                |config| validate_webhook_config(config, EVENT_DEFAULT_DIR),
-                |id, config| {
-                    let args = build_webhook_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
-                    Ok(boxed_target(rustfs_targets::target::webhook::WebhookTarget::new(id, args)?))
-                },
-            ),
-        ),
-        BuiltinTargetDescriptor::new(
-            NOTIFY_AMQP_SUB_SYS,
-            TargetRequestValidator::Amqp(TargetType::NotifyEvent),
-            TargetPluginDescriptor::new(
-                ChannelTargetType::Amqp.as_str(),
-                NOTIFY_AMQP_KEYS,
-                |config| validate_amqp_config(config, EVENT_DEFAULT_DIR),
-                |id, config| {
-                    let args = build_amqp_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
-                    Ok(boxed_target(rustfs_targets::target::amqp::AMQPTarget::new(id, args)?))
-                },
-            ),
-        ),
-        BuiltinTargetDescriptor::new(
-            NOTIFY_KAFKA_SUB_SYS,
-            TargetRequestValidator::Kafka(TargetType::NotifyEvent),
-            TargetPluginDescriptor::new(
-                ChannelTargetType::Kafka.as_str(),
-                NOTIFY_KAFKA_KEYS,
-                |config| validate_kafka_config(config, EVENT_DEFAULT_DIR),
-                |id, config| {
-                    let args = build_kafka_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
-                    Ok(boxed_target(rustfs_targets::target::kafka::KafkaTarget::new(id, args)?))
-                },
-            ),
-        ),
-        BuiltinTargetDescriptor::new(
-            NOTIFY_MQTT_SUB_SYS,
-            TargetRequestValidator::Mqtt,
-            TargetPluginDescriptor::new(
-                ChannelTargetType::Mqtt.as_str(),
-                NOTIFY_MQTT_KEYS,
-                validate_mqtt_config,
-                |id, config| {
-                    let args = build_mqtt_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
-                    Ok(boxed_target(rustfs_targets::target::mqtt::MQTTTarget::new(id, args)?))
-                },
-            ),
-        ),
-        BuiltinTargetDescriptor::new(
-            NOTIFY_MYSQL_SUB_SYS,
-            TargetRequestValidator::MySql(TargetType::NotifyEvent),
-            TargetPluginDescriptor::new(
-                ChannelTargetType::MySql.as_str(),
-                NOTIFY_MYSQL_KEYS,
-                |config| validate_mysql_config(config, EVENT_DEFAULT_DIR),
-                |id, config| {
-                    let args = build_mysql_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
-                    Ok(boxed_target(rustfs_targets::target::mysql::MySqlTarget::new(id, args)?))
-                },
-            ),
-        ),
-        BuiltinTargetDescriptor::new(
-            NOTIFY_NATS_SUB_SYS,
-            TargetRequestValidator::Nats(TargetType::NotifyEvent),
-            TargetPluginDescriptor::new(
-                ChannelTargetType::Nats.as_str(),
-                NOTIFY_NATS_KEYS,
-                |config| validate_nats_config(config, EVENT_DEFAULT_DIR),
-                |id, config| {
-                    let args = build_nats_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
-                    Ok(boxed_target(rustfs_targets::target::nats::NATSTarget::new(id, args)?))
-                },
-            ),
-        ),
-        BuiltinTargetDescriptor::new(
-            NOTIFY_POSTGRES_SUB_SYS,
-            TargetRequestValidator::Postgres(TargetType::NotifyEvent),
-            TargetPluginDescriptor::new(
-                ChannelTargetType::Postgres.as_str(),
-                NOTIFY_POSTGRES_KEYS,
-                |config| validate_postgres_config(config, EVENT_DEFAULT_DIR),
-                |id, config| {
-                    let args = build_postgres_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
-                    Ok(boxed_target(rustfs_targets::target::postgres::PostgresTarget::new(id, args)?))
-                },
-            ),
-        ),
-        BuiltinTargetDescriptor::new(
-            NOTIFY_REDIS_SUB_SYS,
-            TargetRequestValidator::Redis {
-                default_channel: NOTIFY_REDIS_DEFAULT_CHANNEL,
-                target_type: TargetType::NotifyEvent,
-            },
-            TargetPluginDescriptor::new(
-                ChannelTargetType::Redis.as_str(),
-                NOTIFY_REDIS_KEYS,
-                |config| validate_redis_config(config, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL),
-                |id, config| {
-                    let args =
-                        build_redis_args(config, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL, TargetType::NotifyEvent)?;
-                    Ok(boxed_target(rustfs_targets::target::redis::RedisTarget::new(id, args)?))
-                },
-            ),
-        ),
-        BuiltinTargetDescriptor::new(
-            NOTIFY_PULSAR_SUB_SYS,
-            TargetRequestValidator::Pulsar(TargetType::NotifyEvent),
-            TargetPluginDescriptor::new(
-                ChannelTargetType::Pulsar.as_str(),
-                NOTIFY_PULSAR_KEYS,
-                |config| validate_pulsar_config(config, EVENT_DEFAULT_DIR),
-                |id, config| {
-                    let args = build_pulsar_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
-                    Ok(boxed_target(rustfs_targets::target::pulsar::PulsarTarget::new(id, args)?))
-                },
-            ),
-        ),
-    ]
+    builtin_notify_target_descriptors::<Event>()
 }
 
 pub fn builtin_target_plugins() -> Vec<TargetPluginDescriptor<Event>> {

crates/notify/src/integration.rs

@@ -13,66 +13,29 @@
 // limitations under the License.
 
 use crate::notification_system_subscriber::NotificationSystemSubscriberView;
-use crate::notifier::TargetList;
+use crate::notifier::{EventNotifier, TargetList};
+use crate::services::NotifyServices;
 use crate::{
-    Event,
-    error::NotificationError,
-    notifier::EventNotifier,
-    registry::TargetRegistry,
-    rules::{BucketNotificationConfig, ParseConfigError},
-    stream,
+    Event, error::NotificationError, pipeline::LiveEventHistory, registry::TargetRegistry, rule_engine::NotifyRuleEngine,
+    rules::BucketNotificationConfig,
 };
 use hashbrown::HashMap;
-use rustfs_config::notify::{
-    DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_WEBHOOK_ENABLE,
-    ENV_NOTIFY_WEBHOOK_ENDPOINT, NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS,
-    NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
-};
-use rustfs_config::{ENV_NOTIFY_ENABLE, EVENT_DEFAULT_DIR};
+use metrics::{counter, gauge};
+use rustfs_config::notify::{DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY};
 use rustfs_ecstore::config::{Config, KVS};
 use rustfs_s3_common::EventName;
 use rustfs_targets::arn::TargetID;
-use rustfs_targets::store::{Key, Store};
-use rustfs_targets::target::QueuedPayload;
-use rustfs_targets::{StoreError, Target};
-use std::collections::VecDeque;
+use rustfs_targets::{ReplayWorkerManager, RuntimeTargetHealthSnapshot, SharedTarget};
 use std::sync::Arc;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::time::{Duration, Instant};
-use tokio::sync::{RwLock, Semaphore, broadcast, mpsc};
-use tracing::{debug, info, warn};
+use tokio::sync::{RwLock, Semaphore, broadcast};
+use tracing::info;
 
-const MAX_RECENT_LIVE_EVENTS: usize = 1024;
-
-fn notify_configuration_hint() -> String {
-    let webhook_enable_primary = format!("{ENV_NOTIFY_WEBHOOK_ENABLE}_PRIMARY");
-    let webhook_endpoint_primary = format!("{ENV_NOTIFY_WEBHOOK_ENDPOINT}_PRIMARY");
-    format!(
-        "No notify targets configured. Check {ENV_NOTIFY_ENABLE}=true and instance-scoped target env vars (for example {webhook_enable_primary} + {webhook_endpoint_primary} for arn:rustfs:sqs::primary:webhook). If using default queue_dir, ensure {EVENT_DEFAULT_DIR} is writable."
-    )
-}
-
-fn subsystem_target_type(target_type: &str) -> &str {
-    match target_type {
-        NOTIFY_AMQP_SUB_SYS => "amqp",
-        NOTIFY_WEBHOOK_SUB_SYS => "webhook",
-        NOTIFY_KAFKA_SUB_SYS => "kafka",
-        NOTIFY_MQTT_SUB_SYS => "mqtt",
-        NOTIFY_MYSQL_SUB_SYS => "mysql",
-        NOTIFY_NATS_SUB_SYS => "nats",
-        NOTIFY_POSTGRES_SUB_SYS => "postgres",
-        NOTIFY_PULSAR_SUB_SYS => "pulsar",
-        NOTIFY_REDIS_SUB_SYS => "redis",
-        _ => target_type,
-    }
-}
-
-fn runtime_target_id_for_subsystem(target_type: &str, target_name: &str) -> TargetID {
-    TargetID {
-        id: target_name.to_lowercase(),
-        name: subsystem_target_type(target_type).to_string(),
-    }
-}
+const METRIC_NOTIFICATION_CURRENT_SEND_IN_PROGRESS: &str = "rustfs_notification_current_send_in_progress";
+const METRIC_NOTIFICATION_EVENTS_ERRORS_TOTAL: &str = "rustfs_notification_events_errors_total";
+const METRIC_NOTIFICATION_EVENTS_SENT_TOTAL: &str = "rustfs_notification_events_sent_total";
+const METRIC_NOTIFICATION_EVENTS_SKIPPED_TOTAL: &str = "rustfs_notification_events_skipped_total";
 
 #[derive(Clone)]
 pub struct LiveEventBatch {
@@ -81,46 +44,6 @@ pub struct LiveEventBatch {
     pub truncated: bool,
 }
 
-#[derive(Default)]
-struct LiveEventHistory {
-    next_sequence: u64,
-    events: VecDeque<(u64, Arc<Event>)>,
-}
-
-impl LiveEventHistory {
-    fn record(&mut self, event: Arc<Event>) {
-        self.next_sequence = self.next_sequence.saturating_add(1);
-        self.events.push_back((self.next_sequence, event));
-        while self.events.len() > MAX_RECENT_LIVE_EVENTS {
-            self.events.pop_front();
-        }
-    }
-
-    fn snapshot_since(&self, after_sequence: u64, limit: usize) -> LiveEventBatch {
-        let mut events = Vec::new();
-        let mut next_sequence = after_sequence;
-        let mut truncated = false;
-
-        for (sequence, event) in self.events.iter() {
-            if *sequence <= after_sequence {
-                continue;
-            }
-            if events.len() >= limit {
-                truncated = true;
-                break;
-            }
-            next_sequence = *sequence;
-            events.push(event.clone());
-        }
-
-        LiveEventBatch {
-            events,
-            next_sequence,
-            truncated,
-        }
-    }
-}
-
 /// Notify the system of monitoring indicators
 pub struct NotificationMetrics {
     /// The number of events currently being processed
@@ -231,18 +154,7 @@ pub struct NotificationSystem {
     pub registry: Arc<TargetRegistry>,
     /// The current configuration
     pub config: Arc<RwLock<Config>>,
-    /// Cancel sender for managing stream processing tasks
-    stream_cancellers: Arc<RwLock<HashMap<TargetID, mpsc::Sender<()>>>>,
-    /// Concurrent control signal quantity
-    concurrency_limiter: Arc<Semaphore>,
-    /// Monitoring indicators
-    metrics: Arc<NotificationMetrics>,
-    /// Subscriber view
-    subscriber_view: NotificationSystemSubscriberView,
-    /// Live event fan-out for in-process streaming consumers.
-    live_event_sender: broadcast::Sender<Arc<Event>>,
-    /// Recent live event history for peer fan-in consumers.
-    live_event_history: Arc<RwLock<LiveEventHistory>>,
+    services: NotifyServices,
 }
 
 impl NotificationSystem {
@@ -252,107 +164,40 @@ impl NotificationSystem {
             rustfs_utils::get_env_usize(ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY);
         let (live_event_sender, _) = broadcast::channel(1024);
         let metrics = Arc::new(NotificationMetrics::new());
-        NotificationSystem {
-            subscriber_view: NotificationSystemSubscriberView::new(),
-            notifier: Arc::new(EventNotifier::new(metrics.clone())),
-            registry: Arc::new(TargetRegistry::new()),
-            config: Arc::new(RwLock::new(config)),
-            stream_cancellers: Arc::new(RwLock::new(HashMap::new())),
-            concurrency_limiter: Arc::new(Semaphore::new(concurrency_limiter)), // Limit the maximum number of concurrent processing events to 20
+        let subscriber_view = Arc::new(NotificationSystemSubscriberView::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
+        let target_list = notifier.target_list();
+        let registry = Arc::new(TargetRegistry::new());
+        let config = Arc::new(RwLock::new(config));
+        let stream_cancellers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
+        let concurrency_limiter = Arc::new(Semaphore::new(concurrency_limiter)); // Limit the maximum number of concurrent processing events to 20
+        let live_event_history = Arc::new(RwLock::new(LiveEventHistory::default()));
+        let services = NotifyServices::new(
+            notifier.clone(),
+            rule_engine,
+            target_list,
+            registry.clone(),
+            config.clone(),
+            stream_cancellers,
+            concurrency_limiter,
             metrics,
+            subscriber_view,
             live_event_sender,
-            live_event_history: Arc::new(RwLock::new(LiveEventHistory::default())),
+            live_event_history,
+        );
+
+        NotificationSystem {
+            notifier,
+            registry,
+            config,
+            services,
         }
     }
 
-    /// Initializes targets and starts event streams for those with stores.
-    /// Returns a map of (target_id -> cancel_sender) for streams that were started.
-    async fn init_targets_and_start_streams(
-        &self,
-        targets: &[Box<dyn Target<Event> + Send + Sync>],
-    ) -> HashMap<TargetID, mpsc::Sender<()>> {
-        let mut cancellers = HashMap::new();
-        for target in targets {
-            let target_id = target.id();
-            info!("Initializing target: {}", target_id);
-
-            let has_store = target.store().is_some();
-
-            if let Err(e) = target.init().await {
-                warn!("Target {} Initialization failed: {}", target_id, e);
-                // For targets without a store, init failure is fatal — skip.
-                // For store-backed targets, still start the stream so queued events
-                // can be drained when connectivity recovers (send_from_store retries).
-                if !has_store {
-                    continue;
-                }
-                warn!(
-                    "Target {} has a store, starting stream despite init failure — \
-                     connectivity will be retried by send_from_store",
-                    target_id
-                );
-            } else {
-                debug!("Target {} initialized successfully, enabled: {}", target_id, target.is_enabled());
-            }
-
-            if !target.is_enabled() {
-                info!("Target {} is not enabled, event stream processing is skipped", target_id);
-                continue;
-            }
-
-            if let Some(store) = target.store() {
-                info!("Start event stream processing for target {}", target_id);
-
-                let store_clone = store.boxed_clone();
-                let target_arc = Arc::from(target.clone_dyn());
-
-                let cancel_tx = self.enhanced_start_event_stream(
-                    store_clone,
-                    target_arc,
-                    self.metrics.clone(),
-                    self.concurrency_limiter.clone(),
-                );
-
-                let target_id_clone = target_id.clone();
-                cancellers.insert(target_id, cancel_tx);
-                info!("Event stream processing for target {} is started successfully", target_id_clone);
-            } else {
-                info!("Target {} No storage is configured, event stream processing is skipped", target_id);
-            }
-        }
-        cancellers
-    }
-
     /// Initializes the notification system
     pub async fn init(&self) -> Result<(), NotificationError> {
-        info!("Initialize notification system...");
-
-        let config = {
-            let guard = self.config.read().await;
-            debug!(
-                subsystem_count = guard.0.len(),
-                "Initializing notification system with configuration summary"
-            );
-            guard.clone()
-        };
-
-        let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self.registry.create_targets_from_config(&config).await?;
-
-        info!("{} notification targets were created", targets.len());
-        if targets.is_empty() {
-            warn!("{}", notify_configuration_hint());
-        }
-
-        // Initialize targets and start event streams
-        let cancellers = self.init_targets_and_start_streams(&targets).await;
-
-        // Update canceller collection
-        *self.stream_cancellers.write().await = cancellers;
-
-        // Initialize the bucket target
-        self.notifier.init_bucket_targets(targets).await?;
-        info!("Notification system initialized");
-        Ok(())
+        self.services.config_manager.init().await
     }
 
     /// Gets a list of Targets for all currently active (initialized).
@@ -360,7 +205,7 @@ impl NotificationSystem {
     /// # Return
     /// A Vec containing all active Targets `TargetID`.
     pub async fn get_active_targets(&self) -> Vec<TargetID> {
-        self.notifier.target_list().read().await.keys()
+        self.services.runtime_view.get_active_targets().await
     }
 
     /// Gets the complete Target list, including both active and inactive Targets.
@@ -368,67 +213,34 @@ impl NotificationSystem {
     /// # Return
     /// An `Arc<RwLock<TargetList>>` containing all Targets.
     pub async fn get_all_targets(&self) -> Arc<RwLock<TargetList>> {
-        self.notifier.target_list()
+        self.services.runtime_view.get_all_targets()
     }
 
     /// Gets all Target values, including both active and inactive Targets.
     ///
     /// # Return
     /// A Vec containing all Targets.
-    pub async fn get_target_values(&self) -> Vec<Arc<dyn Target<Event> + Send + Sync>> {
-        self.notifier.target_list().read().await.values()
+    pub async fn get_target_values(&self) -> Vec<SharedTarget<Event>> {
+        self.services.runtime_view.get_target_values().await
     }
 
     /// Checks if there are active subscribers for the given bucket and event name.
     pub async fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool {
-        if !self.subscriber_view.has_subscriber(bucket, event) {
-            return false;
-        }
-        self.notifier.has_subscriber(bucket, event).await
+        self.services.bucket_config_manager.has_subscriber(bucket, event).await
     }
 
     /// Returns true when at least one in-process consumer is subscribed to live events.
     pub fn has_live_listeners(&self) -> bool {
-        self.live_event_sender.receiver_count() > 0
+        self.services.pipeline.has_live_listeners()
     }
 
     /// Subscribes to the in-process live event stream.
     pub fn subscribe_live_events(&self) -> broadcast::Receiver<Arc<Event>> {
-        self.live_event_sender.subscribe()
+        self.services.pipeline.subscribe_live_events()
     }
 
     pub async fn recent_live_events_since(&self, after_sequence: u64, limit: usize) -> LiveEventBatch {
-        let history = self.live_event_history.read().await;
-        history.snapshot_since(after_sequence, limit.max(1))
-    }
-
-    async fn update_config_and_reload<F>(&self, mut modifier: F) -> Result<(), NotificationError>
-    where
-        F: FnMut(&mut Config) -> bool, // The closure returns a boolean value indicating whether the configuration has been changed
-    {
-        let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
-            return Err(NotificationError::StorageNotAvailable(
-                "Failed to save target configuration: server storage not initialized".to_string(),
-            ));
-        };
-
-        let mut new_config = rustfs_ecstore::config::com::read_config_without_migrate(store.clone())
-            .await
-            .map_err(|e| NotificationError::ReadConfig(e.to_string()))?;
-
-        if !modifier(&mut new_config) {
-            // If the closure indication has not changed, return in advance
-            info!("Configuration not changed, skipping save and reload.");
-            return Ok(());
-        }
-
-        // Save the modified configuration to storage
-        rustfs_ecstore::config::com::save_server_config(store, &new_config)
-            .await
-            .map_err(|e| NotificationError::SaveConfig(e.to_string()))?;
-
-        info!("Configuration updated. Reloading system...");
-        self.reload_config(new_config).await
+        self.services.pipeline.recent_live_events_since(after_sequence, limit).await
     }
 
     /// Accurately remove a Target and its related resources through TargetID.
@@ -444,28 +256,7 @@ impl NotificationSystem {
     /// # return
     /// If successful, return `Ok(())`.
     pub async fn remove_target(&self, target_id: &TargetID, target_type: &str) -> Result<(), NotificationError> {
-        info!("Attempting to remove target: {}", target_id);
-
-        let ttype = target_type.to_lowercase();
-        let tname = target_id.id.to_lowercase();
-
-        self.update_config_and_reload(|config| {
-            let mut changed = false;
-            if let Some(targets_of_type) = config.0.get_mut(&ttype) {
-                if targets_of_type.remove(&tname).is_some() {
-                    info!("Removed target {} from configuration", target_id);
-                    changed = true;
-                }
-                if targets_of_type.is_empty() {
-                    config.0.remove(&ttype);
-                }
-            }
-            if !changed {
-                warn!("Target {} not found in configuration", target_id);
-            }
-            changed
-        })
-        .await
+        self.services.config_manager.remove_target(target_id, target_type).await
     }
 
     /// Set or update a Target configuration.
@@ -481,14 +272,10 @@ impl NotificationSystem {
     /// If the target configuration is successfully set, it returns Ok(()).
     /// If the target configuration is invalid, it returns Err(NotificationError::Configuration).
     pub async fn set_target_config(&self, target_type: &str, target_name: &str, kvs: KVS) -> Result<(), NotificationError> {
-        info!("Setting config for target {} of type {}", target_name, target_type);
-        let ttype = target_type.to_lowercase();
-        let tname = target_name.to_lowercase();
-        self.update_config_and_reload(|config| {
-            config.0.entry(ttype.clone()).or_default().insert(tname.clone(), kvs.clone());
-            true // The configuration is always modified
-        })
-        .await
+        self.services
+            .config_manager
+            .set_target_config(target_type, target_name, kvs)
+            .await
     }
 
     /// Removes all notification configurations for a bucket.
@@ -498,8 +285,10 @@ impl NotificationSystem {
     /// * `bucket` - The name of the bucket whose notification configuration is to be removed.
     ///
     pub async fn remove_bucket_notification_config(&self, bucket: &str) {
-        self.subscriber_view.clear_bucket(bucket);
-        self.notifier.remove_rules_map(bucket).await;
+        self.services
+            .bucket_config_manager
+            .remove_bucket_notification_config(bucket)
+            .await;
     }
 
     /// Removes a Target configuration.
@@ -515,108 +304,15 @@ impl NotificationSystem {
     /// If the target configuration is successfully removed, it returns Ok(()).
     /// If the target configuration does not exist, it returns Ok(()) without making any changes.
     pub async fn remove_target_config(&self, target_type: &str, target_name: &str) -> Result<(), NotificationError> {
-        info!("Removing config for target {} of type {}", target_name, target_type);
-
-        let ttype = target_type.to_lowercase();
-        let tname = target_name.to_lowercase();
-        let target_id = runtime_target_id_for_subsystem(&ttype, &tname);
-
-        // Deletion is prohibited if bucket rules refer to it
-        if self.notifier.is_target_bound_to_any_bucket(&target_id).await {
-            return Err(NotificationError::Configuration(format!(
-                "Target is still bound to bucket rules and deletion is prohibited: type={} name={}",
-                ttype, tname
-            )));
-        }
-
-        let config_result = self
-            .update_config_and_reload(|config| {
-                let mut changed = false;
-                if let Some(targets) = config.0.get_mut(&ttype) {
-                    if targets.remove(&tname).is_some() {
-                        changed = true;
-                    }
-                    if targets.is_empty() {
-                        config.0.remove(&ttype);
-                    }
-                }
-                if !changed {
-                    info!("Target {} of type {} not found, no changes made.", target_name, target_type);
-                }
-                debug!(
-                    subsystem_count = config.0.len(),
-                    "Target config removal processed and configuration summary updated"
-                );
-                changed
-            })
-            .await;
-
-        if config_result.is_ok() {
-            // Remove from target list
-            let target_list = self.notifier.target_list();
-            let mut target_list_guard = target_list.write().await;
-            let _ = target_list_guard.remove_target_only(&target_id).await;
-        }
-
-        config_result
-    }
-
-    /// Enhanced event stream startup function, including monitoring and concurrency control
-    fn enhanced_start_event_stream(
-        &self,
-        store: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>,
-        target: Arc<dyn Target<Event> + Send + Sync>,
-        metrics: Arc<NotificationMetrics>,
-        semaphore: Arc<Semaphore>,
-    ) -> mpsc::Sender<()> {
-        stream::start_event_stream_with_batching(store, target, metrics, semaphore)
-    }
-
-    /// Update configuration
-    async fn update_config(&self, new_config: Config) {
-        let mut config = self.config.write().await;
-        *config = new_config;
+        self.services
+            .config_manager
+            .remove_target_config(target_type, target_name)
+            .await
     }
 
     /// Reloads the configuration
     pub async fn reload_config(&self, new_config: Config) -> Result<(), NotificationError> {
-        info!("Reload notification configuration starts");
-
-        // Stop all existing streaming services
-        let mut cancellers = self.stream_cancellers.write().await;
-        for (target_id, cancel_tx) in cancellers.drain() {
-            info!("Stop event stream processing for target {}", target_id);
-            let _ = cancel_tx.send(()).await;
-        }
-
-        // Clear the target_list and ensure that reload is a replacement reconstruction
-        self.notifier.remove_all_bucket_targets().await;
-
-        // Update the config
-        self.update_config(new_config.clone()).await;
-
-        // Create new targets from configuration
-        let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self
-            .registry
-            .create_targets_from_config(&new_config)
-            .await
-            .map_err(NotificationError::Target)?;
-
-        info!("{} notification targets were created from the new configuration", targets.len());
-        if targets.is_empty() {
-            warn!("{}", notify_configuration_hint());
-        }
-
-        // Initialize targets and start event streams using shared helper
-        let new_cancellers = self.init_targets_and_start_streams(&targets).await;
-
-        // Update canceler collection
-        *cancellers = new_cancellers;
-
-        // Initialize the bucket target
-        self.notifier.init_bucket_targets(targets).await?;
-        info!("Configuration reloaded end");
-        Ok(())
+        self.services.config_manager.reload_config(new_config).await
     }
 
     /// Loads the bucket notification configuration
@@ -625,93 +321,41 @@ impl NotificationSystem {
         bucket: &str,
         cfg: &BucketNotificationConfig,
     ) -> Result<(), NotificationError> {
-        let arn_list = self.notifier.get_arn_list(&cfg.region).await;
-        if arn_list.is_empty() {
-            return Err(NotificationError::Configuration(notify_configuration_hint()));
-        }
-        info!("Available ARNs: {:?}", arn_list);
-        // Validate the configuration against the available ARNs
-        if let Err(e) = cfg.validate(&cfg.region, &arn_list) {
-            debug!("Bucket notification config validation region:{} failed: {}", &cfg.region, e);
-            if !matches!(e, ParseConfigError::ArnNotFound(_)) {
-                return Err(NotificationError::BucketNotification(e.to_string()));
-            }
-            warn!(
-                bucket = %bucket,
-                region = %cfg.region,
-                error = %e,
-                "Bucket notification config references missing target ARN; keeping compatibility and loading remaining rules"
-            );
-        }
-
-        self.subscriber_view.apply_bucket_config(bucket, cfg);
-        let rules_map = cfg.get_rules_map();
-        self.notifier.add_rules_map(bucket, rules_map.clone()).await;
-        info!("Loaded notification config for bucket: {}", bucket);
-        Ok(())
+        self.services
+            .bucket_config_manager
+            .load_bucket_notification_config(bucket, cfg)
+            .await
     }
 
     /// Sends an event
     pub async fn send_event(&self, event: Arc<Event>) {
-        self.live_event_history.write().await.record(event.clone());
-        let _ = self.live_event_sender.send(event.clone());
-        self.notifier.send(event).await;
+        self.services.pipeline.send_event(event).await;
     }
 
     /// Obtain system status information
     pub fn get_status(&self) -> HashMap<String, String> {
-        let mut status = HashMap::new();
-
-        status.insert("uptime_seconds".to_string(), self.metrics.uptime().as_secs().to_string());
-        status.insert("processing_events".to_string(), self.metrics.processing_count().to_string());
-        status.insert("processed_events".to_string(), self.metrics.processed_count().to_string());
-        status.insert("failed_events".to_string(), self.metrics.failed_count().to_string());
-        status.insert("skipped_events".to_string(), self.metrics.skipped_count().to_string());
-
-        status
+        self.services.status_view.get_status()
     }
 
     pub fn snapshot_metrics(&self) -> NotificationMetricSnapshot {
-        self.metrics.snapshot()
+        self.services.status_view.snapshot_metrics()
     }
 
     pub async fn snapshot_target_metrics(&self) -> Vec<NotificationTargetMetricSnapshot> {
-        let targets = self.notifier.target_list().read().await.values();
-        let mut snapshots = Vec::with_capacity(targets.len());
+        self.services.runtime_view.snapshot_target_metrics().await
+    }
 
-        for target in targets {
-            let delivery = target.delivery_snapshot();
-            let target_id = target.id();
-            snapshots.push(NotificationTargetMetricSnapshot {
-                failed_messages: delivery.failed_messages,
-                queue_length: delivery.queue_length,
-                target_id: target_id.to_string(),
-                target_type: target_id.name,
-                total_messages: delivery.total_messages,
-            });
-        }
+    pub async fn snapshot_target_health(&self) -> Vec<RuntimeTargetHealthSnapshot> {
+        self.services.runtime_view.snapshot_target_health().await
+    }
 
-        snapshots.sort_by(|a, b| a.target_id.cmp(&b.target_id));
-        snapshots
+    pub async fn runtime_status_snapshot(&self) -> rustfs_targets::RuntimeStatusSnapshot {
+        self.services.runtime_view.runtime_status_snapshot().await
     }
 
     // Add a method to shut down the system
     pub async fn shutdown(&self) {
-        info!("Turn off the notification system");
-
-        // Get the number of active targets
-        let active_targets = self.stream_cancellers.read().await.len();
-        info!("Stops {} active event stream processing tasks", active_targets);
-
-        let mut cancellers = self.stream_cancellers.write().await;
-        for (target_id, cancel_tx) in cancellers.drain() {
-            info!("Stop event stream processing for target {}", target_id);
-            let _ = cancel_tx.send(()).await;
-        }
-        // Wait for a short while to make sure the task has a chance to complete
-        tokio::time::sleep(Duration::from_millis(500)).await;
-
-        info!("Notify the system to be shut down completed");
+        self.services.runtime_facade.shutdown().await;
     }
 }
 
@@ -719,6 +363,22 @@ impl Drop for NotificationSystem {
     fn drop(&mut self) {
         // Asynchronous operation cannot be used here, but logs can be recorded.
         info!("Notify the system instance to be destroyed");
+
+        let snapshot = self.snapshot_metrics();
+        for (name, value, is_gauge) in [
+            (METRIC_NOTIFICATION_CURRENT_SEND_IN_PROGRESS, snapshot.current_send_in_progress, true),
+            (METRIC_NOTIFICATION_EVENTS_ERRORS_TOTAL, snapshot.events_errors_total, false),
+            (METRIC_NOTIFICATION_EVENTS_SENT_TOTAL, snapshot.events_sent_total, false),
+            (METRIC_NOTIFICATION_EVENTS_SKIPPED_TOTAL, snapshot.events_skipped_total, false),
+        ] {
+            if is_gauge {
+                gauge!(name).set(value as f64);
+            } else {
+                counter!(name).absolute(value);
+            }
+            info!("shutdown metric {}={}", name, value);
+        }
+
         let status = self.get_status();
         for (key, value) in status {
             info!("key:{}, value:{}", key, value);
@@ -772,58 +432,22 @@ mod tests {
         assert_eq!(batch.events[0].s3.object.key, "one");
     }
 
-    #[test]
-    fn runtime_target_id_for_subsystem_maps_notify_webhook_to_runtime_type() {
-        let target_id = runtime_target_id_for_subsystem(NOTIFY_WEBHOOK_SUB_SYS, "Primary");
-        assert_eq!(target_id.id, "primary");
-        assert_eq!(target_id.name, "webhook");
-    }
+    #[tokio::test]
+    async fn notification_system_exposes_live_event_pipeline() {
+        let system = NotificationSystem::new(Config::default());
+        assert!(!system.has_live_listeners());
 
-    #[test]
-    fn runtime_target_id_for_subsystem_maps_notify_amqp_to_runtime_type() {
-        let target_id = runtime_target_id_for_subsystem(NOTIFY_AMQP_SUB_SYS, "Primary");
-        assert_eq!(target_id.id, "primary");
-        assert_eq!(target_id.name, "amqp");
-    }
+        let _rx = system.subscribe_live_events();
+        assert!(system.has_live_listeners());
 
-    #[test]
-    fn runtime_target_id_for_subsystem_maps_notify_mqtt_to_runtime_type() {
-        let target_id = runtime_target_id_for_subsystem(NOTIFY_MQTT_SUB_SYS, "Analytics");
-        assert_eq!(target_id.id, "analytics");
-        assert_eq!(target_id.name, "mqtt");
-    }
+        system
+            .send_event(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
+            .await;
 
-    #[test]
-    fn runtime_target_id_for_subsystem_maps_notify_kafka_to_runtime_type() {
-        let target_id = runtime_target_id_for_subsystem(NOTIFY_KAFKA_SUB_SYS, "EventBus");
-        assert_eq!(target_id.id, "eventbus");
-        assert_eq!(target_id.name, "kafka");
-    }
-
-    #[test]
-    fn runtime_target_id_for_subsystem_maps_notify_nats_to_runtime_type() {
-        let target_id = runtime_target_id_for_subsystem(NOTIFY_NATS_SUB_SYS, "Bus");
-        assert_eq!(target_id.id, "bus");
-        assert_eq!(target_id.name, "nats");
-    }
-
-    #[test]
-    fn runtime_target_id_for_subsystem_maps_notify_pulsar_to_runtime_type() {
-        let target_id = runtime_target_id_for_subsystem(NOTIFY_PULSAR_SUB_SYS, "Ledger");
-        assert_eq!(target_id.id, "ledger");
-        assert_eq!(target_id.name, "pulsar");
-    }
-
-    #[test]
-    fn runtime_target_id_for_subsystem_maps_notify_redis_to_runtime_type() {
-        let target_id = runtime_target_id_for_subsystem(NOTIFY_REDIS_SUB_SYS, "Primary");
-        assert_eq!(target_id.id, "primary");
-        assert_eq!(target_id.name, "redis");
-    }
-    #[test]
-    fn runtime_target_id_for_subsystem_maps_notify_postgres_to_runtime_type() {
-        let target_id = runtime_target_id_for_subsystem(NOTIFY_POSTGRES_SUB_SYS, "AuditTrail");
-        assert_eq!(target_id.id, "audittrail");
-        assert_eq!(target_id.name, "postgres");
+        let batch = system.recent_live_events_since(0, 16).await;
+        assert_eq!(batch.events.len(), 1);
+        assert_eq!(batch.events[0].s3.object.key, "object");
+        assert_eq!(batch.next_sequence, 1);
+        assert!(!batch.truncated);
     }
 }

crates/notify/src/lib.rs

@@ -18,22 +18,39 @@
 //! It supports sending events to various targets
 //! (like Webhook and MQTT) and includes features like event persistence and retry on failure.
 
+mod bucket_config_manager;
+mod config_manager;
 mod error;
 mod event;
+mod event_bridge;
 pub mod factory;
 mod global;
 pub mod integration;
 mod notification_system_subscriber;
 pub mod notifier;
+mod pipeline;
 pub mod registry;
+mod rule_engine;
 pub mod rules;
-pub mod stream;
+mod runtime_facade;
+mod runtime_view;
+mod services;
+mod status_view;
 
+pub use bucket_config_manager::NotifyBucketConfigManager;
+pub use config_manager::{NotifyConfigManager, runtime_target_id_for_subsystem};
 pub use error::{LifecycleError, NotificationError};
 pub use event::{Event, EventArgs, EventArgsBuilder};
+pub use event_bridge::{LiveEventHistory, NotifyEventBridge};
 pub use global::{
     initialize, initialize_live_events, is_notification_system_initialized, notification_metrics_snapshot, notification_system,
     notification_target_metrics, notifier_global,
 };
 pub use integration::{NotificationMetricSnapshot, NotificationSystem, NotificationTargetMetricSnapshot};
+pub use pipeline::NotifyPipeline;
+pub use rule_engine::NotifyRuleEngine;
 pub use rules::BucketNotificationConfig;
+pub use runtime_facade::NotifyRuntimeFacade;
+pub use runtime_view::NotifyRuntimeView;
+pub use services::NotifyServices;
+pub use status_view::NotifyStatusView;

crates/notify/src/notifier.rs

@@ -12,52 +12,28 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::{
-    error::NotificationError,
-    event::Event,
-    integration::NotificationMetrics,
-    rules::{RulesMap, TargetIdSet},
-};
-use hashbrown::HashMap;
-use percent_encoding::percent_decode_str;
+use crate::{error::NotificationError, event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine};
 use rustfs_config::notify::{DEFAULT_NOTIFY_SEND_CONCURRENCY, ENV_NOTIFY_SEND_CONCURRENCY};
-use rustfs_s3_common::EventName;
-use rustfs_targets::Target;
 use rustfs_targets::arn::TargetID;
 use rustfs_targets::target::EntityTarget;
-use starshard::AsyncShardedHashMap;
+use rustfs_targets::{SharedTarget, Target, TargetRuntimeManager};
 use std::sync::Arc;
 use tokio::sync::{RwLock, Semaphore};
 use tracing::{debug, error, info, instrument, warn};
 
-fn decoded_object_key_for_matching(object_key: &str) -> Option<String> {
-    if !object_key.contains('%') {
-        return None;
-    }
-
-    let decoded = percent_decode_str(object_key).decode_utf8().ok()?;
-    (decoded != object_key).then(|| decoded.into_owned())
-}
-
-fn match_event_targets(rules: &RulesMap, event_name: EventName, object_key: &str) -> TargetIdSet {
-    let mut target_ids = rules.match_rules(event_name, object_key);
-    if let Some(decoded_key) = decoded_object_key_for_matching(object_key) {
-        target_ids.extend(rules.match_rules(event_name, &decoded_key));
-    }
-    target_ids
-}
+pub type SharedNotifyTargetList = Arc<RwLock<TargetList>>;
 
 /// Manages event notification to targets based on rules
 pub struct EventNotifier {
     metrics: Arc<NotificationMetrics>,
-    target_list: Arc<RwLock<TargetList>>,
-    bucket_rules_map: Arc<AsyncShardedHashMap<String, RulesMap, rustc_hash::FxBuildHasher>>,
+    rule_engine: NotifyRuleEngine,
+    target_list: SharedNotifyTargetList,
     send_limiter: Arc<Semaphore>,
 }
 
 impl Default for EventNotifier {
     fn default() -> Self {
-        Self::new(Arc::new(NotificationMetrics::new()))
+        Self::new(Arc::new(NotificationMetrics::new()), NotifyRuleEngine::new())
     }
 }
 
@@ -66,56 +42,25 @@ impl EventNotifier {
     ///
     /// # Returns
     /// Returns a new instance of EventNotifier.
-    pub fn new(metrics: Arc<NotificationMetrics>) -> Self {
+    pub fn new(metrics: Arc<NotificationMetrics>, rule_engine: NotifyRuleEngine) -> Self {
         let max_inflight = rustfs_utils::get_env_usize(ENV_NOTIFY_SEND_CONCURRENCY, DEFAULT_NOTIFY_SEND_CONCURRENCY);
         EventNotifier {
             metrics,
+            rule_engine,
             target_list: Arc::new(RwLock::new(TargetList::new())),
-            bucket_rules_map: Arc::new(AsyncShardedHashMap::new(0)),
             send_limiter: Arc::new(Semaphore::new(max_inflight)),
         }
     }
 
-    /// Checks whether a TargetID is still referenced by any bucket's rules.
-    ///
-    /// # Arguments
-    /// * `target_id` - The TargetID to check.
-    ///
-    /// # Returns
-    /// Returns `true` if the TargetID is bound to any bucket, otherwise `false`.
-    pub async fn is_target_bound_to_any_bucket(&self, target_id: &TargetID) -> bool {
-        // `AsyncShardedHashMap::iter()`: Traverse (bucket_name, rules_map)
-        let items = self.bucket_rules_map.iter().await;
-        for (_bucket, rules_map) in items {
-            if rules_map.contains_target_id(target_id) {
-                return true;
-            }
-        }
-        false
-    }
-
     /// Returns a reference to the target list
     /// This method provides access to the target list for external use.
     ///
     /// # Returns
     /// Returns an `Arc<RwLock<TargetList>>` representing the target list.
-    pub fn target_list(&self) -> Arc<RwLock<TargetList>> {
+    pub fn target_list(&self) -> SharedNotifyTargetList {
         Arc::clone(&self.target_list)
     }
 
-    /// Removes all notification rules for a bucket
-    ///
-    /// # Arguments
-    /// * `bucket` - The name of the bucket for which to remove rules
-    ///
-    /// This method removes all rules associated with the specified bucket name.
-    /// It will log a message indicating the removal of rules.
-    pub async fn remove_rules_map(&self, bucket: &str) {
-        if self.bucket_rules_map.remove(&bucket.to_string()).await.is_some() {
-            info!("Removed all notification rules for bucket: {}", bucket);
-        }
-    }
-
     /// Returns a list of ARNs for the registered targets
     ///
     /// # Arguments
@@ -132,40 +77,6 @@ impl EventNotifier {
             .collect()
     }
 
-    /// Adds a rules map for a bucket
-    ///
-    /// # Arguments
-    /// * `bucket` - The name of the bucket for which to add the rules map
-    /// * `rules_map` - The rules map to add for the bucket
-    pub async fn add_rules_map(&self, bucket: &str, rules_map: RulesMap) {
-        if rules_map.is_empty() {
-            self.bucket_rules_map.remove(&bucket.to_string()).await;
-        } else {
-            self.bucket_rules_map.insert(bucket.to_string(), rules_map).await;
-        }
-        info!("Added rules for bucket: {}", bucket);
-    }
-
-    /// Gets the rules map for a specific bucket.
-    ///
-    /// # Arguments
-    /// * `bucket` - The name of the bucket for which to get the rules map
-    ///
-    /// # Returns
-    /// Returns `Some(RulesMap)` if rules exist for the bucket, otherwise returns `None`.
-    pub async fn get_rules_map(&self, bucket: &str) -> Option<RulesMap> {
-        self.bucket_rules_map.get(&bucket.to_string()).await
-    }
-
-    /// Removes notification rules for a bucket
-    ///
-    /// # Arguments
-    /// * `bucket` - The name of the bucket for which to remove notification rules
-    pub async fn remove_notification(&self, bucket: &str) {
-        self.bucket_rules_map.remove(&bucket.to_string()).await;
-        info!("Removed notification rules for bucket: {}", bucket);
-    }
-
     /// Removes all targets
     pub async fn remove_all_bucket_targets(&self) {
         let mut target_list_guard = self.target_list.write().await;
@@ -175,26 +86,6 @@ impl EventNotifier {
         info!("Removed all targets and their streams");
     }
 
-    /// Checks if there are active subscribers for the given bucket and event name.
-    ///
-    /// # Parameters
-    /// * `bucket_name` - bucket name.
-    /// * `event_name` - Event name.
-    ///
-    /// # Return value
-    /// Return `true` if at least one matching notification rule exists.
-    pub async fn has_subscriber(&self, bucket_name: &str, event_name: &EventName) -> bool {
-        // Rules to check if the bucket exists
-        if let Some(rules_map) = self.bucket_rules_map.get(&bucket_name.to_string()).await {
-            // A composite event (such as ObjectCreatedAll) is expanded to multiple single events.
-            // We need to check whether any of these single events have the rules configured.
-            rules_map.has_subscriber(event_name)
-        } else {
-            // If no bucket is found, no subscribers
-            false
-        }
-    }
-
     /// Sends an event to the appropriate targets based on the bucket rules
     ///
     /// # Arguments
@@ -205,13 +96,7 @@ impl EventNotifier {
         let object_key = &event.s3.object.key;
         let event_name = event.event_name;
 
-        let Some(rules) = self.bucket_rules_map.get(bucket_name).await else {
-            debug!("No rules found for bucket: {}", bucket_name);
-            self.metrics.increment_skipped();
-            return;
-        };
-
-        let target_ids = match_event_targets(&rules, event_name, object_key);
+        let target_ids = self.rule_engine.match_targets(bucket_name, event_name, object_key).await;
         if target_ids.is_empty() {
             debug!("No matching targets for event in bucket: {}", bucket_name);
             self.metrics.increment_skipped();
@@ -287,45 +172,30 @@ impl EventNotifier {
         info!("Event processing initiated for {} targets for bucket: {}", target_ids_len, bucket_name);
     }
 
-    /// Initializes the targets for buckets
-    ///
-    /// # Arguments
-    /// * `targets_to_init` - A vector of boxed targets to initialize
-    ///
-    /// # Returns
-    /// Returns `Ok(())` if initialization is successful, otherwise returns a `NotificationError`.
+    /// Initializes the targets for buckets from shared target handles.
     #[instrument(skip(self, targets_to_init))]
-    pub async fn init_bucket_targets(
-        &self,
-        targets_to_init: Vec<Box<dyn Target<Event> + Send + Sync>>,
-    ) -> Result<(), NotificationError> {
-        // Currently active, simpler logic
-        let mut target_list_guard = self.target_list.write().await; //Gets a write lock for the TargetList
-
-        // Clear existing targets first - rebuild from scratch to ensure consistency with new configuration
+    pub async fn init_bucket_targets_shared(&self, targets_to_init: Vec<SharedTarget<Event>>) -> Result<(), NotificationError> {
+        let mut target_list_guard = self.target_list.write().await;
         target_list_guard.clear();
 
-        for target_boxed in targets_to_init {
-            // Traverse the incoming Box<dyn Target >
-            debug!("init bucket target: {}", target_boxed.name());
-            // TargetList::add method expectations Arc<dyn Target + Send + Sync>
-            // Therefore, you need to convert Box<dyn Target + Send + Sync> to Arc<dyn Target + Send + Sync>
-            let target_arc: Arc<dyn Target<Event> + Send + Sync> = Arc::from(target_boxed);
-            target_list_guard.add(target_arc)?; // Add Arc<dyn Target> to the list
+        for target in targets_to_init {
+            debug!("init bucket target: {}", target.name());
+            target_list_guard.add(target)?;
         }
+
         info!(
-            "Initialized {} targets, list size: {}", // Clearer logs
+            "Initialized {} shared targets, list size: {}",
             target_list_guard.len(),
             target_list_guard.len()
         );
-        Ok(()) // Make sure to return a Result
+        Ok(())
     }
 }
 
 /// A thread-safe list of targets
 pub struct TargetList {
     /// Map of TargetID to Target
-    targets: HashMap<TargetID, Arc<dyn Target<Event> + Send + Sync>>,
+    runtime: TargetRuntimeManager<Event>,
 }
 
 impl Default for TargetList {
@@ -337,7 +207,9 @@ impl Default for TargetList {
 impl TargetList {
     /// Creates a new TargetList
     pub fn new() -> Self {
-        TargetList { targets: HashMap::new() }
+        TargetList {
+            runtime: TargetRuntimeManager::new(),
+        }
     }
 
     /// Adds a target to the list
@@ -349,17 +221,17 @@ impl TargetList {
     /// Returns `Ok(())` if the target was added successfully, or a `NotificationError` if an error occurred.
     pub fn add(&mut self, target: Arc<dyn Target<Event> + Send + Sync>) -> Result<(), NotificationError> {
         let id = target.id();
-        if self.targets.contains_key(&id) {
+        if self.runtime.get_by_target_id(&id).is_some() {
             // Potentially update or log a warning/error if replacing an existing target.
             warn!("Target with ID {} already exists in TargetList. It will be overwritten.", id);
         }
-        self.targets.insert(id, target);
+        self.runtime.add_arc(target);
         Ok(())
     }
 
     /// Clears all targets from the list
     pub fn clear(&mut self) {
-        self.targets.clear();
+        self.runtime.clear();
     }
 
     /// Removes a target by ID. Note: This does not stop its associated event stream.
@@ -370,30 +242,14 @@ impl TargetList {
     ///
     /// # Returns
     /// Returns the removed target if it existed, otherwise `None`.
-    pub async fn remove_target_only(&mut self, id: &TargetID) -> Option<Arc<dyn Target<Event> + Send + Sync>> {
-        if let Some(target_arc) = self.targets.remove(id) {
-            if let Err(e) = target_arc.close().await {
-                // Target's own close logic
-                error!("Failed to close target {} during removal: {}", id, e);
-            }
-            Some(target_arc)
-        } else {
-            None
-        }
+    pub async fn remove_target_only(&mut self, id: &TargetID) -> Option<SharedTarget<Event>> {
+        self.runtime.remove_by_target_id_and_close(id).await
     }
 
     /// Clears all targets from the list. Note: This does not stop their associated event streams.
     /// Stream cancellation should be handled by EventNotifier.
     pub async fn clear_targets_only(&mut self) {
-        let target_ids_to_clear: Vec<TargetID> = self.targets.keys().cloned().collect();
-        for id in target_ids_to_clear {
-            if let Some(target_arc) = self.targets.remove(&id)
-                && let Err(e) = target_arc.close().await
-            {
-                error!("Failed to close target {} during clear: {}", id, e);
-            }
-        }
-        self.targets.clear();
+        self.runtime.clear_and_close().await;
     }
 
     /// Returns a target by ID
@@ -403,34 +259,54 @@ impl TargetList {
     ///
     /// # Returns
     /// Returns the target if it exists, otherwise `None`.
-    pub fn get(&self, id: &TargetID) -> Option<Arc<dyn Target<Event> + Send + Sync>> {
-        self.targets.get(id).cloned()
+    pub fn get(&self, id: &TargetID) -> Option<SharedTarget<Event>> {
+        self.runtime.get_by_target_id(id)
     }
 
     /// Returns all target IDs
     pub fn keys(&self) -> Vec<TargetID> {
-        self.targets.keys().cloned().collect()
+        self.runtime.target_ids()
     }
 
     /// Returns all targets in the list
-    pub fn values(&self) -> Vec<Arc<dyn Target<Event> + Send + Sync>> {
-        self.targets.values().cloned().collect()
+    pub fn values(&self) -> Vec<SharedTarget<Event>> {
+        self.runtime.values()
+    }
+
+    pub fn runtime_snapshots(&self) -> Vec<rustfs_targets::RuntimeTargetSnapshot> {
+        self.runtime.snapshots()
+    }
+
+    pub async fn runtime_health_snapshots(&self) -> Vec<rustfs_targets::RuntimeTargetHealthSnapshot> {
+        self.runtime.health_snapshots().await
+    }
+
+    pub fn runtime_status_snapshot(
+        &self,
+        replay_workers: &rustfs_targets::ReplayWorkerManager,
+    ) -> rustfs_targets::RuntimeStatusSnapshot {
+        self.runtime.status_snapshot(replay_workers)
+    }
+
+    pub fn runtime_mut(&mut self) -> &mut TargetRuntimeManager<Event> {
+        &mut self.runtime
     }
 
     /// Returns the number of targets
     pub fn len(&self) -> usize {
-        self.targets.len()
+        self.runtime.len()
     }
 
     /// is_empty can be derived from len()
     pub fn is_empty(&self) -> bool {
-        self.targets.is_empty()
+        self.runtime.is_empty()
     }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::{rule_engine::NotifyRuleEngine, rules::RulesMap};
     use async_trait::async_trait;
     use rustfs_s3_common::EventName;
     use rustfs_targets::StoreError;
@@ -445,40 +321,57 @@ mod tests {
         atomic::{AtomicUsize, Ordering},
     };
 
-    #[test]
-    fn encoded_event_key_matches_raw_prefix_suffix_filter() {
+    #[tokio::test]
+    async fn encoded_event_key_matches_raw_prefix_suffix_filter() {
         let target_id = TargetID::new("primary".to_string(), "webhook".to_string());
         let mut rules_map = RulesMap::new();
         rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "uploads/*.csv".to_string(), target_id.clone());
 
-        let targets = match_event_targets(&rules_map, EventName::ObjectCreatedPut, "uploads%2Freport.csv");
+        let rule_engine = NotifyRuleEngine::new();
+        rule_engine.set_bucket_rules("test-bucket", rules_map).await;
+
+        let targets = rule_engine
+            .match_targets("test-bucket", EventName::ObjectCreatedPut, "uploads%2Freport.csv")
+            .await;
 
         assert!(targets.contains(&target_id));
     }
 
-    #[test]
-    fn encoded_event_key_matches_raw_and_decoded_rule_targets() {
+    #[tokio::test]
+    async fn encoded_event_key_matches_raw_and_decoded_rule_targets() {
         let raw_target = TargetID::new("raw".to_string(), "webhook".to_string());
         let decoded_target = TargetID::new("decoded".to_string(), "webhook".to_string());
         let mut rules_map = RulesMap::new();
         rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "uploads%2F*.csv".to_string(), raw_target.clone());
         rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "uploads/*.csv".to_string(), decoded_target.clone());
 
-        let targets = match_event_targets(&rules_map, EventName::ObjectCreatedPut, "uploads%2Freport.csv");
+        let rule_engine = NotifyRuleEngine::new();
+        rule_engine.set_bucket_rules("test-bucket", rules_map).await;
+
+        let targets = rule_engine
+            .match_targets("test-bucket", EventName::ObjectCreatedPut, "uploads%2Freport.csv")
+            .await;
 
         assert_eq!(targets.len(), 2);
         assert!(targets.contains(&raw_target));
         assert!(targets.contains(&decoded_target));
     }
 
-    #[test]
-    fn encoded_event_key_does_not_bypass_suffix_filter() {
+    #[tokio::test]
+    async fn encoded_event_key_does_not_bypass_suffix_filter() {
         let target_id = TargetID::new("primary".to_string(), "webhook".to_string());
         let mut rules_map = RulesMap::new();
         rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "uploads/*.csv".to_string(), target_id);
 
-        let root_targets = match_event_targets(&rules_map, EventName::ObjectCreatedPut, "report.csv");
-        let suffix_targets = match_event_targets(&rules_map, EventName::ObjectCreatedPut, "uploads%2Freport.txt");
+        let rule_engine = NotifyRuleEngine::new();
+        rule_engine.set_bucket_rules("test-bucket", rules_map).await;
+
+        let root_targets = rule_engine
+            .match_targets("test-bucket", EventName::ObjectCreatedPut, "report.csv")
+            .await;
+        let suffix_targets = rule_engine
+            .match_targets("test-bucket", EventName::ObjectCreatedPut, "uploads%2Freport.txt")
+            .await;
 
         assert!(root_targets.is_empty());
         assert!(suffix_targets.is_empty());
@@ -547,7 +440,8 @@ mod tests {
 
     #[tokio::test]
     async fn test_send_event_skips_disabled_target() {
-        let notifier = EventNotifier::new(Arc::new(NotificationMetrics::new()));
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = EventNotifier::new(Arc::new(NotificationMetrics::new()), rule_engine.clone());
 
         let enabled_target = TestTarget::new("enabled-target", "webhook", true);
         let disabled_target = TestTarget::new("disabled-target", "webhook", false);
@@ -556,7 +450,7 @@ mod tests {
         rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), enabled_target.id.clone());
         rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), disabled_target.id.clone());
 
-        notifier.add_rules_map("bucket", rules_map).await;
+        rule_engine.set_bucket_rules("bucket", rules_map).await;
         notifier
             .target_list()
             .write()

crates/notify/src/pipeline.rs

@@ -0,0 +1,141 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::{Event, integration::LiveEventBatch, notifier::EventNotifier};
+use std::collections::VecDeque;
+use std::sync::Arc;
+use tokio::sync::{RwLock, broadcast};
+
+const MAX_RECENT_LIVE_EVENTS: usize = 1024;
+
+#[derive(Default)]
+pub struct LiveEventHistory {
+    next_sequence: u64,
+    events: VecDeque<(u64, Arc<Event>)>,
+}
+
+impl LiveEventHistory {
+    pub fn record(&mut self, event: Arc<Event>) {
+        self.next_sequence = self.next_sequence.saturating_add(1);
+        self.events.push_back((self.next_sequence, event));
+        while self.events.len() > MAX_RECENT_LIVE_EVENTS {
+            self.events.pop_front();
+        }
+    }
+
+    pub fn snapshot_since(&self, after_sequence: u64, limit: usize) -> LiveEventBatch {
+        let mut events = Vec::new();
+        let mut next_sequence = after_sequence;
+        let mut truncated = false;
+
+        for (sequence, event) in self.events.iter() {
+            if *sequence <= after_sequence {
+                continue;
+            }
+            if events.len() >= limit {
+                truncated = true;
+                break;
+            }
+            next_sequence = *sequence;
+            events.push(event.clone());
+        }
+
+        LiveEventBatch {
+            events,
+            next_sequence,
+            truncated,
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct NotifyPipeline {
+    notifier: Arc<EventNotifier>,
+    live_event_sender: broadcast::Sender<Arc<Event>>,
+    live_event_history: Arc<RwLock<LiveEventHistory>>,
+}
+
+impl NotifyPipeline {
+    pub fn new(
+        notifier: Arc<EventNotifier>,
+        live_event_sender: broadcast::Sender<Arc<Event>>,
+        live_event_history: Arc<RwLock<LiveEventHistory>>,
+    ) -> Self {
+        Self {
+            notifier,
+            live_event_sender,
+            live_event_history,
+        }
+    }
+
+    pub fn has_live_listeners(&self) -> bool {
+        self.live_event_sender.receiver_count() > 0
+    }
+
+    pub fn subscribe_live_events(&self) -> broadcast::Receiver<Arc<Event>> {
+        self.live_event_sender.subscribe()
+    }
+
+    pub async fn recent_live_events_since(&self, after_sequence: u64, limit: usize) -> LiveEventBatch {
+        let history = self.live_event_history.read().await;
+        history.snapshot_since(after_sequence, limit.max(1))
+    }
+
+    pub async fn send_event(&self, event: Arc<Event>) {
+        self.live_event_history.write().await.record(event.clone());
+        let _ = self.live_event_sender.send(event.clone());
+        self.notifier.send(event).await;
+    }
+}
+
+pub type NotifyEventBridge = NotifyPipeline;
+
+#[cfg(test)]
+mod tests {
+    use super::{LiveEventHistory, NotifyPipeline};
+    use crate::{Event, integration::NotificationMetrics, notifier::EventNotifier, rule_engine::NotifyRuleEngine};
+    use rustfs_s3_common::EventName;
+    use std::sync::Arc;
+    use tokio::sync::{RwLock, broadcast};
+
+    fn build_pipeline() -> NotifyPipeline {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let notifier = Arc::new(EventNotifier::new(metrics, NotifyRuleEngine::new()));
+        let (live_event_sender, _) = broadcast::channel(16);
+        NotifyPipeline::new(notifier, live_event_sender, Arc::new(RwLock::new(LiveEventHistory::default())))
+    }
+
+    #[tokio::test]
+    async fn pipeline_reports_live_listener_subscription_state() {
+        let pipeline = build_pipeline();
+        assert!(!pipeline.has_live_listeners());
+
+        let _rx = pipeline.subscribe_live_events();
+        assert!(pipeline.has_live_listeners());
+    }
+
+    #[tokio::test]
+    async fn pipeline_records_recent_live_events() {
+        let pipeline = build_pipeline();
+        let event = Arc::new(Event::new_test_event("bucket", "one", EventName::ObjectCreatedPut));
+
+        pipeline.send_event(event).await;
+
+        let batch = pipeline.recent_live_events_since(0, 16).await;
+        assert_eq!(batch.next_sequence, 1);
+        assert!(!batch.truncated);
+        assert_eq!(batch.events.len(), 1);
+        assert_eq!(batch.events[0].s3.object.key, "one");
+    }
+}

crates/notify/src/rule_engine.rs

@@ -0,0 +1,133 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::rules::{RulesMap, TargetIdSet};
+use percent_encoding::percent_decode_str;
+use rustfs_s3_common::EventName;
+use rustfs_targets::arn::TargetID;
+use starshard::AsyncShardedHashMap;
+use std::sync::Arc;
+use tracing::info;
+
+fn decoded_object_key_for_matching(object_key: &str) -> Option<String> {
+    if !object_key.contains('%') {
+        return None;
+    }
+
+    let decoded = percent_decode_str(object_key).decode_utf8().ok()?;
+    (decoded != object_key).then(|| decoded.into_owned())
+}
+
+#[derive(Clone)]
+pub struct NotifyRuleEngine {
+    bucket_rules_map: Arc<AsyncShardedHashMap<String, RulesMap, rustc_hash::FxBuildHasher>>,
+}
+
+impl NotifyRuleEngine {
+    pub fn new() -> Self {
+        Self {
+            bucket_rules_map: Arc::new(AsyncShardedHashMap::new(0)),
+        }
+    }
+
+    pub async fn is_target_bound_to_any_bucket(&self, target_id: &TargetID) -> bool {
+        let items = self.bucket_rules_map.iter().await;
+        for (_bucket, rules_map) in items {
+            if rules_map.contains_target_id(target_id) {
+                return true;
+            }
+        }
+        false
+    }
+
+    pub async fn set_bucket_rules(&self, bucket: &str, rules_map: RulesMap) {
+        if rules_map.is_empty() {
+            self.bucket_rules_map.remove(&bucket.to_string()).await;
+        } else {
+            self.bucket_rules_map.insert(bucket.to_string(), rules_map).await;
+        }
+        info!("Updated notification rules for bucket: {}", bucket);
+    }
+
+    pub async fn get_bucket_rules(&self, bucket: &str) -> Option<RulesMap> {
+        self.bucket_rules_map.get(&bucket.to_string()).await
+    }
+
+    pub async fn clear_bucket_rules(&self, bucket: &str) {
+        if self.bucket_rules_map.remove(&bucket.to_string()).await.is_some() {
+            info!("Removed all notification rules for bucket: {}", bucket);
+        }
+    }
+
+    pub async fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool {
+        self.get_bucket_rules(bucket)
+            .await
+            .is_some_and(|rules_map| rules_map.has_subscriber(event))
+    }
+
+    pub async fn match_targets(&self, bucket: &str, event_name: EventName, object_key: &str) -> TargetIdSet {
+        self.get_bucket_rules(bucket)
+            .await
+            .map_or_else(TargetIdSet::new, |rules_map| {
+                let mut target_ids = rules_map.match_rules(event_name, object_key);
+                if let Some(decoded_key) = decoded_object_key_for_matching(object_key) {
+                    target_ids.extend(rules_map.match_rules(event_name, &decoded_key));
+                }
+                target_ids
+            })
+    }
+}
+
+impl Default for NotifyRuleEngine {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::NotifyRuleEngine;
+    use crate::rules::RulesMap;
+    use rustfs_s3_common::EventName;
+    use rustfs_targets::arn::TargetID;
+
+    #[tokio::test]
+    async fn rule_engine_tracks_bucket_rule_lifecycle() {
+        let engine = NotifyRuleEngine::new();
+        let target_id = TargetID::new("primary".to_string(), "webhook".to_string());
+        let mut rules_map = RulesMap::new();
+        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target_id.clone());
+
+        assert!(!engine.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
+        assert!(!engine.is_target_bound_to_any_bucket(&target_id).await);
+
+        engine.set_bucket_rules("bucket", rules_map).await;
+
+        assert!(engine.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
+        assert!(engine.is_target_bound_to_any_bucket(&target_id).await);
+        assert_eq!(
+            engine
+                .match_targets("bucket", EventName::ObjectCreatedPut, "object")
+                .await
+                .into_iter()
+                .collect::<Vec<_>>(),
+            vec![target_id.clone()]
+        );
+
+        engine.clear_bucket_rules("bucket").await;
+
+        assert!(!engine.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
+        assert!(!engine.is_target_bound_to_any_bucket(&target_id).await);
+    }
+}

crates/notify/src/runtime_facade.rs

@@ -0,0 +1,239 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::{Event, NotificationError, integration::NotificationMetrics, notifier::SharedNotifyTargetList};
+use rustfs_targets::{
+    BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter, ReplayEvent, ReplayWorkerManager, RuntimeActivation, Target,
+};
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::{RwLock, Semaphore};
+use tracing::info;
+
+#[derive(Clone)]
+pub struct NotifyRuntimeFacade {
+    target_list: SharedNotifyTargetList,
+    replay_workers: Arc<RwLock<ReplayWorkerManager>>,
+    runtime_adapter: Arc<dyn PluginRuntimeAdapter<Event>>,
+}
+
+impl NotifyRuntimeFacade {
+    pub fn new(
+        target_list: SharedNotifyTargetList,
+        replay_workers: Arc<RwLock<ReplayWorkerManager>>,
+        concurrency_limiter: Arc<Semaphore>,
+        metrics: Arc<NotificationMetrics>,
+    ) -> Self {
+        let replay_metrics = metrics;
+        let runtime_adapter = BuiltinPluginRuntimeAdapter::new(
+            Arc::new(move |event: ReplayEvent<Event>| {
+                let metrics = replay_metrics.clone();
+                Box::pin(async move {
+                    match event {
+                        ReplayEvent::Delivered { .. } => metrics.increment_processed(),
+                        ReplayEvent::RetryableError { .. } => {}
+                        ReplayEvent::Dropped { target, .. }
+                        | ReplayEvent::PermanentFailure { target, .. }
+                        | ReplayEvent::RetryExhausted { target, .. } => {
+                            target.record_final_failure();
+                            metrics.increment_failed();
+                        }
+                        ReplayEvent::UnreadableEntry { .. } => {}
+                    }
+                })
+            }),
+            Arc::new(|target_id, has_replay| {
+                if has_replay {
+                    info!("Event stream processing for target {} is started successfully", target_id);
+                } else {
+                    info!("Target {} has no replay worker to start", target_id);
+                }
+            }),
+            Some(concurrency_limiter),
+            Duration::from_secs(5),
+            Duration::from_millis(500),
+            "Stop event stream processing for target",
+        );
+
+        Self {
+            target_list,
+            replay_workers,
+            runtime_adapter: Arc::new(runtime_adapter),
+        }
+    }
+
+    pub async fn activate_targets_with_replay(
+        &self,
+        targets: Vec<Box<dyn Target<Event> + Send + Sync>>,
+    ) -> RuntimeActivation<Event> {
+        self.runtime_adapter.activate_with_replay(targets).await
+    }
+
+    pub async fn replace_targets(&self, activation: RuntimeActivation<Event>) -> Result<(), NotificationError> {
+        let mut target_list = self.target_list.write().await;
+        let mut replay_workers = self.replay_workers.write().await;
+        self.runtime_adapter
+            .replace_runtime_targets(target_list.runtime_mut(), &mut replay_workers, activation)
+            .await
+            .map_err(NotificationError::Target)?;
+        Ok(())
+    }
+
+    pub async fn stop_replay_workers(&self) {
+        let mut replay_workers = self.replay_workers.write().await;
+        self.runtime_adapter.stop_replay_workers(&mut replay_workers).await;
+    }
+
+    pub async fn shutdown(&self) {
+        info!("Turn off the notification system");
+
+        let active_targets = self.replay_workers.read().await.len();
+        info!("Stops {} active event stream processing tasks", active_targets);
+
+        {
+            let mut target_list = self.target_list.write().await;
+            let mut replay_workers = self.replay_workers.write().await;
+            if let Err(err) = self
+                .runtime_adapter
+                .shutdown(target_list.runtime_mut(), &mut replay_workers)
+                .await
+            {
+                tracing::error!(error = %err, "Failed to shutdown notify runtime cleanly");
+            }
+        }
+        tokio::time::sleep(Duration::from_millis(500)).await;
+
+        info!("Notify the system to be shut down completed");
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::NotifyRuntimeFacade;
+    use crate::{
+        Event, integration::NotificationMetrics, notifier::EventNotifier, rule_engine::NotifyRuleEngine,
+        runtime_view::NotifyRuntimeView,
+    };
+    use async_trait::async_trait;
+    use rustfs_targets::arn::TargetID;
+    use rustfs_targets::store::{Key, Store};
+    use rustfs_targets::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
+    use rustfs_targets::{ReplayWorkerManager, SharedTarget, StoreError, Target, TargetError};
+    use serde::{Serialize, de::DeserializeOwned};
+    use std::sync::Arc;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use tokio::sync::{RwLock, Semaphore};
+
+    #[derive(Clone)]
+    struct TestTarget {
+        close_calls: Arc<AtomicUsize>,
+        id: TargetID,
+    }
+
+    impl TestTarget {
+        fn new(id: &str, name: &str) -> Self {
+            Self {
+                close_calls: Arc::new(AtomicUsize::new(0)),
+                id: TargetID::new(id.to_string(), name.to_string()),
+            }
+        }
+    }
+
+    #[async_trait]
+    impl<E> Target<E> for TestTarget
+    where
+        E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+    {
+        fn id(&self) -> TargetID {
+            self.id.clone()
+        }
+
+        async fn is_active(&self) -> Result<bool, TargetError> {
+            Ok(true)
+        }
+
+        async fn save(&self, _event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        async fn send_raw_from_store(&self, _key: Key, _body: Vec<u8>, _meta: QueuedPayloadMeta) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        async fn close(&self) -> Result<(), TargetError> {
+            self.close_calls.fetch_add(1, Ordering::SeqCst);
+            Ok(())
+        }
+
+        fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
+            None
+        }
+
+        fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
+            Box::new(self.clone())
+        }
+
+        async fn init(&self) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        fn is_enabled(&self) -> bool {
+            true
+        }
+    }
+
+    fn build_facade() -> (NotifyRuntimeFacade, Arc<EventNotifier>, Arc<RwLock<ReplayWorkerManager>>) {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let notifier = Arc::new(EventNotifier::new(metrics.clone(), NotifyRuleEngine::new()));
+        let target_list = notifier.target_list();
+        let replay_workers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
+        let facade = NotifyRuntimeFacade::new(target_list, replay_workers.clone(), Arc::new(Semaphore::new(4)), metrics);
+        (facade, notifier, replay_workers)
+    }
+
+    #[tokio::test]
+    async fn runtime_facade_stops_empty_replay_workers() {
+        let (facade, _, _) = build_facade();
+        facade.stop_replay_workers().await;
+    }
+
+    #[tokio::test]
+    async fn runtime_facade_activates_empty_target_list() {
+        let (facade, _, _) = build_facade();
+        let activation = facade.activate_targets_with_replay(Vec::new()).await;
+
+        assert!(activation.targets.is_empty());
+        assert_eq!(activation.replay_workers.len(), 0);
+    }
+
+    #[tokio::test]
+    async fn runtime_facade_replace_targets_commits_runtime_state() {
+        let (facade, notifier, replay_workers) = build_facade();
+        let target = TestTarget::new("primary", "webhook");
+        let activation = rustfs_targets::RuntimeActivation {
+            replay_workers: ReplayWorkerManager::new(),
+            targets: vec![Arc::new(target) as SharedTarget<Event>],
+        };
+
+        facade
+            .replace_targets(activation)
+            .await
+            .expect("replace_targets should succeed");
+
+        let runtime_view = NotifyRuntimeView::new(notifier.target_list(), replay_workers.clone());
+        let active_targets = runtime_view.get_active_targets().await;
+        assert_eq!(active_targets, vec![TargetID::new("primary".to_string(), "webhook".to_string())]);
+        assert_eq!(replay_workers.read().await.len(), 0);
+    }
+}

crates/notify/src/runtime_view.rs

@@ -0,0 +1,261 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::{Event, NotificationTargetMetricSnapshot, notifier::SharedNotifyTargetList};
+use rustfs_targets::{ReplayWorkerManager, RuntimeTargetHealthSnapshot, SharedTarget, arn::TargetID};
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+#[derive(Clone)]
+pub struct NotifyRuntimeView {
+    target_list: SharedNotifyTargetList,
+    stream_cancellers: Arc<RwLock<ReplayWorkerManager>>,
+}
+
+impl NotifyRuntimeView {
+    pub fn new(target_list: SharedNotifyTargetList, stream_cancellers: Arc<RwLock<ReplayWorkerManager>>) -> Self {
+        Self {
+            target_list,
+            stream_cancellers,
+        }
+    }
+
+    pub async fn get_active_targets(&self) -> Vec<TargetID> {
+        self.target_list.read().await.keys()
+    }
+
+    pub fn get_all_targets(&self) -> SharedNotifyTargetList {
+        self.target_list.clone()
+    }
+
+    pub async fn get_target_values(&self) -> Vec<SharedTarget<Event>> {
+        self.target_list.read().await.values()
+    }
+
+    pub async fn snapshot_target_metrics(&self) -> Vec<NotificationTargetMetricSnapshot> {
+        self.target_list
+            .read()
+            .await
+            .runtime_snapshots()
+            .into_iter()
+            .map(|snapshot| NotificationTargetMetricSnapshot {
+                failed_messages: snapshot.failed_messages,
+                queue_length: snapshot.queue_length,
+                target_id: snapshot.target_id,
+                target_type: snapshot.target_type,
+                total_messages: snapshot.total_messages,
+            })
+            .collect()
+    }
+
+    pub async fn snapshot_target_health(&self) -> Vec<RuntimeTargetHealthSnapshot> {
+        self.target_list.read().await.runtime_health_snapshots().await
+    }
+
+    pub async fn runtime_status_snapshot(&self) -> rustfs_targets::RuntimeStatusSnapshot {
+        let replay_workers = self.stream_cancellers.read().await;
+        let target_list = self.target_list.read().await;
+        target_list.runtime_status_snapshot(&replay_workers)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::NotifyRuntimeView;
+    use crate::{Event, notifier::TargetList};
+    use async_trait::async_trait;
+    use rustfs_targets::arn::TargetID;
+    use rustfs_targets::store::{Key, Store};
+    use rustfs_targets::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliverySnapshot};
+    use rustfs_targets::{ReplayWorkerManager, StoreError, Target, TargetError};
+    use serde::{Serialize, de::DeserializeOwned};
+    use std::sync::{
+        Arc,
+        atomic::{AtomicU64, Ordering},
+    };
+    use tokio::sync::RwLock;
+
+    #[derive(Clone)]
+    struct TestTarget {
+        active: bool,
+        enabled: bool,
+        failed_messages: Arc<AtomicU64>,
+        id: TargetID,
+        total_messages: Arc<AtomicU64>,
+    }
+
+    impl TestTarget {
+        fn new(id: &str, name: &str) -> Self {
+            Self {
+                active: true,
+                enabled: true,
+                failed_messages: Arc::new(AtomicU64::new(0)),
+                id: TargetID::new(id.to_string(), name.to_string()),
+                total_messages: Arc::new(AtomicU64::new(0)),
+            }
+        }
+
+        fn with_active(mut self, active: bool) -> Self {
+            self.active = active;
+            self
+        }
+
+        fn with_enabled(mut self, enabled: bool) -> Self {
+            self.enabled = enabled;
+            self
+        }
+
+        fn record_successes(&self, count: u64) {
+            self.total_messages.store(count, Ordering::Relaxed);
+        }
+
+        fn record_failures(&self, count: u64) {
+            self.failed_messages.store(count, Ordering::Relaxed);
+        }
+    }
+
+    #[async_trait]
+    impl<E> Target<E> for TestTarget
+    where
+        E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+    {
+        fn id(&self) -> TargetID {
+            self.id.clone()
+        }
+
+        async fn is_active(&self) -> Result<bool, TargetError> {
+            Ok(self.active)
+        }
+
+        async fn save(&self, _event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        async fn send_raw_from_store(&self, _key: Key, _body: Vec<u8>, _meta: QueuedPayloadMeta) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        async fn close(&self) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
+            None
+        }
+
+        fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
+            Box::new(self.clone())
+        }
+
+        async fn init(&self) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        fn is_enabled(&self) -> bool {
+            self.enabled
+        }
+
+        fn delivery_snapshot(&self) -> TargetDeliverySnapshot {
+            TargetDeliverySnapshot {
+                failed_messages: self.failed_messages.load(Ordering::Relaxed),
+                queue_length: 0,
+                total_messages: self.total_messages.load(Ordering::Relaxed),
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn runtime_view_reports_empty_runtime_queries() {
+        let runtime_view = NotifyRuntimeView::new(
+            Arc::new(RwLock::new(TargetList::new())),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+        );
+
+        assert!(runtime_view.get_active_targets().await.is_empty());
+        assert!(runtime_view.get_target_values().await.is_empty());
+        assert!(runtime_view.get_all_targets().read().await.is_empty());
+    }
+
+    #[tokio::test]
+    async fn runtime_view_reports_empty_runtime_snapshots() {
+        let runtime_view = NotifyRuntimeView::new(
+            Arc::new(RwLock::new(TargetList::new())),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+        );
+
+        assert!(runtime_view.snapshot_target_metrics().await.is_empty());
+        assert!(runtime_view.snapshot_target_health().await.is_empty());
+
+        let status = runtime_view.runtime_status_snapshot().await;
+        assert_eq!(status.target_count, 0);
+        assert_eq!(status.replay_worker_count, 0);
+    }
+
+    #[tokio::test]
+    async fn runtime_view_reports_non_empty_runtime_queries_and_snapshots() {
+        let target_list = Arc::new(RwLock::new(TargetList::new()));
+        let replay_workers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
+
+        let online = Arc::new(TestTarget::new("primary", "webhook"));
+        online.record_successes(3);
+        online.record_failures(1);
+
+        let disabled = Arc::new(TestTarget::new("backup", "mqtt").with_enabled(false).with_active(false));
+        disabled.record_successes(2);
+
+        {
+            let mut targets = target_list.write().await;
+            targets.add(online.clone() as Arc<dyn Target<Event> + Send + Sync>).unwrap();
+            targets.add(disabled.clone() as Arc<dyn Target<Event> + Send + Sync>).unwrap();
+        }
+
+        let runtime_view = NotifyRuntimeView::new(target_list.clone(), replay_workers.clone());
+
+        let mut active_targets = runtime_view.get_active_targets().await;
+        active_targets.sort();
+        assert_eq!(
+            active_targets,
+            vec![
+                TargetID::new("backup".to_string(), "mqtt".to_string()),
+                TargetID::new("primary".to_string(), "webhook".to_string())
+            ]
+        );
+
+        let target_values = runtime_view.get_target_values().await;
+        assert_eq!(target_values.len(), 2);
+        assert_eq!(runtime_view.get_all_targets().read().await.len(), 2);
+
+        let metric_snapshots = runtime_view.snapshot_target_metrics().await;
+        assert_eq!(metric_snapshots.len(), 2);
+        assert_eq!(metric_snapshots[0].target_id, "backup:mqtt");
+        assert_eq!(metric_snapshots[0].failed_messages, 0);
+        assert_eq!(metric_snapshots[0].total_messages, 2);
+        assert_eq!(metric_snapshots[1].target_id, "primary:webhook");
+        assert_eq!(metric_snapshots[1].failed_messages, 1);
+        assert_eq!(metric_snapshots[1].total_messages, 3);
+
+        let health_snapshots = runtime_view.snapshot_target_health().await;
+        assert_eq!(health_snapshots.len(), 2);
+        assert_eq!(health_snapshots[0].target_id, "backup:mqtt");
+        assert!(!health_snapshots[0].enabled);
+        assert_eq!(health_snapshots[0].state, rustfs_targets::RuntimeTargetHealthState::Disabled);
+        assert_eq!(health_snapshots[1].target_id, "primary:webhook");
+        assert!(health_snapshots[1].enabled);
+        assert_eq!(health_snapshots[1].state, rustfs_targets::RuntimeTargetHealthState::Online);
+
+        let status = runtime_view.runtime_status_snapshot().await;
+        assert_eq!(status.target_count, 2);
+        assert_eq!(status.replay_worker_count, 0);
+    }
+}

crates/notify/src/services.rs

@@ -0,0 +1,120 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::{
+    Event,
+    bucket_config_manager::NotifyBucketConfigManager,
+    config_manager::NotifyConfigManager,
+    integration::NotificationMetrics,
+    notification_system_subscriber::NotificationSystemSubscriberView,
+    notifier::{EventNotifier, SharedNotifyTargetList},
+    pipeline::{LiveEventHistory, NotifyPipeline},
+    registry::TargetRegistry,
+    rule_engine::NotifyRuleEngine,
+    runtime_facade::NotifyRuntimeFacade,
+    runtime_view::NotifyRuntimeView,
+    status_view::NotifyStatusView,
+};
+use rustfs_ecstore::config::Config;
+use rustfs_targets::ReplayWorkerManager;
+use std::sync::Arc;
+use tokio::sync::{RwLock, Semaphore, broadcast};
+
+#[derive(Clone)]
+pub struct NotifyServices {
+    pub bucket_config_manager: NotifyBucketConfigManager,
+    pub config_manager: NotifyConfigManager,
+    pub pipeline: NotifyPipeline,
+    pub runtime_facade: NotifyRuntimeFacade,
+    pub runtime_view: NotifyRuntimeView,
+    pub status_view: NotifyStatusView,
+}
+
+impl NotifyServices {
+    #[allow(clippy::too_many_arguments)]
+    pub fn new(
+        notifier: Arc<EventNotifier>,
+        rule_engine: NotifyRuleEngine,
+        target_list: SharedNotifyTargetList,
+        registry: Arc<TargetRegistry>,
+        config: Arc<RwLock<Config>>,
+        stream_cancellers: Arc<RwLock<ReplayWorkerManager>>,
+        concurrency_limiter: Arc<Semaphore>,
+        metrics: Arc<NotificationMetrics>,
+        subscriber_view: Arc<NotificationSystemSubscriberView>,
+        live_event_sender: broadcast::Sender<Arc<Event>>,
+        live_event_history: Arc<RwLock<LiveEventHistory>>,
+    ) -> Self {
+        let runtime_view = NotifyRuntimeView::new(target_list.clone(), stream_cancellers.clone());
+        let runtime_facade = NotifyRuntimeFacade::new(target_list, stream_cancellers, concurrency_limiter, metrics.clone());
+        let config_manager = NotifyConfigManager::new(config, registry, rule_engine.clone(), runtime_facade.clone());
+        let bucket_config_manager = NotifyBucketConfigManager::new(notifier.clone(), rule_engine, subscriber_view);
+        let pipeline = NotifyPipeline::new(notifier, live_event_sender, live_event_history);
+        let status_view = NotifyStatusView::new(metrics);
+
+        Self {
+            bucket_config_manager,
+            config_manager,
+            pipeline,
+            runtime_facade,
+            runtime_view,
+            status_view,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::NotifyServices;
+    use crate::{
+        integration::NotificationMetrics, notification_system_subscriber::NotificationSystemSubscriberView,
+        notifier::EventNotifier, pipeline::LiveEventHistory, registry::TargetRegistry, rule_engine::NotifyRuleEngine,
+    };
+    use rustfs_ecstore::config::Config;
+    use rustfs_targets::ReplayWorkerManager;
+    use std::sync::Arc;
+    use tokio::sync::{RwLock, Semaphore, broadcast};
+
+    #[tokio::test]
+    async fn services_build_empty_runtime_views() {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
+        let target_list = notifier.target_list();
+        let registry = Arc::new(TargetRegistry::new());
+        let config = Arc::new(RwLock::new(Config::default()));
+        let stream_cancellers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
+        let concurrency_limiter = Arc::new(Semaphore::new(4));
+        let subscriber_view = Arc::new(NotificationSystemSubscriberView::new());
+        let (live_event_sender, _) = broadcast::channel(16);
+        let live_event_history = Arc::new(RwLock::new(LiveEventHistory::default()));
+
+        let services = NotifyServices::new(
+            notifier,
+            rule_engine,
+            target_list,
+            registry,
+            config,
+            stream_cancellers,
+            concurrency_limiter,
+            metrics,
+            subscriber_view,
+            live_event_sender,
+            live_event_history,
+        );
+
+        assert!(services.runtime_view.get_active_targets().await.is_empty());
+        assert_eq!(services.status_view.snapshot_metrics().events_sent_total, 0);
+    }
+}

crates/notify/src/status_view.rs

@@ -0,0 +1,74 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::integration::{NotificationMetricSnapshot, NotificationMetrics};
+use hashbrown::HashMap;
+use std::sync::Arc;
+
+#[derive(Clone)]
+pub struct NotifyStatusView {
+    metrics: Arc<NotificationMetrics>,
+}
+
+impl NotifyStatusView {
+    pub fn new(metrics: Arc<NotificationMetrics>) -> Self {
+        Self { metrics }
+    }
+
+    pub fn get_status(&self) -> HashMap<String, String> {
+        let mut status = HashMap::new();
+
+        status.insert("uptime_seconds".to_string(), self.metrics.uptime().as_secs().to_string());
+        status.insert("processing_events".to_string(), self.metrics.processing_count().to_string());
+        status.insert("processed_events".to_string(), self.metrics.processed_count().to_string());
+        status.insert("failed_events".to_string(), self.metrics.failed_count().to_string());
+        status.insert("skipped_events".to_string(), self.metrics.skipped_count().to_string());
+
+        status
+    }
+
+    pub fn snapshot_metrics(&self) -> NotificationMetricSnapshot {
+        self.metrics.snapshot()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::NotifyStatusView;
+    use crate::integration::NotificationMetrics;
+    use std::sync::Arc;
+
+    #[test]
+    fn status_view_reports_empty_metrics_snapshot() {
+        let status_view = NotifyStatusView::new(Arc::new(NotificationMetrics::new()));
+
+        let snapshot = status_view.snapshot_metrics();
+        assert_eq!(snapshot.current_send_in_progress, 0);
+        assert_eq!(snapshot.events_errors_total, 0);
+        assert_eq!(snapshot.events_sent_total, 0);
+        assert_eq!(snapshot.events_skipped_total, 0);
+    }
+
+    #[test]
+    fn status_view_exposes_status_map_keys() {
+        let status_view = NotifyStatusView::new(Arc::new(NotificationMetrics::new()));
+        let status = status_view.get_status();
+
+        assert!(status.contains_key("uptime_seconds"));
+        assert!(status.contains_key("processing_events"));
+        assert!(status.contains_key("processed_events"));
+        assert!(status.contains_key("failed_events"));
+        assert!(status.contains_key("skipped_events"));
+    }
+}

crates/notify/src/stream.rs

@@ -1,341 +0,0 @@
-// Copyright 2024 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use crate::{Event, integration::NotificationMetrics};
-use rustfs_targets::{
-    StoreError, Target, TargetError,
-    store::{Key, Store, ensure_store_entry_raw_readable},
-    target::QueuedPayload,
-};
-use rustfs_utils::get_env_usize;
-use std::sync::Arc;
-use std::time::{Duration, Instant};
-use tokio::sync::{Semaphore, mpsc};
-use tokio::time::sleep;
-use tracing::{debug, error, info, warn};
-
-/// Streams events from the store to the target with retry logic
-///
-/// # Arguments
-/// - `store`: The event store
-/// - `target`: The target to send events to
-/// - `cancel_rx`: Receiver to listen for cancellation signals
-pub async fn stream_events(
-    store: &mut (dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send),
-    target: &dyn Target<Event>,
-    mut cancel_rx: mpsc::Receiver<()>,
-) {
-    info!("Starting event stream for target: {}", target.name());
-
-    // Retry configuration
-    const MAX_RETRIES: usize = 5;
-    const RETRY_DELAY: Duration = Duration::from_secs(5);
-
-    loop {
-        // Check for cancellation signal
-        if cancel_rx.try_recv().is_ok() {
-            info!("Cancellation received for target: {}", target.name());
-            return;
-        }
-
-        // Get list of events in the store
-        let keys = store.list();
-        if keys.is_empty() {
-            // No events, wait before checking again
-            sleep(Duration::from_secs(1)).await;
-            continue;
-        }
-
-        // Process each event
-        for key in keys {
-            // Check for cancellation before processing each event
-            if cancel_rx.try_recv().is_ok() {
-                info!("Cancellation received during processing for target: {}", target.name());
-                return;
-            }
-
-            let mut retry_count = 0;
-            let mut success = false;
-
-            // Retry logic
-            while retry_count < MAX_RETRIES && !success {
-                match target.send_from_store(key.clone()).await {
-                    Ok(_) => {
-                        info!("Successfully sent event for target: {}", target.name());
-                        // send_from_store deletes the event from store on success
-                        success = true;
-                    }
-                    Err(e) => {
-                        // Handle specific errors
-                        match &e {
-                            TargetError::NotConnected => {
-                                warn!("Target {} not connected, retrying...", target.name());
-                                retry_count += 1;
-                                sleep(RETRY_DELAY).await;
-                            }
-                            TargetError::Timeout(_) => {
-                                warn!("Timeout for target {}, retrying...", target.name());
-                                retry_count += 1;
-                                sleep(Duration::from_secs((retry_count * 5) as u64)).await; // Exponential backoff
-                            }
-                            _ => {
-                                // Permanent error, skip this event
-                                error!("Permanent error for target {}: {}", target.name(), e);
-                                break;
-                            }
-                        }
-                    }
-                }
-            }
-
-            // Remove event from store if successfully sent
-            if retry_count >= MAX_RETRIES && !success {
-                warn!("Max retries exceeded for event {}, target: {}, skipping", key.to_string(), target.name());
-            }
-        }
-
-        // Small delay before next iteration
-        sleep(Duration::from_millis(100)).await;
-    }
-}
-
-/// Starts the event streaming process for a target
-///
-/// # Arguments
-/// - `store`: The event store
-/// - `target`: The target to send events to
-///
-/// # Returns
-/// A sender to signal cancellation of the event stream
-pub fn start_event_stream(
-    mut store: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>,
-    target: Arc<dyn Target<Event> + Send + Sync>,
-) -> mpsc::Sender<()> {
-    let (cancel_tx, cancel_rx) = mpsc::channel(1);
-
-    tokio::spawn(async move {
-        stream_events(&mut *store, &*target, cancel_rx).await;
-        info!("Event stream stopped for target: {}", target.name());
-    });
-
-    cancel_tx
-}
-
-/// Start event stream with batch processing
-///
-/// # Arguments
-/// - `store`: The event store
-/// - `target`: The target to send events to clients
-/// - `metrics`: Metrics for monitoring
-/// - `semaphore`: Semaphore to limit concurrency
-///
-/// # Returns
-/// A sender to signal cancellation of the event stream
-pub fn start_event_stream_with_batching(
-    mut store: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>,
-    target: Arc<dyn Target<Event> + Send + Sync>,
-    metrics: Arc<NotificationMetrics>,
-    semaphore: Arc<Semaphore>,
-) -> mpsc::Sender<()> {
-    let (cancel_tx, cancel_rx) = mpsc::channel(1);
-    debug!("Starting event stream with batching for target: {}", target.name());
-    tokio::spawn(async move {
-        stream_events_with_batching(&mut *store, &*target, cancel_rx, metrics, semaphore).await;
-        info!("Event stream stopped for target: {}", target.name());
-    });
-
-    cancel_tx
-}
-
-/// Event stream processing with batch processing
-///
-/// # Arguments
-/// - `store`: The event store
-/// - `target`: The target to send events to clients
-/// - `cancel_rx`: Receiver to listen for cancellation signals
-/// - `metrics`: Metrics for monitoring
-/// - `semaphore`: Semaphore to limit concurrency
-///
-/// # Notes
-/// This function processes events in batches to improve efficiency.
-pub async fn stream_events_with_batching(
-    store: &mut (dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send),
-    target: &dyn Target<Event>,
-    mut cancel_rx: mpsc::Receiver<()>,
-    metrics: Arc<NotificationMetrics>,
-    semaphore: Arc<Semaphore>,
-) {
-    info!("Starting event stream with batching for target: {}", target.name());
-
-    // Configuration parameters
-    const DEFAULT_BATCH_SIZE: usize = 1;
-    let batch_size = get_env_usize("RUSTFS_EVENT_BATCH_SIZE", DEFAULT_BATCH_SIZE);
-    const BATCH_TIMEOUT: Duration = Duration::from_secs(5);
-    const MAX_RETRIES: usize = 5;
-    const BASE_RETRY_DELAY: Duration = Duration::from_secs(2);
-
-    let mut batch_keys = Vec::with_capacity(batch_size);
-    let mut last_flush = Instant::now();
-
-    loop {
-        // Check the cancel signal
-        if cancel_rx.try_recv().is_ok() {
-            info!("Cancellation received for target: {}", target.name());
-            return;
-        }
-
-        // Get a list of events in storage
-        let keys = store.list();
-        debug!("Found {} keys in store for target: {}", keys.len(), target.name());
-        if keys.is_empty() {
-            // If there is data in the batch and timeout, refresh the batch
-            if !batch_keys.is_empty() && last_flush.elapsed() >= BATCH_TIMEOUT {
-                process_batch(&mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
-                last_flush = Instant::now();
-            }
-
-            // No event, wait before checking
-            tokio::time::sleep(Duration::from_millis(500)).await;
-            continue;
-        }
-
-        // Handle each event
-        for key in keys {
-            // Check the cancel signal again
-            if cancel_rx.try_recv().is_ok() {
-                info!("Cancellation received during processing for target: {}", target.name());
-
-                // Processing collected batches before exiting
-                if !batch_keys.is_empty() {
-                    process_batch(&mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
-                }
-                return;
-            }
-
-            // Skip unreadable entries so a single corrupt file cannot stall the stream.
-            // ensure_store_entry_raw_readable attempts get_raw; on I/O error it calls del() to
-            // remove the corrupt entry before returning Err, so no cleanup is needed here.
-            match ensure_store_entry_raw_readable(&*store, &key) {
-                Ok(true) => {}         // entry is readable, proceed
-                Ok(false) => continue, // entry not found (already removed), skip
-                Err(err) => {
-                    warn!("Skipping unreadable store entry {} for target {}: {}", key, target.name(), err);
-                    continue; // corrupt entry was already deleted by ensure_store_entry_raw_readable
-                }
-            }
-
-            batch_keys.push(key);
-            metrics.increment_processing();
-
-            // If the batch is full or enough time has passed since the last refresh, the batch will be processed
-            if batch_keys.len() >= batch_size || last_flush.elapsed() >= BATCH_TIMEOUT {
-                process_batch(&mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
-                last_flush = Instant::now();
-            }
-        }
-
-        // A small delay will be conducted to check the next round
-        tokio::time::sleep(Duration::from_millis(100)).await;
-    }
-}
-
-/// Processing event batches for targets
-/// # Arguments
-/// - `batch`: The batch of events to process
-/// - `batch_keys`: The corresponding keys of the events in the batch
-/// - `target`: The target to send events to clients
-/// - `max_retries`: Maximum number of retries for sending an event
-/// - `base_delay`: Base delay duration for retries
-/// - `metrics`: Metrics for monitoring
-/// - `semaphore`: Semaphore to limit concurrency
-/// # Notes
-/// This function processes a batch of events, sending each event to the target with retry
-async fn process_batch(
-    batch_keys: &mut Vec<Key>,
-    target: &dyn Target<Event>,
-    max_retries: usize,
-    base_delay: Duration,
-    metrics: &Arc<NotificationMetrics>,
-    semaphore: &Arc<Semaphore>,
-) {
-    debug!("Processing batch of {} events for target: {}", batch_keys.len(), target.name());
-    if batch_keys.is_empty() {
-        return;
-    }
-
-    // Obtain semaphore permission to limit concurrency
-    let permit = match semaphore.clone().acquire_owned().await {
-        Ok(permit) => permit,
-        Err(e) => {
-            error!("Failed to acquire semaphore permit: {}", e);
-            return;
-        }
-    };
-
-    // Handle every event in the batch
-    for key in batch_keys.iter() {
-        let mut retry_count = 0;
-        let mut success = false;
-
-        // Retry logic
-        while retry_count < max_retries && !success {
-            match target.send_from_store(key.clone()).await {
-                Ok(_) => {
-                    debug!("Successfully sent event for target: {}, Key: {}", target.name(), key.to_string());
-                    success = true;
-                    metrics.increment_processed();
-                }
-                Err(e) => match &e {
-                    TargetError::NotConnected => {
-                        warn!("Target {} not connected, retrying...", target.name());
-                        retry_count += 1;
-                        let jitter = Duration::from_millis(key.to_string().len() as u64 % 500);
-                        let backoff = 1u32 << retry_count as u32;
-                        tokio::time::sleep(base_delay * backoff + jitter).await;
-                    }
-                    TargetError::Timeout(_) => {
-                        warn!("Timeout for target {}, retrying...", target.name());
-                        retry_count += 1;
-                        let jitter = Duration::from_millis(key.to_string().len() as u64 % 500);
-                        let backoff = 1u32 << retry_count as u32;
-                        tokio::time::sleep(base_delay * backoff + jitter).await;
-                    }
-                    TargetError::Dropped(reason) => {
-                        warn!("Dropped queued payload for target {}: {}", target.name(), reason);
-                        metrics.increment_failed();
-                        break;
-                    }
-                    _ => {
-                        error!("Permanent error for target {}: {}", target.name(), e);
-                        target.record_final_failure();
-                        metrics.increment_failed();
-                        break;
-                    }
-                },
-            }
-        }
-
-        if retry_count >= max_retries && !success {
-            warn!("Max retries exceeded for event {}, target: {}, skipping", key.to_string(), target.name());
-            target.record_final_failure();
-            metrics.increment_failed();
-        }
-    }
-
-    // Clear processed batches
-    batch_keys.clear();
-
-    // Release semaphore permission (via drop)
-    drop(permit);
-}

crates/targets/AGENTS.md

@@ -2,34 +2,68 @@
 
 Applies to `crates/targets/`.
 
-`rustfs-targets` provides the notification target abstraction layer:
-the `Target<E>` trait, built-in implementations (Webhook, Kafka, MQTT,
-NATS, Pulsar, MySQL, Postgres, Redis), persistent queue store,
-DSN/configuration builders, and the `ChannelTargetType` registry that
-maps target types to their runtime factories.
+`rustfs-targets` is the shared target-plugin foundation for `audit` and
+`notify`. It owns plugin metadata, builtin target descriptors, runtime
+orchestration primitives, and plugin control-plane state modeling.
+
+## Current Module Boundaries
+
+- `manifest.rs`: declarative plugin metadata and marketplace-facing shape.
+  Keep this layer declarative only; do not add runtime execution logic here.
+- `catalog/`: centralized builtin descriptor registration and example external
+  plugin assembly. Keep admin-facing plugin source data here instead of
+  spreading it into handlers.
+- `plugin.rs`: `TargetPluginDescriptor`, `TargetPluginRegistry`,
+  `BuiltinTargetDescriptor`, and admin descriptor metadata.
+- `runtime/`: shared runtime lifecycle and replay orchestration:
+  `TargetRuntimeManager`, `ReplayWorkerManager`, `PluginRuntimeAdapter`,
+  `BuiltinPluginRuntimeAdapter`, and sidecar protocol/runtime MVP types.
+- `control_plane.rs`: install/enable/runtime state models and install policy
+  validation helpers. Keep install/governance state logic centralized here.
+- `config/`, `target/`, `store/`, `check/`: target config normalization,
+  target implementations, queue/store, and endpoint connectivity checks.
+
+## Change Style and Ownership Rules
+
+- Preserve the layering above. Do not move install/runtime/governance logic
+  into admin handlers or manifest structs.
+- Prefer extending shared abstractions (`TargetPluginRegistry`,
+  `PluginRuntimeAdapter`, `TargetRuntimeManager`) over duplicating per-domain
+  orchestration logic.
+- Keep external sidecar behavior scoped to current MVP boundaries unless the
+  task explicitly includes real installer/transport integration.
+- Reuse existing constants/keys from `rustfs_config`; avoid introducing
+  duplicate literals for target field names and subsystem keys.
 
 ## Library Design
 
 - Treat crate code as reusable library code by default.
-- Prefer `thiserror` for library-facing error types.
-- Do not use `unwrap()`, `expect()`, or panic-driven control flow outside tests.
+- Return structured `TargetError`/`StoreError` results; avoid panic-driven
+  control flow outside tests.
+- Keep serialization contracts stable for types re-exported by `lib.rs`.
 
 ## Testing
 
 - Keep unit tests close to the module they test.
-- Keep integration tests under `crates/targets/tests/` directory.
-- Add regression tests for bug fixes and behavior changes.
+- Keep integration tests under `crates/targets/tests/`.
+- Add regression tests for behavior changes in:
+  - plugin manifest/catalog/control-plane contracts
+  - runtime adapter lifecycle behavior
+  - target config normalization and validation
+  - sidecar handshake/policy validation paths
 
 ## Async and Performance
 
 - Keep async paths non-blocking.
-- Move CPU-heavy operations out of async hot paths with `tokio::task::spawn_blocking` when appropriate.
+- Avoid hot-path allocations and repeated config normalization when a cached
+  snapshot can be reused.
+- Use bounded concurrency and timeout guards for runtime and health checks.
 
 ## Integration Tests
 
 Integration tests under `tests/` are `#[ignore]` by default so CI never runs
-them. See the module-level doc comment in each test file for prerequisites
-and run commands:
+them. See module-level doc comments in each test file for prerequisites and
+run commands.
 
 - `tests/mysql_integration.rs` — MySQL 8.0+ / TiDB 8.5+
 - `tests/postgres_integration.rs` — PostgreSQL
@@ -37,4 +71,8 @@ and run commands:
 ## Suggested Validation
 
 - `cargo test -p rustfs-targets`
+- If runtime/plugin contracts changed, run focused tests under:
+  - `cargo test -p rustfs-targets plugin`
+  - `cargo test -p rustfs-targets runtime`
+  - `cargo test -p rustfs-targets control_plane`
 - Full gate before commit: `make pre-commit`

crates/targets/src/catalog/builtin.rs

@@ -0,0 +1,426 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::plugin::{
+    BuiltinTargetAdminDescriptor, BuiltinTargetDescriptor, TargetAdminMetadata, TargetPluginDescriptor, TargetRequestValidator,
+    boxed_target,
+};
+use crate::target::{ChannelTargetType, TargetType};
+use crate::{Target, TargetError};
+use rustfs_config::audit::{
+    AUDIT_AMQP_KEYS, AUDIT_KAFKA_KEYS, AUDIT_MQTT_KEYS, AUDIT_MYSQL_KEYS, AUDIT_NATS_KEYS, AUDIT_POSTGRES_KEYS,
+    AUDIT_PULSAR_KEYS, AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_REDIS_KEYS, AUDIT_WEBHOOK_KEYS,
+};
+use rustfs_config::notify::{
+    NOTIFY_AMQP_KEYS, NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS,
+    NOTIFY_MYSQL_KEYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_KEYS,
+    NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_DEFAULT_CHANNEL, NOTIFY_REDIS_KEYS,
+    NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
+};
+use rustfs_config::{
+    AUDIT_DEFAULT_DIR, EVENT_DEFAULT_DIR,
+    audit::{
+        AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_MYSQL_SUB_SYS, AUDIT_NATS_SUB_SYS,
+        AUDIT_POSTGRES_SUB_SYS, AUDIT_PULSAR_SUB_SYS, AUDIT_REDIS_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS,
+    },
+};
+use rustfs_ecstore::config::KVS;
+use serde::Serialize;
+use serde::de::DeserializeOwned;
+
+use crate::config::{
+    build_amqp_args, build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args,
+    build_pulsar_args, build_redis_args, build_webhook_args, validate_amqp_config, validate_kafka_config, validate_mqtt_config,
+    validate_mysql_config, validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config,
+    validate_webhook_config,
+};
+
+type BoxedTarget<E> = Box<dyn Target<E> + Send + Sync>;
+
+fn build_descriptor<E, Create, Validate>(
+    subsystem: &'static str,
+    request_validator: TargetRequestValidator,
+    target_type: &'static str,
+    valid_fields: &'static [&'static str],
+    validate_config: Validate,
+    create_target: Create,
+) -> BuiltinTargetDescriptor<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+    Create: Fn(String, &KVS) -> Result<BoxedTarget<E>, TargetError> + Send + Sync + 'static,
+    Validate: Fn(&KVS) -> Result<(), TargetError> + Send + Sync + 'static,
+{
+    BuiltinTargetDescriptor::new(
+        subsystem,
+        request_validator,
+        TargetPluginDescriptor::new(target_type, valid_fields, validate_config, create_target),
+    )
+}
+
+fn build_admin_descriptor(
+    subsystem: &'static str,
+    request_validator: TargetRequestValidator,
+    target_type: &'static str,
+    valid_fields: &'static [&'static str],
+) -> BuiltinTargetAdminDescriptor {
+    BuiltinTargetAdminDescriptor::new(
+        crate::manifest::builtin_target_manifest(target_type),
+        valid_fields,
+        TargetAdminMetadata::new(subsystem, request_validator),
+    )
+}
+
+pub fn builtin_audit_target_admin_descriptors() -> Vec<BuiltinTargetAdminDescriptor> {
+    vec![
+        build_admin_descriptor(
+            AUDIT_AMQP_SUB_SYS,
+            TargetRequestValidator::Amqp(TargetType::AuditLog),
+            ChannelTargetType::Amqp.as_str(),
+            AUDIT_AMQP_KEYS,
+        ),
+        build_admin_descriptor(
+            AUDIT_WEBHOOK_SUB_SYS,
+            TargetRequestValidator::Webhook,
+            ChannelTargetType::Webhook.as_str(),
+            AUDIT_WEBHOOK_KEYS,
+        ),
+        build_admin_descriptor(
+            AUDIT_MQTT_SUB_SYS,
+            TargetRequestValidator::Mqtt,
+            ChannelTargetType::Mqtt.as_str(),
+            AUDIT_MQTT_KEYS,
+        ),
+        build_admin_descriptor(
+            AUDIT_NATS_SUB_SYS,
+            TargetRequestValidator::Nats(TargetType::AuditLog),
+            ChannelTargetType::Nats.as_str(),
+            AUDIT_NATS_KEYS,
+        ),
+        build_admin_descriptor(
+            AUDIT_PULSAR_SUB_SYS,
+            TargetRequestValidator::Pulsar(TargetType::AuditLog),
+            ChannelTargetType::Pulsar.as_str(),
+            AUDIT_PULSAR_KEYS,
+        ),
+        build_admin_descriptor(
+            AUDIT_KAFKA_SUB_SYS,
+            TargetRequestValidator::Kafka(TargetType::AuditLog),
+            ChannelTargetType::Kafka.as_str(),
+            AUDIT_KAFKA_KEYS,
+        ),
+        build_admin_descriptor(
+            AUDIT_REDIS_SUB_SYS,
+            TargetRequestValidator::Redis {
+                default_channel: AUDIT_REDIS_DEFAULT_CHANNEL,
+                target_type: TargetType::AuditLog,
+            },
+            ChannelTargetType::Redis.as_str(),
+            AUDIT_REDIS_KEYS,
+        ),
+        build_admin_descriptor(
+            AUDIT_MYSQL_SUB_SYS,
+            TargetRequestValidator::MySql(TargetType::AuditLog),
+            ChannelTargetType::MySql.as_str(),
+            AUDIT_MYSQL_KEYS,
+        ),
+        build_admin_descriptor(
+            AUDIT_POSTGRES_SUB_SYS,
+            TargetRequestValidator::Postgres(TargetType::AuditLog),
+            ChannelTargetType::Postgres.as_str(),
+            AUDIT_POSTGRES_KEYS,
+        ),
+    ]
+}
+
+pub fn builtin_audit_target_descriptors<E>() -> Vec<BuiltinTargetDescriptor<E>>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    vec![
+        build_descriptor(
+            AUDIT_AMQP_SUB_SYS,
+            TargetRequestValidator::Amqp(TargetType::AuditLog),
+            ChannelTargetType::Amqp.as_str(),
+            AUDIT_AMQP_KEYS,
+            |config| validate_amqp_config(config, AUDIT_DEFAULT_DIR),
+            |id, config| {
+                let args = build_amqp_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
+                Ok(boxed_target(crate::target::amqp::AMQPTarget::<E>::new(id, args)?))
+            },
+        ),
+        build_descriptor(
+            AUDIT_WEBHOOK_SUB_SYS,
+            TargetRequestValidator::Webhook,
+            ChannelTargetType::Webhook.as_str(),
+            AUDIT_WEBHOOK_KEYS,
+            |config| validate_webhook_config(config, AUDIT_DEFAULT_DIR),
+            |id, config| {
+                let args = build_webhook_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
+                Ok(boxed_target(crate::target::webhook::WebhookTarget::<E>::new(id, args)?))
+            },
+        ),
+        build_descriptor(
+            AUDIT_MQTT_SUB_SYS,
+            TargetRequestValidator::Mqtt,
+            ChannelTargetType::Mqtt.as_str(),
+            AUDIT_MQTT_KEYS,
+            validate_mqtt_config,
+            |id, config| {
+                let args = build_mqtt_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
+                Ok(boxed_target(crate::target::mqtt::MQTTTarget::<E>::new(id, args)?))
+            },
+        ),
+        build_descriptor(
+            AUDIT_NATS_SUB_SYS,
+            TargetRequestValidator::Nats(TargetType::AuditLog),
+            ChannelTargetType::Nats.as_str(),
+            AUDIT_NATS_KEYS,
+            |config| validate_nats_config(config, AUDIT_DEFAULT_DIR),
+            |id, config| {
+                let args = build_nats_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
+                Ok(boxed_target(crate::target::nats::NATSTarget::<E>::new(id, args)?))
+            },
+        ),
+        build_descriptor(
+            AUDIT_PULSAR_SUB_SYS,
+            TargetRequestValidator::Pulsar(TargetType::AuditLog),
+            ChannelTargetType::Pulsar.as_str(),
+            AUDIT_PULSAR_KEYS,
+            |config| validate_pulsar_config(config, AUDIT_DEFAULT_DIR),
+            |id, config| {
+                let args = build_pulsar_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
+                Ok(boxed_target(crate::target::pulsar::PulsarTarget::<E>::new(id, args)?))
+            },
+        ),
+        build_descriptor(
+            AUDIT_KAFKA_SUB_SYS,
+            TargetRequestValidator::Kafka(TargetType::AuditLog),
+            ChannelTargetType::Kafka.as_str(),
+            AUDIT_KAFKA_KEYS,
+            |config| validate_kafka_config(config, AUDIT_DEFAULT_DIR),
+            |id, config| {
+                let args = build_kafka_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
+                Ok(boxed_target(crate::target::kafka::KafkaTarget::<E>::new(id, args)?))
+            },
+        ),
+        build_descriptor(
+            AUDIT_REDIS_SUB_SYS,
+            TargetRequestValidator::Redis {
+                default_channel: AUDIT_REDIS_DEFAULT_CHANNEL,
+                target_type: TargetType::AuditLog,
+            },
+            ChannelTargetType::Redis.as_str(),
+            AUDIT_REDIS_KEYS,
+            |config| validate_redis_config(config, AUDIT_DEFAULT_DIR, AUDIT_REDIS_DEFAULT_CHANNEL),
+            |id, config| {
+                let args = build_redis_args(config, AUDIT_DEFAULT_DIR, AUDIT_REDIS_DEFAULT_CHANNEL, TargetType::AuditLog)?;
+                Ok(boxed_target(crate::target::redis::RedisTarget::<E>::new(id, args)?))
+            },
+        ),
+        build_descriptor(
+            AUDIT_MYSQL_SUB_SYS,
+            TargetRequestValidator::MySql(TargetType::AuditLog),
+            ChannelTargetType::MySql.as_str(),
+            AUDIT_MYSQL_KEYS,
+            |config| validate_mysql_config(config, AUDIT_DEFAULT_DIR),
+            |id, config| {
+                let args = build_mysql_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
+                Ok(boxed_target(crate::target::mysql::MySqlTarget::<E>::new(id, args)?))
+            },
+        ),
+        build_descriptor(
+            AUDIT_POSTGRES_SUB_SYS,
+            TargetRequestValidator::Postgres(TargetType::AuditLog),
+            ChannelTargetType::Postgres.as_str(),
+            AUDIT_POSTGRES_KEYS,
+            |config| validate_postgres_config(config, AUDIT_DEFAULT_DIR),
+            |id, config| {
+                let args = build_postgres_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
+                Ok(boxed_target(crate::target::postgres::PostgresTarget::<E>::new(id, args)?))
+            },
+        ),
+    ]
+}
+
+pub fn builtin_notify_target_admin_descriptors() -> Vec<BuiltinTargetAdminDescriptor> {
+    vec![
+        build_admin_descriptor(
+            NOTIFY_WEBHOOK_SUB_SYS,
+            TargetRequestValidator::Webhook,
+            ChannelTargetType::Webhook.as_str(),
+            NOTIFY_WEBHOOK_KEYS,
+        ),
+        build_admin_descriptor(
+            NOTIFY_AMQP_SUB_SYS,
+            TargetRequestValidator::Amqp(TargetType::NotifyEvent),
+            ChannelTargetType::Amqp.as_str(),
+            NOTIFY_AMQP_KEYS,
+        ),
+        build_admin_descriptor(
+            NOTIFY_KAFKA_SUB_SYS,
+            TargetRequestValidator::Kafka(TargetType::NotifyEvent),
+            ChannelTargetType::Kafka.as_str(),
+            NOTIFY_KAFKA_KEYS,
+        ),
+        build_admin_descriptor(
+            NOTIFY_MQTT_SUB_SYS,
+            TargetRequestValidator::Mqtt,
+            ChannelTargetType::Mqtt.as_str(),
+            NOTIFY_MQTT_KEYS,
+        ),
+        build_admin_descriptor(
+            NOTIFY_MYSQL_SUB_SYS,
+            TargetRequestValidator::MySql(TargetType::NotifyEvent),
+            ChannelTargetType::MySql.as_str(),
+            NOTIFY_MYSQL_KEYS,
+        ),
+        build_admin_descriptor(
+            NOTIFY_NATS_SUB_SYS,
+            TargetRequestValidator::Nats(TargetType::NotifyEvent),
+            ChannelTargetType::Nats.as_str(),
+            NOTIFY_NATS_KEYS,
+        ),
+        build_admin_descriptor(
+            NOTIFY_POSTGRES_SUB_SYS,
+            TargetRequestValidator::Postgres(TargetType::NotifyEvent),
+            ChannelTargetType::Postgres.as_str(),
+            NOTIFY_POSTGRES_KEYS,
+        ),
+        build_admin_descriptor(
+            NOTIFY_REDIS_SUB_SYS,
+            TargetRequestValidator::Redis {
+                default_channel: NOTIFY_REDIS_DEFAULT_CHANNEL,
+                target_type: TargetType::NotifyEvent,
+            },
+            ChannelTargetType::Redis.as_str(),
+            NOTIFY_REDIS_KEYS,
+        ),
+        build_admin_descriptor(
+            NOTIFY_PULSAR_SUB_SYS,
+            TargetRequestValidator::Pulsar(TargetType::NotifyEvent),
+            ChannelTargetType::Pulsar.as_str(),
+            NOTIFY_PULSAR_KEYS,
+        ),
+    ]
+}
+
+pub fn builtin_notify_target_descriptors<E>() -> Vec<BuiltinTargetDescriptor<E>>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    vec![
+        build_descriptor(
+            NOTIFY_WEBHOOK_SUB_SYS,
+            TargetRequestValidator::Webhook,
+            ChannelTargetType::Webhook.as_str(),
+            NOTIFY_WEBHOOK_KEYS,
+            |config| validate_webhook_config(config, EVENT_DEFAULT_DIR),
+            |id, config| {
+                let args = build_webhook_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
+                Ok(boxed_target(crate::target::webhook::WebhookTarget::<E>::new(id, args)?))
+            },
+        ),
+        build_descriptor(
+            NOTIFY_AMQP_SUB_SYS,
+            TargetRequestValidator::Amqp(TargetType::NotifyEvent),
+            ChannelTargetType::Amqp.as_str(),
+            NOTIFY_AMQP_KEYS,
+            |config| validate_amqp_config(config, EVENT_DEFAULT_DIR),
+            |id, config| {
+                let args = build_amqp_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
+                Ok(boxed_target(crate::target::amqp::AMQPTarget::<E>::new(id, args)?))
+            },
+        ),
+        build_descriptor(
+            NOTIFY_KAFKA_SUB_SYS,
+            TargetRequestValidator::Kafka(TargetType::NotifyEvent),
+            ChannelTargetType::Kafka.as_str(),
+            NOTIFY_KAFKA_KEYS,
+            |config| validate_kafka_config(config, EVENT_DEFAULT_DIR),
+            |id, config| {
+                let args = build_kafka_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
+                Ok(boxed_target(crate::target::kafka::KafkaTarget::<E>::new(id, args)?))
+            },
+        ),
+        build_descriptor(
+            NOTIFY_MQTT_SUB_SYS,
+            TargetRequestValidator::Mqtt,
+            ChannelTargetType::Mqtt.as_str(),
+            NOTIFY_MQTT_KEYS,
+            validate_mqtt_config,
+            |id, config| {
+                let args = build_mqtt_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
+                Ok(boxed_target(crate::target::mqtt::MQTTTarget::<E>::new(id, args)?))
+            },
+        ),
+        build_descriptor(
+            NOTIFY_MYSQL_SUB_SYS,
+            TargetRequestValidator::MySql(TargetType::NotifyEvent),
+            ChannelTargetType::MySql.as_str(),
+            NOTIFY_MYSQL_KEYS,
+            |config| validate_mysql_config(config, EVENT_DEFAULT_DIR),
+            |id, config| {
+                let args = build_mysql_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
+                Ok(boxed_target(crate::target::mysql::MySqlTarget::<E>::new(id, args)?))
+            },
+        ),
+        build_descriptor(
+            NOTIFY_NATS_SUB_SYS,
+            TargetRequestValidator::Nats(TargetType::NotifyEvent),
+            ChannelTargetType::Nats.as_str(),
+            NOTIFY_NATS_KEYS,
+            |config| validate_nats_config(config, EVENT_DEFAULT_DIR),
+            |id, config| {
+                let args = build_nats_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
+                Ok(boxed_target(crate::target::nats::NATSTarget::<E>::new(id, args)?))
+            },
+        ),
+        build_descriptor(
+            NOTIFY_POSTGRES_SUB_SYS,
+            TargetRequestValidator::Postgres(TargetType::NotifyEvent),
+            ChannelTargetType::Postgres.as_str(),
+            NOTIFY_POSTGRES_KEYS,
+            |config| validate_postgres_config(config, EVENT_DEFAULT_DIR),
+            |id, config| {
+                let args = build_postgres_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
+                Ok(boxed_target(crate::target::postgres::PostgresTarget::<E>::new(id, args)?))
+            },
+        ),
+        build_descriptor(
+            NOTIFY_REDIS_SUB_SYS,
+            TargetRequestValidator::Redis {
+                default_channel: NOTIFY_REDIS_DEFAULT_CHANNEL,
+                target_type: TargetType::NotifyEvent,
+            },
+            ChannelTargetType::Redis.as_str(),
+            NOTIFY_REDIS_KEYS,
+            |config| validate_redis_config(config, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL),
+            |id, config| {
+                let args = build_redis_args(config, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL, TargetType::NotifyEvent)?;
+                Ok(boxed_target(crate::target::redis::RedisTarget::<E>::new(id, args)?))
+            },
+        ),
+        build_descriptor(
+            NOTIFY_PULSAR_SUB_SYS,
+            TargetRequestValidator::Pulsar(TargetType::NotifyEvent),
+            ChannelTargetType::Pulsar.as_str(),
+            NOTIFY_PULSAR_KEYS,
+            |config| validate_pulsar_config(config, EVENT_DEFAULT_DIR),
+            |id, config| {
+                let args = build_pulsar_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
+                Ok(boxed_target(crate::target::pulsar::PulsarTarget::<E>::new(id, args)?))
+            },
+        ),
+    ]
+}

crates/targets/src/catalog/mod.rs

@@ -0,0 +1,105 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod builtin;
+
+use crate::control_plane::external_target_plugin_installation;
+use crate::domain::TargetDomain;
+use crate::manifest::{
+    TargetPluginArtifactManifest, TargetPluginDistributionManifest, TargetPluginEntrypointKind,
+    TargetPluginExternalRuntimeContract, TargetPluginManifest, TargetPluginMarketplaceManifest, TargetPluginRuntimeTransport,
+    installable_target_marketplace_manifest,
+};
+use crate::runtime::sidecar::SidecarPluginRuntime;
+use crate::runtime::sidecar_protocol::{SIDECAR_RUNTIME_PROTOCOL_VERSION, SidecarHandshake, SidecarPluginCapability};
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ExampleInstallableTargetPlugin {
+    pub manifest: TargetPluginMarketplaceManifest,
+    pub installation: crate::TargetPluginInstallation,
+    pub runtime: SidecarPluginRuntime,
+    pub valid_fields: Vec<String>,
+}
+
+pub fn example_external_webhook_plugin() -> ExampleInstallableTargetPlugin {
+    let base = TargetPluginManifest {
+        plugin_id: "external:webhook-sidecar",
+        display_name: "Webhook Sidecar",
+        provider: "rustfs-labs",
+        version: "1.0.0",
+        target_type: "webhook",
+        supported_domains: &[TargetDomain::Notify],
+        secret_fields: &["auth_token"],
+    };
+    let manifest = installable_target_marketplace_manifest(
+        base,
+        TargetPluginEntrypointKind::Sidecar,
+        TargetPluginExternalRuntimeContract {
+            protocol_version: SIDECAR_RUNTIME_PROTOCOL_VERSION,
+            transport: TargetPluginRuntimeTransport::Grpc,
+        },
+        TargetPluginDistributionManifest {
+            artifacts: &[TargetPluginArtifactManifest {
+                artifact_id: "sidecar-linux-amd64",
+                target_triple: "x86_64-unknown-linux-gnu",
+                download_uri: "https://plugins.example.test/webhook-sidecar.tar.zst",
+                digest_sha256: "0123456789abcdef0123456789abcdef",
+                size_bytes: 8192,
+            }],
+        },
+    );
+
+    let handshake = SidecarHandshake {
+        protocol_version: SIDECAR_RUNTIME_PROTOCOL_VERSION.to_string(),
+        plugin_id: base.plugin_id.to_string(),
+        plugin_version: base.version.to_string(),
+        supported_domains: vec![TargetDomain::Notify],
+        capabilities: vec![
+            SidecarPluginCapability::HealthCheck,
+            SidecarPluginCapability::SendEvent,
+            SidecarPluginCapability::Shutdown,
+        ],
+    };
+    let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", handshake);
+    runtime
+        .enable(base.plugin_id, TargetDomain::Notify)
+        .expect("example sidecar plugin handshake should validate");
+
+    ExampleInstallableTargetPlugin {
+        manifest,
+        installation: external_target_plugin_installation(
+            base.version,
+            "0123456789abcdef0123456789abcdef",
+            "sidecar-linux-amd64",
+            Some("2026-05-13T20:00:00Z".to_string()),
+        ),
+        runtime,
+        valid_fields: vec!["endpoint".to_string(), "auth_token".to_string()],
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::example_external_webhook_plugin;
+
+    #[test]
+    fn example_external_plugin_exposes_installation_and_runtime_metadata() {
+        let example = example_external_webhook_plugin();
+
+        assert_eq!(example.manifest.plugin_id, "external:webhook-sidecar");
+        assert_eq!(example.installation.install_state, crate::TargetPluginInstallState::Installed);
+        assert!(example.runtime.healthy);
+        assert_eq!(example.valid_fields, vec!["endpoint".to_string(), "auth_token".to_string()]);
+    }
+}

crates/targets/src/config/instance.rs

@@ -0,0 +1,346 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::loader::collect_merged_target_configs_from_env;
+use crate::domain::TargetDomain;
+use rustfs_ecstore::config::{Config, KVS};
+use std::collections::HashSet;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct TargetPluginInstanceCompatDescriptor<'a> {
+    pub domain: TargetDomain,
+    pub plugin_id: &'a str,
+    pub target_type: &'a str,
+    pub subsystem: &'a str,
+    pub route_prefix: &'a str,
+    pub valid_fields: &'a [&'a str],
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TargetInstanceSourceClass {
+    Config,
+    Env,
+    Mixed,
+}
+
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
+pub struct TargetInstanceSourceHints {
+    pub has_file_default: bool,
+    pub has_file_instance: bool,
+    pub has_env_default: bool,
+    pub has_env_instance: bool,
+}
+
+impl TargetInstanceSourceHints {
+    #[inline]
+    pub fn has_config_source(self) -> bool {
+        self.has_file_default || self.has_file_instance
+    }
+
+    #[inline]
+    pub fn has_env_source(self) -> bool {
+        self.has_env_default || self.has_env_instance
+    }
+
+    #[inline]
+    pub fn classification(self) -> TargetInstanceSourceClass {
+        match (self.has_config_source(), self.has_env_source()) {
+            (true, true) => TargetInstanceSourceClass::Mixed,
+            (true, false) => TargetInstanceSourceClass::Config,
+            (false, true) => TargetInstanceSourceClass::Env,
+            (false, false) => TargetInstanceSourceClass::Config,
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TargetPluginInstanceRecord {
+    pub domain: TargetDomain,
+    pub plugin_id: String,
+    pub target_type: String,
+    pub subsystem: String,
+    pub instance_id: String,
+    pub enabled: bool,
+    pub source_hints: TargetInstanceSourceHints,
+    pub effective_config: KVS,
+}
+
+pub type LegacyTargetInstanceDescriptor<'a> = TargetPluginInstanceCompatDescriptor<'a>;
+pub type TargetPluginInstance = TargetPluginInstanceRecord;
+
+pub fn normalize_target_plugin_instances(
+    config: &Config,
+    descriptor: &TargetPluginInstanceCompatDescriptor<'_>,
+) -> Vec<TargetPluginInstanceRecord> {
+    normalize_target_plugin_instances_from_env(config, descriptor, std::env::vars())
+}
+
+pub fn normalize_target_plugin_instances_from_env<I>(
+    config: &Config,
+    descriptor: &TargetPluginInstanceCompatDescriptor<'_>,
+    env_vars: I,
+) -> Vec<TargetPluginInstanceRecord>
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    let valid_fields = descriptor
+        .valid_fields
+        .iter()
+        .map(|field| (*field).to_string())
+        .collect::<HashSet<_>>();
+
+    collect_merged_target_configs_from_env(
+        config,
+        descriptor.subsystem,
+        descriptor.route_prefix,
+        descriptor.target_type,
+        &valid_fields,
+        env_vars,
+    )
+    .into_iter()
+    .map(|record| TargetPluginInstanceRecord {
+        domain: descriptor.domain,
+        plugin_id: descriptor.plugin_id.to_string(),
+        target_type: descriptor.target_type.to_string(),
+        subsystem: descriptor.subsystem.to_string(),
+        instance_id: record.instance_id,
+        enabled: record.enabled,
+        source_hints: TargetInstanceSourceHints {
+            has_file_default: record.has_file_default,
+            has_file_instance: record.has_file_instance,
+            has_env_default: record.has_env_default,
+            has_env_instance: record.has_env_instance,
+        },
+        effective_config: record.effective_config,
+    })
+    .collect()
+}
+
+pub fn normalize_legacy_target_instances(
+    config: &Config,
+    descriptor: &LegacyTargetInstanceDescriptor<'_>,
+) -> Vec<TargetPluginInstance> {
+    normalize_target_plugin_instances(config, descriptor)
+}
+
+pub fn normalize_legacy_target_instances_from_env<I>(
+    config: &Config,
+    descriptor: &LegacyTargetInstanceDescriptor<'_>,
+    env_vars: I,
+) -> Vec<TargetPluginInstance>
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    normalize_target_plugin_instances_from_env(config, descriptor, env_vars)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{
+        TargetInstanceSourceClass, TargetPluginInstanceCompatDescriptor, normalize_legacy_target_instances_from_env,
+        normalize_target_plugin_instances_from_env,
+    };
+    use crate::domain::TargetDomain;
+    use crate::manifest::builtin_target_manifest;
+    use rustfs_config::audit::{AUDIT_ROUTE_PREFIX, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS};
+    use rustfs_config::notify::{NOTIFY_ROUTE_PREFIX, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS};
+    use rustfs_config::{ENABLE_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_LIMIT};
+    use rustfs_ecstore::config::{Config, KVS};
+    use std::collections::HashMap;
+
+    fn notify_webhook_descriptor() -> TargetPluginInstanceCompatDescriptor<'static> {
+        TargetPluginInstanceCompatDescriptor {
+            domain: TargetDomain::Notify,
+            plugin_id: builtin_target_manifest("webhook").plugin_id,
+            target_type: "webhook",
+            subsystem: NOTIFY_WEBHOOK_SUB_SYS,
+            route_prefix: NOTIFY_ROUTE_PREFIX,
+            valid_fields: NOTIFY_WEBHOOK_KEYS,
+        }
+    }
+
+    fn audit_webhook_descriptor() -> TargetPluginInstanceCompatDescriptor<'static> {
+        TargetPluginInstanceCompatDescriptor {
+            domain: TargetDomain::Audit,
+            plugin_id: builtin_target_manifest("webhook").plugin_id,
+            target_type: "webhook",
+            subsystem: AUDIT_WEBHOOK_SUB_SYS,
+            route_prefix: AUDIT_ROUTE_PREFIX,
+            valid_fields: AUDIT_WEBHOOK_KEYS,
+        }
+    }
+
+    #[test]
+    fn normalize_notify_instances_merges_file_and_env_sources() {
+        let mut cfg = Config(HashMap::new());
+        let mut subsystem = HashMap::new();
+
+        let mut default_kvs = KVS::new();
+        default_kvs.insert(ENABLE_KEY.to_string(), "on".to_string());
+        default_kvs.insert(WEBHOOK_QUEUE_LIMIT.to_string(), "10".to_string());
+        subsystem.insert("_".to_string(), default_kvs);
+
+        let mut primary = KVS::new();
+        primary.insert(WEBHOOK_ENDPOINT.to_string(), "https://example.com/primary".to_string());
+        subsystem.insert("primary".to_string(), primary);
+
+        cfg.0.insert(NOTIFY_WEBHOOK_SUB_SYS.to_string(), subsystem);
+
+        let instances = normalize_legacy_target_instances_from_env(
+            &cfg,
+            &notify_webhook_descriptor(),
+            vec![
+                ("RUSTFS_NOTIFY_WEBHOOK_QUEUE_LIMIT".to_string(), "42".to_string()),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_SECONDARY".to_string(), "on".to_string()),
+                (
+                    "RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_SECONDARY".to_string(),
+                    "https://example.com/secondary".to_string(),
+                ),
+            ],
+        );
+
+        assert_eq!(instances.len(), 2);
+
+        let primary = instances
+            .iter()
+            .find(|instance| instance.instance_id == "primary")
+            .expect("primary notify instance should be normalized");
+        assert_eq!(primary.domain, TargetDomain::Notify);
+        assert_eq!(primary.plugin_id, "builtin:webhook");
+        assert!(primary.enabled);
+        assert_eq!(primary.effective_config.lookup(WEBHOOK_QUEUE_LIMIT).as_deref(), Some("42"));
+        assert_eq!(
+            primary.effective_config.lookup(WEBHOOK_ENDPOINT).as_deref(),
+            Some("https://example.com/primary")
+        );
+        assert_eq!(primary.source_hints.classification(), TargetInstanceSourceClass::Mixed);
+        assert!(primary.source_hints.has_file_default);
+        assert!(primary.source_hints.has_file_instance);
+        assert!(primary.source_hints.has_env_default);
+        assert!(!primary.source_hints.has_env_instance);
+
+        let secondary = instances
+            .iter()
+            .find(|instance| instance.instance_id == "secondary")
+            .expect("secondary env notify instance should be normalized");
+        assert!(secondary.enabled);
+        assert_eq!(
+            secondary.effective_config.lookup(WEBHOOK_ENDPOINT).as_deref(),
+            Some("https://example.com/secondary")
+        );
+        assert_eq!(secondary.effective_config.lookup(WEBHOOK_QUEUE_LIMIT).as_deref(), Some("42"));
+        assert_eq!(secondary.source_hints.classification(), TargetInstanceSourceClass::Mixed);
+        assert!(secondary.source_hints.has_file_default);
+        assert!(!secondary.source_hints.has_file_instance);
+        assert!(secondary.source_hints.has_env_default);
+        assert!(secondary.source_hints.has_env_instance);
+    }
+
+    #[test]
+    fn normalize_audit_instances_preserves_domain_and_subsystem() {
+        let mut cfg = Config(HashMap::new());
+        let mut subsystem = HashMap::new();
+
+        let mut default_kvs = KVS::new();
+        default_kvs.insert(ENABLE_KEY.to_string(), "off".to_string());
+        subsystem.insert("_".to_string(), default_kvs);
+
+        let mut primary = KVS::new();
+        primary.insert(ENABLE_KEY.to_string(), "on".to_string());
+        primary.insert(WEBHOOK_ENDPOINT.to_string(), "https://example.com/audit".to_string());
+        subsystem.insert("primary".to_string(), primary);
+
+        cfg.0.insert(AUDIT_WEBHOOK_SUB_SYS.to_string(), subsystem);
+
+        let instances = normalize_legacy_target_instances_from_env(&cfg, &audit_webhook_descriptor(), Vec::new());
+
+        assert_eq!(instances.len(), 1);
+        let primary = &instances[0];
+        assert_eq!(primary.domain, TargetDomain::Audit);
+        assert_eq!(primary.target_type, "webhook");
+        assert_eq!(primary.subsystem, AUDIT_WEBHOOK_SUB_SYS);
+        assert_eq!(primary.instance_id, "primary");
+        assert!(primary.enabled);
+        assert_eq!(
+            primary.effective_config.lookup(WEBHOOK_ENDPOINT).as_deref(),
+            Some("https://example.com/audit")
+        );
+        assert_eq!(primary.source_hints.classification(), TargetInstanceSourceClass::Config);
+    }
+
+    #[test]
+    fn normalize_instances_keeps_disabled_records() {
+        let cfg = Config(HashMap::new());
+
+        let instances = normalize_legacy_target_instances_from_env(
+            &cfg,
+            &notify_webhook_descriptor(),
+            vec![
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_DISABLED".to_string(), "off".to_string()),
+                (
+                    "RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_DISABLED".to_string(),
+                    "https://example.com/disabled".to_string(),
+                ),
+            ],
+        );
+
+        assert_eq!(instances.len(), 1);
+        let disabled = &instances[0];
+        assert_eq!(disabled.instance_id, "disabled");
+        assert!(!disabled.enabled);
+        assert_eq!(disabled.source_hints.classification(), TargetInstanceSourceClass::Env);
+        assert!(disabled.source_hints.has_env_instance);
+    }
+
+    #[test]
+    fn normalize_instances_excludes_default_only_entries() {
+        let mut cfg = Config(HashMap::new());
+        let mut subsystem = HashMap::new();
+
+        let mut default_kvs = KVS::new();
+        default_kvs.insert(ENABLE_KEY.to_string(), "on".to_string());
+        default_kvs.insert(WEBHOOK_QUEUE_LIMIT.to_string(), "99".to_string());
+        subsystem.insert("_".to_string(), default_kvs);
+
+        cfg.0.insert(NOTIFY_WEBHOOK_SUB_SYS.to_string(), subsystem);
+
+        let instances = normalize_legacy_target_instances_from_env(
+            &cfg,
+            &notify_webhook_descriptor(),
+            vec![("RUSTFS_NOTIFY_WEBHOOK_QUEUE_LIMIT".to_string(), "100".to_string())],
+        );
+
+        assert!(instances.is_empty());
+    }
+
+    #[test]
+    fn compatibility_wrapper_matches_canonical_instance_model() {
+        let mut cfg = Config(HashMap::new());
+        let mut subsystem = HashMap::new();
+
+        let mut primary = KVS::new();
+        primary.insert(ENABLE_KEY.to_string(), "on".to_string());
+        primary.insert(WEBHOOK_ENDPOINT.to_string(), "https://example.com/primary".to_string());
+        subsystem.insert("primary".to_string(), primary);
+        cfg.0.insert(NOTIFY_WEBHOOK_SUB_SYS.to_string(), subsystem);
+
+        let descriptor = notify_webhook_descriptor();
+        let env = vec![("RUSTFS_NOTIFY_WEBHOOK_QUEUE_LIMIT".to_string(), "7".to_string())];
+
+        let canonical = normalize_target_plugin_instances_from_env(&cfg, &descriptor, env.clone());
+        let compatibility = normalize_legacy_target_instances_from_env(&cfg, &descriptor, env);
+
+        assert_eq!(canonical, compatibility);
+    }
+}

crates/targets/src/config/loader.rs

@@ -13,10 +13,9 @@
 // limitations under the License.
 
 use super::common::{is_target_enabled, split_env_field_and_instance};
-use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, EnableState};
+use rustfs_config::{DEFAULT_DELIMITER, ENV_PREFIX};
 use rustfs_ecstore::config::{Config, KVS};
 use std::collections::{HashMap, HashSet};
-use std::str::FromStr;
 use tracing::{debug, warn};
 
 pub fn collect_target_configs(
@@ -72,6 +71,17 @@ fn redacted_target_config(config: &KVS) -> Vec<(String, String)> {
         .collect()
 }
 
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(crate) struct MergedTargetConfigRecord {
+    pub instance_id: String,
+    pub effective_config: KVS,
+    pub enabled: bool,
+    pub has_file_default: bool,
+    pub has_file_instance: bool,
+    pub has_env_default: bool,
+    pub has_env_instance: bool,
+}
+
 pub fn collect_env_target_instance_ids(route_prefix: &str, target_type: &str, valid_fields: &HashSet<String>) -> HashSet<String> {
     collect_env_target_instance_ids_from_env(route_prefix, target_type, valid_fields, std::env::vars())
 }
@@ -113,25 +123,40 @@ pub fn collect_target_configs_from_env<I>(
 where
     I: IntoIterator<Item = (String, String)>,
 {
-    let all_env: Vec<(String, String)> = env_vars.into_iter().filter(|(key, _)| key.starts_with(ENV_PREFIX)).collect();
-    let section_name = format!("{route_prefix}{target_type}").to_lowercase();
-    let file_configs = config.0.get(&section_name).cloned().unwrap_or_default();
-    let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default();
+    collect_merged_target_configs_from_env(
+        config,
+        &format!("{route_prefix}{target_type}").to_lowercase(),
+        route_prefix,
+        target_type,
+        valid_fields,
+        env_vars,
+    )
+    .into_iter()
+    .filter(|record| record.enabled)
+    .map(|record| (record.instance_id, record.effective_config))
+    .collect()
+}
+
+pub(crate) fn collect_merged_target_configs_from_env<I>(
+    config: &Config,
+    section_name: &str,
+    route_prefix: &str,
+    target_type: &str,
+    valid_fields: &HashSet<String>,
+    env_vars: I,
+) -> Vec<MergedTargetConfigRecord>
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    let all_env: Vec<(String, String)> = env_vars.into_iter().filter(|(key, _)| key.starts_with(ENV_PREFIX)).collect();
+    let file_configs = config.0.get(section_name).cloned().unwrap_or_default();
+    let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default();
+    let has_file_default = file_configs.contains_key(DEFAULT_DELIMITER);
 
-    let enable_prefix =
-        format!("{ENV_PREFIX}{route_prefix}{target_type}{DEFAULT_DELIMITER}{ENABLE_KEY}{DEFAULT_DELIMITER}").to_uppercase();
     let env_prefix = format!("{ENV_PREFIX}{route_prefix}{target_type}{DEFAULT_DELIMITER}").to_uppercase();
 
-    let mut instance_ids_from_env = HashSet::new();
     let mut env_overrides: HashMap<String, KVS> = HashMap::new();
     for (key, value) in &all_env {
-        if EnableState::from_str(value).ok().map(|s| s.is_enabled()).unwrap_or(false)
-            && let Some(id) = key.strip_prefix(&enable_prefix)
-            && !id.is_empty()
-        {
-            instance_ids_from_env.insert(id.to_lowercase());
-        }
-
         let Some(rest) = key.strip_prefix(&env_prefix) else {
             continue;
         };
@@ -158,6 +183,7 @@ where
     }
 
     let mut effective_default = default_cfg;
+    let has_env_default = env_overrides.contains_key(DEFAULT_DELIMITER);
     if let Some(default_env_cfg) = env_overrides.remove(DEFAULT_DELIMITER) {
         effective_default.extend(default_env_cfg);
     }
@@ -167,16 +193,25 @@ where
         .filter(|key| key.as_str() != DEFAULT_DELIMITER)
         .cloned()
         .collect();
-    all_instance_ids.extend(instance_ids_from_env);
+    all_instance_ids.extend(
+        env_overrides
+            .iter()
+            .filter(|(instance_id, env_cfg)| {
+                instance_id.as_str() != DEFAULT_DELIMITER && env_cfg.lookup(rustfs_config::ENABLE_KEY).is_some()
+            })
+            .map(|(instance_id, _)| instance_id.clone()),
+    );
     all_instance_ids.sort();
     all_instance_ids.dedup();
 
     let mut merged_configs = Vec::new();
     for id in all_instance_ids {
         let mut merged_config = effective_default.clone();
+        let has_file_instance = file_configs.contains_key(&id);
         if let Some(file_instance_cfg) = file_configs.get(&id) {
             merged_config.extend(file_instance_cfg.clone());
         }
+        let has_env_instance = env_overrides.contains_key(&id);
         if let Some(env_instance_cfg) = env_overrides.get(&id) {
             merged_config.extend(env_instance_cfg.clone());
         }
@@ -185,9 +220,15 @@ where
             let redacted_config = redacted_target_config(&merged_config);
             debug!(instance_id = %id, ?redacted_config, "Merged target configuration");
         }
-        if is_target_enabled(&merged_config) {
-            merged_configs.push((id, merged_config));
-        }
+        merged_configs.push(MergedTargetConfigRecord {
+            instance_id: id,
+            enabled: is_target_enabled(&merged_config),
+            effective_config: merged_config,
+            has_file_default,
+            has_file_instance,
+            has_env_default,
+            has_env_instance,
+        });
     }
 
     merged_configs
@@ -273,6 +314,29 @@ mod tests {
         assert_eq!(configs[0].1.lookup(WEBHOOK_ENDPOINT).as_deref(), Some("https://example.com/from-env"));
     }
 
+    #[test]
+    fn collect_target_configs_does_not_materialize_env_only_instance_without_enable_flag() {
+        let mut cfg = Config(HashMap::new());
+        let mut subsystem = HashMap::new();
+        let mut default_kvs = KVS::new();
+        default_kvs.insert(ENABLE_KEY.to_string(), "on".to_string());
+        subsystem.insert("_".to_string(), default_kvs);
+        cfg.0.insert("notify_webhook".to_string(), subsystem);
+
+        let configs = collect_target_configs_from_env(
+            &cfg,
+            NOTIFY_ROUTE_PREFIX,
+            "webhook",
+            &HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]),
+            vec![(
+                "RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_SECONDARY".to_string(),
+                "https://example.com/secondary".to_string(),
+            )],
+        );
+
+        assert!(configs.is_empty());
+    }
+
     #[test]
     fn collect_env_target_instance_ids_handles_keys_with_internal_underscores() {
         let ids = collect_env_target_instance_ids_from_env(

crates/targets/src/config/mod.rs

@@ -13,9 +13,15 @@
 // limitations under the License.
 
 mod common;
+mod instance;
 mod loader;
 mod target_args;
 
+pub use instance::{
+    LegacyTargetInstanceDescriptor, TargetInstanceSourceClass, TargetInstanceSourceHints, TargetPluginInstance,
+    TargetPluginInstanceCompatDescriptor, TargetPluginInstanceRecord, normalize_legacy_target_instances,
+    normalize_legacy_target_instances_from_env, normalize_target_plugin_instances, normalize_target_plugin_instances_from_env,
+};
 pub use loader::{
     collect_env_target_instance_ids, collect_env_target_instance_ids_from_env, collect_target_configs,
     collect_target_configs_from_env,

crates/targets/src/control_plane.rs

@@ -0,0 +1,424 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::manifest::{
+    TargetPluginDistributionManifest, TargetPluginExternalRuntimeContract, TargetPluginManifest, TargetPluginRuntimeTransport,
+};
+use crate::runtime::sidecar_protocol::SIDECAR_RUNTIME_PROTOCOL_VERSION;
+use serde::{Deserialize, Serialize};
+use url::Url;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TargetPluginInstallState {
+    NotInstalled,
+    Installed,
+    InstallFailed,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TargetPluginEnableState {
+    Enabled,
+    Disabled,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TargetPluginRuntimeState {
+    Running,
+    Offline,
+    Error,
+    Unknown,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub struct TargetPluginRevision {
+    pub version: String,
+    pub digest_sha256: Option<String>,
+    pub source: String,
+    pub installed_at: Option<String>,
+    pub artifact_id: Option<String>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub struct TargetPluginInstallation {
+    pub install_state: TargetPluginInstallState,
+    pub current_revision: Option<TargetPluginRevision>,
+    pub previous_revision: Option<TargetPluginRevision>,
+    pub validation_error: Option<String>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub struct TargetPluginOperationalState {
+    pub install_state: TargetPluginInstallState,
+    pub enable_state: TargetPluginEnableState,
+    pub runtime_state: TargetPluginRuntimeState,
+}
+
+pub fn builtin_target_plugin_installation(manifest: &TargetPluginManifest) -> TargetPluginInstallation {
+    TargetPluginInstallation {
+        install_state: TargetPluginInstallState::Installed,
+        current_revision: Some(TargetPluginRevision {
+            version: manifest.version.to_string(),
+            digest_sha256: None,
+            source: "builtin".to_string(),
+            installed_at: None,
+            artifact_id: None,
+        }),
+        previous_revision: None,
+        validation_error: None,
+    }
+}
+
+pub fn external_target_plugin_installation(
+    version: impl Into<String>,
+    digest_sha256: impl Into<String>,
+    artifact_id: impl Into<String>,
+    installed_at: Option<String>,
+) -> TargetPluginInstallation {
+    TargetPluginInstallation {
+        install_state: TargetPluginInstallState::Installed,
+        current_revision: Some(TargetPluginRevision {
+            version: version.into(),
+            digest_sha256: Some(digest_sha256.into()),
+            source: "external".to_string(),
+            installed_at,
+            artifact_id: Some(artifact_id.into()),
+        }),
+        previous_revision: None,
+        validation_error: None,
+    }
+}
+
+pub fn failed_external_target_plugin_installation(
+    version: impl Into<String>,
+    artifact_id: impl Into<String>,
+    validation_error: impl Into<String>,
+) -> TargetPluginInstallation {
+    TargetPluginInstallation {
+        install_state: TargetPluginInstallState::InstallFailed,
+        current_revision: Some(TargetPluginRevision {
+            version: version.into(),
+            digest_sha256: None,
+            source: "external".to_string(),
+            installed_at: None,
+            artifact_id: Some(artifact_id.into()),
+        }),
+        previous_revision: None,
+        validation_error: Some(validation_error.into()),
+    }
+}
+
+pub fn rollback_target_plugin_installation(
+    current: TargetPluginRevision,
+    previous: TargetPluginRevision,
+) -> TargetPluginInstallation {
+    TargetPluginInstallation {
+        install_state: TargetPluginInstallState::Installed,
+        current_revision: Some(previous),
+        previous_revision: Some(current),
+        validation_error: None,
+    }
+}
+
+pub fn builtin_target_plugin_operational_state(
+    enabled: bool,
+    runtime_state: TargetPluginRuntimeState,
+) -> TargetPluginOperationalState {
+    TargetPluginOperationalState {
+        install_state: TargetPluginInstallState::Installed,
+        enable_state: if enabled {
+            TargetPluginEnableState::Enabled
+        } else {
+            TargetPluginEnableState::Disabled
+        },
+        runtime_state,
+    }
+}
+
+pub fn runtime_state_from_status_label(status: &str) -> TargetPluginRuntimeState {
+    if status.eq_ignore_ascii_case("online") {
+        TargetPluginRuntimeState::Running
+    } else if status.eq_ignore_ascii_case("offline") {
+        TargetPluginRuntimeState::Offline
+    } else if status.eq_ignore_ascii_case("error") {
+        TargetPluginRuntimeState::Error
+    } else {
+        TargetPluginRuntimeState::Unknown
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TargetPluginInstallPolicy {
+    pub allowed_providers: Vec<String>,
+    pub allowed_download_hosts: Vec<String>,
+    pub require_https: bool,
+    pub require_signature: bool,
+}
+
+impl Default for TargetPluginInstallPolicy {
+    fn default() -> Self {
+        Self {
+            allowed_providers: vec!["rustfs".to_string(), "rustfs-labs".to_string()],
+            allowed_download_hosts: vec!["plugins.example.test".to_string()],
+            require_https: true,
+            require_signature: false,
+        }
+    }
+}
+
+pub fn validate_external_plugin_installation(
+    manifest: &TargetPluginManifest,
+    runtime_contract: &TargetPluginExternalRuntimeContract,
+    distribution: Option<TargetPluginDistributionManifest>,
+    policy: &TargetPluginInstallPolicy,
+) -> Result<(), String> {
+    if !policy.allowed_providers.iter().any(|provider| provider == manifest.provider) {
+        return Err(format!("provider {} is not allowed by install policy", manifest.provider));
+    }
+
+    if runtime_contract.transport == TargetPluginRuntimeTransport::Grpc
+        && runtime_contract.protocol_version != SIDECAR_RUNTIME_PROTOCOL_VERSION
+    {
+        return Err(format!(
+            "sidecar runtime protocol mismatch: expected {}, got {}",
+            SIDECAR_RUNTIME_PROTOCOL_VERSION, runtime_contract.protocol_version
+        ));
+    }
+
+    if policy.require_signature {
+        return Err(
+            "signature verification is required by install policy but manifests do not expose signatures yet".to_string(),
+        );
+    }
+
+    let distribution = distribution.ok_or_else(|| "external plugin is missing distribution metadata".to_string())?;
+    if distribution.artifacts.is_empty() {
+        return Err("external plugin distribution has no artifacts".to_string());
+    }
+
+    for artifact in distribution.artifacts {
+        let parsed_uri = Url::parse(artifact.download_uri)
+            .map_err(|err| format!("invalid artifact download uri {}: {}", artifact.download_uri, err))?;
+        if policy.require_https && parsed_uri.scheme() != "https" {
+            return Err(format!(
+                "artifact {} must use https download uri, got {}",
+                artifact.artifact_id, artifact.download_uri
+            ));
+        }
+        let host = parsed_uri
+            .host_str()
+            .ok_or_else(|| format!("artifact {} download uri has no host", artifact.artifact_id))?;
+        if !policy.allowed_download_hosts.iter().any(|allowed| allowed == host) {
+            return Err(format!("artifact {} download host {} is not allowed", artifact.artifact_id, host));
+        }
+        if artifact.size_bytes == 0 {
+            return Err(format!("artifact {} must declare a non-zero size", artifact.artifact_id));
+        }
+        if artifact.digest_sha256.len() < 16 || !artifact.digest_sha256.chars().all(|ch| ch.is_ascii_hexdigit()) {
+            return Err(format!(
+                "artifact {} has invalid digest_sha256 {}",
+                artifact.artifact_id, artifact.digest_sha256
+            ));
+        }
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{
+        TargetPluginEnableState, TargetPluginInstallPolicy, TargetPluginInstallState, TargetPluginRevision,
+        TargetPluginRuntimeState, builtin_target_plugin_installation, builtin_target_plugin_operational_state,
+        external_target_plugin_installation, failed_external_target_plugin_installation, rollback_target_plugin_installation,
+        runtime_state_from_status_label, validate_external_plugin_installation,
+    };
+    use crate::manifest::{
+        TargetPluginArtifactManifest, TargetPluginDistributionManifest, TargetPluginExternalRuntimeContract,
+        TargetPluginManifest, TargetPluginRuntimeTransport, builtin_target_manifest,
+    };
+
+    #[test]
+    fn builtin_installation_maps_to_virtual_installed_revision() {
+        let installation = builtin_target_plugin_installation(&builtin_target_manifest("webhook"));
+
+        assert_eq!(installation.install_state, TargetPluginInstallState::Installed);
+        assert_eq!(
+            installation
+                .current_revision
+                .as_ref()
+                .expect("builtin installation should expose current revision")
+                .source,
+            "builtin"
+        );
+        assert_eq!(
+            installation
+                .current_revision
+                .as_ref()
+                .expect("builtin installation should expose current revision")
+                .artifact_id,
+            None
+        );
+        assert!(installation.previous_revision.is_none());
+        assert_eq!(installation.validation_error, None);
+    }
+
+    #[test]
+    fn builtin_operational_state_tracks_enablement_and_runtime() {
+        let enabled = builtin_target_plugin_operational_state(true, TargetPluginRuntimeState::Running);
+        let disabled = builtin_target_plugin_operational_state(false, TargetPluginRuntimeState::Offline);
+
+        assert_eq!(enabled.install_state, TargetPluginInstallState::Installed);
+        assert_eq!(enabled.enable_state, TargetPluginEnableState::Enabled);
+        assert_eq!(enabled.runtime_state, TargetPluginRuntimeState::Running);
+
+        assert_eq!(disabled.enable_state, TargetPluginEnableState::Disabled);
+        assert_eq!(disabled.runtime_state, TargetPluginRuntimeState::Offline);
+    }
+
+    #[test]
+    fn runtime_state_from_status_maps_known_labels() {
+        assert_eq!(runtime_state_from_status_label("online"), TargetPluginRuntimeState::Running);
+        assert_eq!(runtime_state_from_status_label("offline"), TargetPluginRuntimeState::Offline);
+        assert_eq!(runtime_state_from_status_label("error"), TargetPluginRuntimeState::Error);
+        assert_eq!(runtime_state_from_status_label("unexpected"), TargetPluginRuntimeState::Unknown);
+    }
+
+    #[test]
+    fn external_installation_captures_revision_metadata() {
+        let installation = external_target_plugin_installation(
+            "1.2.3",
+            "0123456789abcdef",
+            "sidecar-linux-amd64",
+            Some("2026-05-13T12:00:00Z".to_string()),
+        );
+
+        let revision = installation
+            .current_revision
+            .as_ref()
+            .expect("external installation should expose current revision");
+        assert_eq!(installation.install_state, TargetPluginInstallState::Installed);
+        assert_eq!(revision.source, "external");
+        assert_eq!(revision.digest_sha256.as_deref(), Some("0123456789abcdef"));
+        assert_eq!(revision.artifact_id.as_deref(), Some("sidecar-linux-amd64"));
+        assert_eq!(installation.validation_error, None);
+    }
+
+    #[test]
+    fn rollback_swaps_current_and_previous_revisions() {
+        let current = TargetPluginRevision {
+            version: "2.0.0".to_string(),
+            digest_sha256: Some("new-digest".to_string()),
+            source: "external".to_string(),
+            installed_at: Some("2026-05-13T12:05:00Z".to_string()),
+            artifact_id: Some("sidecar-linux-amd64-v2".to_string()),
+        };
+        let previous = TargetPluginRevision {
+            version: "1.9.0".to_string(),
+            digest_sha256: Some("old-digest".to_string()),
+            source: "external".to_string(),
+            installed_at: Some("2026-05-13T11:55:00Z".to_string()),
+            artifact_id: Some("sidecar-linux-amd64-v1".to_string()),
+        };
+
+        let installation = rollback_target_plugin_installation(current.clone(), previous.clone());
+
+        assert_eq!(installation.current_revision, Some(previous));
+        assert_eq!(installation.previous_revision, Some(current));
+        assert_eq!(installation.validation_error, None);
+    }
+
+    #[test]
+    fn failed_external_installation_preserves_error_context() {
+        let installation =
+            failed_external_target_plugin_installation("1.2.3", "sidecar-linux-amd64", "digest mismatch during install");
+
+        assert_eq!(installation.install_state, TargetPluginInstallState::InstallFailed);
+        assert_eq!(installation.validation_error.as_deref(), Some("digest mismatch during install"));
+    }
+
+    #[test]
+    fn validate_external_installation_accepts_allowed_https_artifact() {
+        let manifest = TargetPluginManifest {
+            plugin_id: "external:webhook-sidecar",
+            display_name: "Webhook Sidecar",
+            provider: "rustfs-labs",
+            version: "1.0.0",
+            target_type: "webhook",
+            supported_domains: &[],
+            secret_fields: &[],
+        };
+        let distribution = TargetPluginDistributionManifest {
+            artifacts: &[TargetPluginArtifactManifest {
+                artifact_id: "sidecar-linux-amd64",
+                target_triple: "x86_64-unknown-linux-gnu",
+                download_uri: "https://plugins.example.test/webhook-sidecar.tar.zst",
+                digest_sha256: "0123456789abcdef0123456789abcdef",
+                size_bytes: 8192,
+            }],
+        };
+        let policy = TargetPluginInstallPolicy::default();
+
+        let result = validate_external_plugin_installation(
+            &manifest,
+            &TargetPluginExternalRuntimeContract {
+                protocol_version: crate::SIDECAR_RUNTIME_PROTOCOL_VERSION,
+                transport: TargetPluginRuntimeTransport::Grpc,
+            },
+            Some(distribution),
+            &policy,
+        );
+
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn validate_external_installation_rejects_disallowed_provider() {
+        let manifest = TargetPluginManifest {
+            plugin_id: "external:webhook-sidecar",
+            display_name: "Webhook Sidecar",
+            provider: "unknown-vendor",
+            version: "1.0.0",
+            target_type: "webhook",
+            supported_domains: &[],
+            secret_fields: &[],
+        };
+        let policy = TargetPluginInstallPolicy::default();
+
+        let result = validate_external_plugin_installation(
+            &manifest,
+            &TargetPluginExternalRuntimeContract {
+                protocol_version: crate::SIDECAR_RUNTIME_PROTOCOL_VERSION,
+                transport: TargetPluginRuntimeTransport::Grpc,
+            },
+            Some(TargetPluginDistributionManifest {
+                artifacts: &[TargetPluginArtifactManifest {
+                    artifact_id: "sidecar-linux-amd64",
+                    target_triple: "x86_64-unknown-linux-gnu",
+                    download_uri: "https://plugins.example.test/webhook-sidecar.tar.zst",
+                    digest_sha256: "0123456789abcdef0123456789abcdef",
+                    size_bytes: 8192,
+                }],
+            }),
+            &policy,
+        );
+
+        assert!(result.is_err());
+    }
+}

crates/targets/src/domain.rs

@@ -0,0 +1,43 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::target::TargetType;
+use serde::{Deserialize, Serialize};
+
+/// Logical target domains supported by RustFS target plugins.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TargetDomain {
+    Notify,
+    Audit,
+}
+
+impl TargetDomain {
+    #[inline]
+    pub fn runtime_target_type(self) -> TargetType {
+        match self {
+            TargetDomain::Notify => TargetType::NotifyEvent,
+            TargetDomain::Audit => TargetType::AuditLog,
+        }
+    }
+}
+
+impl From<TargetType> for TargetDomain {
+    fn from(value: TargetType) -> Self {
+        match value {
+            TargetType::NotifyEvent => TargetDomain::Notify,
+            TargetType::AuditLog => TargetDomain::Audit,
+        }
+    }
+}

crates/targets/src/lib.rs

@@ -13,10 +13,15 @@
 // limitations under the License.
 
 pub mod arn;
+pub mod catalog;
 mod check;
 pub mod config;
+pub mod control_plane;
+pub mod domain;
 pub mod error;
+pub mod manifest;
 pub mod plugin;
+pub mod runtime;
 pub mod store;
 pub mod sys;
 pub mod target;
@@ -26,8 +31,36 @@ pub use check::{
     check_mysql_server_available, check_nats_server_available, check_postgres_server_available, check_pulsar_broker_available,
     check_redis_server_available,
 };
+pub use config::{
+    LegacyTargetInstanceDescriptor, TargetInstanceSourceClass, TargetInstanceSourceHints, TargetPluginInstance,
+    TargetPluginInstanceCompatDescriptor, TargetPluginInstanceRecord, normalize_legacy_target_instances,
+    normalize_legacy_target_instances_from_env, normalize_target_plugin_instances, normalize_target_plugin_instances_from_env,
+};
+pub use control_plane::{
+    TargetPluginEnableState, TargetPluginInstallState, TargetPluginInstallation, TargetPluginOperationalState,
+    TargetPluginRevision, TargetPluginRuntimeState, builtin_target_plugin_installation, builtin_target_plugin_operational_state,
+    external_target_plugin_installation, rollback_target_plugin_installation, runtime_state_from_status_label,
+};
+pub use domain::TargetDomain;
 pub use error::{StoreError, TargetError};
-pub use plugin::{BuiltinTargetDescriptor, TargetPluginDescriptor, TargetPluginRegistry, TargetRequestValidator, boxed_target};
+pub use manifest::{
+    TargetPluginArtifactManifest, TargetPluginDistributionManifest, TargetPluginEntrypointKind,
+    TargetPluginExternalRuntimeContract, TargetPluginManifest, TargetPluginMarketplaceManifest, TargetPluginPackaging,
+    TargetPluginRuntimeTransport, builtin_target_marketplace_manifest, installable_target_marketplace_manifest,
+};
+pub use plugin::{
+    BuiltinTargetAdminDescriptor, BuiltinTargetDescriptor, TargetAdminMetadata, TargetPluginDescriptor, TargetPluginRegistry,
+    TargetRequestValidator, boxed_target,
+};
+pub use runtime::{
+    ReplayEvent, ReplayWorkerManager, RuntimeActivation, RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot,
+    RuntimeTargetHealthState, RuntimeTargetSnapshot, SharedTarget, TargetRuntimeManager, activate_targets_with_replay,
+    adapter::{BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter},
+    init_target_and_optionally_start_replay,
+    sidecar::SidecarPluginRuntime,
+    sidecar_protocol::{SIDECAR_RUNTIME_PROTOCOL_VERSION, SidecarHandshake, SidecarPluginCapability},
+    start_replay_worker,
+};
 pub use rustfs_s3_common::EventName;
 use serde::{Deserialize, Serialize};
 pub use sys::user_agent::*;

crates/targets/src/manifest.rs

@@ -0,0 +1,314 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::domain::TargetDomain;
+use rustfs_config::{
+    AMQP_PASSWORD, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, MQTT_PASSWORD,
+    MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MYSQL_DSN_STRING, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY,
+    NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TOKEN, POSTGRES_DSN_STRING,
+    POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, PULSAR_AUTH_TOKEN, PULSAR_PASSWORD, REDIS_PASSWORD, REDIS_TLS_CLIENT_CERT,
+    REDIS_TLS_CLIENT_KEY, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY,
+};
+
+/// Shared plugin manifest metadata for a target implementation.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct TargetPluginManifest {
+    pub plugin_id: &'static str,
+    pub display_name: &'static str,
+    pub provider: &'static str,
+    pub version: &'static str,
+    pub target_type: &'static str,
+    pub supported_domains: &'static [TargetDomain],
+    pub secret_fields: &'static [&'static str],
+}
+
+/// Declares how a plugin is packaged relative to the RustFS process boundary.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TargetPluginPackaging {
+    Builtin,
+    External,
+}
+
+/// Declares what kind of entrypoint a plugin would use when instantiated.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TargetPluginEntrypointKind {
+    Builtin,
+    Sidecar,
+    Wasm,
+}
+
+/// Declares the transport boundary RustFS would use to communicate with a
+/// plugin runtime without committing to any concrete loader implementation.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TargetPluginRuntimeTransport {
+    InProcess,
+    Grpc,
+    WasmHost,
+}
+
+/// Declarative external runtime contract for future installable plugins.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct TargetPluginExternalRuntimeContract {
+    pub protocol_version: &'static str,
+    pub transport: TargetPluginRuntimeTransport,
+}
+
+/// Declarative distribution metadata for an installable target plugin.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct TargetPluginArtifactManifest {
+    pub artifact_id: &'static str,
+    pub target_triple: &'static str,
+    pub download_uri: &'static str,
+    pub digest_sha256: &'static str,
+    pub size_bytes: u64,
+}
+
+/// Declarative distribution metadata for an installable target plugin.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct TargetPluginDistributionManifest {
+    pub artifacts: &'static [TargetPluginArtifactManifest],
+}
+
+/// Marketplace-oriented manifest metadata that is explicit about future
+/// installable plugin boundaries without introducing any loading behavior.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct TargetPluginMarketplaceManifest {
+    pub plugin_id: &'static str,
+    pub display_name: &'static str,
+    pub provider: &'static str,
+    pub version: &'static str,
+    pub target_type: &'static str,
+    pub supported_domains: &'static [TargetDomain],
+    pub secret_fields: &'static [&'static str],
+    pub packaging: TargetPluginPackaging,
+    pub entrypoint_kind: TargetPluginEntrypointKind,
+    pub api_compatibility_version: &'static str,
+    pub runtime_contract: TargetPluginExternalRuntimeContract,
+    pub distribution: Option<TargetPluginDistributionManifest>,
+}
+
+const BUILTIN_PLUGIN_API_COMPATIBILITY_VERSION: &str = "rustfs.target-plugin.v1";
+const BUILTIN_PLUGIN_RUNTIME_PROTOCOL_VERSION: &str = "rustfs.target-runtime.v1";
+
+const SUPPORTED_BUILTIN_DOMAINS: &[TargetDomain] = &[TargetDomain::Audit, TargetDomain::Notify];
+const NO_SECRET_FIELDS: &[&str] = &[];
+
+const WEBHOOK_SECRET_FIELDS: &[&str] = &[WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY];
+const MQTT_SECRET_FIELDS: &[&str] = &[MQTT_PASSWORD, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY];
+const KAFKA_SECRET_FIELDS: &[&str] = &[KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY];
+const AMQP_SECRET_FIELDS: &[&str] = &[AMQP_PASSWORD, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY];
+const NATS_SECRET_FIELDS: &[&str] = &[
+    NATS_PASSWORD,
+    NATS_TOKEN,
+    NATS_CREDENTIALS_FILE,
+    NATS_TLS_CLIENT_CERT,
+    NATS_TLS_CLIENT_KEY,
+];
+const PULSAR_SECRET_FIELDS: &[&str] = &[PULSAR_AUTH_TOKEN, PULSAR_PASSWORD];
+const MYSQL_SECRET_FIELDS: &[&str] = &[MYSQL_DSN_STRING, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY];
+const REDIS_SECRET_FIELDS: &[&str] = &[REDIS_PASSWORD, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY];
+const POSTGRES_SECRET_FIELDS: &[&str] = &[POSTGRES_DSN_STRING, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY];
+
+#[inline]
+pub fn builtin_target_manifest(target_type: &'static str) -> TargetPluginManifest {
+    let (display_name, secret_fields) = match target_type {
+        "webhook" => ("Webhook", WEBHOOK_SECRET_FIELDS),
+        "mqtt" => ("MQTT", MQTT_SECRET_FIELDS),
+        "kafka" => ("Kafka", KAFKA_SECRET_FIELDS),
+        "amqp" => ("AMQP", AMQP_SECRET_FIELDS),
+        "nats" => ("NATS", NATS_SECRET_FIELDS),
+        "pulsar" => ("Pulsar", PULSAR_SECRET_FIELDS),
+        "mysql" => ("MySQL", MYSQL_SECRET_FIELDS),
+        "redis" => ("Redis", REDIS_SECRET_FIELDS),
+        "postgres" => ("Postgres", POSTGRES_SECRET_FIELDS),
+        _ => ("Custom Target", NO_SECRET_FIELDS),
+    };
+
+    TargetPluginManifest {
+        plugin_id: builtin_plugin_id(target_type),
+        display_name,
+        provider: "rustfs",
+        version: env!("CARGO_PKG_VERSION"),
+        target_type,
+        supported_domains: SUPPORTED_BUILTIN_DOMAINS,
+        secret_fields,
+    }
+}
+
+#[inline]
+pub fn builtin_target_marketplace_manifest(target_type: &'static str) -> TargetPluginMarketplaceManifest {
+    TargetPluginMarketplaceManifest::from(builtin_target_manifest(target_type))
+}
+
+impl From<TargetPluginManifest> for TargetPluginMarketplaceManifest {
+    fn from(value: TargetPluginManifest) -> Self {
+        Self {
+            plugin_id: value.plugin_id,
+            display_name: value.display_name,
+            provider: value.provider,
+            version: value.version,
+            target_type: value.target_type,
+            supported_domains: value.supported_domains,
+            secret_fields: value.secret_fields,
+            packaging: TargetPluginPackaging::Builtin,
+            entrypoint_kind: TargetPluginEntrypointKind::Builtin,
+            api_compatibility_version: BUILTIN_PLUGIN_API_COMPATIBILITY_VERSION,
+            runtime_contract: TargetPluginExternalRuntimeContract {
+                protocol_version: BUILTIN_PLUGIN_RUNTIME_PROTOCOL_VERSION,
+                transport: TargetPluginRuntimeTransport::InProcess,
+            },
+            distribution: None,
+        }
+    }
+}
+
+#[inline]
+pub fn installable_target_marketplace_manifest(
+    base: TargetPluginManifest,
+    entrypoint_kind: TargetPluginEntrypointKind,
+    runtime_contract: TargetPluginExternalRuntimeContract,
+    distribution: TargetPluginDistributionManifest,
+) -> TargetPluginMarketplaceManifest {
+    TargetPluginMarketplaceManifest {
+        plugin_id: base.plugin_id,
+        display_name: base.display_name,
+        provider: base.provider,
+        version: base.version,
+        target_type: base.target_type,
+        supported_domains: base.supported_domains,
+        secret_fields: base.secret_fields,
+        packaging: TargetPluginPackaging::External,
+        entrypoint_kind,
+        api_compatibility_version: BUILTIN_PLUGIN_API_COMPATIBILITY_VERSION,
+        runtime_contract,
+        distribution: Some(distribution),
+    }
+}
+
+#[inline]
+fn builtin_plugin_id(target_type: &'static str) -> &'static str {
+    match target_type {
+        "webhook" => "builtin:webhook",
+        "mqtt" => "builtin:mqtt",
+        "kafka" => "builtin:kafka",
+        "amqp" => "builtin:amqp",
+        "nats" => "builtin:nats",
+        "pulsar" => "builtin:pulsar",
+        "mysql" => "builtin:mysql",
+        "redis" => "builtin:redis",
+        "postgres" => "builtin:postgres",
+        _ => "custom:target",
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{
+        TargetPluginArtifactManifest, TargetPluginDistributionManifest, TargetPluginEntrypointKind,
+        TargetPluginExternalRuntimeContract, TargetPluginMarketplaceManifest, TargetPluginPackaging,
+        TargetPluginRuntimeTransport, builtin_target_manifest, builtin_target_marketplace_manifest,
+        installable_target_marketplace_manifest,
+    };
+    use crate::domain::TargetDomain;
+    use rustfs_config::{WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY};
+
+    #[test]
+    fn builtin_webhook_manifest_marks_secret_fields() {
+        let manifest = builtin_target_manifest("webhook");
+
+        assert_eq!(manifest.plugin_id, "builtin:webhook");
+        assert_eq!(manifest.display_name, "Webhook");
+        assert!(manifest.secret_fields.contains(&WEBHOOK_AUTH_TOKEN));
+        assert!(manifest.secret_fields.contains(&WEBHOOK_CLIENT_CERT));
+        assert!(manifest.secret_fields.contains(&WEBHOOK_CLIENT_KEY));
+    }
+
+    #[test]
+    fn builtin_manifest_derives_marketplace_boundary_metadata() {
+        let manifest = builtin_target_marketplace_manifest("webhook");
+
+        assert_eq!(manifest.plugin_id, "builtin:webhook");
+        assert_eq!(manifest.display_name, "Webhook");
+        assert_eq!(manifest.target_type, "webhook");
+        assert_eq!(manifest.packaging, TargetPluginPackaging::Builtin);
+        assert_eq!(manifest.entrypoint_kind, TargetPluginEntrypointKind::Builtin);
+        assert_eq!(manifest.api_compatibility_version, "rustfs.target-plugin.v1");
+        assert_eq!(
+            manifest.runtime_contract,
+            TargetPluginExternalRuntimeContract {
+                protocol_version: "rustfs.target-runtime.v1",
+                transport: TargetPluginRuntimeTransport::InProcess,
+            }
+        );
+        assert_eq!(manifest.distribution, None);
+    }
+
+    #[test]
+    fn marketplace_manifest_preserves_supported_domains() {
+        let manifest = builtin_target_marketplace_manifest("kafka");
+
+        assert_eq!(manifest.supported_domains, &[TargetDomain::Audit, TargetDomain::Notify]);
+    }
+
+    #[test]
+    fn marketplace_manifest_from_builtin_manifest_is_stable() {
+        let base = builtin_target_manifest("redis");
+        let derived = TargetPluginMarketplaceManifest::from(base);
+
+        assert_eq!(derived.plugin_id, "builtin:redis");
+        assert_eq!(derived.target_type, "redis");
+        assert_eq!(derived.packaging, TargetPluginPackaging::Builtin);
+        assert_eq!(derived.entrypoint_kind, TargetPluginEntrypointKind::Builtin);
+        assert_eq!(derived.runtime_contract.transport, TargetPluginRuntimeTransport::InProcess);
+        assert_eq!(derived.distribution, None);
+    }
+
+    #[test]
+    fn installable_manifest_expresses_external_boundary_declaratively() {
+        let base = builtin_target_manifest("webhook");
+        let manifest = installable_target_marketplace_manifest(
+            base,
+            TargetPluginEntrypointKind::Sidecar,
+            TargetPluginExternalRuntimeContract {
+                protocol_version: "rustfs.target-runtime.v1",
+                transport: TargetPluginRuntimeTransport::Grpc,
+            },
+            TargetPluginDistributionManifest {
+                artifacts: &[TargetPluginArtifactManifest {
+                    artifact_id: "sidecar-linux-amd64",
+                    target_triple: "x86_64-unknown-linux-gnu",
+                    download_uri: "https://plugins.example.test/webhook-plugin.tar.zst",
+                    digest_sha256: "0123456789abcdef",
+                    size_bytes: 4096,
+                }],
+            },
+        );
+
+        assert_eq!(manifest.packaging, TargetPluginPackaging::External);
+        assert_eq!(manifest.entrypoint_kind, TargetPluginEntrypointKind::Sidecar);
+        assert_eq!(manifest.runtime_contract.transport, TargetPluginRuntimeTransport::Grpc);
+        assert_eq!(
+            manifest.distribution,
+            Some(TargetPluginDistributionManifest {
+                artifacts: &[TargetPluginArtifactManifest {
+                    artifact_id: "sidecar-linux-amd64",
+                    target_triple: "x86_64-unknown-linux-gnu",
+                    download_uri: "https://plugins.example.test/webhook-plugin.tar.zst",
+                    digest_sha256: "0123456789abcdef",
+                    size_bytes: 4096,
+                }],
+            })
+        );
+    }
+}

crates/targets/src/plugin.rs

@@ -12,7 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::{Target, TargetError, config::collect_target_configs};
+use crate::{
+    PluginRuntimeAdapter, RuntimeActivation, Target, TargetError,
+    config::collect_target_configs,
+    manifest::{TargetPluginManifest, builtin_target_manifest},
+};
 use hashbrown::HashMap;
 use rustfs_ecstore::config::{Config, KVS};
 use serde::Serialize;
@@ -41,12 +45,70 @@ pub enum TargetRequestValidator {
     },
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct TargetAdminMetadata {
+    subsystem: &'static str,
+    request_validator: TargetRequestValidator,
+}
+
+impl TargetAdminMetadata {
+    pub fn new(subsystem: &'static str, request_validator: TargetRequestValidator) -> Self {
+        Self {
+            subsystem,
+            request_validator,
+        }
+    }
+
+    #[inline]
+    pub fn subsystem(&self) -> &'static str {
+        self.subsystem
+    }
+
+    #[inline]
+    pub fn request_validator(&self) -> TargetRequestValidator {
+        self.request_validator
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct BuiltinTargetAdminDescriptor {
+    manifest: TargetPluginManifest,
+    valid_fields: &'static [&'static str],
+    admin: TargetAdminMetadata,
+}
+
+impl BuiltinTargetAdminDescriptor {
+    pub fn new(manifest: TargetPluginManifest, valid_fields: &'static [&'static str], admin: TargetAdminMetadata) -> Self {
+        Self {
+            manifest,
+            valid_fields,
+            admin,
+        }
+    }
+
+    #[inline]
+    pub fn manifest(&self) -> &TargetPluginManifest {
+        &self.manifest
+    }
+
+    #[inline]
+    pub fn valid_fields(&self) -> &'static [&'static str] {
+        self.valid_fields
+    }
+
+    #[inline]
+    pub fn admin_metadata(&self) -> TargetAdminMetadata {
+        self.admin
+    }
+}
+
 #[derive(Clone)]
 pub struct TargetPluginDescriptor<E>
 where
     E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
 {
     create_target: TargetCreateFn<E>,
+    manifest: TargetPluginManifest,
     target_type: &'static str,
     valid_fields: &'static [&'static str],
     valid_fields_set: Arc<HashSet<String>>,
@@ -63,13 +125,27 @@ where
         validate_config: Validate,
         create_target: Create,
     ) -> Self
+    where
+        Create: Fn(String, &KVS) -> Result<BoxedTarget<E>, TargetError> + Send + Sync + 'static,
+        Validate: Fn(&KVS) -> Result<(), TargetError> + Send + Sync + 'static,
+    {
+        Self::with_manifest(builtin_target_manifest(target_type), valid_fields, validate_config, create_target)
+    }
+
+    pub fn with_manifest<Create, Validate>(
+        manifest: TargetPluginManifest,
+        valid_fields: &'static [&'static str],
+        validate_config: Validate,
+        create_target: Create,
+    ) -> Self
     where
         Create: Fn(String, &KVS) -> Result<BoxedTarget<E>, TargetError> + Send + Sync + 'static,
         Validate: Fn(&KVS) -> Result<(), TargetError> + Send + Sync + 'static,
     {
         Self {
             create_target: Arc::new(create_target),
-            target_type,
+            manifest,
+            target_type: manifest.target_type,
             valid_fields,
             valid_fields_set: Arc::new(valid_fields.iter().map(|field| (*field).to_string()).collect()),
             validate_config: Arc::new(validate_config),
@@ -81,6 +157,11 @@ where
         self.target_type
     }
 
+    #[inline]
+    pub fn manifest(&self) -> &TargetPluginManifest {
+        &self.manifest
+    }
+
     #[inline]
     pub fn valid_fields(&self) -> &'static [&'static str] {
         self.valid_fields
@@ -108,8 +189,7 @@ where
     E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
 {
     plugin: TargetPluginDescriptor<E>,
-    request_validator: TargetRequestValidator,
-    subsystem: &'static str,
+    admin: TargetAdminMetadata,
 }
 
 impl<E> BuiltinTargetDescriptor<E>
@@ -119,8 +199,7 @@ where
     pub fn new(subsystem: &'static str, request_validator: TargetRequestValidator, plugin: TargetPluginDescriptor<E>) -> Self {
         Self {
             plugin,
-            request_validator,
-            subsystem,
+            admin: TargetAdminMetadata::new(subsystem, request_validator),
         }
     }
 
@@ -129,14 +208,32 @@ where
         &self.plugin
     }
 
+    #[inline]
+    pub fn admin_metadata(&self) -> TargetAdminMetadata {
+        self.admin
+    }
+
     #[inline]
     pub fn request_validator(&self) -> TargetRequestValidator {
-        self.request_validator
+        self.admin.request_validator()
     }
 
     #[inline]
     pub fn subsystem(&self) -> &'static str {
-        self.subsystem
+        self.admin.subsystem()
+    }
+}
+
+impl<E> From<BuiltinTargetDescriptor<E>> for BuiltinTargetAdminDescriptor
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    fn from(descriptor: BuiltinTargetDescriptor<E>) -> Self {
+        Self::new(
+            *descriptor.plugin().manifest(),
+            descriptor.plugin().valid_fields(),
+            descriptor.admin_metadata(),
+        )
     }
 }
 
@@ -220,6 +317,19 @@ where
         info!(count = successful_targets.len(), "All target processing completed");
         Ok(successful_targets)
     }
+
+    pub async fn create_activation_from_config<A>(
+        &self,
+        config: &Config,
+        route_prefix: &str,
+        adapter: &A,
+    ) -> Result<RuntimeActivation<E>, TargetError>
+    where
+        A: PluginRuntimeAdapter<E> + ?Sized,
+    {
+        let targets = self.create_targets_from_config(config, route_prefix).await?;
+        Ok(adapter.activate_with_replay(targets).await)
+    }
 }
 
 pub fn boxed_target<E, T>(target: T) -> BoxedTarget<E>
@@ -229,3 +339,106 @@ where
 {
     Box::new(target)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::{TargetPluginDescriptor, TargetPluginRegistry};
+    use crate::runtime::adapter::BuiltinPluginRuntimeAdapter;
+    use crate::store::{Key, Store};
+    use crate::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
+    use crate::{StoreError, Target, TargetError};
+    use async_trait::async_trait;
+    use rustfs_config::ENABLE_KEY;
+    use rustfs_ecstore::config::{Config, KVS};
+    use serde::{Serialize, de::DeserializeOwned};
+    use std::collections::HashMap;
+    use std::sync::Arc;
+    use std::time::Duration;
+
+    #[derive(Clone)]
+    struct TestTarget {
+        id: crate::arn::TargetID,
+    }
+
+    #[async_trait]
+    impl<E> Target<E> for TestTarget
+    where
+        E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+    {
+        fn id(&self) -> crate::arn::TargetID {
+            self.id.clone()
+        }
+
+        async fn is_active(&self) -> Result<bool, TargetError> {
+            Ok(true)
+        }
+
+        async fn save(&self, _event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        async fn send_raw_from_store(&self, _key: Key, _body: Vec<u8>, _meta: QueuedPayloadMeta) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        async fn close(&self) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
+            None
+        }
+
+        fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
+            Box::new(self.clone())
+        }
+
+        fn is_enabled(&self) -> bool {
+            true
+        }
+    }
+
+    fn builtin_adapter() -> BuiltinPluginRuntimeAdapter<String> {
+        BuiltinPluginRuntimeAdapter::new(
+            Arc::new(|_event| Box::pin(async {})),
+            Arc::new(|_target_id, _has_replay| {}),
+            None,
+            Duration::from_millis(10),
+            Duration::from_millis(10),
+            "stopping plugin registry test replay worker",
+        )
+    }
+
+    #[tokio::test]
+    async fn registry_creates_activation_from_config_via_runtime_adapter() {
+        let mut registry = TargetPluginRegistry::new();
+        registry.register(TargetPluginDescriptor::new(
+            "test",
+            &[ENABLE_KEY, "endpoint"],
+            |_config| Ok(()),
+            |id, _config| {
+                Ok(Box::new(TestTarget {
+                    id: crate::arn::TargetID::new(id, "test".to_string()),
+                }))
+            },
+        ));
+
+        let mut cfg = Config(HashMap::new());
+        let mut section = HashMap::new();
+        let mut primary = KVS::new();
+        primary.insert(ENABLE_KEY.to_string(), "on".to_string());
+        primary.insert("endpoint".to_string(), "https://example.com/hook".to_string());
+        section.insert("primary".to_string(), primary);
+        cfg.0.insert("notify_test".to_string(), section);
+
+        let adapter = builtin_adapter();
+        let activation = registry
+            .create_activation_from_config(&cfg, "notify_", &adapter)
+            .await
+            .expect("activation should be created through runtime adapter");
+
+        assert_eq!(activation.targets.len(), 1);
+        assert_eq!(activation.targets[0].id().to_string(), "primary:test");
+        assert!(activation.replay_workers.is_empty());
+    }
+}

crates/targets/src/runtime/adapter.rs

@@ -0,0 +1,346 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::{
+    ReplayEvent, ReplayWorkerManager, RuntimeActivation, RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot,
+    TargetRuntimeManager, activate_targets_with_replay, init_target_and_optionally_start_replay, start_replay_worker,
+};
+use crate::{Target, TargetError};
+use async_trait::async_trait;
+use serde::Serialize;
+use serde::de::DeserializeOwned;
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::Semaphore;
+
+type ReplayHook<E> = Arc<dyn Fn(ReplayEvent<E>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
+type ReplayStartObserver = Arc<dyn Fn(&str, bool) + Send + Sync>;
+
+/// Shared runtime contract for target plugins.
+#[async_trait]
+pub trait PluginRuntimeAdapter<E>: Send + Sync
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    async fn activate_with_replay(&self, targets: Vec<Box<dyn Target<E> + Send + Sync>>) -> RuntimeActivation<E>;
+
+    async fn replace_runtime_targets(
+        &self,
+        runtime: &mut TargetRuntimeManager<E>,
+        replay_workers: &mut ReplayWorkerManager,
+        activation: RuntimeActivation<E>,
+    ) -> Result<(), TargetError>;
+
+    async fn stop_replay_workers(&self, replay_workers: &mut ReplayWorkerManager);
+
+    fn snapshot_runtime_status(
+        &self,
+        runtime: &TargetRuntimeManager<E>,
+        replay_workers: &ReplayWorkerManager,
+    ) -> RuntimeStatusSnapshot;
+
+    async fn snapshot_runtime_health(&self, runtime: &TargetRuntimeManager<E>) -> Vec<RuntimeTargetHealthSnapshot>;
+
+    async fn shutdown(
+        &self,
+        runtime: &mut TargetRuntimeManager<E>,
+        replay_workers: &mut ReplayWorkerManager,
+    ) -> Result<(), TargetError>;
+}
+
+/// Built-in in-process runtime adapter that preserves the current replay and
+/// activation behavior while presenting a stable runtime contract to callers.
+#[derive(Clone)]
+pub struct BuiltinPluginRuntimeAdapter<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    replay_hook: ReplayHook<E>,
+    replay_start_observer: ReplayStartObserver,
+    replay_semaphore: Option<Arc<Semaphore>>,
+    batch_timeout: Duration,
+    idle_sleep: Duration,
+    stop_log_prefix: Arc<str>,
+}
+
+impl<E> BuiltinPluginRuntimeAdapter<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    pub fn new(
+        replay_hook: ReplayHook<E>,
+        replay_start_observer: ReplayStartObserver,
+        replay_semaphore: Option<Arc<Semaphore>>,
+        batch_timeout: Duration,
+        idle_sleep: Duration,
+        stop_log_prefix: impl Into<Arc<str>>,
+    ) -> Self {
+        Self {
+            replay_hook,
+            replay_start_observer,
+            replay_semaphore,
+            batch_timeout,
+            idle_sleep,
+            stop_log_prefix: stop_log_prefix.into(),
+        }
+    }
+}
+
+#[async_trait]
+impl<E> PluginRuntimeAdapter<E> for BuiltinPluginRuntimeAdapter<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    async fn activate_with_replay(&self, targets: Vec<Box<dyn Target<E> + Send + Sync>>) -> RuntimeActivation<E> {
+        let replay_hook = Arc::clone(&self.replay_hook);
+        let replay_start_observer = Arc::clone(&self.replay_start_observer);
+        let replay_semaphore = self.replay_semaphore.clone();
+        let batch_timeout = self.batch_timeout;
+        let idle_sleep = self.idle_sleep;
+
+        activate_targets_with_replay(targets, move |target| {
+            let replay_hook = Arc::clone(&replay_hook);
+            let replay_start_observer = Arc::clone(&replay_start_observer);
+            let replay_semaphore = replay_semaphore.clone();
+
+            async move {
+                init_target_and_optionally_start_replay(
+                    target,
+                    move |target_id, has_replay| replay_start_observer(target_id, has_replay),
+                    move |store, target| {
+                        start_replay_worker(
+                            store,
+                            target,
+                            Arc::clone(&replay_hook),
+                            replay_semaphore.clone(),
+                            batch_timeout,
+                            idle_sleep,
+                        )
+                    },
+                )
+                .await
+            }
+        })
+        .await
+    }
+
+    async fn replace_runtime_targets(
+        &self,
+        runtime: &mut TargetRuntimeManager<E>,
+        replay_workers: &mut ReplayWorkerManager,
+        activation: RuntimeActivation<E>,
+    ) -> Result<(), TargetError> {
+        self.stop_replay_workers(replay_workers).await;
+        runtime.clear_and_close().await;
+
+        for target in activation.targets {
+            runtime.add_arc(target);
+        }
+
+        *replay_workers = activation.replay_workers;
+        Ok(())
+    }
+
+    async fn stop_replay_workers(&self, replay_workers: &mut ReplayWorkerManager) {
+        replay_workers.stop_all(&self.stop_log_prefix).await;
+    }
+
+    fn snapshot_runtime_status(
+        &self,
+        runtime: &TargetRuntimeManager<E>,
+        replay_workers: &ReplayWorkerManager,
+    ) -> RuntimeStatusSnapshot {
+        runtime.status_snapshot(replay_workers)
+    }
+
+    async fn snapshot_runtime_health(&self, runtime: &TargetRuntimeManager<E>) -> Vec<RuntimeTargetHealthSnapshot> {
+        runtime.health_snapshots().await
+    }
+
+    async fn shutdown(
+        &self,
+        runtime: &mut TargetRuntimeManager<E>,
+        replay_workers: &mut ReplayWorkerManager,
+    ) -> Result<(), TargetError> {
+        self.stop_replay_workers(replay_workers).await;
+        runtime.clear_and_close().await;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter};
+    use crate::arn::TargetID;
+    use crate::store::{Key, QueueStore, Store};
+    use crate::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
+    use crate::{StoreError, Target, TargetError};
+    use async_trait::async_trait;
+    use serde::{Serialize, de::DeserializeOwned};
+    use std::sync::Arc;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::time::Duration;
+    use tempfile::tempdir;
+
+    #[derive(Clone)]
+    struct TestTarget {
+        close_calls: Arc<AtomicUsize>,
+        id: TargetID,
+        init_fails: bool,
+        store: Option<QueueStore<QueuedPayload>>,
+    }
+
+    impl TestTarget {
+        fn new(id: &str, name: &str) -> Self {
+            Self {
+                close_calls: Arc::new(AtomicUsize::new(0)),
+                id: TargetID::new(id.to_string(), name.to_string()),
+                init_fails: false,
+                store: None,
+            }
+        }
+
+        fn with_failed_init(mut self) -> Self {
+            self.init_fails = true;
+            self
+        }
+
+        fn with_store(mut self) -> Self {
+            let dir = tempdir().expect("tempdir should be created for queue store tests");
+            let store = QueueStore::<QueuedPayload>::new(dir.path(), 16, ".queue");
+            store.open().expect("queue store should open");
+            self.store = Some(store);
+            self
+        }
+    }
+
+    #[async_trait]
+    impl<E> Target<E> for TestTarget
+    where
+        E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+    {
+        fn id(&self) -> TargetID {
+            self.id.clone()
+        }
+
+        async fn is_active(&self) -> Result<bool, TargetError> {
+            Ok(true)
+        }
+
+        async fn save(&self, _event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        async fn send_raw_from_store(&self, _key: Key, _body: Vec<u8>, _meta: QueuedPayloadMeta) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        async fn close(&self) -> Result<(), TargetError> {
+            self.close_calls.fetch_add(1, Ordering::SeqCst);
+            Ok(())
+        }
+
+        fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
+            self.store
+                .as_ref()
+                .map(|store| store as &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync))
+        }
+
+        fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
+            Box::new(self.clone())
+        }
+
+        async fn init(&self) -> Result<(), TargetError> {
+            if self.init_fails {
+                return Err(TargetError::Configuration("forced init failure".to_string()));
+            }
+            Ok(())
+        }
+
+        fn is_enabled(&self) -> bool {
+            true
+        }
+    }
+
+    fn builtin_adapter() -> BuiltinPluginRuntimeAdapter<String> {
+        BuiltinPluginRuntimeAdapter::new(
+            Arc::new(|_event| Box::pin(async {})),
+            Arc::new(|_target_id, _has_replay| {}),
+            None,
+            Duration::from_millis(10),
+            Duration::from_millis(10),
+            "stopping test replay worker",
+        )
+    }
+
+    #[tokio::test]
+    async fn builtin_adapter_handles_empty_target_activation() {
+        let adapter = builtin_adapter();
+        let activation = adapter.activate_with_replay(Vec::new()).await;
+
+        assert!(activation.targets.is_empty());
+        assert!(activation.replay_workers.is_empty());
+    }
+
+    #[tokio::test]
+    async fn builtin_adapter_skips_non_store_target_when_init_fails() {
+        let adapter = builtin_adapter();
+        let target = TestTarget::new("primary", "webhook").with_failed_init();
+
+        let activation = adapter.activate_with_replay(vec![Box::new(target)]).await;
+
+        assert!(activation.targets.is_empty());
+        assert!(activation.replay_workers.is_empty());
+    }
+
+    #[tokio::test]
+    async fn builtin_adapter_keeps_store_backed_target_when_init_fails() {
+        let adapter = builtin_adapter();
+        let target = TestTarget::new("primary", "webhook").with_failed_init().with_store();
+
+        let activation = adapter.activate_with_replay(vec![Box::new(target)]).await;
+
+        assert_eq!(activation.targets.len(), 1);
+        assert_eq!(activation.replay_workers.len(), 1);
+    }
+
+    #[tokio::test]
+    async fn builtin_adapter_shutdown_clears_runtime_and_replay_workers() {
+        let adapter = builtin_adapter();
+        let target = TestTarget::new("primary", "webhook");
+        let close_calls = Arc::clone(&target.close_calls);
+        let mut runtime = crate::runtime::TargetRuntimeManager::new();
+        let mut replay_workers = crate::runtime::ReplayWorkerManager::new();
+
+        let activation = adapter.activate_with_replay(vec![Box::new(target)]).await;
+        adapter
+            .replace_runtime_targets(&mut runtime, &mut replay_workers, activation)
+            .await
+            .expect("replace_runtime_targets should succeed");
+
+        assert_eq!(runtime.len(), 1);
+        assert_eq!(replay_workers.len(), 0);
+
+        adapter
+            .shutdown(&mut runtime, &mut replay_workers)
+            .await
+            .expect("shutdown should succeed");
+
+        assert!(runtime.is_empty());
+        assert!(replay_workers.is_empty());
+        assert_eq!(close_calls.load(Ordering::SeqCst), 1);
+    }
+}

crates/targets/src/runtime/mod.rs

@@ -0,0 +1,641 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod adapter;
+pub mod sidecar;
+pub mod sidecar_protocol;
+
+use crate::Target;
+use crate::arn::TargetID;
+use crate::store::{Key, Store, ensure_store_entry_raw_readable};
+use crate::target::QueuedPayload;
+use crate::target::TargetDeliverySnapshot;
+use crate::{StoreError, TargetError};
+use serde::Serialize;
+use serde::de::DeserializeOwned;
+use std::sync::Arc;
+use std::{collections::HashMap, fmt::Debug};
+use std::{future::Future, pin::Pin, time::Duration};
+use tokio::sync::{Semaphore, mpsc};
+
+/// Shared target trait object used by the runtime manager.
+pub type SharedTarget<E> = Arc<dyn Target<E> + Send + Sync>;
+type ReplayHook<E> = Arc<dyn Fn(ReplayEvent<E>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
+
+#[derive(Debug, Default)]
+pub struct ReplayWorkerManager {
+    cancellers: HashMap<String, mpsc::Sender<()>>,
+}
+
+impl ReplayWorkerManager {
+    pub fn new() -> Self {
+        Self {
+            cancellers: HashMap::new(),
+        }
+    }
+
+    pub fn insert(&mut self, target_id: String, cancel_tx: mpsc::Sender<()>) {
+        self.cancellers.insert(target_id, cancel_tx);
+    }
+
+    pub fn len(&self) -> usize {
+        self.cancellers.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.cancellers.is_empty()
+    }
+
+    pub fn snapshot(&self, target_count: usize) -> RuntimeStatusSnapshot {
+        RuntimeStatusSnapshot {
+            replay_worker_count: self.len(),
+            target_count,
+        }
+    }
+
+    pub async fn stop_all(&mut self, log_prefix: &str) {
+        for (target_id, cancel_tx) in self.cancellers.drain() {
+            tracing::info!(target_id = %target_id, "{log_prefix}");
+            let _ = cancel_tx.send(()).await;
+        }
+    }
+}
+
+pub struct RuntimeActivation<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    pub replay_workers: ReplayWorkerManager,
+    pub targets: Vec<SharedTarget<E>>,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct RuntimeStatusSnapshot {
+    pub replay_worker_count: usize,
+    pub target_count: usize,
+}
+
+/// A read-only runtime snapshot for a target instance.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct RuntimeTargetSnapshot {
+    pub failed_messages: u64,
+    pub queue_length: u64,
+    pub target_id: String,
+    pub target_type: String,
+    pub total_messages: u64,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum RuntimeTargetHealthState {
+    Disabled,
+    Error,
+    Offline,
+    Online,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct RuntimeTargetHealthSnapshot {
+    pub enabled: bool,
+    pub error_message: Option<String>,
+    pub state: RuntimeTargetHealthState,
+    pub target_id: String,
+    pub target_type: String,
+}
+
+pub enum ReplayEvent<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    Delivered {
+        key: Key,
+        target: SharedTarget<E>,
+    },
+    RetryableError {
+        error: TargetError,
+        key: Key,
+        retry_count: usize,
+        target: SharedTarget<E>,
+    },
+    Dropped {
+        key: Key,
+        reason: String,
+        target: SharedTarget<E>,
+    },
+    PermanentFailure {
+        error: TargetError,
+        key: Key,
+        target: SharedTarget<E>,
+    },
+    RetryExhausted {
+        key: Key,
+        target: SharedTarget<E>,
+    },
+    UnreadableEntry {
+        error: StoreError,
+        key: Key,
+        target: SharedTarget<E>,
+    },
+}
+
+/// Shared runtime container for managing instantiated targets.
+///
+/// This intentionally focuses on low-risk shared lifecycle primitives first:
+/// add/remove/close/list/snapshot. Replay workers and reload orchestration can
+/// be layered on top in later phases.
+pub struct TargetRuntimeManager<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    targets: HashMap<String, SharedTarget<E>>,
+}
+
+impl<E> Default for TargetRuntimeManager<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl<E> Debug for TargetRuntimeManager<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("TargetRuntimeManager")
+            .field("target_count", &self.targets.len())
+            .finish()
+    }
+}
+
+impl<E> TargetRuntimeManager<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    pub fn new() -> Self {
+        Self { targets: HashMap::new() }
+    }
+
+    pub fn add_arc(&mut self, target: SharedTarget<E>) -> Option<SharedTarget<E>> {
+        let key = target.id().to_string();
+        self.targets.insert(key, target)
+    }
+
+    pub fn add_boxed(&mut self, target: Box<dyn Target<E> + Send + Sync>) -> Option<SharedTarget<E>> {
+        self.add_arc(Arc::from(target))
+    }
+
+    pub fn get(&self, key: &str) -> Option<SharedTarget<E>> {
+        self.targets.get(key).cloned()
+    }
+
+    pub fn get_by_target_id(&self, target_id: &TargetID) -> Option<SharedTarget<E>> {
+        self.get(&target_id.to_string())
+    }
+
+    pub fn remove(&mut self, key: &str) -> Option<SharedTarget<E>> {
+        self.targets.remove(key)
+    }
+
+    pub fn remove_by_target_id(&mut self, target_id: &TargetID) -> Option<SharedTarget<E>> {
+        self.remove(&target_id.to_string())
+    }
+
+    pub fn clear(&mut self) {
+        self.targets.clear();
+    }
+
+    pub async fn remove_and_close(&mut self, key: &str) -> Option<SharedTarget<E>> {
+        let target = self.targets.remove(key)?;
+        if let Err(err) = target.close().await {
+            tracing::error!(target_id = %key, error = %err, "Failed to close target during removal");
+        }
+        Some(target)
+    }
+
+    pub async fn remove_by_target_id_and_close(&mut self, target_id: &TargetID) -> Option<SharedTarget<E>> {
+        self.remove_and_close(&target_id.to_string()).await
+    }
+
+    pub async fn clear_and_close(&mut self) {
+        let target_ids: Vec<String> = self.targets.keys().cloned().collect();
+        for target_id in target_ids {
+            let _ = self.remove_and_close(&target_id).await;
+        }
+        self.targets.clear();
+    }
+
+    pub fn target_ids(&self) -> Vec<TargetID> {
+        self.targets.values().map(|target| target.id()).collect()
+    }
+
+    pub fn keys(&self) -> Vec<String> {
+        self.targets.keys().cloned().collect()
+    }
+
+    pub fn values(&self) -> Vec<SharedTarget<E>> {
+        self.targets.values().cloned().collect()
+    }
+
+    pub fn len(&self) -> usize {
+        self.targets.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.targets.is_empty()
+    }
+
+    pub fn snapshots(&self) -> Vec<RuntimeTargetSnapshot> {
+        let mut snapshots = Vec::with_capacity(self.targets.len());
+        for target in self.targets.values() {
+            let delivery = target.delivery_snapshot();
+            let target_id = target.id();
+            snapshots.push(snapshot_from_delivery(target_id, delivery));
+        }
+        snapshots.sort_by(|a, b| a.target_id.cmp(&b.target_id));
+        snapshots
+    }
+
+    pub fn status_snapshot(&self, replay_workers: &ReplayWorkerManager) -> RuntimeStatusSnapshot {
+        replay_workers.snapshot(self.len())
+    }
+
+    pub async fn health_snapshots(&self) -> Vec<RuntimeTargetHealthSnapshot> {
+        let mut snapshots = Vec::with_capacity(self.targets.len());
+        for target in self.targets.values() {
+            let enabled = target.is_enabled();
+            let target_id = target.id();
+            let (state, error_message) = if !enabled {
+                (RuntimeTargetHealthState::Disabled, None)
+            } else {
+                match target.is_active().await {
+                    Ok(true) => (RuntimeTargetHealthState::Online, None),
+                    Ok(false) => (RuntimeTargetHealthState::Offline, None),
+                    Err(err) => (RuntimeTargetHealthState::Error, Some(err.to_string())),
+                }
+            };
+
+            snapshots.push(RuntimeTargetHealthSnapshot {
+                enabled,
+                error_message,
+                state,
+                target_id: target_id.to_string(),
+                target_type: target_id.name,
+            });
+        }
+        snapshots.sort_by(|a, b| a.target_id.cmp(&b.target_id));
+        snapshots
+    }
+}
+
+fn snapshot_from_delivery(target_id: TargetID, delivery: TargetDeliverySnapshot) -> RuntimeTargetSnapshot {
+    RuntimeTargetSnapshot {
+        failed_messages: delivery.failed_messages,
+        queue_length: delivery.queue_length,
+        target_id: target_id.to_string(),
+        target_type: target_id.name,
+        total_messages: delivery.total_messages,
+    }
+}
+
+pub async fn init_target_and_optionally_start_replay<E, F, G>(
+    target: Box<dyn Target<E> + Send + Sync>,
+    on_replay_start: F,
+    start_replay: G,
+) -> Option<(SharedTarget<E>, Option<mpsc::Sender<()>>)>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+    F: FnOnce(&str, bool),
+    G: FnOnce(Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>, SharedTarget<E>) -> mpsc::Sender<()>,
+{
+    let target_id = target.id().to_string();
+    let has_store = target.store().is_some();
+
+    if let Err(err) = target.init().await {
+        tracing::error!(target_id = %target_id, error = %err, "Failed to initialize target");
+        if !has_store {
+            return None;
+        }
+        tracing::warn!(
+            target_id = %target_id,
+            "Proceeding with store-backed target despite init failure"
+        );
+    }
+
+    let shared: SharedTarget<E> = Arc::from(target);
+    if !shared.is_enabled() {
+        on_replay_start(&target_id, false);
+        return Some((shared, None));
+    }
+
+    let cancel = shared
+        .store()
+        .map(|store| start_replay(store.boxed_clone(), Arc::clone(&shared)));
+    on_replay_start(&target_id, cancel.is_some());
+    Some((shared, cancel))
+}
+
+pub async fn activate_targets_with_replay<E, F, Fut>(
+    targets: Vec<Box<dyn Target<E> + Send + Sync>>,
+    mut activate_one: F,
+) -> RuntimeActivation<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+    F: FnMut(Box<dyn Target<E> + Send + Sync>) -> Fut,
+    Fut: Future<Output = Option<(SharedTarget<E>, Option<mpsc::Sender<()>>)>>,
+{
+    let mut replay_workers = ReplayWorkerManager::new();
+    let mut shared_targets = Vec::new();
+
+    for target in targets {
+        if let Some((shared_target, cancel_tx)) = activate_one(target).await {
+            let target_id = shared_target.id().to_string();
+            if let Some(cancel_tx) = cancel_tx {
+                replay_workers.insert(target_id, cancel_tx);
+            }
+            shared_targets.push(shared_target);
+        }
+    }
+
+    RuntimeActivation {
+        replay_workers,
+        targets: shared_targets,
+    }
+}
+
+pub fn start_replay_worker<E>(
+    mut store: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>,
+    target: SharedTarget<E>,
+    hook: ReplayHook<E>,
+    semaphore: Option<Arc<Semaphore>>,
+    batch_timeout: Duration,
+    idle_sleep: Duration,
+) -> mpsc::Sender<()>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    let (cancel_tx, cancel_rx) = mpsc::channel(1);
+
+    tokio::spawn(async move {
+        stream_replay_worker(&mut *store, target, cancel_rx, hook, semaphore, batch_timeout, idle_sleep).await;
+    });
+
+    cancel_tx
+}
+
+async fn stream_replay_worker<E>(
+    store: &mut (dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send),
+    target: SharedTarget<E>,
+    mut cancel_rx: mpsc::Receiver<()>,
+    hook: ReplayHook<E>,
+    semaphore: Option<Arc<Semaphore>>,
+    batch_timeout: Duration,
+    idle_sleep: Duration,
+) where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    const MAX_RETRIES: usize = 5;
+    const BASE_RETRY_DELAY: Duration = Duration::from_secs(2);
+
+    let mut batch_keys = Vec::with_capacity(1);
+    let mut last_flush = tokio::time::Instant::now();
+
+    loop {
+        if cancel_rx.try_recv().is_ok() {
+            return;
+        }
+
+        let keys = store.list();
+        if keys.is_empty() {
+            if !batch_keys.is_empty() && last_flush.elapsed() >= batch_timeout {
+                process_replay_batch(&mut batch_keys, target.clone(), &hook, semaphore.clone()).await;
+                last_flush = tokio::time::Instant::now();
+            }
+            tokio::time::sleep(idle_sleep).await;
+            continue;
+        }
+
+        for key in keys {
+            if cancel_rx.try_recv().is_ok() {
+                if !batch_keys.is_empty() {
+                    process_replay_batch(&mut batch_keys, target.clone(), &hook, semaphore.clone()).await;
+                }
+                return;
+            }
+
+            match ensure_store_entry_raw_readable(&*store, &key) {
+                Ok(true) => {}
+                Ok(false) => continue,
+                Err(err) => {
+                    hook(ReplayEvent::UnreadableEntry {
+                        error: err,
+                        key,
+                        target: target.clone(),
+                    })
+                    .await;
+                    continue;
+                }
+            }
+
+            batch_keys.push(key);
+            if !batch_keys.is_empty() || last_flush.elapsed() >= batch_timeout {
+                process_replay_batch(&mut batch_keys, target.clone(), &hook, semaphore.clone()).await;
+                last_flush = tokio::time::Instant::now();
+            }
+        }
+
+        tokio::time::sleep(Duration::from_millis(100)).await;
+    }
+
+    async fn process_replay_batch<E>(
+        batch_keys: &mut Vec<Key>,
+        target: SharedTarget<E>,
+        hook: &ReplayHook<E>,
+        semaphore: Option<Arc<Semaphore>>,
+    ) where
+        E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+    {
+        if batch_keys.is_empty() {
+            return;
+        }
+
+        let _permit = match semaphore {
+            Some(ref semaphore) => match semaphore.clone().acquire_owned().await {
+                Ok(permit) => Some(permit),
+                Err(err) => {
+                    tracing::error!(error = %err, "Failed to acquire replay semaphore permit");
+                    return;
+                }
+            },
+            None => None,
+        };
+
+        for key in batch_keys.iter() {
+            let mut retry_count = 0usize;
+            let mut success = false;
+
+            while retry_count < MAX_RETRIES && !success {
+                match target.send_from_store(key.clone()).await {
+                    Ok(_) => {
+                        hook(ReplayEvent::Delivered {
+                            key: key.clone(),
+                            target: target.clone(),
+                        })
+                        .await;
+                        success = true;
+                    }
+                    Err(err) => match err {
+                        TargetError::NotConnected | TargetError::Timeout(_) => {
+                            retry_count += 1;
+                            hook(ReplayEvent::RetryableError {
+                                error: err,
+                                key: key.clone(),
+                                retry_count,
+                                target: target.clone(),
+                            })
+                            .await;
+
+                            let jitter = Duration::from_millis(key.to_string().len() as u64 % 500);
+                            let backoff = 1u32 << retry_count as u32;
+                            tokio::time::sleep(BASE_RETRY_DELAY * backoff + jitter).await;
+                        }
+                        TargetError::Dropped(reason) => {
+                            hook(ReplayEvent::Dropped {
+                                key: key.clone(),
+                                reason,
+                                target: target.clone(),
+                            })
+                            .await;
+                            break;
+                        }
+                        other => {
+                            hook(ReplayEvent::PermanentFailure {
+                                error: other,
+                                key: key.clone(),
+                                target: target.clone(),
+                            })
+                            .await;
+                            break;
+                        }
+                    },
+                }
+            }
+
+            if retry_count >= MAX_RETRIES && !success {
+                hook(ReplayEvent::RetryExhausted {
+                    key: key.clone(),
+                    target: target.clone(),
+                })
+                .await;
+            }
+        }
+
+        batch_keys.clear();
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::TargetRuntimeManager;
+    use crate::StoreError;
+    use crate::arn::TargetID;
+    use crate::store::{Key, Store};
+    use crate::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
+    use crate::{Target, TargetError};
+    use async_trait::async_trait;
+    use serde::{Serialize, de::DeserializeOwned};
+    use std::sync::Arc;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+
+    #[derive(Clone)]
+    struct TestTarget {
+        id: TargetID,
+        close_calls: Arc<AtomicUsize>,
+    }
+
+    impl TestTarget {
+        fn new(id: &str, name: &str) -> Self {
+            Self {
+                id: TargetID::new(id.to_string(), name.to_string()),
+                close_calls: Arc::new(AtomicUsize::new(0)),
+            }
+        }
+    }
+
+    #[async_trait]
+    impl<E> Target<E> for TestTarget
+    where
+        E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+    {
+        fn id(&self) -> TargetID {
+            self.id.clone()
+        }
+
+        async fn is_active(&self) -> Result<bool, TargetError> {
+            Ok(true)
+        }
+
+        async fn save(&self, _event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        async fn send_raw_from_store(&self, _key: Key, _body: Vec<u8>, _meta: QueuedPayloadMeta) -> Result<(), TargetError> {
+            Ok(())
+        }
+
+        async fn close(&self) -> Result<(), TargetError> {
+            self.close_calls.fetch_add(1, Ordering::SeqCst);
+            Ok(())
+        }
+
+        fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
+            None
+        }
+
+        fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
+            Box::new(self.clone())
+        }
+
+        fn is_enabled(&self) -> bool {
+            true
+        }
+    }
+
+    #[tokio::test]
+    async fn runtime_manager_removes_and_closes_target() {
+        let mut manager = TargetRuntimeManager::<String>::new();
+        let target = TestTarget::new("primary", "webhook");
+        let close_calls = Arc::clone(&target.close_calls);
+
+        manager.add_boxed(Box::new(target));
+        assert_eq!(manager.len(), 1);
+
+        let removed = manager.remove_and_close("primary:webhook").await;
+        assert!(removed.is_some());
+        assert_eq!(manager.len(), 0);
+        assert_eq!(close_calls.load(Ordering::SeqCst), 1);
+    }
+
+    #[test]
+    fn runtime_manager_snapshots_targets() {
+        let mut manager = TargetRuntimeManager::<String>::new();
+        manager.add_boxed(Box::new(TestTarget::new("primary", "webhook")));
+
+        let snapshots = manager.snapshots();
+        assert_eq!(snapshots.len(), 1);
+        assert_eq!(snapshots[0].target_id, "primary:webhook");
+        assert_eq!(snapshots[0].target_type, "webhook");
+    }
+}

crates/targets/src/runtime/sidecar.rs

@@ -0,0 +1,171 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::TargetDomain;
+use crate::runtime::sidecar_protocol::SidecarHandshake;
+use serde::{Deserialize, Serialize};
+use std::time::Duration;
+
+const DEFAULT_FAILURE_THRESHOLD: usize = 3;
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub struct SidecarPluginRuntime {
+    pub endpoint: String,
+    pub handshake: SidecarHandshake,
+    pub healthy: bool,
+    pub failure_count: usize,
+    pub degraded_to_builtin: bool,
+    pub last_error: Option<String>,
+}
+
+impl SidecarPluginRuntime {
+    pub fn new(endpoint: impl Into<String>, handshake: SidecarHandshake) -> Self {
+        Self {
+            endpoint: endpoint.into(),
+            handshake,
+            healthy: false,
+            failure_count: 0,
+            degraded_to_builtin: false,
+            last_error: None,
+        }
+    }
+
+    pub fn enable(&mut self, expected_plugin_id: &str, required_domain: TargetDomain) -> Result<(), String> {
+        self.handshake.validate(expected_plugin_id)?;
+        if !self.handshake.supported_domains.contains(&required_domain) {
+            return Err(format!(
+                "sidecar plugin {} does not support required domain {:?}",
+                self.handshake.plugin_id, required_domain
+            ));
+        }
+
+        self.healthy = true;
+        self.degraded_to_builtin = false;
+        self.last_error = None;
+        self.failure_count = 0;
+        Ok(())
+    }
+
+    pub fn mark_unhealthy(&mut self) {
+        self.healthy = false;
+    }
+
+    pub fn record_failure(&mut self, error: impl Into<String>) {
+        self.failure_count = self.failure_count.saturating_add(1);
+        self.healthy = false;
+        self.last_error = Some(error.into());
+        if self.failure_count >= DEFAULT_FAILURE_THRESHOLD {
+            self.degraded_to_builtin = true;
+        }
+    }
+
+    pub fn send_with_timeout(&mut self, operation_timeout: Duration, simulated_latency: Duration) -> Result<(), String> {
+        if simulated_latency > operation_timeout {
+            self.record_failure(format!(
+                "sidecar send timeout after {:?} (budget {:?})",
+                simulated_latency, operation_timeout
+            ));
+            return Err(self
+                .last_error
+                .clone()
+                .unwrap_or_else(|| "sidecar timeout without recorded error".to_string()));
+        }
+        self.healthy = true;
+        self.last_error = None;
+        Ok(())
+    }
+
+    pub fn shutdown(&mut self) {
+        self.healthy = false;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::SidecarPluginRuntime;
+    use crate::TargetDomain;
+    use crate::runtime::sidecar_protocol::{SIDECAR_RUNTIME_PROTOCOL_VERSION, SidecarHandshake, SidecarPluginCapability};
+    use std::time::Duration;
+
+    fn notify_sidecar_handshake() -> SidecarHandshake {
+        SidecarHandshake {
+            protocol_version: SIDECAR_RUNTIME_PROTOCOL_VERSION.to_string(),
+            plugin_id: "external:webhook".to_string(),
+            plugin_version: "1.2.3".to_string(),
+            supported_domains: vec![TargetDomain::Notify],
+            capabilities: vec![
+                SidecarPluginCapability::HealthCheck,
+                SidecarPluginCapability::SendEvent,
+                SidecarPluginCapability::Shutdown,
+            ],
+        }
+    }
+
+    #[test]
+    fn sidecar_runtime_enable_marks_runtime_healthy() {
+        let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", notify_sidecar_handshake());
+
+        runtime
+            .enable("external:webhook", TargetDomain::Notify)
+            .expect("sidecar runtime should enable");
+
+        assert!(runtime.healthy);
+    }
+
+    #[test]
+    fn sidecar_runtime_enable_rejects_domain_mismatch() {
+        let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", notify_sidecar_handshake());
+
+        let result = runtime.enable("external:webhook", TargetDomain::Audit);
+
+        assert!(result.is_err());
+        assert!(!runtime.healthy);
+    }
+
+    #[test]
+    fn sidecar_runtime_shutdown_marks_runtime_unhealthy() {
+        let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", notify_sidecar_handshake());
+        runtime
+            .enable("external:webhook", TargetDomain::Notify)
+            .expect("sidecar runtime should enable");
+
+        runtime.shutdown();
+
+        assert!(!runtime.healthy);
+    }
+
+    #[test]
+    fn sidecar_runtime_degrades_to_builtin_after_failure_threshold() {
+        let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", notify_sidecar_handshake());
+
+        runtime.record_failure("send failed");
+        runtime.record_failure("send failed again");
+        runtime.record_failure("send failed third time");
+
+        assert!(runtime.degraded_to_builtin);
+        assert!(!runtime.healthy);
+        assert_eq!(runtime.failure_count, 3);
+    }
+
+    #[test]
+    fn sidecar_runtime_send_timeout_records_last_error() {
+        let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", notify_sidecar_handshake());
+
+        let result = runtime.send_with_timeout(Duration::from_millis(50), Duration::from_millis(75));
+
+        assert!(result.is_err());
+        assert_eq!(runtime.last_error.as_deref(), Some("sidecar send timeout after 75ms (budget 50ms)"));
+    }
+}

crates/targets/src/runtime/sidecar_protocol.rs

@@ -0,0 +1,106 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::TargetDomain;
+use serde::{Deserialize, Serialize};
+
+pub const SIDECAR_RUNTIME_PROTOCOL_VERSION: &str = "rustfs.target-runtime.v1";
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum SidecarPluginCapability {
+    HealthCheck,
+    SendEvent,
+    Shutdown,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub struct SidecarHandshake {
+    pub protocol_version: String,
+    pub plugin_id: String,
+    pub plugin_version: String,
+    pub supported_domains: Vec<TargetDomain>,
+    pub capabilities: Vec<SidecarPluginCapability>,
+}
+
+impl SidecarHandshake {
+    pub fn validate(&self, expected_plugin_id: &str) -> Result<(), String> {
+        if self.protocol_version != SIDECAR_RUNTIME_PROTOCOL_VERSION {
+            return Err(format!(
+                "unsupported sidecar protocol version: expected {}, got {}",
+                SIDECAR_RUNTIME_PROTOCOL_VERSION, self.protocol_version
+            ));
+        }
+
+        if self.plugin_id != expected_plugin_id {
+            return Err(format!(
+                "sidecar plugin id mismatch: expected {}, got {}",
+                expected_plugin_id, self.plugin_id
+            ));
+        }
+
+        for capability in [
+            SidecarPluginCapability::HealthCheck,
+            SidecarPluginCapability::SendEvent,
+            SidecarPluginCapability::Shutdown,
+        ] {
+            if !self.capabilities.contains(&capability) {
+                return Err(format!("sidecar handshake missing required capability: {:?}", capability));
+            }
+        }
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{SIDECAR_RUNTIME_PROTOCOL_VERSION, SidecarHandshake, SidecarPluginCapability};
+    use crate::TargetDomain;
+
+    #[test]
+    fn sidecar_handshake_accepts_expected_contract() {
+        let handshake = SidecarHandshake {
+            protocol_version: SIDECAR_RUNTIME_PROTOCOL_VERSION.to_string(),
+            plugin_id: "external:webhook".to_string(),
+            plugin_version: "1.2.3".to_string(),
+            supported_domains: vec![TargetDomain::Notify],
+            capabilities: vec![
+                SidecarPluginCapability::HealthCheck,
+                SidecarPluginCapability::SendEvent,
+                SidecarPluginCapability::Shutdown,
+            ],
+        };
+
+        assert!(handshake.validate("external:webhook").is_ok());
+    }
+
+    #[test]
+    fn sidecar_handshake_rejects_protocol_mismatch() {
+        let handshake = SidecarHandshake {
+            protocol_version: "rustfs.target-runtime.v0".to_string(),
+            plugin_id: "external:webhook".to_string(),
+            plugin_version: "1.2.3".to_string(),
+            supported_domains: vec![TargetDomain::Notify],
+            capabilities: vec![
+                SidecarPluginCapability::HealthCheck,
+                SidecarPluginCapability::SendEvent,
+                SidecarPluginCapability::Shutdown,
+            ],
+        };
+
+        assert!(handshake.validate("external:webhook").is_err());
+    }
+}

crates/targets/src/target/amqp.rs

@@ -19,13 +19,14 @@
 //! body through `send_raw_from_store`.
 
 use crate::{
-    StoreError, Target, TargetLog,
+    StoreError, Target,
     arn::TargetID,
     error::TargetError,
-    store::{Key, QueueStore, Store},
+    store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetType, queue_store_subdir_name,
+        TargetType, build_queued_payload_with_records, is_connectivity_error, open_target_queue_store,
+        persist_queued_payload_to_store,
     },
 };
 use async_trait::async_trait;
@@ -39,11 +40,11 @@ use rustfs_config::{AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY};
 use serde::Serialize;
 use serde::de::DeserializeOwned;
 use std::fmt;
-use std::path::{Path, PathBuf};
+use std::path::Path;
 use std::sync::Arc;
 use std::time::Duration;
 use tokio::sync::Mutex as AsyncMutex;
-use tracing::{error, info, instrument, warn};
+use tracing::{info, instrument, warn};
 use url::Url;
 
 #[derive(Clone)]
@@ -315,22 +316,14 @@ where
     pub fn new(id: String, args: AMQPArgs) -> Result<Self, TargetError> {
         args.validate()?;
         let target_id = TargetID::new(id, ChannelTargetType::Amqp.as_str().to_string());
-        let queue_store = if !args.queue_dir.is_empty() {
-            let base_path = PathBuf::from(&args.queue_dir);
-            let specific_queue_path = base_path.join(queue_store_subdir_name(ChannelTargetType::Amqp.as_str(), &target_id.id));
-            let extension = match args.target_type {
-                TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION,
-                TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION,
-            };
-            let store = QueueStore::<QueuedPayload>::new(specific_queue_path, args.queue_limit, extension);
-            if let Err(e) = store.open() {
-                error!(target_id = %target_id, error = %e, "Failed to open store for AMQP target");
-                return Err(TargetError::Storage(format!("{e}")));
-            }
-            Some(Box::new(store) as Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>)
-        } else {
-            None
-        };
+        let queue_store = open_target_queue_store(
+            &args.queue_dir,
+            args.queue_limit,
+            args.target_type,
+            ChannelTargetType::Amqp.as_str(),
+            &target_id,
+            "Failed to open store for AMQP target",
+        )?;
 
         Ok(Self {
             id: target_id,
@@ -344,22 +337,7 @@ where
     }
 
     fn build_queued_payload(&self, event: &EntityTarget<E>) -> Result<QueuedPayload, TargetError> {
-        let object_name = crate::target::decode_object_name(&event.object_name)?;
-        let key = format!("{}/{}", event.bucket_name, object_name);
-        let log = TargetLog {
-            event_name: event.event_name,
-            key,
-            records: vec![event.clone()],
-        };
-        let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?;
-        let meta = QueuedPayloadMeta::new(
-            event.event_name,
-            event.bucket_name.clone(),
-            event.object_name.clone(),
-            "application/json",
-            body.len(),
-        );
-        Ok(QueuedPayload::new(meta, body))
+        build_queued_payload_with_records(event, vec![event.clone()])
     }
 
     async fn get_or_connect(&self) -> Result<Arc<AMQPConnection>, TargetError> {
@@ -453,16 +431,9 @@ where
         };
 
         if let Some(store) = &self.store {
-            let encoded = match queued.encode() {
-                Ok(encoded) => encoded,
-                Err(err) => {
-                    self.delivery_counters.record_final_failure();
-                    return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}")));
-                }
-            };
-            if let Err(e) = store.put_raw(&encoded) {
+            if let Err(e) = persist_queued_payload_to_store(store.as_ref(), &queued) {
                 self.delivery_counters.record_final_failure();
-                return Err(TargetError::Storage(format!("Failed to save event to store: {e}")));
+                return Err(e);
             }
             Ok(())
         } else {
@@ -505,10 +476,7 @@ where
         }
         match self.get_or_connect().await {
             Ok(_) => Ok(()),
-            Err(err)
-                if self.store.is_some()
-                    && matches!(err, TargetError::Network(_) | TargetError::Timeout(_) | TargetError::NotConnected) =>
-            {
+            Err(err) if self.store.is_some() && is_connectivity_error(&err) => {
                 warn!(target_id = %self.id, error = %err, "AMQP init failed; events will buffer in store");
                 Ok(())
             }
@@ -535,6 +503,7 @@ mod tests {
     use super::*;
     use rustfs_s3_common::EventName;
     use serde_json::json;
+    use std::path::PathBuf;
     use std::sync::Arc;
     use uuid::Uuid;
 

crates/targets/src/target/kafka.rs

@@ -13,23 +13,22 @@
 // limitations under the License.
 
 use crate::{
-    StoreError, Target, TargetLog,
+    StoreError, Target,
     arn::TargetID,
     error::TargetError,
-    store::{Key, QueueStore, Store},
+    store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetType, queue_store_subdir_name,
+        TargetType, build_queued_payload, invalidate_cache_on_connectivity_error, open_target_queue_store,
+        persist_queued_payload_to_store,
     },
 };
 use async_trait::async_trait;
-use rustfs_config::audit::AUDIT_STORE_EXTENSION;
-use rustfs_config::notify::NOTIFY_STORE_EXTENSION;
 use rustfs_kafka_async::error::{ConnectionError, Error as KafkaError};
 use rustfs_kafka_async::{AsyncProducer, AsyncProducerConfig, Record, RequiredAcks, SecurityConfig};
 use serde::Serialize;
 use serde::de::DeserializeOwned;
-use std::{marker::PhantomData, path::PathBuf, sync::Arc, time::Duration};
+use std::{marker::PhantomData, sync::Arc, time::Duration};
 use tokio::sync::Mutex;
 use tracing::{debug, error, info, instrument, warn};
 
@@ -123,10 +122,6 @@ where
         }
     }
 
-    fn is_connection_error(err: &TargetError) -> bool {
-        matches!(err, TargetError::NotConnected | TargetError::Timeout(_) | TargetError::Network(_))
-    }
-
     /// Creates a new KafkaTarget
     #[instrument(skip(args), fields(target_id = %id))]
     pub fn new(id: String, args: KafkaArgs) -> Result<Self, TargetError> {
@@ -134,25 +129,14 @@ where
 
         let target_id = TargetID::new(id, ChannelTargetType::Kafka.as_str().to_string());
 
-        let queue_store = if !args.queue_dir.is_empty() {
-            let queue_dir =
-                PathBuf::from(&args.queue_dir).join(queue_store_subdir_name(ChannelTargetType::Kafka.as_str(), &target_id.id));
-
-            let extension = match args.target_type {
-                TargetType::AuditLog => AUDIT_STORE_EXTENSION,
-                TargetType::NotifyEvent => NOTIFY_STORE_EXTENSION,
-            };
-
-            let store = QueueStore::<QueuedPayload>::new(queue_dir, args.queue_limit, extension);
-            if let Err(e) = store.open() {
-                error!("Failed to open store for Kafka target {}: {}", target_id.id, e);
-                return Err(TargetError::Storage(format!("{e}")));
-            }
-
-            Some(Box::new(store) as Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>)
-        } else {
-            None
-        };
+        let queue_store = open_target_queue_store(
+            &args.queue_dir,
+            args.queue_limit,
+            args.target_type,
+            ChannelTargetType::Kafka.as_str(),
+            &target_id,
+            "Failed to open store for Kafka target",
+        )?;
 
         info!(target_id = %target_id.id, "Kafka target created");
         Ok(KafkaTarget {
@@ -211,26 +195,7 @@ where
 
     /// Serializes the event and builds a QueuedPayload
     fn build_queued_payload(&self, event: &EntityTarget<E>) -> Result<QueuedPayload, TargetError> {
-        let object_name = crate::target::decode_object_name(&event.object_name)?;
-        let key = format!("{}/{}", event.bucket_name, object_name);
-
-        let log = TargetLog {
-            event_name: event.event_name,
-            key,
-            records: vec![event.data.clone()],
-        };
-
-        let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?;
-
-        let meta = QueuedPayloadMeta::new(
-            event.event_name,
-            event.bucket_name.clone(),
-            event.object_name.clone(),
-            "application/json",
-            body.len(),
-        );
-
-        Ok(QueuedPayload::new(meta, body))
+        build_queued_payload(event)
     }
 
     /// Sends the raw body to Kafka
@@ -249,9 +214,7 @@ where
 
         if let Err(err) = producer.send(&Record::from_value(&self.args.topic, body.as_slice())).await {
             let mapped = Self::map_kafka_error(err, "Failed to send message to Kafka");
-            if Self::is_connection_error(&mapped) {
-                self.invalidate_cached_producer().await;
-            }
+            invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_producer()).await;
             return Err(mapped);
         }
 
@@ -297,16 +260,9 @@ where
         };
 
         if let Some(store) = &self.store {
-            let encoded = match queued.encode() {
-                Ok(encoded) => encoded,
-                Err(err) => {
-                    self.delivery_counters.record_final_failure();
-                    return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}")));
-                }
-            };
-            if let Err(e) = store.put_raw(&encoded) {
+            if let Err(e) = persist_queued_payload_to_store(store.as_ref(), &queued) {
                 self.delivery_counters.record_final_failure();
-                return Err(TargetError::Storage(format!("Failed to save event to store: {e}")));
+                return Err(e);
             }
             debug!("Event saved to store for Kafka target: {}", self.id);
             Ok(())

crates/targets/src/target/mod.rs

@@ -13,15 +13,17 @@
 // limitations under the License.
 
 use crate::arn::TargetID;
-use crate::store::{Key, Store};
+use crate::store::{Key, QueueStore, Store};
 use crate::{StoreError, TargetError, TargetLog};
 use async_trait::async_trait;
 use rustfs_s3_common::EventName;
 use serde::de::DeserializeOwned;
 use serde::{Deserialize, Serialize};
 use std::fmt::Formatter;
+use std::future::Future;
+use std::path::PathBuf;
 use std::sync::Arc;
-use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::time::{SystemTime, UNIX_EPOCH};
 use tracing::{debug, warn};
 
@@ -50,6 +52,8 @@ pub struct TargetDeliveryCounters {
     total_messages: AtomicU64,
 }
 
+pub(crate) type BoxedQueuedStore = Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>;
+
 impl TargetDeliveryCounters {
     #[inline]
     pub fn record_success(&self) {
@@ -408,6 +412,17 @@ pub fn decode_object_name(encoded: &str) -> Result<String, TargetError> {
 pub(crate) fn build_queued_payload<E>(event: &EntityTarget<E>) -> Result<QueuedPayload, TargetError>
 where
     E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    build_queued_payload_with_records(event, vec![event.data.clone()])
+}
+
+pub(crate) fn build_queued_payload_with_records<E, R>(
+    event: &EntityTarget<E>,
+    records: Vec<R>,
+) -> Result<QueuedPayload, TargetError>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+    R: Serialize,
 {
     let object_name = decode_object_name(&event.object_name)?;
     let key = format!("{}/{}", event.bucket_name, object_name);
@@ -415,7 +430,7 @@ where
     let log = TargetLog {
         event_name: event.event_name,
         key,
-        records: vec![event.data.clone()],
+        records,
     };
 
     let body = serde_json::to_vec(&log).map_err(|err| TargetError::Serialization(format!("Failed to serialize event: {err}")))?;
@@ -430,6 +445,68 @@ where
     Ok(QueuedPayload::new(meta, body))
 }
 
+pub(crate) fn open_target_queue_store(
+    queue_dir: &str,
+    queue_limit: u64,
+    target_type: TargetType,
+    target_type_label: &str,
+    target_id: &TargetID,
+    open_context: &str,
+) -> Result<Option<BoxedQueuedStore>, TargetError> {
+    fn boxed_queue_store(store: QueueStore<QueuedPayload>) -> BoxedQueuedStore {
+        Box::new(store)
+    }
+
+    if queue_dir.is_empty() {
+        return Ok(None);
+    }
+
+    let queue_dir = PathBuf::from(queue_dir).join(queue_store_subdir_name(target_type_label, &target_id.id));
+    let extension = match target_type {
+        TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION,
+        TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION,
+    };
+    let store = QueueStore::<QueuedPayload>::new(queue_dir, queue_limit, extension);
+    store
+        .open()
+        .map_err(|err| TargetError::Storage(format!("{open_context}: {err}")))?;
+
+    Ok(Some(boxed_queue_store(store)))
+}
+
+pub(crate) fn persist_queued_payload_to_store(
+    store: &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync),
+    queued: &QueuedPayload,
+) -> Result<(), TargetError> {
+    let encoded = queued
+        .encode()
+        .map_err(|err| TargetError::Storage(format!("Failed to encode queued payload: {err}")))?;
+    store
+        .put_raw(&encoded)
+        .map(|_| ())
+        .map_err(|err| TargetError::Storage(format!("Failed to save event to store: {err}")))
+}
+
+pub(crate) fn is_connectivity_error(err: &TargetError) -> bool {
+    matches!(err, TargetError::NotConnected | TargetError::Timeout(_) | TargetError::Network(_))
+}
+
+pub(crate) async fn invalidate_cache_on_connectivity_error<F, Fut>(err: &TargetError, invalidate: F)
+where
+    F: FnOnce() -> Fut,
+    Fut: Future<Output = ()>,
+{
+    if is_connectivity_error(err) {
+        invalidate().await;
+    }
+}
+
+pub(crate) fn mark_target_disconnected_on_connectivity_error(connected: &AtomicBool, err: &TargetError) {
+    if is_connectivity_error(err) {
+        connected.store(false, Ordering::SeqCst);
+    }
+}
+
 pub(crate) fn delete_stored_payload(
     store: &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync),
     key: &Key,
@@ -457,6 +534,90 @@ pub(crate) fn ensure_rustls_provider_installed() {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::fs;
+    use std::sync::Mutex;
+    use uuid::Uuid;
+
+    #[derive(Clone)]
+    struct MockQueuedStore {
+        fail_put_raw: bool,
+        writes: Arc<Mutex<Vec<Vec<u8>>>>,
+    }
+
+    impl MockQueuedStore {
+        fn new(fail_put_raw: bool) -> Self {
+            Self {
+                fail_put_raw,
+                writes: Arc::new(Mutex::new(Vec::new())),
+            }
+        }
+    }
+
+    impl Store<QueuedPayload> for MockQueuedStore {
+        type Error = StoreError;
+        type Key = Key;
+
+        fn open(&self) -> Result<(), Self::Error> {
+            Ok(())
+        }
+
+        fn put(&self, _item: Arc<QueuedPayload>) -> Result<Self::Key, Self::Error> {
+            Err(StoreError::Internal("not implemented in mock".to_string()))
+        }
+
+        fn put_multiple(&self, _items: Vec<QueuedPayload>) -> Result<Self::Key, Self::Error> {
+            Err(StoreError::Internal("not implemented in mock".to_string()))
+        }
+
+        fn put_raw(&self, data: &[u8]) -> Result<Self::Key, Self::Error> {
+            if self.fail_put_raw {
+                return Err(StoreError::Internal("mock put_raw failed".to_string()));
+            }
+            self.writes.lock().expect("mock writes lock poisoned").push(data.to_vec());
+            Ok(Key {
+                name: "mock".to_string(),
+                extension: ".json".to_string(),
+                item_count: 1,
+                compress: false,
+            })
+        }
+
+        fn get(&self, _key: &Self::Key) -> Result<QueuedPayload, Self::Error> {
+            Err(StoreError::Internal("not implemented in mock".to_string()))
+        }
+
+        fn get_multiple(&self, _key: &Self::Key) -> Result<Vec<QueuedPayload>, Self::Error> {
+            Err(StoreError::Internal("not implemented in mock".to_string()))
+        }
+
+        fn get_raw(&self, _key: &Self::Key) -> Result<Vec<u8>, Self::Error> {
+            Err(StoreError::Internal("not implemented in mock".to_string()))
+        }
+
+        fn del(&self, _key: &Self::Key) -> Result<(), Self::Error> {
+            Err(StoreError::Internal("not implemented in mock".to_string()))
+        }
+
+        fn delete(&self) -> Result<(), Self::Error> {
+            Err(StoreError::Internal("not implemented in mock".to_string()))
+        }
+
+        fn list(&self) -> Vec<Self::Key> {
+            Vec::new()
+        }
+
+        fn len(&self) -> usize {
+            0
+        }
+
+        fn is_empty(&self) -> bool {
+            true
+        }
+
+        fn boxed_clone(&self) -> Box<dyn Store<QueuedPayload, Error = Self::Error, Key = Self::Key> + Send + Sync> {
+            Box::new(self.clone())
+        }
+    }
 
     #[test]
     fn channel_target_type_amqp_uses_runtime_name() {
@@ -501,6 +662,139 @@ mod tests {
         assert_eq!(value["Records"][0], "payload-data");
     }
 
+    #[test]
+    fn build_queued_payload_with_records_preserves_custom_record_shape() {
+        let event = EntityTarget {
+            object_name: "object.txt".to_string(),
+            bucket_name: "bucket-a".to_string(),
+            event_name: EventName::ObjectCreatedPut,
+            data: "ignored".to_string(),
+        };
+
+        let payload = build_queued_payload_with_records(&event, vec![event.clone()]).unwrap();
+        let value: serde_json::Value = serde_json::from_slice(&payload.body).unwrap();
+
+        assert_eq!(value["Records"][0]["bucket_name"], "bucket-a");
+        assert_eq!(value["Records"][0]["object_name"], "object.txt");
+        assert_eq!(value["Records"][0]["data"], "ignored");
+    }
+
+    #[test]
+    fn open_target_queue_store_returns_none_when_queue_dir_empty() {
+        let target_id = TargetID::new("target-a".to_string(), ChannelTargetType::Webhook.as_str().to_string());
+        let store = open_target_queue_store(
+            "",
+            100,
+            TargetType::NotifyEvent,
+            ChannelTargetType::Webhook.as_str(),
+            &target_id,
+            "open failed",
+        )
+        .unwrap();
+        assert!(store.is_none());
+    }
+
+    #[test]
+    fn open_target_queue_store_adds_context_on_open_error() {
+        let base = std::env::temp_dir().join(format!("rustfs-target-store-file-{}", Uuid::new_v4()));
+        fs::write(&base, b"not-a-directory").expect("failed to create file base");
+        let target_id = TargetID::new("target-a".to_string(), ChannelTargetType::Kafka.as_str().to_string());
+
+        let result = open_target_queue_store(
+            base.to_str().unwrap(),
+            100,
+            TargetType::NotifyEvent,
+            ChannelTargetType::Kafka.as_str(),
+            &target_id,
+            "custom open context",
+        );
+
+        match result {
+            Ok(_) => panic!("expected open_target_queue_store to fail on file base path"),
+            Err(err) => assert!(err.to_string().contains("custom open context")),
+        }
+        let _ = fs::remove_file(base);
+    }
+
+    #[test]
+    fn persist_queued_payload_to_store_writes_encoded_payload() {
+        let store = MockQueuedStore::new(false);
+        let meta = QueuedPayloadMeta::new(
+            EventName::ObjectCreatedPut,
+            "bucket-a".to_string(),
+            "obj.txt".to_string(),
+            "application/json",
+            7,
+        );
+        let queued = QueuedPayload::new(meta, br#"{"x":1}"#.to_vec());
+
+        persist_queued_payload_to_store(&store, &queued).unwrap();
+
+        let writes = store.writes.lock().expect("mock writes lock poisoned");
+        assert_eq!(writes.len(), 1);
+        let decoded = QueuedPayload::decode(&writes[0]).unwrap();
+        assert_eq!(decoded.body, br#"{"x":1}"#);
+    }
+
+    #[test]
+    fn persist_queued_payload_to_store_maps_store_error() {
+        let store = MockQueuedStore::new(true);
+        let meta = QueuedPayloadMeta::new(
+            EventName::ObjectCreatedPut,
+            "bucket-a".to_string(),
+            "obj.txt".to_string(),
+            "application/json",
+            7,
+        );
+        let queued = QueuedPayload::new(meta, br#"{"x":1}"#.to_vec());
+
+        let err = persist_queued_payload_to_store(&store, &queued).expect_err("expected put_raw failure");
+        assert!(err.to_string().contains("Failed to save event to store"));
+    }
+
+    #[test]
+    fn is_connectivity_error_classifies_target_errors() {
+        assert!(is_connectivity_error(&TargetError::NotConnected));
+        assert!(is_connectivity_error(&TargetError::Timeout("timeout".to_string())));
+        assert!(is_connectivity_error(&TargetError::Network("network".to_string())));
+        assert!(!is_connectivity_error(&TargetError::Storage("storage".to_string())));
+        assert!(!is_connectivity_error(&TargetError::Serialization("serialization".to_string())));
+    }
+
+    #[tokio::test]
+    async fn invalidate_cache_on_connectivity_error_only_runs_for_connectivity_failures() {
+        let marker = Arc::new(AtomicBool::new(false));
+        invalidate_cache_on_connectivity_error(&TargetError::NotConnected, {
+            let marker = Arc::clone(&marker);
+            move || async move {
+                marker.store(true, Ordering::SeqCst);
+            }
+        })
+        .await;
+        assert!(marker.load(Ordering::SeqCst));
+
+        marker.store(false, Ordering::SeqCst);
+        invalidate_cache_on_connectivity_error(&TargetError::Request("request failed".to_string()), {
+            let marker = Arc::clone(&marker);
+            move || async move {
+                marker.store(true, Ordering::SeqCst);
+            }
+        })
+        .await;
+        assert!(!marker.load(Ordering::SeqCst));
+    }
+
+    #[test]
+    fn mark_target_disconnected_on_connectivity_error_only_marks_connectivity_failures() {
+        let connected = AtomicBool::new(true);
+        mark_target_disconnected_on_connectivity_error(&connected, &TargetError::Timeout("timeout".to_string()));
+        assert!(!connected.load(Ordering::SeqCst));
+
+        connected.store(true, Ordering::SeqCst);
+        mark_target_disconnected_on_connectivity_error(&connected, &TargetError::Request("request failed".to_string()));
+        assert!(connected.load(Ordering::SeqCst));
+    }
+
     #[test]
     fn queued_payload_decode_rejects_invalid_magic() {
         let err = QueuedPayload::decode(b"bad-payload").unwrap_err();

crates/targets/src/target/mqtt.rs

@@ -13,13 +13,14 @@
 // limitations under the License.
 
 use crate::{
-    StoreError, Target, TargetLog,
+    StoreError, Target,
     arn::TargetID,
     error::TargetError,
-    store::{Key, QueueStore, Store},
+    store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetType, queue_store_subdir_name,
+        TargetType, build_queued_payload_with_records, mark_target_disconnected_on_connectivity_error, open_target_queue_store,
+        persist_queued_payload_to_store,
     },
 };
 use async_trait::async_trait;
@@ -37,7 +38,7 @@ use serde::de::DeserializeOwned;
 use std::sync::Arc;
 use std::{
     marker::PhantomData,
-    path::{Path, PathBuf},
+    path::Path,
     sync::atomic::{AtomicBool, Ordering},
     time::Duration,
 };
@@ -502,30 +503,14 @@ where
     pub fn new(id: String, args: MQTTArgs) -> Result<Self, TargetError> {
         args.validate()?;
         let target_id = TargetID::new(id, ChannelTargetType::Mqtt.as_str().to_string());
-        let queue_store = if !args.queue_dir.is_empty() {
-            let base_path = PathBuf::from(&args.queue_dir);
-            let unique_dir_name = queue_store_subdir_name(ChannelTargetType::Mqtt.as_str(), &target_id.id);
-            // Ensure the directory name is valid for filesystem
-            let specific_queue_path = base_path.join(unique_dir_name);
-            debug!(target_id = %target_id, path = %specific_queue_path.display(), "Initializing queue store for MQTT target");
-            let extension = match args.target_type {
-                TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION,
-                TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION,
-            };
-
-            let store = QueueStore::<QueuedPayload>::new(specific_queue_path, args.queue_limit, extension);
-            if let Err(e) = store.open() {
-                error!(
-                    target_id = %target_id,
-                    error = %e,
-                    "Failed to open store for MQTT target"
-                );
-                return Err(TargetError::Storage(format!("{e}")));
-            }
-            Some(Box::new(store) as Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>)
-        } else {
-            None
-        };
+        let queue_store = open_target_queue_store(
+            &args.queue_dir,
+            args.queue_limit,
+            args.target_type,
+            ChannelTargetType::Mqtt.as_str(),
+            &target_id,
+            "Failed to open store for MQTT target",
+        )?;
 
         let (cancel_tx, cancel_rx) = mpsc::channel(1);
         let bg_task_manager = Arc::new(BgTaskManager {
@@ -631,25 +616,7 @@ where
     }
 
     fn build_queued_payload(&self, event: &EntityTarget<E>) -> Result<QueuedPayload, TargetError> {
-        let object_name = crate::target::decode_object_name(&event.object_name)?;
-
-        let key = format!("{}/{}", event.bucket_name, object_name);
-
-        let log = TargetLog {
-            event_name: event.event_name,
-            key,
-            records: vec![event.clone()],
-        };
-
-        let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?;
-        let meta = QueuedPayloadMeta::new(
-            event.event_name,
-            event.bucket_name.clone(),
-            event.object_name.clone(),
-            "application/json",
-            body.len(),
-        );
-        Ok(QueuedPayload::new(meta, body))
+        build_queued_payload_with_records(event, vec![event.clone()])
     }
 
     #[instrument(skip(self, body, meta), fields(target_id = %self.id))]
@@ -673,9 +640,10 @@ where
             .await
             .map_err(|e| {
                 if e.to_string().contains("Connection") || e.to_string().contains("Timeout") {
-                    self.connected.store(false, Ordering::SeqCst);
                     warn!(target_id = %self.id, error = %e, "Publish failed due to connection issue, marking as not connected.");
-                    TargetError::NotConnected
+                    let err = TargetError::NotConnected;
+                    mark_target_disconnected_on_connectivity_error(&self.connected, &err);
+                    err
                 } else {
                     TargetError::Request(format!("Failed to publish message: {e}"))
                 }
@@ -899,14 +867,7 @@ where
 
         if let Some(store) = &self.store {
             debug!(target_id = %self.id, "Event saved to store start");
-            let encoded = match queued.encode() {
-                Ok(encoded) => encoded,
-                Err(err) => {
-                    self.delivery_counters.record_final_failure();
-                    return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}")));
-                }
-            };
-            match store.put_raw(&encoded) {
+            match persist_queued_payload_to_store(store.as_ref(), &queued) {
                 Ok(_) => {
                     debug!(target_id = %self.id, "Event saved to store for MQTT target successfully.");
                     Ok(())
@@ -914,7 +875,7 @@ where
                 Err(e) => {
                     error!(target_id = %self.id, error = %e, "Failed to save event to store");
                     self.delivery_counters.record_final_failure();
-                    Err(TargetError::Storage(format!("Failed to save event to store: {e}")))
+                    Err(e)
                 }
             }
         } else {

crates/targets/src/target/mysql.rs

@@ -16,15 +16,15 @@ use crate::{
     StoreError, Target,
     arn::TargetID,
     error::TargetError,
-    store::{Key, QueueStore, Store},
+    store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetType, build_queued_payload, delete_stored_payload, queue_store_subdir_name,
+        TargetType, build_queued_payload, delete_stored_payload, is_connectivity_error, open_target_queue_store,
+        persist_queued_payload_to_store,
     },
 };
 use async_trait::async_trait;
 use mysql_async::{Conn, Opts, OptsBuilder, Pool, PoolConstraints, PoolOpts, SslOpts, prelude::Queryable};
-use rustfs_config::notify::NOTIFY_STORE_EXTENSION;
 use rustfs_config::{MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY};
 use serde::Serialize;
 use serde::de::DeserializeOwned;
@@ -494,25 +494,14 @@ where
 
         let target_id = TargetID::new(id, ChannelTargetType::MySql.as_str().to_string());
 
-        // If `queue_dir` is non-empty, a `QueueStore` is created for persistent at-least-once delivery.
-        let queue_store = if !args.queue_dir.is_empty() {
-            let queue_dir =
-                PathBuf::from(&args.queue_dir).join(queue_store_subdir_name(ChannelTargetType::MySql.as_str(), &target_id.id));
-
-            let extension = match args.target_type {
-                TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION,
-                TargetType::NotifyEvent => NOTIFY_STORE_EXTENSION,
-            };
-
-            let store = QueueStore::<QueuedPayload>::new(queue_dir, args.queue_limit, extension);
-            if let Err(e) = store.open() {
-                return Err(TargetError::Storage(format!("Failed to open MySQL queue store: {e}")));
-            }
-
-            Some(Box::new(store) as Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>)
-        } else {
-            None
-        };
+        let queue_store = open_target_queue_store(
+            &args.queue_dir,
+            args.queue_limit,
+            args.target_type,
+            ChannelTargetType::MySql.as_str(),
+            &target_id,
+            "Failed to open MySQL queue store",
+        )?;
 
         info!(target_id = %target_id.id, table = %args.table, "MySQL target created");
 
@@ -733,18 +722,9 @@ where
         };
 
         if let Some(store) = &self.store {
-            // persist the event to a local queue before attempting to insert into MySQL. This will allow us to guarantee at-least-once delivery even if the database is temporarily unreachable or if the process crashes after acknowledging receipt but before writing to the database.
-            let encoded = match queued.encode() {
-                Ok(encoded) => encoded,
-                Err(err) => {
-                    self.delivery_counters.record_final_failure();
-                    return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}")));
-                }
-            };
-
-            if let Err(e) = store.put_raw(&encoded) {
+            if let Err(e) = persist_queued_payload_to_store(store.as_ref(), &queued) {
                 self.delivery_counters.record_final_failure();
-                return Err(TargetError::Storage(format!("Failed to save event to store: {e}")));
+                return Err(e);
             }
 
             debug!("Event saved to queue store for MySQL target: {}", self.id);
@@ -789,12 +769,8 @@ where
         }
 
         if let Err(e) = self.insert_event(&body, &meta).await {
-            if matches!(e, TargetError::NotConnected) {
+            if is_connectivity_error(&e) {
                 warn!(target_id = %self.id, "MySQL not reachable, event remains in queue store");
-                return Err(TargetError::NotConnected);
-            }
-            if matches!(e, TargetError::Timeout(_)) {
-                warn!(target_id = %self.id, "MySQL timeout, event remains in queue store");
                 return Err(e);
             }
             error!(target_id = %self.id, error = %e, "Failed to send event from store");

crates/targets/src/target/nats.rs

@@ -13,13 +13,13 @@
 // limitations under the License.
 
 use crate::{
-    StoreError, Target, TargetLog,
+    StoreError, Target,
     arn::TargetID,
     error::TargetError,
-    store::{Key, QueueStore, Store},
+    store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetType, queue_store_subdir_name,
+        TargetType, build_queued_payload_with_records, open_target_queue_store, persist_queued_payload_to_store,
     },
 };
 use async_trait::async_trait;
@@ -30,7 +30,7 @@ use std::path::{Path, PathBuf};
 use std::str::FromStr;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
-use tracing::{error, info, instrument};
+use tracing::{info, instrument};
 
 #[derive(Debug, Clone)]
 pub struct NATSArgs {
@@ -195,22 +195,14 @@ where
     pub fn new(id: String, args: NATSArgs) -> Result<Self, TargetError> {
         args.validate()?;
         let target_id = TargetID::new(id, ChannelTargetType::Nats.as_str().to_string());
-        let queue_store = if !args.queue_dir.is_empty() {
-            let base_path = PathBuf::from(&args.queue_dir);
-            let specific_queue_path = base_path.join(queue_store_subdir_name(ChannelTargetType::Nats.as_str(), &target_id.id));
-            let extension = match args.target_type {
-                TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION,
-                TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION,
-            };
-            let store = QueueStore::<QueuedPayload>::new(specific_queue_path, args.queue_limit, extension);
-            if let Err(e) = store.open() {
-                error!(target_id = %target_id, error = %e, "Failed to open store for NATS target");
-                return Err(TargetError::Storage(format!("{e}")));
-            }
-            Some(Box::new(store) as Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>)
-        } else {
-            None
-        };
+        let queue_store = open_target_queue_store(
+            &args.queue_dir,
+            args.queue_limit,
+            args.target_type,
+            ChannelTargetType::Nats.as_str(),
+            &target_id,
+            "Failed to open store for NATS target",
+        )?;
 
         Ok(Self {
             id: target_id,
@@ -241,22 +233,7 @@ where
     }
 
     fn build_queued_payload(&self, event: &EntityTarget<E>) -> Result<QueuedPayload, TargetError> {
-        let object_name = crate::target::decode_object_name(&event.object_name)?;
-        let key = format!("{}/{}", event.bucket_name, object_name);
-        let log = TargetLog {
-            event_name: event.event_name,
-            key,
-            records: vec![event.clone()],
-        };
-        let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?;
-        let meta = QueuedPayloadMeta::new(
-            event.event_name,
-            event.bucket_name.clone(),
-            event.object_name.clone(),
-            "application/json",
-            body.len(),
-        );
-        Ok(QueuedPayload::new(meta, body))
+        build_queued_payload_with_records(event, vec![event.clone()])
     }
 
     async fn send_body(&self, body: Vec<u8>) -> Result<(), TargetError> {
@@ -298,16 +275,9 @@ where
         };
 
         if let Some(store) = &self.store {
-            let encoded = match queued.encode() {
-                Ok(encoded) => encoded,
-                Err(err) => {
-                    self.delivery_counters.record_final_failure();
-                    return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}")));
-                }
-            };
-            if let Err(e) = store.put_raw(&encoded) {
+            if let Err(e) = persist_queued_payload_to_store(store.as_ref(), &queued) {
                 self.delivery_counters.record_final_failure();
-                return Err(TargetError::Storage(format!("Failed to save event to store: {e}")));
+                return Err(e);
             }
             Ok(())
         } else {

crates/targets/src/target/postgres.rs

@@ -29,10 +29,10 @@ use crate::{
     StoreError, Target,
     arn::TargetID,
     error::TargetError,
-    store::{Key, QueueStore, Store},
+    store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetType, build_queued_payload, queue_store_subdir_name,
+        TargetType, build_queued_payload, open_target_queue_store, persist_queued_payload_to_store,
     },
 };
 use async_trait::async_trait;
@@ -44,11 +44,11 @@ use serde::Serialize;
 use serde::de::DeserializeOwned;
 use std::fmt;
 use std::io::BufReader;
-use std::path::{Path, PathBuf};
+use std::path::Path;
 use std::sync::Arc;
 use tokio_postgres::Config;
 use tokio_postgres_rustls::MakeRustlsConnect;
-use tracing::{error, info, instrument, warn};
+use tracing::{info, instrument, warn};
 use url::Url;
 use uuid::Uuid;
 
@@ -585,23 +585,14 @@ where
         let target_id = TargetID::new(id, ChannelTargetType::Postgres.as_str().to_string());
         let pool = build_pool(&args)?;
 
-        let queue_store = if !args.queue_dir.is_empty() {
-            let base_path = PathBuf::from(&args.queue_dir);
-            let specific_queue_path =
-                base_path.join(queue_store_subdir_name(ChannelTargetType::Postgres.as_str(), &target_id.id));
-            let extension = match args.target_type {
-                TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION,
-                TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION,
-            };
-            let store = QueueStore::<QueuedPayload>::new(specific_queue_path, args.queue_limit, extension);
-            if let Err(e) = store.open() {
-                error!(target_id = %target_id, error = %e, "Failed to open store for PostgreSQL target");
-                return Err(TargetError::Storage(format!("{e}")));
-            }
-            Some(Box::new(store) as Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>)
-        } else {
-            None
-        };
+        let queue_store = open_target_queue_store(
+            &args.queue_dir,
+            args.queue_limit,
+            args.target_type,
+            ChannelTargetType::Postgres.as_str(),
+            &target_id,
+            "Failed to open store for PostgreSQL target",
+        )?;
 
         Ok(Self {
             id: target_id,
@@ -712,16 +703,9 @@ where
         };
 
         if let Some(store) = &self.store {
-            let encoded = match queued.encode() {
-                Ok(encoded) => encoded,
-                Err(err) => {
-                    self.delivery_counters.record_final_failure();
-                    return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}")));
-                }
-            };
-            if let Err(e) = store.put_raw(&encoded) {
+            if let Err(e) = persist_queued_payload_to_store(store.as_ref(), &queued) {
                 self.delivery_counters.record_final_failure();
-                return Err(TargetError::Storage(format!("Failed to save event to store: {e}")));
+                return Err(e);
             }
             Ok(())
         } else {

crates/targets/src/target/pulsar.rs

@@ -13,24 +13,24 @@
 // limitations under the License.
 
 use crate::{
-    StoreError, Target, TargetLog,
+    StoreError, Target,
     arn::TargetID,
     error::TargetError,
-    store::{Key, QueueStore, Store},
+    store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetType, queue_store_subdir_name,
+        TargetType, build_queued_payload_with_records, open_target_queue_store, persist_queued_payload_to_store,
     },
 };
 use async_trait::async_trait;
 use pulsar::{Authentication, Producer, Pulsar, TokioExecutor};
 use serde::Serialize;
 use serde::de::DeserializeOwned;
-use std::path::{Path, PathBuf};
+use std::path::Path;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
 use tokio::sync::Mutex as AsyncMutex;
-use tracing::{error, info, instrument};
+use tracing::{info, instrument};
 use url::Url;
 
 #[derive(Debug, Clone)]
@@ -186,22 +186,14 @@ where
     pub fn new(id: String, args: PulsarArgs) -> Result<Self, TargetError> {
         args.validate()?;
         let target_id = TargetID::new(id, ChannelTargetType::Pulsar.as_str().to_string());
-        let queue_store = if !args.queue_dir.is_empty() {
-            let base_path = PathBuf::from(&args.queue_dir);
-            let specific_queue_path = base_path.join(queue_store_subdir_name(ChannelTargetType::Pulsar.as_str(), &target_id.id));
-            let extension = match args.target_type {
-                TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION,
-                TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION,
-            };
-            let store = QueueStore::<QueuedPayload>::new(specific_queue_path, args.queue_limit, extension);
-            if let Err(e) = store.open() {
-                error!(target_id = %target_id, error = %e, "Failed to open store for Pulsar target");
-                return Err(TargetError::Storage(format!("{e}")));
-            }
-            Some(Box::new(store) as Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>)
-        } else {
-            None
-        };
+        let queue_store = open_target_queue_store(
+            &args.queue_dir,
+            args.queue_limit,
+            args.target_type,
+            ChannelTargetType::Pulsar.as_str(),
+            &target_id,
+            "Failed to open store for Pulsar target",
+        )?;
 
         Ok(Self {
             id: target_id,
@@ -249,22 +241,7 @@ where
     }
 
     fn build_queued_payload(&self, event: &EntityTarget<E>) -> Result<QueuedPayload, TargetError> {
-        let object_name = crate::target::decode_object_name(&event.object_name)?;
-        let key = format!("{}/{}", event.bucket_name, object_name);
-        let log = TargetLog {
-            event_name: event.event_name,
-            key,
-            records: vec![event.clone()],
-        };
-        let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?;
-        let meta = QueuedPayloadMeta::new(
-            event.event_name,
-            event.bucket_name.clone(),
-            event.object_name.clone(),
-            "application/json",
-            body.len(),
-        );
-        Ok(QueuedPayload::new(meta, body))
+        build_queued_payload_with_records(event, vec![event.clone()])
     }
 
     async fn send_body(&self, body: Vec<u8>) -> Result<(), TargetError> {
@@ -317,16 +294,9 @@ where
         };
 
         if let Some(store) = &self.store {
-            let encoded = match queued.encode() {
-                Ok(encoded) => encoded,
-                Err(err) => {
-                    self.delivery_counters.record_final_failure();
-                    return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}")));
-                }
-            };
-            if let Err(e) = store.put_raw(&encoded) {
+            if let Err(e) = persist_queued_payload_to_store(store.as_ref(), &queued) {
                 self.delivery_counters.record_final_failure();
-                return Err(TargetError::Storage(format!("Failed to save event to store: {e}")));
+                return Err(e);
             }
             Ok(())
         } else {

crates/targets/src/target/redis.rs

@@ -16,10 +16,11 @@ use crate::{
     StoreError, Target,
     arn::TargetID,
     error::TargetError,
-    store::{Key, QueueStore, Store},
+    store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetType, build_queued_payload, queue_store_subdir_name,
+        TargetType, build_queued_payload, invalidate_cache_on_connectivity_error, is_connectivity_error,
+        mark_target_disconnected_on_connectivity_error, open_target_queue_store, persist_queued_payload_to_store,
     },
 };
 use async_trait::async_trait;
@@ -33,12 +34,12 @@ use rustfs_config::{REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, R
 use serde::Serialize;
 use serde::de::DeserializeOwned;
 use std::fmt;
-use std::path::{Path, PathBuf};
+use std::path::Path;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::time::Duration;
 use tokio::sync::Mutex;
-use tracing::{debug, error, info, instrument, warn};
+use tracing::{debug, info, instrument, warn};
 use url::Url;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -320,22 +321,14 @@ where
         let target_id = TargetID::new(id, ChannelTargetType::Redis.as_str().to_string());
         let publisher_client = build_redis_client(&args)?;
 
-        let queue_store = if !args.queue_dir.is_empty() {
-            let base_path = PathBuf::from(&args.queue_dir);
-            let specific_queue_path = base_path.join(queue_store_subdir_name(ChannelTargetType::Redis.as_str(), &target_id.id));
-            let extension = match args.target_type {
-                TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION,
-                TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION,
-            };
-            let store = QueueStore::<QueuedPayload>::new(specific_queue_path, args.queue_limit, extension);
-            if let Err(e) = store.open() {
-                error!(target_id = %target_id, error = %e, "Failed to open store for Redis target");
-                return Err(TargetError::Storage(format!("{e}")));
-            }
-            Some(Box::new(store) as Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>)
-        } else {
-            None
-        };
+        let queue_store = open_target_queue_store(
+            &args.queue_dir,
+            args.queue_limit,
+            args.target_type,
+            ChannelTargetType::Redis.as_str(),
+            &target_id,
+            "Failed to open store for Redis target",
+        )?;
 
         info!(target_id = %target_id, "Redis target created");
         Ok(Self {
@@ -391,9 +384,7 @@ where
             Ok(_) => Ok(()),
             Err(err) => {
                 let mapped = map_redis_error(err);
-                if is_retryable_target_error(&mapped) {
-                    self.invalidate_cached_publisher().await;
-                }
+                invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_publisher()).await;
                 Err(mapped)
             }
         }
@@ -437,9 +428,7 @@ where
                 }
                 Err(err) => {
                     let mapped = map_redis_error(err);
-                    if is_retryable_target_error(&mapped) {
-                        self.invalidate_cached_publisher().await;
-                    }
+                    invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_publisher()).await;
 
                     warn!(
                         target_id = %self.id,
@@ -450,7 +439,7 @@ where
                         "Redis publish attempt failed"
                     );
 
-                    if !is_retryable_target_error(&mapped) || attempt >= self.args.max_retry_attempts {
+                    if !is_connectivity_error(&mapped) || attempt >= self.args.max_retry_attempts {
                         last_error = Some(mapped);
                         break;
                     }
@@ -492,14 +481,15 @@ where
                 Ok(true)
             }
             Ok(Err(err)) => {
-                self.invalidate_cached_publisher().await;
-                self.connected.store(false, Ordering::SeqCst);
+                invalidate_cache_on_connectivity_error(&err, || self.invalidate_cached_publisher()).await;
+                mark_target_disconnected_on_connectivity_error(&self.connected, &err);
                 Err(err)
             }
             Err(_) => {
-                self.invalidate_cached_publisher().await;
-                self.connected.store(false, Ordering::SeqCst);
-                Err(TargetError::Timeout("Redis connection timed out".to_string()))
+                let timeout_err = TargetError::Timeout("Redis connection timed out".to_string());
+                invalidate_cache_on_connectivity_error(&timeout_err, || self.invalidate_cached_publisher()).await;
+                mark_target_disconnected_on_connectivity_error(&self.connected, &timeout_err);
+                Err(timeout_err)
             }
         }
     }
@@ -514,17 +504,9 @@ where
         };
 
         if let Some(store) = &self.store {
-            let encoded = match queued.encode() {
-                Ok(encoded) => encoded,
-                Err(err) => {
-                    self.delivery_counters.record_final_failure();
-                    return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}")));
-                }
-            };
-
-            if let Err(e) = store.put_raw(&encoded) {
+            if let Err(e) = persist_queued_payload_to_store(store.as_ref(), &queued) {
                 self.delivery_counters.record_final_failure();
-                return Err(TargetError::Storage(format!("Failed to save event to store: {e}")));
+                return Err(e);
             }
 
             debug!(target_id = %self.id, "Event saved to store for Redis target");
@@ -556,14 +538,14 @@ where
         }
 
         if let Err(err) = self.init_inner().await {
-            if matches!(err, TargetError::NotConnected | TargetError::Timeout(_) | TargetError::Network(_)) {
+            if is_connectivity_error(&err) {
                 warn!(target_id = %self.id, error = %err, "Redis target not ready; queued event remains in store");
             }
             return Err(err);
         }
 
         if let Err(err) = self.send_body(body, &meta).await {
-            if matches!(err, TargetError::NotConnected | TargetError::Timeout(_) | TargetError::Network(_)) {
+            if is_connectivity_error(&err) {
                 warn!(target_id = %self.id, error = %err, "Failed to send Redis event from store: target not connected. Event remains queued.");
             }
             return Err(err);
@@ -734,10 +716,6 @@ fn map_redis_error(err: RedisError) -> TargetError {
     }
 }
 
-fn is_retryable_target_error(err: &TargetError) -> bool {
-    matches!(err, TargetError::NotConnected | TargetError::Timeout(_) | TargetError::Network(_))
-}
-
 fn compute_retry_delay(attempt: usize, min_delay: Duration, max_delay: Duration) -> Duration {
     let shift = attempt.saturating_sub(1).min(16) as u32;
     let factor = 1u32 << shift;

crates/targets/src/target/webhook.rs

@@ -13,24 +13,21 @@
 // limitations under the License.
 
 use crate::{
-    StoreError, Target, TargetLog,
+    StoreError, Target,
     arn::TargetID,
     error::TargetError,
-    store::{Key, QueueStore, Store},
+    store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetType, queue_store_subdir_name,
+        TargetType, build_queued_payload, open_target_queue_store, persist_queued_payload_to_store,
     },
 };
 use async_trait::async_trait;
 use reqwest::{Client, StatusCode, Url};
-use rustfs_config::audit::AUDIT_STORE_EXTENSION;
-use rustfs_config::notify::NOTIFY_STORE_EXTENSION;
 use serde::Serialize;
 use serde::de::DeserializeOwned;
 use std::{
     marker::PhantomData,
-    path::PathBuf,
     sync::{
         Arc,
         atomic::{AtomicBool, Ordering},
@@ -151,28 +148,14 @@ where
         // Build HTTP client using the helper function
         let http_client = Arc::new(Self::build_http_client(&args)?);
 
-        // Build storage
-        let queue_store = if !args.queue_dir.is_empty() {
-            let queue_dir =
-                PathBuf::from(&args.queue_dir).join(queue_store_subdir_name(ChannelTargetType::Webhook.as_str(), &target_id.id));
-
-            let extension = match args.target_type {
-                TargetType::AuditLog => AUDIT_STORE_EXTENSION,
-                TargetType::NotifyEvent => NOTIFY_STORE_EXTENSION,
-            };
-
-            let store = QueueStore::<QueuedPayload>::new(queue_dir, args.queue_limit, extension);
-
-            if let Err(e) = store.open() {
-                error!("Failed to open store for Webhook target {}: {}", target_id.id, e);
-                return Err(TargetError::Storage(format!("{e}")));
-            }
-
-            // Make sure that the Store trait implemented by QueueStore matches the expected error type
-            Some(Box::new(store) as Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>)
-        } else {
-            None
-        };
+        let queue_store = open_target_queue_store(
+            &args.queue_dir,
+            args.queue_limit,
+            args.target_type,
+            ChannelTargetType::Webhook.as_str(),
+            &target_id,
+            "Failed to open store for Webhook target",
+        )?;
 
         // Create a cancel channel
         let (cancel_sender, _) = mpsc::channel(1);
@@ -302,22 +285,7 @@ where
     }
 
     fn build_queued_payload(&self, event: &EntityTarget<E>) -> Result<QueuedPayload, TargetError> {
-        let object_name = crate::target::decode_object_name(&event.object_name)?;
-        let key = format!("{}/{}", event.bucket_name, object_name);
-        let log = TargetLog {
-            event_name: event.event_name,
-            key,
-            records: vec![event.data.clone()],
-        };
-        let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?;
-        let meta = QueuedPayloadMeta::new(
-            event.event_name,
-            event.bucket_name.clone(),
-            event.object_name.clone(),
-            "application/json",
-            body.len(),
-        );
-        Ok(QueuedPayload::new(meta, body))
+        build_queued_payload(event)
     }
 
     async fn send_body(&self, body: Vec<u8>, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
@@ -408,16 +376,9 @@ where
         };
 
         if let Some(store) = &self.store {
-            let encoded = match queued.encode() {
-                Ok(encoded) => encoded,
-                Err(err) => {
-                    self.delivery_counters.record_final_failure();
-                    return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}")));
-                }
-            };
-            if let Err(e) = store.put_raw(&encoded) {
+            if let Err(e) = persist_queued_payload_to_store(store.as_ref(), &queued) {
                 self.delivery_counters.record_final_failure();
-                return Err(TargetError::Storage(format!("Failed to save event to store: {e}")));
+                return Err(e);
             }
             debug!("Event saved to store for target: {}", self.id);
             Ok(())

rustfs/src/admin/handlers/audit.rs

@@ -14,12 +14,12 @@
 
 use crate::admin::{
     auth::validate_admin_request,
+    handlers::audit_runtime_config::{load_server_config_from_store, update_audit_config_and_reload},
     handlers::target_descriptor::{
-        AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, allowed_target_keys,
-        build_json_response, collect_validated_key_values as shared_collect_validated_key_values,
+        AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, build_enabled_target_kvs,
+        build_json_response, collect_runtime_statuses, extract_supported_target_params,
         merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason,
-        target_mutation_block_reason as shared_target_mutation_block_reason, target_service_name, target_spec,
-        validate_target_request,
+        target_mutation_block_reason as shared_target_mutation_block_reason,
     },
     router::{AdminOperation, Operation, S3Router},
 };
@@ -27,23 +27,19 @@ use crate::auth::{check_key_valid, get_session_token};
 use crate::server::{
     ADMIN_PREFIX, RemoteAddr, is_audit_module_enabled, refresh_audit_module_enabled, refresh_persisted_module_switches_from_store,
 };
-use futures::stream::{FuturesUnordered, StreamExt};
 use http::StatusCode;
 use hyper::Method;
 use matchit::Params;
-use rustfs_audit::factory::builtin_target_descriptors as builtin_audit_target_descriptors;
-use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState};
+use rustfs_audit::audit_system;
 use rustfs_config::audit::AUDIT_ROUTE_PREFIX;
-use rustfs_config::{AUDIT_DEFAULT_DIR, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE};
+use rustfs_config::{AUDIT_DEFAULT_DIR, MAX_ADMIN_REQUEST_BODY_SIZE};
 use rustfs_ecstore::config::Config;
 use rustfs_policy::policy::action::{Action, AdminAction};
+use rustfs_targets::catalog::builtin::builtin_audit_target_admin_descriptors;
 use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
 use serde::{Deserialize, Serialize};
-use std::collections::{HashMap, HashSet};
-use std::sync::Arc;
+use std::collections::HashMap;
 use std::sync::LazyLock;
-use tokio::sync::Semaphore;
-use tokio::time::{Duration, timeout};
 use tracing::{Span, warn};
 
 pub fn register_audit_target_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
@@ -93,7 +89,7 @@ struct AuditEndpointsResponse {
 }
 
 static AUDIT_TARGET_SPECS: LazyLock<Vec<AdminTargetSpec>> = LazyLock::new(|| {
-    builtin_audit_target_descriptors()
+    builtin_audit_target_admin_descriptors()
         .into_iter()
         .map(|descriptor| admin_target_spec_from_builtin(&descriptor))
         .collect()
@@ -113,18 +109,6 @@ async fn authorize_audit_admin_request(req: &S3Request<Body>, action: AdminActio
     validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await
 }
 
-fn has_any_audit_targets(config: &Config) -> bool {
-    for spec in audit_target_specs() {
-        let Some(targets) = config.0.get(spec.subsystem) else {
-            continue;
-        };
-        if targets.keys().any(|key| key != DEFAULT_DELIMITER) {
-            return true;
-        }
-    }
-    false
-}
-
 fn audit_target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
     shared_target_mutation_block_reason(
         audit_target_specs(),
@@ -160,85 +144,7 @@ fn merge_audit_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey,
 }
 
 fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, &'a str)> {
-    let target_type = params
-        .get("target_type")
-        .ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'target_type'"))?;
-    if target_service_name(audit_target_specs(), target_type).is_none() {
-        return Err(s3_error!(InvalidArgument, "unsupported audit target type: '{}'", target_type));
-    }
-    let target_name = params
-        .get("target_name")
-        .ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'target_name'"))?;
-    Ok((target_type, target_name))
-}
-
-async fn load_server_config_from_store() -> S3Result<Config> {
-    let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
-        return Ok(Config::new());
-    };
-
-    rustfs_ecstore::config::com::read_config_without_migrate(store)
-        .await
-        .map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))
-}
-
-async fn apply_audit_runtime_config(config: Config) -> S3Result<()> {
-    let has_targets = has_any_audit_targets(&config);
-
-    if let Some(system) = audit_system() {
-        match system.get_state().await {
-            AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => {
-                if has_targets {
-                    system
-                        .reload_config(config)
-                        .await
-                        .map_err(|e| s3_error!(InternalError, "failed to reload audit config: {}", e))?;
-                } else {
-                    system
-                        .close()
-                        .await
-                        .map_err(|e| s3_error!(InternalError, "failed to stop audit system: {}", e))?;
-                }
-            }
-            AuditSystemState::Stopped | AuditSystemState::Stopping => {
-                if has_targets {
-                    system
-                        .start(config)
-                        .await
-                        .map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?;
-                }
-            }
-        }
-    } else if has_targets {
-        start_global_audit_system(config)
-            .await
-            .map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?;
-    }
-
-    Ok(())
-}
-
-async fn update_audit_config_and_reload<F>(mut modifier: F) -> S3Result<()>
-where
-    F: FnMut(&mut Config) -> bool,
-{
-    let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
-        return Err(s3_error!(InternalError, "server storage not initialized"));
-    };
-
-    let mut config = rustfs_ecstore::config::com::read_config_without_migrate(store.clone())
-        .await
-        .map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?;
-
-    if !modifier(&mut config) {
-        return Ok(());
-    }
-
-    rustfs_ecstore::config::com::save_server_config(store, &config)
-        .await
-        .map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?;
-
-    apply_audit_runtime_config(config).await
+    extract_supported_target_params(audit_target_specs(), params, "audit")
 }
 
 pub struct AuditTargetConfig {}
@@ -269,28 +175,16 @@ impl Operation for AuditTargetConfig {
             .map_err(|e| s3_error!(InvalidArgument, "invalid json body for audit target config: {}", e))?;
 
         let specs = audit_target_specs();
-        let allowed_keys: HashSet<&str> = allowed_target_keys(specs, target_type);
-
-        let kv_map = shared_collect_validated_key_values(
+        let kvs = build_enabled_target_kvs(
+            specs,
             audit_body.key_values.iter().map(|kv| (kv.key.as_str(), kv.value.as_str())),
-            &allowed_keys,
             target_type,
+            AUDIT_DEFAULT_DIR,
             "audit target",
-        )?;
+        )
+        .await?;
 
-        let spec = target_spec(specs, target_type)
-            .ok_or_else(|| s3_error!(InvalidArgument, "unsupported audit target type: '{}'", target_type))?;
-        timeout(Duration::from_secs(10), validate_target_request(spec, &kv_map, AUDIT_DEFAULT_DIR))
-            .await
-            .map_err(|_| s3_error!(InvalidArgument, "audit target validation timed out"))??;
-
-        let mut kvs = rustfs_ecstore::config::KVS::new();
-        for (key, value) in kv_map {
-            kvs.insert(key, value);
-        }
-        kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
-
-        update_audit_config_and_reload(|config| {
+        update_audit_config_and_reload(audit_target_specs(), |config| {
             config
                 .0
                 .entry(target_type.to_lowercase())
@@ -315,25 +209,7 @@ impl Operation for ListAuditTargets {
 
         let mut runtime_statuses = HashMap::new();
         if let Some(system) = audit_system() {
-            let targets = system.get_target_values().await;
-            let semaphore = Arc::new(Semaphore::new(10));
-            let mut futures = FuturesUnordered::new();
-
-            for target in targets {
-                let sem = Arc::clone(&semaphore);
-                futures.push(async move {
-                    let _permit = sem.acquire().await;
-                    let status = match timeout(Duration::from_secs(3), target.is_active()).await {
-                        Ok(Ok(true)) => "online",
-                        _ => "offline",
-                    };
-                    ((target.id().id, target.id().name), status.to_string())
-                });
-            }
-
-            while let Some((key, status)) = futures.next().await {
-                runtime_statuses.insert(key, status);
-            }
+            runtime_statuses = collect_runtime_statuses(system.get_target_values().await).await;
         }
 
         let config = load_server_config_from_store().await?;
@@ -363,7 +239,7 @@ impl Operation for RemoveAuditTarget {
             return Err(s3_error!(InvalidRequest, "{reason}"));
         }
 
-        update_audit_config_and_reload(|config| {
+        update_audit_config_and_reload(audit_target_specs(), |config| {
             let mut changed = false;
             if let Some(targets) = config.0.get_mut(&target_type.to_lowercase()) {
                 if targets.remove(&target_name.to_lowercase()).is_some() {
@@ -384,9 +260,10 @@ impl Operation for RemoveAuditTarget {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::admin::handlers::target_descriptor::collect_validated_key_values as shared_collect_validated_key_values;
     use matchit::Router;
-    use rustfs_config::ENV_PREFIX;
     use rustfs_config::audit::{AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS};
+    use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX};
     use rustfs_ecstore::config::{KV, KVS};
     use serial_test::serial;
     use std::collections::{HashMap, HashSet};

rustfs/src/admin/handlers/audit_runtime_config.rs

@@ -0,0 +1,130 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::admin::handlers::target_descriptor::AdminTargetSpec;
+use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState};
+use rustfs_config::DEFAULT_DELIMITER;
+use rustfs_ecstore::config::Config;
+use s3s::{S3Result, s3_error};
+
+pub(crate) async fn load_server_config_from_store() -> S3Result<Config> {
+    let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
+        return Ok(Config::new());
+    };
+
+    rustfs_ecstore::config::com::read_config_without_migrate(store)
+        .await
+        .map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))
+}
+
+fn has_any_audit_targets(specs: &[AdminTargetSpec], config: &Config) -> bool {
+    specs.iter().any(|spec| {
+        config
+            .0
+            .get(spec.subsystem)
+            .is_some_and(|targets| targets.keys().any(|key| key != DEFAULT_DELIMITER))
+    })
+}
+
+pub(crate) async fn apply_audit_runtime_config(specs: &[AdminTargetSpec], config: Config) -> S3Result<()> {
+    let has_targets = has_any_audit_targets(specs, &config);
+
+    if let Some(system) = audit_system() {
+        match system.get_state().await {
+            AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => {
+                if has_targets {
+                    system
+                        .reload_config(config)
+                        .await
+                        .map_err(|e| s3_error!(InternalError, "failed to reload audit config: {}", e))?;
+                } else {
+                    system
+                        .close()
+                        .await
+                        .map_err(|e| s3_error!(InternalError, "failed to stop audit system: {}", e))?;
+                }
+            }
+            AuditSystemState::Stopped | AuditSystemState::Stopping => {
+                if has_targets {
+                    system
+                        .start(config)
+                        .await
+                        .map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?;
+                }
+            }
+        }
+    } else if has_targets {
+        start_global_audit_system(config)
+            .await
+            .map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?;
+    }
+
+    Ok(())
+}
+
+pub(crate) async fn update_audit_config_and_reload<F>(specs: &[AdminTargetSpec], mut modifier: F) -> S3Result<()>
+where
+    F: FnMut(&mut Config) -> bool,
+{
+    let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
+        return Err(s3_error!(InternalError, "server storage not initialized"));
+    };
+
+    let mut config = rustfs_ecstore::config::com::read_config_without_migrate(store.clone())
+        .await
+        .map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?;
+
+    if !modifier(&mut config) {
+        return Ok(());
+    }
+
+    rustfs_ecstore::config::com::save_server_config(store, &config)
+        .await
+        .map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?;
+
+    apply_audit_runtime_config(specs, config).await
+}
+
+pub(crate) async fn set_audit_target_config(
+    specs: &[AdminTargetSpec],
+    subsystem: &str,
+    target_name: &str,
+    kvs: rustfs_ecstore::config::KVS,
+) -> S3Result<()> {
+    update_audit_config_and_reload(specs, |config| {
+        config
+            .0
+            .entry(subsystem.to_lowercase())
+            .or_default()
+            .insert(target_name.to_lowercase(), kvs.clone());
+        true
+    })
+    .await
+}
+
+pub(crate) async fn remove_audit_target_config(specs: &[AdminTargetSpec], subsystem: &str, target_name: &str) -> S3Result<()> {
+    update_audit_config_and_reload(specs, |config| {
+        let mut changed = false;
+        if let Some(targets) = config.0.get_mut(&subsystem.to_lowercase()) {
+            if targets.remove(&target_name.to_lowercase()).is_some() {
+                changed = true;
+            }
+            if targets.is_empty() {
+                config.0.remove(&subsystem.to_lowercase());
+            }
+        }
+        changed
+    })
+    .await
+}

rustfs/src/admin/handlers/event.rs

@@ -14,12 +14,12 @@
 
 use crate::admin::{
     auth::validate_admin_request,
+    handlers::notify_runtime_access::{get_notification_system, load_notification_config_snapshot},
     handlers::target_descriptor::{
-        AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, allowed_target_keys,
-        build_json_response, collect_validated_key_values as shared_collect_validated_key_values,
+        AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, build_enabled_target_kvs,
+        build_json_response, collect_runtime_statuses, extract_supported_target_params,
         merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason,
-        target_mutation_block_reason as shared_target_mutation_block_reason, target_service_name, target_spec,
-        validate_target_request,
+        target_mutation_block_reason as shared_target_mutation_block_reason,
     },
     router::{AdminOperation, Operation, S3Router},
 };
@@ -28,22 +28,18 @@ use crate::server::{
     ADMIN_PREFIX, RemoteAddr, is_notify_module_enabled, refresh_notify_module_enabled,
     refresh_persisted_module_switches_from_store,
 };
-use futures::stream::{FuturesUnordered, StreamExt};
 use http::StatusCode;
 use hyper::Method;
 use matchit::Params;
 use rustfs_config::notify::NOTIFY_ROUTE_PREFIX;
-use rustfs_config::{ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE};
+use rustfs_config::{EVENT_DEFAULT_DIR, MAX_ADMIN_REQUEST_BODY_SIZE};
 use rustfs_ecstore::config::Config;
-use rustfs_notify::factory::builtin_target_descriptors as builtin_notification_target_descriptors;
 use rustfs_policy::policy::action::{Action, AdminAction};
+use rustfs_targets::catalog::builtin::builtin_notify_target_admin_descriptors;
 use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
 use serde::{Deserialize, Serialize};
-use std::collections::{HashMap, HashSet};
-use std::sync::Arc;
+use std::collections::HashMap;
 use std::sync::LazyLock;
-use tokio::sync::Semaphore;
-use tokio::time::{Duration, timeout};
 use tracing::{Span, info, warn};
 
 pub fn register_notification_target_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
@@ -99,7 +95,7 @@ struct NotificationEndpointsResponse {
 }
 
 static NOTIFICATION_TARGET_SPECS: LazyLock<Vec<AdminTargetSpec>> = LazyLock::new(|| {
-    builtin_notification_target_descriptors()
+    builtin_notify_target_admin_descriptors()
         .into_iter()
         .map(|descriptor| admin_target_spec_from_builtin(&descriptor))
         .collect()
@@ -121,10 +117,6 @@ async fn authorize_notification_admin_request(req: &S3Request<Body>, action: Adm
     validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await
 }
 
-fn get_notification_system() -> S3Result<Arc<rustfs_notify::NotificationSystem>> {
-    rustfs_notify::notification_system().ok_or_else(|| s3_error!(InternalError, "notification system not initialized"))
-}
-
 fn target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
     shared_target_mutation_block_reason(
         notification_target_specs(),
@@ -180,8 +172,7 @@ impl Operation for NotificationTarget {
         if let Some(reason) = notification_target_operation_block_reason("managing notification targets from the console").await {
             return Err(s3_error!(InvalidRequest, "{reason}"));
         }
-        let ns = get_notification_system()?;
-        let config_snapshot = ns.config.read().await.clone();
+        let (ns, config_snapshot) = load_notification_config_snapshot().await?;
         if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) {
             return Err(s3_error!(InvalidRequest, "{reason}"));
         }
@@ -196,28 +187,17 @@ impl Operation for NotificationTarget {
             .map_err(|e| s3_error!(InvalidArgument, "invalid json body for target config: {}", e))?;
 
         let specs = notification_target_specs();
-        let allowed_keys: HashSet<&str> = allowed_target_keys(specs, target_type);
-
-        let kv_map = shared_collect_validated_key_values(
+        let kvs = build_enabled_target_kvs(
+            specs,
             notification_body
                 .key_values
                 .iter()
                 .map(|kv| (kv.key.as_str(), kv.value.as_str())),
-            &allowed_keys,
             target_type,
+            EVENT_DEFAULT_DIR,
             "target",
-        )?;
-        let spec = target_spec(specs, target_type)
-            .ok_or_else(|| s3_error!(InvalidArgument, "unsupported target type: '{}'", target_type))?;
-        timeout(Duration::from_secs(10), validate_target_request(spec, &kv_map, EVENT_DEFAULT_DIR))
-            .await
-            .map_err(|_| s3_error!(InvalidArgument, "target validation timed out"))??;
-
-        let mut kvs = rustfs_ecstore::config::KVS::new();
-        for (key, value) in kv_map {
-            kvs.insert(key, value);
-        }
-        kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
+        )
+        .await?;
 
         info!("Setting target config for type '{}', name '{}'", target_type, target_name);
         ns.set_target_config(target_type, target_name, kvs)
@@ -235,29 +215,8 @@ impl Operation for ListNotificationTargets {
         let span = Span::current();
         let _enter = span.enter();
         authorize_notification_admin_request(&req, AdminAction::GetBucketTargetAction).await?;
-        let ns = get_notification_system()?;
-
-        let targets = ns.get_target_values().await;
-        let semaphore = Arc::new(Semaphore::new(10));
-        let mut futures = FuturesUnordered::new();
-
-        for target in targets {
-            let sem = Arc::clone(&semaphore);
-            futures.push(async move {
-                let _permit = sem.acquire().await;
-                let status = match timeout(Duration::from_secs(3), target.is_active()).await {
-                    Ok(Ok(true)) => "online",
-                    _ => "offline",
-                };
-                ((target.id().id, target.id().name), status.to_string())
-            });
-        }
-
-        let mut runtime_statuses = HashMap::new();
-        while let Some((key, status)) = futures.next().await {
-            runtime_statuses.insert(key, status);
-        }
-        let config = ns.config.read().await.clone();
+        let (ns, config) = load_notification_config_snapshot().await?;
+        let runtime_statuses = collect_runtime_statuses(ns.get_target_values().await).await;
         let notification_endpoints = merge_notification_endpoints(&config, runtime_statuses);
 
         let data = serde_json::to_vec(&NotificationEndpointsResponse { notification_endpoints })
@@ -283,30 +242,15 @@ impl Operation for ListTargetsArns {
         }
         let ns = get_notification_system()?;
 
-        let targets = ns.get_target_values().await;
         let region = req
             .region
             .clone()
             .ok_or_else(|| s3_error!(InvalidRequest, "region not found"))?;
-        let semaphore = Arc::new(Semaphore::new(10));
-        let mut futures = FuturesUnordered::new();
-
-        for target in targets {
-            let sem = Arc::clone(&semaphore);
-            futures.push(async move {
-                let _permit = sem.acquire().await;
-                let status = match timeout(Duration::from_secs(3), target.is_active()).await {
-                    Ok(Ok(true)) => "online",
-                    _ => "offline",
-                };
-                (target.id(), status.to_string())
-            });
-        }
-
-        let mut target_statuses = Vec::new();
-        while let Some(target_status) = futures.next().await {
-            target_statuses.push(target_status);
-        }
+        let target_statuses = collect_runtime_statuses(ns.get_target_values().await)
+            .await
+            .into_iter()
+            .map(|((account_id, service), status)| (rustfs_targets::arn::TargetID::new(account_id, service), status))
+            .collect();
 
         let data_target_arn_list = collect_online_target_arns(region.as_str(), target_statuses);
 
@@ -329,8 +273,7 @@ impl Operation for RemoveNotificationTarget {
         if let Some(reason) = notification_target_operation_block_reason("managing notification targets from the console").await {
             return Err(s3_error!(InvalidRequest, "{reason}"));
         }
-        let ns = get_notification_system()?;
-        let config_snapshot = ns.config.read().await.clone();
+        let (ns, config_snapshot) = load_notification_config_snapshot().await?;
         if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) {
             return Err(s3_error!(InvalidRequest, "{reason}"));
         }
@@ -344,27 +287,19 @@ impl Operation for RemoveNotificationTarget {
     }
 }
 
-fn extract_param<'a>(params: &'a Params<'_, '_>, key: &str) -> S3Result<&'a str> {
-    params
-        .get(key)
-        .ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: '{}'", key))
-}
-
 fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, &'a str)> {
-    let target_type = extract_param(params, "target_type")?;
-    if target_service_name(notification_target_specs(), target_type).is_none() {
-        return Err(s3_error!(InvalidArgument, "unsupported target type: '{}'", target_type));
-    }
-    let target_name = extract_param(params, "target_name")?;
-    Ok((target_type, target_name))
+    extract_supported_target_params(notification_target_specs(), params, "notification")
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::admin::handlers::target_descriptor::{
+        allowed_target_keys, collect_validated_key_values as shared_collect_validated_key_values,
+    };
     use matchit::Router;
-    use rustfs_config::DEFAULT_DELIMITER;
     use rustfs_config::notify::{NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
+    use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY};
     use rustfs_ecstore::config::{KV, KVS};
     use rustfs_targets::arn::TargetID;
     use serial_test::serial;

rustfs/src/admin/handlers/mod.rs

@@ -14,6 +14,7 @@
 
 pub mod account_info;
 pub mod audit;
+mod audit_runtime_config;
 pub mod bucket_meta;
 pub mod event;
 pub mod group;
@@ -27,7 +28,10 @@ pub mod kms_keys;
 pub mod kms_management;
 pub mod metrics;
 pub mod module_switch;
+mod notify_runtime_access;
 pub mod oidc;
+pub mod plugins_catalog;
+pub mod plugins_instances;
 pub mod policies;
 pub mod pools;
 pub mod profile;
@@ -57,6 +61,11 @@ mod tests {
         let _account_handler = account_info::AccountInfoHandler {};
         let _list_audit_targets = audit::ListAuditTargets {};
         let _get_module_switches = module_switch::GetModuleSwitchesHandler {};
+        let _get_plugin_catalog = plugins_catalog::GetPluginCatalogHandler {};
+        let _list_plugin_instances = plugins_instances::ListPluginInstancesHandler {};
+        let _get_plugin_instance = plugins_instances::GetPluginInstanceHandler {};
+        let _put_plugin_instance = plugins_instances::PutPluginInstanceHandler {};
+        let _delete_plugin_instance = plugins_instances::DeletePluginInstanceHandler {};
         let _update_module_switches = module_switch::UpdateModuleSwitchesHandler {};
         let _service_handler = system::ServiceHandle {};
         let _server_info_handler = system::ServerInfoHandler {};

rustfs/src/admin/handlers/notify_runtime_access.rs

@@ -0,0 +1,47 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rustfs_ecstore::config::Config;
+use s3s::{S3Result, s3_error};
+use std::sync::Arc;
+
+pub(crate) fn get_notification_system() -> S3Result<Arc<rustfs_notify::NotificationSystem>> {
+    rustfs_notify::notification_system().ok_or_else(|| s3_error!(InternalError, "notification system not initialized"))
+}
+
+pub(crate) async fn load_notification_config_snapshot() -> S3Result<(Arc<rustfs_notify::NotificationSystem>, Config)> {
+    let system = get_notification_system()?;
+    let config = system.config.read().await.clone();
+    Ok((system, config))
+}
+
+pub(crate) async fn set_notification_target_config(
+    subsystem: &str,
+    target_name: &str,
+    kvs: rustfs_ecstore::config::KVS,
+) -> S3Result<()> {
+    let system = get_notification_system()?;
+    system
+        .set_target_config(subsystem, target_name, kvs)
+        .await
+        .map_err(|e| s3_error!(InternalError, "failed to set notification target config: {}", e))
+}
+
+pub(crate) async fn remove_notification_target_config(subsystem: &str, target_name: &str) -> S3Result<()> {
+    let system = get_notification_system()?;
+    system
+        .remove_target_config(subsystem, target_name)
+        .await
+        .map_err(|e| s3_error!(InternalError, "failed to remove notification target config: {}", e))
+}

rustfs/src/admin/handlers/plugins_catalog.rs

@@ -0,0 +1,265 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::admin::{
+    auth::validate_admin_request,
+    plugin_contract::{
+        PluginCatalogDomainEntry, PluginCatalogEntry, PluginCatalogResponse, PluginContractDomain, PluginContractEntrypointKind,
+        PluginContractPackaging, PluginDistributionContract, PluginRuntimeContract,
+    },
+    router::{AdminOperation, Operation, S3Router},
+};
+use crate::auth::{check_key_valid, get_session_token};
+use crate::server::{ADMIN_PREFIX, RemoteAddr};
+use http::{HeaderMap, HeaderValue, StatusCode};
+use hyper::Method;
+use matchit::Params;
+use rustfs_policy::policy::action::{Action, AdminAction};
+use rustfs_targets::catalog::{
+    builtin::builtin_audit_target_admin_descriptors, builtin::builtin_notify_target_admin_descriptors,
+};
+use rustfs_targets::{
+    BuiltinTargetAdminDescriptor, builtin_target_marketplace_manifest, builtin_target_plugin_installation,
+    catalog::example_external_webhook_plugin,
+};
+use s3s::header::CONTENT_TYPE;
+use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
+use serde::Serialize;
+use std::collections::HashMap;
+
+pub fn register_plugin_catalog_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
+    r.insert(
+        Method::GET,
+        format!("{}{}", ADMIN_PREFIX, "/v4/plugins/catalog").as_str(),
+        AdminOperation(&GetPluginCatalogHandler {}),
+    )?;
+
+    Ok(())
+}
+
+fn target_domain_name_from_subsystem(subsystem: &str) -> PluginContractDomain {
+    if subsystem.starts_with("audit_") {
+        PluginContractDomain::Audit
+    } else {
+        PluginContractDomain::Notify
+    }
+}
+
+fn build_catalog_response() -> PluginCatalogResponse {
+    let mut plugins: HashMap<&'static str, PluginCatalogEntry> = HashMap::new();
+
+    for descriptor in builtin_notify_target_admin_descriptors()
+        .into_iter()
+        .chain(builtin_audit_target_admin_descriptors())
+    {
+        merge_catalog_descriptor(&mut plugins, &descriptor);
+    }
+
+    let mut plugins = plugins.into_values().collect::<Vec<_>>();
+    plugins.push(example_external_webhook_plugin_entry());
+    plugins.sort_by(|a, b| a.target_type.cmp(&b.target_type));
+    for plugin in &mut plugins {
+        plugin.supported_domains.sort();
+        plugin.domain_configs.sort_by_key(|a| a.domain);
+    }
+
+    PluginCatalogResponse { plugins }
+}
+
+fn example_external_webhook_plugin_entry() -> PluginCatalogEntry {
+    let example = example_external_webhook_plugin();
+    let manifest = example.manifest;
+
+    PluginCatalogEntry {
+        plugin_id: manifest.plugin_id.to_string(),
+        target_type: manifest.target_type.to_string(),
+        display_name: manifest.display_name.to_string(),
+        provider: manifest.provider.to_string(),
+        version: manifest.version.to_string(),
+        packaging: PluginContractPackaging::from(manifest.packaging),
+        entrypoint_kind: PluginContractEntrypointKind::from(manifest.entrypoint_kind),
+        api_compatibility_version: manifest.api_compatibility_version.to_string(),
+        runtime_contract: PluginRuntimeContract::from(manifest.runtime_contract),
+        distribution: manifest.distribution.map(PluginDistributionContract::from),
+        supported_domains: manifest.supported_domains.iter().copied().map(Into::into).collect(),
+        secret_fields: manifest.secret_fields.iter().map(|field| (*field).to_string()).collect(),
+        domain_configs: vec![PluginCatalogDomainEntry {
+            domain: PluginContractDomain::Notify,
+            subsystem: "notify_webhook".to_string(),
+            valid_fields: example.valid_fields,
+        }],
+        installation: Some(example.installation.into()),
+    }
+}
+
+fn merge_catalog_descriptor(plugins: &mut HashMap<&'static str, PluginCatalogEntry>, descriptor: &BuiltinTargetAdminDescriptor) {
+    let manifest = descriptor.manifest();
+    let marketplace = builtin_target_marketplace_manifest(manifest.target_type);
+    let domain = target_domain_name_from_subsystem(descriptor.admin_metadata().subsystem());
+    let domain_entry = PluginCatalogDomainEntry {
+        domain,
+        subsystem: descriptor.admin_metadata().subsystem().to_string(),
+        valid_fields: descriptor.valid_fields().iter().map(|field| (*field).to_string()).collect(),
+    };
+
+    let entry = plugins.entry(manifest.plugin_id).or_insert_with(|| PluginCatalogEntry {
+        plugin_id: manifest.plugin_id.to_string(),
+        target_type: manifest.target_type.to_string(),
+        display_name: manifest.display_name.to_string(),
+        provider: manifest.provider.to_string(),
+        version: manifest.version.to_string(),
+        packaging: PluginContractPackaging::from(marketplace.packaging),
+        entrypoint_kind: PluginContractEntrypointKind::from(marketplace.entrypoint_kind),
+        api_compatibility_version: marketplace.api_compatibility_version.to_string(),
+        runtime_contract: PluginRuntimeContract::from(marketplace.runtime_contract),
+        distribution: marketplace.distribution.map(PluginDistributionContract::from),
+        supported_domains: manifest.supported_domains.iter().copied().map(Into::into).collect(),
+        secret_fields: manifest.secret_fields.iter().map(|field| (*field).to_string()).collect(),
+        domain_configs: Vec::new(),
+        installation: Some(builtin_target_plugin_installation(manifest).into()),
+    });
+
+    if !entry.domain_configs.iter().any(|existing| existing.domain == domain) {
+        entry.domain_configs.push(domain_entry);
+    }
+}
+
+async fn authorize_plugin_catalog_request(req: &S3Request<Body>) -> S3Result<()> {
+    let Some(input_cred) = &req.credentials else {
+        return Err(s3_error!(InvalidRequest, "authentication required"));
+    };
+
+    let (cred, owner) =
+        check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
+
+    validate_admin_request(
+        &req.headers,
+        &cred,
+        owner,
+        false,
+        vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)],
+        req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0)),
+    )
+    .await
+}
+
+fn build_json_response(
+    status: StatusCode,
+    body: &impl Serialize,
+    request_id: Option<&HeaderValue>,
+) -> S3Result<S3Response<(StatusCode, Body)>> {
+    let data = serde_json::to_vec(body).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
+    let mut header = HeaderMap::new();
+    header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
+    if let Some(value) = request_id {
+        header.insert("x-request-id", value.clone());
+    }
+    Ok(S3Response::with_headers((status, Body::from(data)), header))
+}
+
+pub struct GetPluginCatalogHandler {}
+
+#[async_trait::async_trait]
+impl Operation for GetPluginCatalogHandler {
+    async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        authorize_plugin_catalog_request(&req).await?;
+        build_json_response(StatusCode::OK, &build_catalog_response(), req.headers.get("x-request-id"))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::build_catalog_response;
+    use crate::admin::plugin_contract::{
+        PluginContractDomain, PluginContractEntrypointKind, PluginContractPackaging, PluginRuntimeTransport,
+    };
+
+    #[test]
+    fn plugin_catalog_handlers_require_admin_authorization_contract() {
+        let src = include_str!("plugins_catalog.rs");
+        let handler_block = extract_block_between_markers(src, "impl Operation for GetPluginCatalogHandler", "#[cfg(test)]");
+
+        assert!(
+            handler_block.contains("authorize_plugin_catalog_request(&req).await?;"),
+            "plugin catalog GET should require admin authorization"
+        );
+    }
+
+    #[test]
+    fn plugin_catalog_contains_representative_builtin_targets() {
+        let response = build_catalog_response();
+
+        let webhook = response
+            .plugins
+            .iter()
+            .find(|plugin| plugin.plugin_id == "builtin:webhook")
+            .expect("builtin webhook plugin should be present");
+        assert_eq!(webhook.target_type, "webhook");
+        assert_eq!(webhook.display_name, "Webhook");
+        assert_eq!(webhook.packaging, PluginContractPackaging::Builtin);
+        assert_eq!(webhook.entrypoint_kind, PluginContractEntrypointKind::Builtin);
+        assert_eq!(webhook.api_compatibility_version, "rustfs.target-plugin.v1");
+        assert_eq!(webhook.runtime_contract.protocol_version, "rustfs.target-runtime.v1");
+        assert_eq!(webhook.runtime_contract.transport, PluginRuntimeTransport::InProcess);
+        assert_eq!(webhook.distribution, None);
+        assert!(webhook.supported_domains.contains(&PluginContractDomain::Audit));
+        assert!(webhook.supported_domains.contains(&PluginContractDomain::Notify));
+        assert!(
+            webhook
+                .domain_configs
+                .iter()
+                .any(|domain| domain.subsystem == "audit_webhook")
+        );
+        assert!(
+            webhook
+                .domain_configs
+                .iter()
+                .any(|domain| domain.subsystem == "notify_webhook")
+        );
+
+        let kafka = response
+            .plugins
+            .iter()
+            .find(|plugin| plugin.plugin_id == "builtin:kafka")
+            .expect("builtin kafka plugin should be present");
+        assert_eq!(kafka.target_type, "kafka");
+        assert!(kafka.domain_configs.iter().any(|domain| domain.subsystem == "audit_kafka"));
+        assert!(kafka.domain_configs.iter().any(|domain| domain.subsystem == "notify_kafka"));
+    }
+
+    #[test]
+    fn plugin_catalog_exposes_secret_fields_only_as_metadata() {
+        let response = build_catalog_response();
+        let webhook = response
+            .plugins
+            .iter()
+            .find(|plugin| plugin.plugin_id == "builtin:webhook")
+            .expect("builtin webhook plugin should be present");
+
+        assert!(webhook.secret_fields.contains(&"auth_token".to_string()));
+        assert!(!webhook.secret_fields.iter().any(|field| field.contains("https://")));
+        assert!(!webhook.secret_fields.iter().any(|field| field.contains("password=")));
+    }
+
+    fn extract_block_between_markers<'a>(src: &'a str, start_marker: &str, end_marker: &str) -> &'a str {
+        let start = src
+            .find(start_marker)
+            .unwrap_or_else(|| panic!("Expected marker `{start_marker}` in source"));
+        let after_start = &src[start..];
+        let end = after_start
+            .find(end_marker)
+            .unwrap_or_else(|| panic!("Expected end marker `{end_marker}` in source"));
+        &after_start[..end]
+    }
+}

rustfs/src/admin/handlers/target_descriptor.rs

@@ -12,22 +12,28 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use futures::StreamExt;
+use futures::future::BoxFuture;
 use hashbrown::HashSet as HbHashSet;
 use http::{HeaderMap, HeaderValue, StatusCode};
 use rustfs_config::{
-    AMQP_QUEUE_DIR, ENABLE_KEY, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_TOPIC, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_TLS_CA,
-    MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME,
-    MQTT_WS_PATH_ALLOWLIST, MYSQL_QUEUE_DIR, POSTGRES_QUEUE_DIR, REDIS_QUEUE_DIR,
+    AMQP_QUEUE_DIR, ENABLE_KEY, EnableState, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_TOPIC, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS,
+    MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC,
+    MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_QUEUE_DIR, POSTGRES_QUEUE_DIR, REDIS_QUEUE_DIR,
 };
-use rustfs_ecstore::config::Config;
+use rustfs_ecstore::config::{Config, KVS};
+use rustfs_targets::SharedTarget;
 use rustfs_targets::{
-    BuiltinTargetDescriptor, TargetError, TargetRequestValidator, check_amqp_broker_available, check_kafka_broker_available,
-    check_mqtt_broker_available_with_tls, check_mysql_server_available, check_nats_server_available,
-    check_postgres_server_available, check_pulsar_broker_available, check_redis_server_available,
+    BuiltinTargetAdminDescriptor, TargetAdminMetadata, TargetDomain, TargetError, TargetRequestValidator,
+    check_amqp_broker_available, check_kafka_broker_available, check_mqtt_broker_available_with_tls,
+    check_mysql_server_available, check_nats_server_available, check_postgres_server_available, check_pulsar_broker_available,
+    check_redis_server_available,
     config::{
-        build_amqp_args, build_kafka_args, build_mysql_args, build_nats_args, build_postgres_args, build_pulsar_args,
-        build_redis_args, collect_env_target_instance_ids, validate_redis_config,
+        TargetPluginInstanceCompatDescriptor, TargetPluginInstanceRecord, build_amqp_args, build_kafka_args, build_mysql_args,
+        build_nats_args, build_postgres_args, build_pulsar_args, build_redis_args, normalize_target_plugin_instances,
+        validate_redis_config,
     },
+    manifest::builtin_target_manifest,
     target::{TargetType, mqtt::MQTTTlsConfig},
 };
 use s3s::{Body, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
@@ -36,12 +42,14 @@ use std::collections::{HashMap, HashSet};
 use std::io::{Error, ErrorKind};
 use std::path::Path;
 use std::sync::Arc;
-use tokio::time::{Duration, sleep};
+use tokio::sync::Semaphore;
+use tokio::time::{Duration, sleep, timeout};
 use url::Url;
 
 pub(crate) type EndpointKey = (String, String);
 type AdminRequestValidatorFn =
-    Arc<dyn Fn(&HashMap<String, String>, &str) -> futures::future::BoxFuture<'static, S3Result<()>> + Send + Sync>;
+    Arc<dyn for<'a> Fn(&'a HashMap<String, String>, &'a str) -> BoxFuture<'a, S3Result<()>> + Send + Sync>;
+type DomainScopedValidatorFn = for<'a> fn(&'a HashMap<String, String>, &'a str, TargetDomain) -> BoxFuture<'a, S3Result<()>>;
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
 #[serde(rename_all = "lowercase")]
@@ -59,28 +67,26 @@ pub(crate) struct MergedTargetEndpoint {
     pub source: TargetEndpointSource,
 }
 
-#[derive(Clone, Copy, Debug)]
-pub(crate) enum TargetDomain {
-    Notify,
-    Audit,
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(crate) struct TargetInstanceReadModel {
+    pub canonical_id: String,
+    pub plugin_id: String,
+    pub domain: TargetDomain,
+    pub subsystem: String,
+    pub account_id: String,
+    pub service: String,
+    pub status: String,
+    pub runtime_present: bool,
+    pub source: TargetEndpointSource,
+    pub enabled: bool,
+    pub config: KVS,
 }
 
-impl TargetDomain {
-    pub(crate) fn runtime_target_type(self) -> TargetType {
-        match self {
-            TargetDomain::Notify => TargetType::NotifyEvent,
-            TargetDomain::Audit => TargetType::AuditLog,
-        }
-    }
-}
-
-impl From<TargetType> for TargetDomain {
-    fn from(value: TargetType) -> Self {
-        match value {
-            TargetType::NotifyEvent => TargetDomain::Notify,
-            TargetType::AuditLog => TargetDomain::Audit,
-        }
-    }
+struct TargetEndpointSnapshot {
+    normalized_instances: Vec<TargetPluginInstanceRecord>,
+    configured_keys: Vec<EndpointKey>,
+    config_targets: HbHashSet<EndpointKey>,
+    env_targets: HbHashSet<EndpointKey>,
 }
 
 #[derive(Clone)]
@@ -91,69 +97,55 @@ pub(crate) struct AdminTargetSpec {
     validator: AdminRequestValidatorFn,
 }
 
-pub(crate) fn admin_target_spec_from_builtin<E>(descriptor: &BuiltinTargetDescriptor<E>) -> AdminTargetSpec
-where
-    E: Send + Sync + 'static + Clone + serde::Serialize + serde::de::DeserializeOwned,
-{
+pub(crate) fn admin_target_spec_from_builtin(descriptor: &BuiltinTargetAdminDescriptor) -> AdminTargetSpec {
+    let admin = descriptor.admin_metadata();
     AdminTargetSpec {
-        subsystem: descriptor.subsystem(),
-        service: descriptor.plugin().target_type(),
-        valid_keys: descriptor.plugin().valid_fields(),
-        validator: match descriptor.request_validator() {
-            TargetRequestValidator::Webhook => Arc::new(validate_webhook_request_entry),
-            TargetRequestValidator::Mqtt => Arc::new(validate_mqtt_request_entry),
-            TargetRequestValidator::Amqp(target_type) => {
-                if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
-                    Arc::new(validate_audit_amqp_request_entry)
-                } else {
-                    Arc::new(validate_notify_amqp_request_entry)
-                }
-            }
-            TargetRequestValidator::Kafka(target_type) => {
-                if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
-                    Arc::new(validate_audit_kafka_request_entry)
-                } else {
-                    Arc::new(validate_notify_kafka_request_entry)
-                }
-            }
-            TargetRequestValidator::MySql(target_type) => {
-                Arc::new(move |kv_map, default_queue_dir| validate_mysql_request_entry(kv_map, default_queue_dir, target_type))
-            }
-            TargetRequestValidator::Nats(target_type) => {
-                if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
-                    Arc::new(validate_audit_nats_request_entry)
-                } else {
-                    Arc::new(validate_notify_nats_request_entry)
-                }
-            }
-            TargetRequestValidator::Postgres(target_type) => {
-                if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
-                    Arc::new(validate_audit_postgres_request_entry)
-                } else {
-                    Arc::new(validate_notify_postgres_request_entry)
-                }
-            }
-            TargetRequestValidator::Pulsar(target_type) => {
-                if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
-                    Arc::new(validate_audit_pulsar_request_entry)
-                } else {
-                    Arc::new(validate_notify_pulsar_request_entry)
-                }
-            }
-            TargetRequestValidator::Redis {
-                default_channel,
-                target_type,
-            } => {
-                if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
-                    validate_audit_redis_request_entry(default_channel)
-                } else {
-                    validate_notify_redis_request_entry(default_channel)
-                }
-            }
-        },
+        subsystem: admin.subsystem(),
+        service: descriptor.manifest().target_type,
+        valid_keys: descriptor.valid_fields(),
+        validator: validator_from_metadata(admin),
     }
 }
 
+fn validator_from_metadata(metadata: TargetAdminMetadata) -> AdminRequestValidatorFn {
+    match metadata.request_validator() {
+        TargetRequestValidator::Webhook => Arc::new(validate_webhook_request_entry),
+        TargetRequestValidator::Mqtt => Arc::new(validate_mqtt_request_entry),
+        TargetRequestValidator::Amqp(target_type) => {
+            domain_request_validator(TargetDomain::from(target_type), validate_amqp_request)
+        }
+        TargetRequestValidator::Kafka(target_type) => {
+            domain_request_validator(TargetDomain::from(target_type), validate_kafka_request)
+        }
+        TargetRequestValidator::MySql(target_type) => {
+            Arc::new(move |kv_map, default_queue_dir| validate_mysql_request_entry(kv_map, default_queue_dir, target_type))
+        }
+        TargetRequestValidator::Nats(target_type) => {
+            domain_request_validator(TargetDomain::from(target_type), validate_nats_request)
+        }
+        TargetRequestValidator::Postgres(target_type) => {
+            domain_request_validator(TargetDomain::from(target_type), validate_postgres_request)
+        }
+        TargetRequestValidator::Pulsar(target_type) => {
+            domain_request_validator(TargetDomain::from(target_type), validate_pulsar_request)
+        }
+        TargetRequestValidator::Redis {
+            default_channel,
+            target_type,
+        } => redis_request_validator(TargetDomain::from(target_type), default_channel),
+    }
+}
+
+fn domain_request_validator(domain: TargetDomain, validator: DomainScopedValidatorFn) -> AdminRequestValidatorFn {
+    Arc::new(move |kv_map, default_queue_dir| validator(kv_map, default_queue_dir, domain))
+}
+
+fn redis_request_validator(domain: TargetDomain, default_channel: &'static str) -> AdminRequestValidatorFn {
+    Arc::new(move |kv_map, default_queue_dir| {
+        Box::pin(validate_redis_request(kv_map, default_queue_dir, domain, default_channel))
+    })
+}
+
 impl std::fmt::Debug for AdminTargetSpec {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("AdminTargetSpec")
@@ -182,54 +174,26 @@ pub(crate) fn target_service_name(specs: &[AdminTargetSpec], target_type: &str)
     target_spec(specs, target_type).map(|spec| spec.service)
 }
 
-pub(crate) fn collect_configured_endpoint_keys(specs: &[AdminTargetSpec], config: &Config) -> Vec<EndpointKey> {
-    let mut endpoints = Vec::new();
-    for spec in specs {
-        let Some(targets) = config.0.get(spec.subsystem) else {
-            continue;
-        };
-
-        for (target_name, kvs) in targets {
-            if target_name == rustfs_config::DEFAULT_DELIMITER {
-                continue;
-            }
-            let enabled = kvs.lookup(ENABLE_KEY).as_deref().map(config_enable_is_on).unwrap_or(false);
-            if enabled {
-                endpoints.push((target_name.clone(), spec.service.to_string()));
-            }
-        }
+pub(crate) fn extract_supported_target_params<'a>(
+    specs: &[AdminTargetSpec],
+    params: &'a matchit::Params<'_, '_>,
+    unsupported_target_label: &str,
+) -> S3Result<(&'a str, &'a str)> {
+    let target_type = params
+        .get("target_type")
+        .ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'target_type'"))?;
+    if target_service_name(specs, target_type).is_none() {
+        return Err(s3_error!(
+            InvalidArgument,
+            "unsupported {} target type: '{}'",
+            unsupported_target_label,
+            target_type
+        ));
     }
-    endpoints
-}
-
-pub(crate) fn collect_config_entry_keys(specs: &[AdminTargetSpec], config: &Config) -> HbHashSet<EndpointKey> {
-    let mut endpoints = HbHashSet::new();
-    for spec in specs {
-        let Some(targets) = config.0.get(spec.subsystem) else {
-            continue;
-        };
-
-        for target_name in targets.keys() {
-            if target_name == rustfs_config::DEFAULT_DELIMITER {
-                continue;
-            }
-            endpoints.insert(normalized_endpoint_key(target_name, spec.service));
-        }
-    }
-    endpoints
-}
-
-pub(crate) fn collect_env_endpoint_keys(specs: &[AdminTargetSpec], route_prefix: &str) -> HbHashSet<EndpointKey> {
-    let mut endpoints = HbHashSet::new();
-    for spec in specs {
-        let valid_keys = spec.valid_keys.iter().map(|key| (*key).to_string()).collect::<HashSet<_>>();
-        for instance_id in collect_env_target_instance_ids(route_prefix, spec.service, &valid_keys) {
-            if instance_id != rustfs_config::DEFAULT_DELIMITER && !instance_id.is_empty() {
-                endpoints.insert(normalized_endpoint_key(&instance_id, spec.service));
-            }
-        }
-    }
-    endpoints
+    let target_name = params
+        .get("target_name")
+        .ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'target_name'"))?;
+    Ok((target_type, target_name))
 }
 
 pub(crate) fn classify_endpoint_source(
@@ -237,7 +201,11 @@ pub(crate) fn classify_endpoint_source(
     env_targets: &HbHashSet<EndpointKey>,
     key: &EndpointKey,
 ) -> TargetEndpointSource {
-    match (config_targets.contains(key), env_targets.contains(key)) {
+    classify_endpoint_source_flags(config_targets.contains(key), env_targets.contains(key))
+}
+
+fn classify_endpoint_source_flags(has_config_source: bool, has_env_source: bool) -> TargetEndpointSource {
+    match (has_config_source, has_env_source) {
         (true, true) => TargetEndpointSource::Mixed,
         (true, false) => TargetEndpointSource::Config,
         (false, true) => TargetEndpointSource::Env,
@@ -252,11 +220,10 @@ pub(crate) fn endpoint_source(
     target_type: &str,
     target_name: &str,
 ) -> TargetEndpointSource {
-    let config_targets = collect_config_entry_keys(specs, config);
-    let env_targets = collect_env_endpoint_keys(specs, route_prefix);
+    let snapshot = collect_endpoint_snapshot(specs, route_prefix, config);
     let service = target_service_name(specs, target_type).unwrap_or_default();
     let key = normalized_endpoint_key(target_name, service);
-    classify_endpoint_source(&config_targets, &env_targets, &key)
+    classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, &key)
 }
 
 pub(crate) fn target_mutation_block_reason(
@@ -301,6 +268,33 @@ pub(crate) fn build_json_response(
     S3Response::with_headers((status, body), header)
 }
 
+pub(crate) async fn collect_runtime_statuses<E>(targets: Vec<SharedTarget<E>>) -> HashMap<EndpointKey, String>
+where
+    E: Send + Sync + 'static + Clone + serde::Serialize + serde::de::DeserializeOwned,
+{
+    let semaphore = Arc::new(Semaphore::new(10));
+    let mut futures = futures::stream::FuturesUnordered::new();
+
+    for target in targets {
+        let sem = Arc::clone(&semaphore);
+        futures.push(async move {
+            let _permit = sem.acquire().await;
+            let status = match tokio::time::timeout(Duration::from_secs(3), target.is_active()).await {
+                Ok(Ok(true)) => "online",
+                _ => "offline",
+            };
+            ((target.id().id, target.id().name), status.to_string())
+        });
+    }
+
+    let mut runtime_statuses = HashMap::new();
+    while let Some((key, status)) = futures.next().await {
+        runtime_statuses.insert(key, status);
+    }
+
+    runtime_statuses
+}
+
 pub(crate) fn merge_target_endpoints(
     specs: &[AdminTargetSpec],
     route_prefix: &str,
@@ -309,9 +303,7 @@ pub(crate) fn merge_target_endpoints(
 ) -> Vec<MergedTargetEndpoint> {
     let mut endpoints = Vec::new();
     let mut seen = HashSet::new();
-    let configured_keys = collect_configured_endpoint_keys(specs, config);
-    let config_targets = collect_config_entry_keys(specs, config);
-    let env_targets = collect_env_endpoint_keys(specs, route_prefix);
+    let snapshot = collect_endpoint_snapshot(specs, route_prefix, config);
     let mut normalized_runtime_statuses: HashMap<EndpointKey, (String, String, String)> = HashMap::new();
 
     for ((account_id, service), status) in runtime_statuses {
@@ -321,7 +313,7 @@ pub(crate) fn merge_target_endpoints(
             .or_insert((account_id, service, status));
     }
 
-    for key in configured_keys {
+    for key in snapshot.configured_keys {
         let normalized = normalized_endpoint_key(&key.0, &key.1);
         if !seen.insert(normalized.clone()) {
             continue;
@@ -336,7 +328,7 @@ pub(crate) fn merge_target_endpoints(
             account_id: key.0,
             service: key.1,
             status,
-            source: classify_endpoint_source(&config_targets, &env_targets, &normalized),
+            source: classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, &normalized),
         });
     }
 
@@ -346,12 +338,12 @@ pub(crate) fn merge_target_endpoints(
                 account_id,
                 service,
                 status,
-                source: classify_endpoint_source(&config_targets, &env_targets, &normalized),
+                source: classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, &normalized),
             });
         }
     }
 
-    for key in &env_targets {
+    for key in &snapshot.env_targets {
         if !seen.insert(key.clone()) {
             continue;
         }
@@ -360,7 +352,7 @@ pub(crate) fn merge_target_endpoints(
             account_id: key.0.clone(),
             service: key.1.clone(),
             status: "offline".to_string(),
-            source: classify_endpoint_source(&config_targets, &env_targets, key),
+            source: classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, key),
         });
     }
 
@@ -368,6 +360,96 @@ pub(crate) fn merge_target_endpoints(
     endpoints
 }
 
+pub(crate) fn canonical_target_instance_id(plugin_id: &str, domain: TargetDomain, instance_id: &str) -> String {
+    format!("{plugin_id}:{}:{}", canonical_domain_label(domain), instance_id.to_lowercase())
+}
+
+pub(crate) fn collect_target_instances(
+    specs: &[AdminTargetSpec],
+    route_prefix: &str,
+    config: &Config,
+    runtime_statuses: HashMap<EndpointKey, String>,
+) -> Vec<TargetInstanceReadModel> {
+    let mut instances = Vec::new();
+    let mut seen = HashSet::new();
+    let mut normalized_runtime_statuses: HashMap<EndpointKey, (String, String, String)> = HashMap::new();
+    let domain = inferred_target_domain(route_prefix);
+    let snapshot = collect_endpoint_snapshot(specs, route_prefix, config);
+
+    for ((account_id, service), status) in runtime_statuses {
+        let normalized = normalized_endpoint_key(&account_id, &service);
+        normalized_runtime_statuses
+            .entry(normalized)
+            .or_insert((account_id, service, status));
+    }
+
+    for instance in snapshot.normalized_instances {
+        let key = normalized_endpoint_key(&instance.instance_id, &instance.target_type);
+        if !seen.insert(key.clone()) {
+            continue;
+        }
+
+        let runtime_present = normalized_runtime_statuses.contains_key(&key);
+        let status = normalized_runtime_statuses
+            .remove(&key)
+            .map(|(_, _, status)| status)
+            .unwrap_or_else(|| "offline".to_string());
+        let source = classify_endpoint_source_flags(instance_has_config_entry(&instance), instance_has_env_entry(&instance));
+
+        instances.push(TargetInstanceReadModel {
+            canonical_id: canonical_target_instance_id(&instance.plugin_id, domain, &instance.instance_id),
+            plugin_id: instance.plugin_id,
+            domain,
+            subsystem: instance.subsystem,
+            account_id: instance.instance_id,
+            service: instance.target_type,
+            status,
+            runtime_present,
+            source,
+            enabled: instance.enabled,
+            config: instance.effective_config,
+        });
+    }
+
+    for (normalized, (account_id, service, status)) in normalized_runtime_statuses {
+        if !seen.insert(normalized) {
+            continue;
+        }
+
+        let (plugin_id, subsystem): (String, String) = target_spec_by_service(specs, &service)
+            .map(|spec| (builtin_target_manifest(spec.service).plugin_id.to_string(), spec.subsystem.to_string()))
+            .unwrap_or_else(|| ("custom:target".to_string(), format!("{}_{}", canonical_domain_label(domain), service)));
+        instances.push(TargetInstanceReadModel {
+            canonical_id: canonical_target_instance_id(&plugin_id, domain, &account_id),
+            plugin_id,
+            domain,
+            subsystem,
+            account_id,
+            service,
+            status,
+            runtime_present: true,
+            source: TargetEndpointSource::Runtime,
+            enabled: true,
+            config: KVS::new(),
+        });
+    }
+
+    instances.sort_by(|a, b| a.service.cmp(&b.service).then_with(|| a.account_id.cmp(&b.account_id)));
+    instances
+}
+
+pub(crate) fn find_target_instance(
+    specs: &[AdminTargetSpec],
+    route_prefix: &str,
+    config: &Config,
+    runtime_statuses: HashMap<EndpointKey, String>,
+    canonical_id: &str,
+) -> Option<TargetInstanceReadModel> {
+    collect_target_instances(specs, route_prefix, config, runtime_statuses)
+        .into_iter()
+        .find(|instance| instance.canonical_id == canonical_id)
+}
+
 pub(crate) fn allowed_target_keys(specs: &[AdminTargetSpec], target_type: &str) -> HashSet<&'static str> {
     target_spec(specs, target_type)
         .map(|spec| spec.valid_keys.iter().copied().collect())
@@ -435,8 +517,109 @@ pub(crate) async fn validate_target_request(
     spec.validate_request(kv_map, default_queue_dir).await
 }
 
-fn config_enable_is_on(value: &str) -> bool {
-    matches!(value.trim().to_ascii_lowercase().as_str(), "on" | "true" | "yes" | "1")
+pub(crate) async fn build_enabled_target_kvs<'a, I>(
+    specs: &[AdminTargetSpec],
+    key_values: I,
+    target_type: &str,
+    default_queue_dir: &str,
+    target_label: &str,
+) -> S3Result<KVS>
+where
+    I: IntoIterator<Item = (&'a str, &'a str)>,
+{
+    let allowed_keys = allowed_target_keys(specs, target_type);
+    let kv_map = collect_validated_key_values(key_values, &allowed_keys, target_type, target_label)?;
+    let spec = target_spec(specs, target_type)
+        .ok_or_else(|| s3_error!(InvalidArgument, "unsupported target type: '{}'", target_type))?;
+    timeout(Duration::from_secs(10), validate_target_request(spec, &kv_map, default_queue_dir))
+        .await
+        .map_err(|_| s3_error!(InvalidArgument, "target validation timed out"))??;
+
+    let mut kvs = KVS::new();
+    for (key, value) in kv_map {
+        kvs.insert(key, value);
+    }
+    kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
+    Ok(kvs)
+}
+
+fn instance_has_config_entry(instance: &TargetPluginInstanceRecord) -> bool {
+    instance.source_hints.has_file_instance
+}
+
+fn instance_has_env_entry(instance: &TargetPluginInstanceRecord) -> bool {
+    instance.source_hints.has_env_instance
+}
+
+fn normalized_target_instances(
+    specs: &[AdminTargetSpec],
+    route_prefix: &str,
+    config: &Config,
+) -> Vec<TargetPluginInstanceRecord> {
+    specs
+        .iter()
+        .flat_map(|spec| {
+            normalize_target_plugin_instances(
+                config,
+                &TargetPluginInstanceCompatDescriptor {
+                    domain: inferred_target_domain(route_prefix),
+                    plugin_id: builtin_target_manifest(spec.service).plugin_id,
+                    target_type: spec.service,
+                    subsystem: spec.subsystem,
+                    route_prefix,
+                    valid_fields: spec.valid_keys,
+                },
+            )
+        })
+        .collect()
+}
+
+fn inferred_target_domain(route_prefix: &str) -> TargetDomain {
+    match route_prefix {
+        rustfs_config::notify::NOTIFY_ROUTE_PREFIX => TargetDomain::Notify,
+        rustfs_config::audit::AUDIT_ROUTE_PREFIX => TargetDomain::Audit,
+        _ => TargetDomain::Notify,
+    }
+}
+
+fn canonical_domain_label(domain: TargetDomain) -> &'static str {
+    match domain {
+        TargetDomain::Notify => "notify",
+        TargetDomain::Audit => "audit",
+    }
+}
+
+fn target_spec_by_service<'a>(specs: &'a [AdminTargetSpec], service: &str) -> Option<&'a AdminTargetSpec> {
+    specs.iter().find(|spec| spec.service == service)
+}
+
+fn collect_endpoint_snapshot(specs: &[AdminTargetSpec], route_prefix: &str, config: &Config) -> TargetEndpointSnapshot {
+    let normalized_instances = normalized_target_instances(specs, route_prefix, config);
+    let mut configured_keys = Vec::new();
+    let mut config_targets = HbHashSet::new();
+    let mut env_targets = HbHashSet::new();
+
+    for instance in &normalized_instances {
+        let key = normalized_endpoint_key(&instance.instance_id, &instance.target_type);
+
+        if instance_has_config_entry(instance) {
+            config_targets.insert(key.clone());
+            if instance.enabled {
+                configured_keys.push((instance.instance_id.clone(), instance.target_type.clone()));
+            }
+        }
+
+        if instance_has_env_entry(instance) {
+            env_targets.insert(key);
+        }
+    }
+
+    TargetEndpointSnapshot {
+        normalized_instances,
+        configured_keys,
+        config_targets,
+        env_targets,
+    }
 }
 
 async fn retry_with_backoff<F, Fut, T>(mut operation: F, max_attempts: usize, base_delay: Duration) -> Result<T, Error>
@@ -489,12 +672,11 @@ async fn validate_webhook_request(kv_map: &HashMap<String, String>) -> S3Result<
     Ok(())
 }
 
-fn validate_webhook_request_entry(
-    kv_map: &HashMap<String, String>,
-    _default_queue_dir: &str,
-) -> futures::future::BoxFuture<'static, S3Result<()>> {
-    let kv_map = kv_map.clone();
-    Box::pin(async move { validate_webhook_request(&kv_map).await })
+fn validate_webhook_request_entry<'a>(
+    kv_map: &'a HashMap<String, String>,
+    _default_queue_dir: &'a str,
+) -> BoxFuture<'a, S3Result<()>> {
+    Box::pin(validate_webhook_request(kv_map))
 }
 
 async fn validate_mqtt_request(kv_map: &HashMap<String, String>) -> S3Result<()> {
@@ -541,131 +723,34 @@ async fn validate_mqtt_request(kv_map: &HashMap<String, String>) -> S3Result<()>
     Ok(())
 }
 
-fn validate_mqtt_request_entry(
-    kv_map: &HashMap<String, String>,
-    _default_queue_dir: &str,
-) -> futures::future::BoxFuture<'static, S3Result<()>> {
-    let kv_map = kv_map.clone();
-    Box::pin(async move { validate_mqtt_request(&kv_map).await })
+fn validate_mqtt_request_entry<'a>(
+    kv_map: &'a HashMap<String, String>,
+    _default_queue_dir: &'a str,
+) -> BoxFuture<'a, S3Result<()>> {
+    Box::pin(validate_mqtt_request(kv_map))
 }
 
-fn validate_notify_nats_request_entry(
-    kv_map: &HashMap<String, String>,
-    default_queue_dir: &str,
-) -> futures::future::BoxFuture<'static, S3Result<()>> {
-    let kv_map = kv_map.clone();
-    let default_queue_dir = default_queue_dir.to_string();
-    Box::pin(async move { validate_nats_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await })
-}
-
-fn validate_audit_nats_request_entry(
-    kv_map: &HashMap<String, String>,
-    default_queue_dir: &str,
-) -> futures::future::BoxFuture<'static, S3Result<()>> {
-    let kv_map = kv_map.clone();
-    let default_queue_dir = default_queue_dir.to_string();
-    Box::pin(async move { validate_nats_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await })
-}
-
-fn validate_notify_kafka_request_entry(
-    kv_map: &HashMap<String, String>,
-    default_queue_dir: &str,
-) -> futures::future::BoxFuture<'static, S3Result<()>> {
-    let kv_map = kv_map.clone();
-    let default_queue_dir = default_queue_dir.to_string();
-    Box::pin(async move { validate_kafka_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await })
-}
-
-fn validate_audit_kafka_request_entry(
-    kv_map: &HashMap<String, String>,
-    default_queue_dir: &str,
-) -> futures::future::BoxFuture<'static, S3Result<()>> {
-    let kv_map = kv_map.clone();
-    let default_queue_dir = default_queue_dir.to_string();
-    Box::pin(async move { validate_kafka_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await })
-}
-
-fn validate_notify_amqp_request_entry(
-    kv_map: &HashMap<String, String>,
-    default_queue_dir: &str,
-) -> futures::future::BoxFuture<'static, S3Result<()>> {
-    let kv_map = kv_map.clone();
-    let default_queue_dir = default_queue_dir.to_string();
-    Box::pin(async move { validate_amqp_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await })
-}
-
-fn validate_audit_amqp_request_entry(
-    kv_map: &HashMap<String, String>,
-    default_queue_dir: &str,
-) -> futures::future::BoxFuture<'static, S3Result<()>> {
-    let kv_map = kv_map.clone();
-    let default_queue_dir = default_queue_dir.to_string();
-    Box::pin(async move { validate_amqp_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await })
-}
-
-fn validate_notify_pulsar_request_entry(
-    kv_map: &HashMap<String, String>,
-    default_queue_dir: &str,
-) -> futures::future::BoxFuture<'static, S3Result<()>> {
-    let kv_map = kv_map.clone();
-    let default_queue_dir = default_queue_dir.to_string();
-    Box::pin(async move { validate_pulsar_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await })
-}
-
-fn validate_audit_pulsar_request_entry(
-    kv_map: &HashMap<String, String>,
-    default_queue_dir: &str,
-) -> futures::future::BoxFuture<'static, S3Result<()>> {
-    let kv_map = kv_map.clone();
-    let default_queue_dir = default_queue_dir.to_string();
-    Box::pin(async move { validate_pulsar_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await })
-}
-
-fn validate_mysql_request_entry(
-    kv_map: &HashMap<String, String>,
-    default_queue_dir: &str,
+fn validate_mysql_request_entry<'a>(
+    kv_map: &'a HashMap<String, String>,
+    default_queue_dir: &'a str,
     target_type: TargetType,
-) -> futures::future::BoxFuture<'static, S3Result<()>> {
-    let kv_map = kv_map.clone();
-    let default_queue_dir = default_queue_dir.to_string();
-    Box::pin(async move { validate_mysql_request(&kv_map, &default_queue_dir, target_type).await })
+) -> BoxFuture<'a, S3Result<()>> {
+    Box::pin(validate_mysql_request(kv_map, default_queue_dir, target_type))
 }
 
-fn validate_notify_postgres_request_entry(
+fn validate_nats_request<'a>(
+    kv_map: &'a HashMap<String, String>,
+    default_queue_dir: &'a str,
+    domain: TargetDomain,
+) -> BoxFuture<'a, S3Result<()>> {
+    Box::pin(async move { validate_nats_request_impl(kv_map, default_queue_dir, domain).await })
+}
+
+async fn validate_nats_request_impl(
     kv_map: &HashMap<String, String>,
     default_queue_dir: &str,
-) -> futures::future::BoxFuture<'static, S3Result<()>> {
-    let kv_map = kv_map.clone();
-    let default_queue_dir = default_queue_dir.to_string();
-    Box::pin(async move { validate_postgres_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await })
-}
-
-fn validate_audit_postgres_request_entry(
-    kv_map: &HashMap<String, String>,
-    default_queue_dir: &str,
-) -> futures::future::BoxFuture<'static, S3Result<()>> {
-    let kv_map = kv_map.clone();
-    let default_queue_dir = default_queue_dir.to_string();
-    Box::pin(async move { validate_postgres_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await })
-}
-
-fn validate_notify_redis_request_entry(default_channel: &'static str) -> AdminRequestValidatorFn {
-    Arc::new(move |kv_map: &HashMap<String, String>, default_queue_dir: &str| {
-        let kv_map = kv_map.clone();
-        let default_queue_dir = default_queue_dir.to_string();
-        Box::pin(async move { validate_redis_request(&kv_map, &default_queue_dir, TargetDomain::Notify, default_channel).await })
-    })
-}
-
-fn validate_audit_redis_request_entry(default_channel: &'static str) -> AdminRequestValidatorFn {
-    Arc::new(move |kv_map: &HashMap<String, String>, default_queue_dir: &str| {
-        let kv_map = kv_map.clone();
-        let default_queue_dir = default_queue_dir.to_string();
-        Box::pin(async move { validate_redis_request(&kv_map, &default_queue_dir, TargetDomain::Audit, default_channel).await })
-    })
-}
-
-async fn validate_nats_request(kv_map: &HashMap<String, String>, default_queue_dir: &str, domain: TargetDomain) -> S3Result<()> {
+    domain: TargetDomain,
+) -> S3Result<()> {
     if let Some(queue_dir) = kv_map.get("queue_dir") {
         validate_queue_dir(queue_dir.as_str()).await?;
     }
@@ -677,7 +762,19 @@ async fn validate_nats_request(kv_map: &HashMap<String, String>, default_queue_d
     })
 }
 
-async fn validate_kafka_request(kv_map: &HashMap<String, String>, default_queue_dir: &str, domain: TargetDomain) -> S3Result<()> {
+fn validate_kafka_request<'a>(
+    kv_map: &'a HashMap<String, String>,
+    default_queue_dir: &'a str,
+    domain: TargetDomain,
+) -> BoxFuture<'a, S3Result<()>> {
+    Box::pin(async move { validate_kafka_request_impl(kv_map, default_queue_dir, domain).await })
+}
+
+async fn validate_kafka_request_impl(
+    kv_map: &HashMap<String, String>,
+    default_queue_dir: &str,
+    domain: TargetDomain,
+) -> S3Result<()> {
     if let Some(queue_dir) = kv_map.get(KAFKA_QUEUE_DIR) {
         validate_queue_dir(queue_dir.as_str()).await?;
     }
@@ -697,7 +794,19 @@ async fn validate_kafka_request(kv_map: &HashMap<String, String>, default_queue_
     })
 }
 
-async fn validate_amqp_request(kv_map: &HashMap<String, String>, default_queue_dir: &str, domain: TargetDomain) -> S3Result<()> {
+fn validate_amqp_request<'a>(
+    kv_map: &'a HashMap<String, String>,
+    default_queue_dir: &'a str,
+    domain: TargetDomain,
+) -> BoxFuture<'a, S3Result<()>> {
+    Box::pin(async move { validate_amqp_request_impl(kv_map, default_queue_dir, domain).await })
+}
+
+async fn validate_amqp_request_impl(
+    kv_map: &HashMap<String, String>,
+    default_queue_dir: &str,
+    domain: TargetDomain,
+) -> S3Result<()> {
     if let Some(queue_dir) = kv_map.get(AMQP_QUEUE_DIR) {
         validate_queue_dir(queue_dir.as_str()).await?;
     }
@@ -709,7 +818,15 @@ async fn validate_amqp_request(kv_map: &HashMap<String, String>, default_queue_d
     })
 }
 
-async fn validate_pulsar_request(
+fn validate_pulsar_request<'a>(
+    kv_map: &'a HashMap<String, String>,
+    default_queue_dir: &'a str,
+    domain: TargetDomain,
+) -> BoxFuture<'a, S3Result<()>> {
+    Box::pin(async move { validate_pulsar_request_impl(kv_map, default_queue_dir, domain).await })
+}
+
+async fn validate_pulsar_request_impl(
     kv_map: &HashMap<String, String>,
     default_queue_dir: &str,
     domain: TargetDomain,
@@ -742,7 +859,15 @@ async fn validate_mysql_request(
     })
 }
 
-async fn validate_postgres_request(
+fn validate_postgres_request<'a>(
+    kv_map: &'a HashMap<String, String>,
+    default_queue_dir: &'a str,
+    domain: TargetDomain,
+) -> BoxFuture<'a, S3Result<()>> {
+    Box::pin(async move { validate_postgres_request_impl(kv_map, default_queue_dir, domain).await })
+}
+
+async fn validate_postgres_request_impl(
     kv_map: &HashMap<String, String>,
     default_queue_dir: &str,
     domain: TargetDomain,

rustfs/src/admin/mod.rs

@@ -15,6 +15,7 @@
 mod auth;
 pub mod console;
 pub mod handlers;
+mod plugin_contract;
 pub mod router;
 pub mod service;
 pub mod site_replication_identity;
@@ -26,8 +27,8 @@ mod console_test;
 mod route_registration_test;
 
 use handlers::{
-    audit, bucket_meta, heal, health, kms, module_switch, oidc, pools, profile_admin, quota, rebalance, replication,
-    site_replication, sts, system, tier, user,
+    audit, bucket_meta, heal, health, kms, module_switch, oidc, plugins_catalog, plugins_instances, pools, profile_admin, quota,
+    rebalance, replication, site_replication, sts, system, tier, user,
 };
 use router::{AdminOperation, S3Router};
 use s3s::route::S3Route;
@@ -58,6 +59,8 @@ pub fn make_admin_route(console_enabled: bool) -> std::io::Result<impl S3Route>
     bucket_meta::register_bucket_meta_route(&mut r)?;
     audit::register_audit_target_route(&mut r)?;
     module_switch::register_module_switch_route(&mut r)?;
+    plugins_catalog::register_plugin_catalog_route(&mut r)?;
+    plugins_instances::register_plugin_instance_route(&mut r)?;
 
     replication::register_replication_route(&mut r)?;
     site_replication::register_site_replication_route(&mut r)?;

rustfs/src/admin/plugin_contract.rs

@@ -0,0 +1,577 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rustfs_targets::{
+    TargetDomain, TargetPluginArtifactManifest, TargetPluginDistributionManifest, TargetPluginEnableState,
+    TargetPluginEntrypointKind, TargetPluginExternalRuntimeContract, TargetPluginInstallState, TargetPluginInstallation,
+    TargetPluginOperationalState, TargetPluginPackaging, TargetPluginRuntimeState, TargetPluginRuntimeTransport,
+};
+use serde::Serialize;
+use std::collections::HashMap;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub(crate) enum PluginContractDomain {
+    Audit,
+    Notify,
+}
+
+impl From<TargetDomain> for PluginContractDomain {
+    fn from(value: TargetDomain) -> Self {
+        match value {
+            TargetDomain::Audit => Self::Audit,
+            TargetDomain::Notify => Self::Notify,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub(crate) enum PluginContractPackaging {
+    Builtin,
+    External,
+}
+
+impl From<TargetPluginPackaging> for PluginContractPackaging {
+    fn from(value: TargetPluginPackaging) -> Self {
+        match value {
+            TargetPluginPackaging::Builtin => Self::Builtin,
+            TargetPluginPackaging::External => Self::External,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub(crate) enum PluginContractEntrypointKind {
+    Builtin,
+    Sidecar,
+    Wasm,
+}
+
+impl From<TargetPluginEntrypointKind> for PluginContractEntrypointKind {
+    fn from(value: TargetPluginEntrypointKind) -> Self {
+        match value {
+            TargetPluginEntrypointKind::Builtin => Self::Builtin,
+            TargetPluginEntrypointKind::Sidecar => Self::Sidecar,
+            TargetPluginEntrypointKind::Wasm => Self::Wasm,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub(crate) enum PluginRuntimeTransport {
+    InProcess,
+    Grpc,
+    WasmHost,
+}
+
+impl From<TargetPluginRuntimeTransport> for PluginRuntimeTransport {
+    fn from(value: TargetPluginRuntimeTransport) -> Self {
+        match value {
+            TargetPluginRuntimeTransport::InProcess => Self::InProcess,
+            TargetPluginRuntimeTransport::Grpc => Self::Grpc,
+            TargetPluginRuntimeTransport::WasmHost => Self::WasmHost,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub(crate) enum PluginInstallState {
+    NotInstalled,
+    Installed,
+    InstallFailed,
+}
+
+impl From<TargetPluginInstallState> for PluginInstallState {
+    fn from(value: TargetPluginInstallState) -> Self {
+        match value {
+            TargetPluginInstallState::NotInstalled => Self::NotInstalled,
+            TargetPluginInstallState::Installed => Self::Installed,
+            TargetPluginInstallState::InstallFailed => Self::InstallFailed,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub(crate) enum PluginEnableState {
+    Enabled,
+    Disabled,
+}
+
+impl From<TargetPluginEnableState> for PluginEnableState {
+    fn from(value: TargetPluginEnableState) -> Self {
+        match value {
+            TargetPluginEnableState::Enabled => Self::Enabled,
+            TargetPluginEnableState::Disabled => Self::Disabled,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub(crate) enum PluginOperationalRuntimeState {
+    Running,
+    Offline,
+    Error,
+    Unknown,
+}
+
+impl From<TargetPluginRuntimeState> for PluginOperationalRuntimeState {
+    fn from(value: TargetPluginRuntimeState) -> Self {
+        match value {
+            TargetPluginRuntimeState::Running => Self::Running,
+            TargetPluginRuntimeState::Offline => Self::Offline,
+            TargetPluginRuntimeState::Error => Self::Error,
+            TargetPluginRuntimeState::Unknown => Self::Unknown,
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub(crate) struct PluginRevisionContract {
+    pub version: String,
+    pub digest_sha256: Option<String>,
+    pub source: String,
+    pub installed_at: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub artifact_id: Option<String>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub(crate) struct PluginInstallationContract {
+    pub install_state: PluginInstallState,
+    pub current_revision: Option<PluginRevisionContract>,
+    pub previous_revision: Option<PluginRevisionContract>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub validation_error: Option<String>,
+}
+
+impl From<TargetPluginInstallation> for PluginInstallationContract {
+    fn from(value: TargetPluginInstallation) -> Self {
+        Self {
+            install_state: PluginInstallState::from(value.install_state),
+            current_revision: value.current_revision.map(|revision| PluginRevisionContract {
+                version: revision.version,
+                digest_sha256: revision.digest_sha256,
+                source: revision.source,
+                installed_at: revision.installed_at,
+                artifact_id: revision.artifact_id,
+            }),
+            previous_revision: value.previous_revision.map(|revision| PluginRevisionContract {
+                version: revision.version,
+                digest_sha256: revision.digest_sha256,
+                source: revision.source,
+                installed_at: revision.installed_at,
+                artifact_id: revision.artifact_id,
+            }),
+            validation_error: value.validation_error,
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub(crate) struct PluginOperationalStateContract {
+    pub install_state: PluginInstallState,
+    pub enable_state: PluginEnableState,
+    pub runtime_state: PluginOperationalRuntimeState,
+}
+
+impl From<TargetPluginOperationalState> for PluginOperationalStateContract {
+    fn from(value: TargetPluginOperationalState) -> Self {
+        Self {
+            install_state: PluginInstallState::from(value.install_state),
+            enable_state: PluginEnableState::from(value.enable_state),
+            runtime_state: PluginOperationalRuntimeState::from(value.runtime_state),
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub(crate) struct PluginRuntimeContract {
+    pub protocol_version: String,
+    pub transport: PluginRuntimeTransport,
+}
+
+impl From<TargetPluginExternalRuntimeContract> for PluginRuntimeContract {
+    fn from(value: TargetPluginExternalRuntimeContract) -> Self {
+        Self {
+            protocol_version: value.protocol_version.to_string(),
+            transport: PluginRuntimeTransport::from(value.transport),
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub(crate) struct PluginArtifactContract {
+    pub artifact_id: String,
+    pub target_triple: String,
+    pub download_uri: String,
+    pub digest_sha256: String,
+    pub size_bytes: u64,
+}
+
+impl From<TargetPluginArtifactManifest> for PluginArtifactContract {
+    fn from(value: TargetPluginArtifactManifest) -> Self {
+        Self {
+            artifact_id: value.artifact_id.to_string(),
+            target_triple: value.target_triple.to_string(),
+            download_uri: value.download_uri.to_string(),
+            digest_sha256: value.digest_sha256.to_string(),
+            size_bytes: value.size_bytes,
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub(crate) struct PluginDistributionContract {
+    pub artifacts: Vec<PluginArtifactContract>,
+}
+
+impl From<TargetPluginDistributionManifest> for PluginDistributionContract {
+    fn from(value: TargetPluginDistributionManifest) -> Self {
+        Self {
+            artifacts: value.artifacts.iter().copied().map(PluginArtifactContract::from).collect(),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub(crate) enum PluginInstanceSource {
+    Config,
+    Env,
+    Mixed,
+    Runtime,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub(crate) struct PluginCatalogDomainEntry {
+    pub domain: PluginContractDomain,
+    pub subsystem: String,
+    pub valid_fields: Vec<String>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub(crate) struct PluginCatalogEntry {
+    pub plugin_id: String,
+    pub target_type: String,
+    pub display_name: String,
+    pub provider: String,
+    pub version: String,
+    pub packaging: PluginContractPackaging,
+    pub entrypoint_kind: PluginContractEntrypointKind,
+    pub api_compatibility_version: String,
+    pub runtime_contract: PluginRuntimeContract,
+    pub distribution: Option<PluginDistributionContract>,
+    pub supported_domains: Vec<PluginContractDomain>,
+    pub secret_fields: Vec<String>,
+    pub domain_configs: Vec<PluginCatalogDomainEntry>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub installation: Option<PluginInstallationContract>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+pub(crate) struct PluginCatalogResponse {
+    pub plugins: Vec<PluginCatalogEntry>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub(crate) struct PluginInstanceEntry {
+    pub id: String,
+    pub plugin_id: String,
+    pub domain: PluginContractDomain,
+    pub subsystem: String,
+    pub account_id: String,
+    pub service: String,
+    pub status: String,
+    pub source: PluginInstanceSource,
+    pub enabled: bool,
+    pub config: HashMap<String, String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub operational_state: Option<PluginOperationalStateContract>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub diagnostic_codes: Vec<PluginInstanceDiagnosticCode>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub(crate) enum PluginInstanceDiagnosticCode {
+    ModuleDisabled,
+    InstanceDisabled,
+    EnvironmentManaged,
+    MixedSource,
+    NotLoadedInRuntime,
+    RuntimeOffline,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub(crate) struct PluginInstanceDiagnostic {
+    pub code: PluginInstanceDiagnosticCode,
+    pub message: String,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub(crate) struct PluginInstanceDiagnosticCount {
+    pub code: PluginInstanceDiagnosticCode,
+    pub count: usize,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+pub(crate) struct PluginInstanceDetail {
+    #[serde(flatten)]
+    pub instance: PluginInstanceEntry,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub diagnostics: Vec<PluginInstanceDiagnostic>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+pub(crate) struct PluginInstancesResponse {
+    pub instances: Vec<PluginInstanceEntry>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub diagnostic_counts: Vec<PluginInstanceDiagnosticCount>,
+    pub truncated: bool,
+    pub next_marker: Option<String>,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{
+        PluginArtifactContract, PluginCatalogDomainEntry, PluginCatalogEntry, PluginCatalogResponse, PluginContractDomain,
+        PluginContractEntrypointKind, PluginContractPackaging, PluginDistributionContract, PluginInstanceDetail,
+        PluginInstanceDiagnostic, PluginInstanceDiagnosticCode, PluginInstanceDiagnosticCount, PluginInstanceEntry,
+        PluginInstanceSource, PluginInstancesResponse, PluginRuntimeContract, PluginRuntimeTransport,
+    };
+    use serde_json::json;
+    use std::collections::HashMap;
+
+    #[test]
+    fn plugin_catalog_contract_serializes_stable_json_shape() {
+        let response = PluginCatalogResponse {
+            plugins: vec![PluginCatalogEntry {
+                plugin_id: "builtin:webhook".to_string(),
+                target_type: "webhook".to_string(),
+                display_name: "Webhook".to_string(),
+                provider: "rustfs".to_string(),
+                version: "1.0.0".to_string(),
+                packaging: PluginContractPackaging::Builtin,
+                entrypoint_kind: PluginContractEntrypointKind::Builtin,
+                api_compatibility_version: "rustfs.target-plugin.v1".to_string(),
+                runtime_contract: PluginRuntimeContract {
+                    protocol_version: "rustfs.target-runtime.v1".to_string(),
+                    transport: PluginRuntimeTransport::InProcess,
+                },
+                distribution: None,
+                supported_domains: vec![PluginContractDomain::Audit, PluginContractDomain::Notify],
+                secret_fields: vec!["auth_token".to_string()],
+                domain_configs: vec![PluginCatalogDomainEntry {
+                    domain: PluginContractDomain::Notify,
+                    subsystem: "notify_webhook".to_string(),
+                    valid_fields: vec!["endpoint".to_string(), "auth_token".to_string()],
+                }],
+                installation: None,
+            }],
+        };
+
+        let value = serde_json::to_value(response).expect("catalog response should serialize");
+        assert_eq!(
+            value,
+            json!({
+                "plugins": [{
+                    "plugin_id": "builtin:webhook",
+                    "target_type": "webhook",
+                    "display_name": "Webhook",
+                    "provider": "rustfs",
+                    "version": "1.0.0",
+                    "packaging": "builtin",
+                    "entrypoint_kind": "builtin",
+                    "api_compatibility_version": "rustfs.target-plugin.v1",
+                    "runtime_contract": {
+                        "protocol_version": "rustfs.target-runtime.v1",
+                        "transport": "in_process"
+                    },
+                    "distribution": null,
+                    "supported_domains": ["audit", "notify"],
+                    "secret_fields": ["auth_token"],
+                    "domain_configs": [{
+                        "domain": "notify",
+                        "subsystem": "notify_webhook",
+                        "valid_fields": ["endpoint", "auth_token"]
+                    }]
+                }]
+            })
+        );
+    }
+
+    #[test]
+    fn plugin_instance_contract_serializes_stable_json_shape() {
+        let response = PluginInstancesResponse {
+            instances: vec![PluginInstanceEntry {
+                id: "builtin:webhook:notify:primary".to_string(),
+                plugin_id: "builtin:webhook".to_string(),
+                domain: PluginContractDomain::Notify,
+                subsystem: "notify_webhook".to_string(),
+                account_id: "primary".to_string(),
+                service: "webhook".to_string(),
+                status: "offline".to_string(),
+                source: PluginInstanceSource::Config,
+                enabled: true,
+                config: HashMap::from([
+                    ("enable".to_string(), "on".to_string()),
+                    ("endpoint".to_string(), "https://example.com/hook".to_string()),
+                ]),
+                operational_state: None,
+                diagnostic_codes: vec![PluginInstanceDiagnosticCode::NotLoadedInRuntime],
+            }],
+            diagnostic_counts: vec![PluginInstanceDiagnosticCount {
+                code: PluginInstanceDiagnosticCode::NotLoadedInRuntime,
+                count: 1,
+            }],
+            truncated: false,
+            next_marker: None,
+        };
+
+        let value = serde_json::to_value(response).expect("instance response should serialize");
+        assert_eq!(
+            value,
+            json!({
+                "instances": [{
+                    "id": "builtin:webhook:notify:primary",
+                    "plugin_id": "builtin:webhook",
+                    "domain": "notify",
+                    "subsystem": "notify_webhook",
+                    "account_id": "primary",
+                    "service": "webhook",
+                    "status": "offline",
+                    "source": "config",
+                    "enabled": true,
+                    "config": {
+                        "enable": "on",
+                        "endpoint": "https://example.com/hook"
+                    },
+                    "diagnostic_codes": ["not_loaded_in_runtime"]
+                }],
+                "diagnostic_counts": [{
+                    "code": "not_loaded_in_runtime",
+                    "count": 1
+                }],
+                "truncated": false,
+                "next_marker": null
+            })
+        );
+    }
+
+    #[test]
+    fn plugin_instance_detail_contract_serializes_diagnostics_when_present() {
+        let detail = PluginInstanceDetail {
+            instance: PluginInstanceEntry {
+                id: "builtin:webhook:notify:primary".to_string(),
+                plugin_id: "builtin:webhook".to_string(),
+                domain: PluginContractDomain::Notify,
+                subsystem: "notify_webhook".to_string(),
+                account_id: "primary".to_string(),
+                service: "webhook".to_string(),
+                status: "offline".to_string(),
+                source: PluginInstanceSource::Config,
+                enabled: true,
+                config: HashMap::from([("endpoint".to_string(), "https://example.com/hook".to_string())]),
+                operational_state: None,
+                diagnostic_codes: vec![PluginInstanceDiagnosticCode::NotLoadedInRuntime],
+            },
+            diagnostics: vec![PluginInstanceDiagnostic {
+                code: PluginInstanceDiagnosticCode::NotLoadedInRuntime,
+                message: "plugin instance is enabled in config but not currently loaded in runtime".to_string(),
+            }],
+        };
+
+        let value = serde_json::to_value(detail).expect("instance detail should serialize");
+        assert_eq!(
+            value,
+            json!({
+                "id": "builtin:webhook:notify:primary",
+                "plugin_id": "builtin:webhook",
+                "domain": "notify",
+                "subsystem": "notify_webhook",
+                "account_id": "primary",
+                "service": "webhook",
+                "status": "offline",
+                "source": "config",
+                "enabled": true,
+                "config": {
+                    "endpoint": "https://example.com/hook"
+                },
+                "diagnostic_codes": ["not_loaded_in_runtime"],
+                "diagnostics": [{
+                    "code": "not_loaded_in_runtime",
+                    "message": "plugin instance is enabled in config but not currently loaded in runtime"
+                }]
+            })
+        );
+    }
+
+    #[test]
+    fn plugin_catalog_distribution_contract_serializes_when_present() {
+        let entry = PluginCatalogEntry {
+            plugin_id: "external:webhook".to_string(),
+            target_type: "webhook".to_string(),
+            display_name: "Webhook+".to_string(),
+            provider: "example".to_string(),
+            version: "1.2.3".to_string(),
+            packaging: PluginContractPackaging::Builtin,
+            entrypoint_kind: PluginContractEntrypointKind::Sidecar,
+            api_compatibility_version: "rustfs.target-plugin.v1".to_string(),
+            runtime_contract: PluginRuntimeContract {
+                protocol_version: "rustfs.target-runtime.v1".to_string(),
+                transport: PluginRuntimeTransport::Grpc,
+            },
+            distribution: Some(PluginDistributionContract {
+                artifacts: vec![PluginArtifactContract {
+                    artifact_id: "sidecar-linux-amd64".to_string(),
+                    target_triple: "x86_64-unknown-linux-gnu".to_string(),
+                    download_uri: "https://plugins.example.test/webhook.tar.zst".to_string(),
+                    digest_sha256: "0123456789abcdef".to_string(),
+                    size_bytes: 4096,
+                }],
+            }),
+            supported_domains: vec![PluginContractDomain::Notify],
+            secret_fields: Vec::new(),
+            domain_configs: Vec::new(),
+            installation: None,
+        };
+
+        let value = serde_json::to_value(entry).expect("catalog entry should serialize");
+        assert_eq!(value["distribution"]["artifacts"][0]["artifact_id"], "sidecar-linux-amd64");
+        assert_eq!(value["distribution"]["artifacts"][0]["target_triple"], "x86_64-unknown-linux-gnu");
+        assert_eq!(
+            value["distribution"]["artifacts"][0]["download_uri"],
+            "https://plugins.example.test/webhook.tar.zst"
+        );
+        assert_eq!(value["distribution"]["artifacts"][0]["digest_sha256"], "0123456789abcdef");
+        assert_eq!(value["distribution"]["artifacts"][0]["size_bytes"], 4096);
+    }
+}

rustfs/src/admin/route_registration_test.rs

@@ -14,8 +14,8 @@
 
 use crate::admin::{
     handlers::{
-        audit, bucket_meta, heal, health, kms, module_switch, oidc, pools, profile_admin, quota, rebalance, replication,
-        site_replication, sts, system, tier, user,
+        audit, bucket_meta, heal, health, kms, module_switch, oidc, plugins_catalog, plugins_instances, pools, profile_admin,
+        quota, rebalance, replication, site_replication, sts, system, tier, user,
     },
     router::{AdminOperation, S3Router},
 };
@@ -54,6 +54,8 @@ fn register_admin_routes(router: &mut S3Router<AdminOperation>) {
     bucket_meta::register_bucket_meta_route(router).expect("register bucket meta route");
     audit::register_audit_target_route(router).expect("register audit target route");
     module_switch::register_module_switch_route(router).expect("register module switch route");
+    plugins_catalog::register_plugin_catalog_route(router).expect("register plugin catalog route");
+    plugins_instances::register_plugin_instance_route(router).expect("register plugin instances route");
     replication::register_replication_route(router).expect("register replication route");
     site_replication::register_site_replication_route(router).expect("register site replication route");
     profile_admin::register_profiling_route(router).expect("register profile route");
@@ -101,6 +103,11 @@ fn test_register_routes_cover_representative_admin_paths() {
     assert_route(&router, Method::GET, &admin_path("/v3/audit/target/list"));
     assert_route(&router, Method::GET, &admin_path("/v3/module-switches"));
     assert_route(&router, Method::PUT, &admin_path("/v3/module-switches"));
+    assert_route(&router, Method::GET, &admin_path("/v4/plugins/catalog"));
+    assert_route(&router, Method::GET, &admin_path("/v4/plugins/instances"));
+    assert_route(&router, Method::GET, &admin_path("/v4/plugins/instances/example-id"));
+    assert_route(&router, Method::PUT, &admin_path("/v4/plugins/instances/example-id"));
+    assert_route(&router, Method::DELETE, &admin_path("/v4/plugins/instances/example-id"));
     assert_route(&router, Method::PUT, &admin_path("/v3/audit/target/audit_webhook/test-audit"));
     assert_route(&router, Method::DELETE, &admin_path("/v3/audit/target/audit_webhook/test-audit/reset"));
     assert_route(&router, Method::GET, &admin_path("/v3/accountinfo"));

rustfs/src/admin/router.rs

@@ -510,7 +510,7 @@ fn build_object_lambda_get_request(req: &S3Request<Body>, bucket: &str, object:
         })
         .transpose()?;
     let version_id = query_value_exact(&filtered_uri, "versionId").filter(|value| !value.is_empty());
-    let range = parse_optional_header(&req.headers, http::header::RANGE)?
+    let range = parse_optional_header(&req.headers, header::RANGE)?
         .map(|value| Range::parse(&value).map_err(|_| s3_error!(InvalidArgument, "Range header is invalid")))
         .transpose()?;
 
@@ -520,13 +520,10 @@ fn build_object_lambda_get_request(req: &S3Request<Body>, bucket: &str, object:
         .part_number(part_number)
         .version_id(version_id)
         .range(range)
-        .if_match(parse_optional_etag_condition_header::<IfMatch>(&req.headers, http::header::IF_MATCH)?)
-        .if_none_match(parse_optional_etag_condition_header::<IfNoneMatch>(
-            &req.headers,
-            http::header::IF_NONE_MATCH,
-        )?)
-        .if_modified_since(parse_optional_timestamp_header(&req.headers, http::header::IF_MODIFIED_SINCE)?)
-        .if_unmodified_since(parse_optional_timestamp_header(&req.headers, http::header::IF_UNMODIFIED_SINCE)?);
+        .if_match(parse_optional_etag_condition_header::<IfMatch>(&req.headers, header::IF_MATCH)?)
+        .if_none_match(parse_optional_etag_condition_header::<IfNoneMatch>(&req.headers, header::IF_NONE_MATCH)?)
+        .if_modified_since(parse_optional_timestamp_header(&req.headers, header::IF_MODIFIED_SINCE)?)
+        .if_unmodified_since(parse_optional_timestamp_header(&req.headers, header::IF_UNMODIFIED_SINCE)?);
 
     builder = builder.sse_customer_algorithm(parse_optional_header(
         &req.headers,

rustfs/src/server/audit.rs

@@ -72,13 +72,7 @@ pub async fn start_audit_system() -> AuditResult<()> {
 
     // 1. Get the global configuration loaded by ecstore
     let server_config = match server_config_from_context() {
-        Some(config) => {
-            info!(
-                target: "rustfs::main::start_audit_system",
-                "Global server configuration loads successfully: {:?}", config
-            );
-            config
-        }
+        Some(config) => config,
         None => {
             warn!(
                 target: "rustfs::main::start_audit_system",
@@ -92,9 +86,8 @@ pub async fn start_audit_system() -> AuditResult<()> {
         target: "rustfs::main::start_audit_system",
         "The global server configuration is loaded"
     );
-    // 2. Check if the notify subsystem exists in the configuration, and skip initialization if it doesn't
-    let has_targets = has_any_audit_targets(&server_config);
-    if !has_targets {
+    // 2. Check if the audit subsystem exists in the configuration, and skip initialization if it doesn't
+    if !has_any_audit_targets(&server_config) {
         info!(
             target: "rustfs::main::start_audit_system",
             "Audit subsystem targets are not configured, and audit system initialization is skipped."
@@ -107,35 +100,16 @@ pub async fn start_audit_system() -> AuditResult<()> {
         "Audit subsystem configuration detected and started initializing the audit system."
     );
 
-    if let Some(system) = audit_system() {
-        match system.get_state().await {
-            AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => {
-                // Match notify behavior: prefer reloading the existing singleton
-                // instead of constructing a second lifecycle path on re-enable.
-                match system.reload_config(server_config).await {
-                    Ok(()) => {
-                        info!(
-                            target: "rustfs::main::start_audit_system",
-                            "Audit system reloaded successfully with time: {}.",
-                            jiff::Zoned::now()
-                        );
-                        Ok(())
-                    }
-                    Err(e) => {
-                        warn!(
-                            target: "rustfs::main::start_audit_system",
-                            "Audit system reload failed: {:?}",
-                            e
-                        );
-                        Err(e)
-                    }
-                }
-            }
-            AuditSystemState::Stopped | AuditSystemState::Stopping => match system.start(server_config).await {
+    let system = audit_system().unwrap_or_else(init_audit_system);
+    match system.get_state().await {
+        AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => {
+            // Match notify behavior: prefer reloading the existing singleton
+            // instead of constructing a second lifecycle path on re-enable.
+            match system.reload_config(server_config).await {
                 Ok(()) => {
                     info!(
                         target: "rustfs::main::start_audit_system",
-                        "Audit system started successfully with time: {}.",
+                        "Audit system reloaded successfully with time: {}.",
                         jiff::Zoned::now()
                     );
                     Ok(())
@@ -143,16 +117,14 @@ pub async fn start_audit_system() -> AuditResult<()> {
                 Err(e) => {
                     warn!(
                         target: "rustfs::main::start_audit_system",
-                        "Audit system startup failed: {:?}",
+                        "Audit system reload failed: {:?}",
                         e
                     );
                     Err(e)
                 }
-            },
+            }
         }
-    } else {
-        let system = init_audit_system();
-        match system.start(server_config).await {
+        AuditSystemState::Stopped | AuditSystemState::Stopping => match system.start(server_config).await {
             Ok(()) => {
                 info!(
                     target: "rustfs::main::start_audit_system",
@@ -169,7 +141,7 @@ pub async fn start_audit_system() -> AuditResult<()> {
                 );
                 Err(e)
             }
-        }
+        },
     }
 }
 

rustfs/src/server/event.rs

@@ -17,6 +17,7 @@ use crate::app::context::resolve_server_config;
 use rustfs_ecstore::event_notification::{EventArgs as EcstoreEventArgs, register_event_dispatch_hook};
 use rustfs_notify::EventArgs as NotifyEventArgs;
 use rustfs_s3_common::EventName;
+use std::net::SocketAddr;
 use std::sync::atomic::{AtomicBool, Ordering};
 use tokio::spawn;
 use tracing::{error, info, instrument, warn};
@@ -39,13 +40,7 @@ pub fn is_notify_module_enabled() -> bool {
 
 fn convert_ecstore_event_args(args: EcstoreEventArgs) -> NotifyEventArgs {
     let version_id = args.object.version_id.map(|v| v.to_string()).unwrap_or_default();
-    let (host, port) = match args.host.rsplit_once(':') {
-        Some((host, port)) => match port.parse::<u16>() {
-            Ok(port) => (host.to_string(), port),
-            Err(_) => (args.host, 0),
-        },
-        None => (args.host, 0),
-    };
+    let (host, port) = parse_host_and_port(args.host);
     let req_params = args.req_params.into_iter().collect();
     let resp_elements = args.resp_elements.into_iter().collect();
 
@@ -62,6 +57,24 @@ fn convert_ecstore_event_args(args: EcstoreEventArgs) -> NotifyEventArgs {
     }
 }
 
+fn parse_host_and_port(host: String) -> (String, u16) {
+    if let Ok(addr) = host.parse::<SocketAddr>() {
+        return (addr.ip().to_string(), addr.port());
+    }
+
+    if host.chars().filter(|&c| c == ':').count() != 1 {
+        return (host, 0);
+    }
+
+    match host.split_once(':') {
+        Some((base, port)) if !base.is_empty() => match port.parse::<u16>() {
+            Ok(port) => (base.to_string(), port),
+            Err(_) => (host, 0),
+        },
+        _ => (host, 0),
+    }
+}
+
 fn install_ecstore_event_dispatch_hook() {
     let installed = register_event_dispatch_hook(|args| {
         let notify_args = convert_ecstore_event_args(args);
@@ -75,6 +88,23 @@ fn install_ecstore_event_dispatch_hook() {
     }
 }
 
+fn ensure_live_events_initialized() -> bool {
+    if rustfs_notify::notification_system().is_some() {
+        return true;
+    }
+
+    match rustfs_notify::initialize_live_events() {
+        Ok(()) => {
+            install_ecstore_event_dispatch_hook();
+            true
+        }
+        Err(e) => {
+            error!("Failed to initialize live event stream support: {}", e);
+            false
+        }
+    }
+}
+
 /// Shuts down the event notifier system gracefully
 pub async fn shutdown_event_notifier() {
     info!("Shutting down event notifier system...");
@@ -110,17 +140,11 @@ pub async fn init_event_notifier() {
             "Notify module is disabled, initializing live event stream support only. Set {}=true to enable notification targets.",
             rustfs_config::ENV_NOTIFY_ENABLE
         );
-        if rustfs_notify::notification_system().is_none() {
-            match rustfs_notify::initialize_live_events() {
-                Ok(()) => {
-                    install_ecstore_event_dispatch_hook();
-                    info!(
-                        target: "rustfs::main::init_event_notifier",
-                        "Live event stream support initialized successfully."
-                    );
-                }
-                Err(e) => error!("Failed to initialize live event stream support: {}", e),
-            }
+        if ensure_live_events_initialized() {
+            info!(
+                target: "rustfs::main::init_event_notifier",
+                "Live event stream support initialized successfully."
+            );
         }
         return;
     }
@@ -155,13 +179,49 @@ pub async fn init_event_notifier() {
                 "Event notifier system reloaded successfully."
             );
         }
-    } else if let Err(e) = rustfs_notify::initialize(server_config).await {
-        error!("Failed to initialize event notifier system: {}", e);
     } else {
-        install_ecstore_event_dispatch_hook();
-        info!(
-            target: "rustfs::main::init_event_notifier",
-            "Event notifier system initialized successfully."
-        );
+        match rustfs_notify::initialize(server_config).await {
+            Ok(()) => {
+                install_ecstore_event_dispatch_hook();
+                info!(
+                    target: "rustfs::main::init_event_notifier",
+                    "Event notifier system initialized successfully."
+                );
+            }
+            Err(e) => error!("Failed to initialize event notifier system: {}", e),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::parse_host_and_port;
+
+    #[test]
+    fn parse_host_and_port_with_ipv4_and_port() {
+        let (host, port) = parse_host_and_port("127.0.0.1:9000".to_string());
+        assert_eq!(host, "127.0.0.1");
+        assert_eq!(port, 9000);
+    }
+
+    #[test]
+    fn parse_host_and_port_with_bracketed_ipv6_and_port() {
+        let (host, port) = parse_host_and_port("[::1]:9000".to_string());
+        assert_eq!(host, "::1");
+        assert_eq!(port, 9000);
+    }
+
+    #[test]
+    fn parse_host_and_port_with_ipv6_without_port() {
+        let (host, port) = parse_host_and_port("::1".to_string());
+        assert_eq!(host, "::1");
+        assert_eq!(port, 0);
+    }
+
+    #[test]
+    fn parse_host_and_port_with_hostname_and_port() {
+        let (host, port) = parse_host_and_port("localhost:9001".to_string());
+        assert_eq!(host, "localhost");
+        assert_eq!(port, 9001);
     }
 }